From 599dadd4f0f579b225e0a97a3f48f94dc6ef918c Mon Sep 17 00:00:00 2001 From: David Date: Sat, 5 Sep 2026 07:53:53 -0500 Subject: [PATCH 1/2] PTF: family descriptors, provenance-based eligibility, deterministic order Four changes to ApplyPtfPatches, each fixing something that only shows up once a mod patches something bigger than a keybind file. Files no family describes behave exactly as before: the seven-attribute identity is still the fallback, so the three config files currently merged are untouched by all of this. 1. EXTENSION-LESS BASE PATHS. `wildcard.rfind('.')` returning npos meant "give up", so any file without a dot could never be patched. `levels//whdata_1` is one -- the registry that says which profile streams a layer. A file should not be unpatchable because of how it is spelled. 2. ELIGIBILITY IS PROVENANCE, NOT JUST THE NAME. Every `__*` the pak filesystem returns was merged, and that filesystem is the base game plus every mod, flattened. The base game ships a great many files that already carry a `__` -- counted on a retail 1.5.6 install, restricted to the directories a content mod patches: 5,902 files across 3,245 distinct suffixes muz 411 zena 371 kuman 200 bandita 96 straz 68 autotests 20 Warhorse publish these as reserved modids for exactly this reason. A player who installs a mod called `muz` should not thereby merge 411 of Warhorse's own build fixtures into their tables. `ModIndex` (new) answers it exactly: read mod_order.txt, walk each enabled mod's paks, and record which names each ships. A part is eligible when a MOD ships it AND its suffix names an enabled mod. No reserved-name list to maintain, nothing to go stale at a patch. The index reads only zip central directories and needs no external library. It cannot be taken from the enumeration instead: ICryPak::FindFirst fills attrib/times/size/name with no pak field, and attrib is 0x80000001 for all 184 hits measured across the base game and three separate mods -- it separates pak from loose and nothing finer. 3. ORDER IS mod_order.txt, NOT ENUMERATION ORDER. Patches were applied in whatever order FindFirst returned them, so two mods touching one row resolved differently on different machines, and differently from what the player's load order says. Candidates are now collected, filtered, stable-sorted by the modid's position, and the position is logged. 4. IDENTITY COMES FROM A FAMILY DESCRIPTOR. The shipped rule compares seven hardcoded attribute names, which spans the three keybind files it was written for and cannot express anything else: whdata_1 keys on a CHILD ELEMENT, waitinglinks on a pair of attributes plus a child, and a Libs/Tables row on that table's primary key. A fixed list is silently wrong rather than absent -- skill_check_difficulty.xml carries none of the seven, so every row keys identically and rows pair by position. ptf_families.h is generated from a table that is checked offline against shipped artifacts: merge(vanilla, diff(vanilla, override)) must reproduce that override byte for byte, over 12 files and 11 negative controls. The rules arrive here already falsified against something real rather than as a design. Also: glob matching where `*` never crosses a separator. With fnmatch semantics a rule written for a graph NODE also catches every EDGE inside it, the edges key on an attribute they do not carry, and a contested rewiring comes out silently applied. IXmlNode: getContent declared at slot [49] and [47] annotated as getParent, both measured in game -- [49] is the trivial getter of the content member at +0x48, confirmed by reading real element text rather than by elimination. The node header is 0x58, not the 0x48 the clone implementation's allocation size suggests. Co-Authored-By: Claude Opus 5 (1M context) --- Projects/PTFextender/src/mod_index.cpp | 245 +++++++++++++++++++++ Projects/PTFextender/src/mod_index.h | 80 +++++++ Projects/PTFextender/src/plugin.cpp | 278 +++++++++++++++++++++--- Projects/PTFextender/src/ptf_families.h | 93 ++++++++ include/Offsets/vtables/IXmlNode.h | 20 +- 5 files changed, 686 insertions(+), 30 deletions(-) create mode 100644 Projects/PTFextender/src/mod_index.cpp create mode 100644 Projects/PTFextender/src/mod_index.h create mode 100644 Projects/PTFextender/src/ptf_families.h diff --git a/Projects/PTFextender/src/mod_index.cpp b/Projects/PTFextender/src/mod_index.cpp new file mode 100644 index 00000000..7408f1fd --- /dev/null +++ b/Projects/PTFextender/src/mod_index.cpp @@ -0,0 +1,245 @@ +#include "mod_index.h" + +#include + +#include +#include +#include +#include + +namespace ptf { +namespace { + +std::string Lower(const std::string& s) +{ + std::string out(s); + std::transform(out.begin(), out.end(), out.begin(), + [](unsigned char c) { return static_cast(::tolower(c)); }); + return out; +} + +std::string BareName(const std::string& path) +{ + const size_t slash = path.find_last_of("/\\"); + return (slash == std::string::npos) ? path : path.substr(slash + 1); +} + +// ---------------------------------------------------------------- zip reading +// +// Only the central directory, and only the names. A pak is a plain zip: the End Of Central +// Directory record is the last thing in the file, and it points at a run of 0x02014b50 headers each +// carrying a name length and a name. Nothing here decompresses anything. +// +// The EOCD sits at the very end unless the archive has a trailing comment, so it is found by +// scanning backwards over the last 64 KB -- the largest a comment may be. + +#pragma pack(push, 1) +struct EOCD { + uint32_t sig; // 0x06054b50 + uint16_t disk, cdDisk; + uint16_t entriesHere, entriesTotal; + uint32_t cdSize, cdOffset; + uint16_t commentLen; +}; +struct CDHeader { + uint32_t sig; // 0x02014b50 + uint16_t madeBy, needed, flags, method; + uint16_t modTime, modDate; + uint32_t crc, compSize, uncompSize; + uint16_t nameLen, extraLen, commentLen; + uint16_t diskStart, internalAttr; + uint32_t externalAttr, localOffset; +}; +#pragma pack(pop) + +bool ReadAt(HANDLE h, long long offset, void* buf, DWORD n) +{ + LARGE_INTEGER li; + li.QuadPart = offset; + if (!::SetFilePointerEx(h, li, nullptr, FILE_BEGIN)) { + return false; + } + DWORD got = 0; + return ::ReadFile(h, buf, n, &got, nullptr) && got == n; +} + +} // namespace + +void ModIndex::IndexPak(const std::string& pakPath, const std::string& modid) +{ + HANDLE h = ::CreateFileA(pakPath.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (h == INVALID_HANDLE_VALUE) { + return; + } + LARGE_INTEGER size{}; + if (!::GetFileSizeEx(h, &size) || size.QuadPart < static_cast(sizeof(EOCD))) { + ::CloseHandle(h); + return; + } + + // scan back over at most 64 KB + the EOCD itself for the signature + const long long tailLen = (std::min)(static_cast(64 * 1024 + sizeof(EOCD)), + size.QuadPart); + std::vector tail(static_cast(tailLen)); + if (!ReadAt(h, size.QuadPart - tailLen, tail.data(), static_cast(tailLen))) { + ::CloseHandle(h); + return; + } + long long eocdAt = -1; + for (long long i = tailLen - static_cast(sizeof(EOCD)); i >= 0; --i) { + uint32_t sig = 0; + std::memcpy(&sig, tail.data() + i, sizeof(sig)); + if (sig == 0x06054b50u) { + eocdAt = i; + break; + } + } + if (eocdAt < 0) { + ::CloseHandle(h); + return; + } + EOCD eocd{}; + std::memcpy(&eocd, tail.data() + eocdAt, sizeof(eocd)); + + // ZIP64 archives put 0xFFFFFFFF here. No mod pak measured is near 4 GB, and reading a truncated + // directory would silently under-report provenance -- which fails OPEN, letting base files + // through. Skip the pak instead, and let the caller see it in the pak count. + if (eocd.cdOffset == 0xFFFFFFFFu || eocd.cdSize == 0xFFFFFFFFu || + eocd.entriesTotal == 0xFFFFu) { + ::CloseHandle(h); + return; + } + + std::vector cd(eocd.cdSize); + if (eocd.cdSize == 0 || !ReadAt(h, eocd.cdOffset, cd.data(), eocd.cdSize)) { + ::CloseHandle(h); + return; + } + ::CloseHandle(h); + + size_t at = 0; + for (uint16_t i = 0; i < eocd.entriesTotal; ++i) { + if (at + sizeof(CDHeader) > cd.size()) { + break; + } + CDHeader hdr{}; + std::memcpy(&hdr, cd.data() + at, sizeof(hdr)); + if (hdr.sig != 0x02014b50u) { + break; + } + const size_t nameAt = at + sizeof(CDHeader); + if (nameAt + hdr.nameLen > cd.size()) { + break; + } + std::string name(reinterpret_cast(cd.data() + nameAt), hdr.nameLen); + if (!name.empty() && name.back() != '/' && name.back() != '\\') { + // BARE NAME, because that is what the find record carries. Two mods shipping the same + // bare name in different directories is not a case this has to separate: eligibility + // asks "did a mod ship a file called this", and the merge then loads it by full path. + m_owner[Lower(BareName(name))] = modid; + } + at = nameAt + hdr.nameLen + hdr.extraLen + hdr.commentLen; + } + ++m_paks; +} + +bool ModIndex::Build(const std::string& gameRoot) +{ + m_order.clear(); + m_rank.clear(); + m_owner.clear(); + m_paks = 0; + + const std::string modsDir = gameRoot + "\\Mods"; + const std::string orderPath = modsDir + "\\mod_order.txt"; + + HANDLE h = ::CreateFileA(orderPath.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (h == INVALID_HANDLE_VALUE) { + return false; + } + LARGE_INTEGER size{}; + ::GetFileSizeEx(h, &size); + std::string text; + if (size.QuadPart > 0 && size.QuadPart < (1 << 20)) { + text.resize(static_cast(size.QuadPart)); + DWORD got = 0; + ::ReadFile(h, &text[0], static_cast(text.size()), &got, nullptr); + text.resize(got); + } + ::CloseHandle(h); + + // A BOM lands on the FIRST modid and nowhere else, so the list reads as correct while exactly + // one mod fails to match its folder. Windows PowerShell writes one for `-Encoding utf8`, and + // this file is hand-edited as often as it is generated. + if (text.size() >= 3 && static_cast(text[0]) == 0xEF && + static_cast(text[1]) == 0xBB && static_cast(text[2]) == 0xBF) { + text.erase(0, 3); + } + + size_t pos = 0; + while (pos <= text.size()) { + const size_t nl = text.find('\n', pos); + std::string line = text.substr(pos, (nl == std::string::npos ? text.size() : nl) - pos); + pos = (nl == std::string::npos) ? text.size() + 1 : nl + 1; + while (!line.empty() && (line.back() == '\r' || line.back() == ' ' || line.back() == '\t')) { + line.pop_back(); + } + size_t b = 0; + while (b < line.size() && (line[b] == ' ' || line[b] == '\t')) { + ++b; + } + line = line.substr(b); + if (line.empty() || line[0] == '#') { + continue; + } + m_rank[Lower(line)] = static_cast(m_order.size()); + m_order.push_back(line); + } + + // Walk each enabled mod for its paks. A mod listed with no folder on disk simply contributes + // nothing -- that is a real state (`temptation` is listed on this install and has no folder), + // not an error. + for (const std::string& modid : m_order) { + std::vector dirs{modsDir + "\\" + modid}; + while (!dirs.empty()) { + const std::string dir = dirs.back(); + dirs.pop_back(); + WIN32_FIND_DATAA fd{}; + HANDLE fh = ::FindFirstFileA((dir + "\\*").c_str(), &fd); + if (fh == INVALID_HANDLE_VALUE) { + continue; + } + do { + const std::string name = fd.cFileName; + if (name == "." || name == "..") { + continue; + } + const std::string full = dir + "\\" + name; + if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + dirs.push_back(full); + } else if (name.size() > 4 && + Lower(name.substr(name.size() - 4)) == ".pak") { + IndexPak(full, modid); + } + } while (::FindNextFileA(fh, &fd)); + ::FindClose(fh); + } + } + return true; +} + +int ModIndex::Rank(const std::string& modid) const +{ + const auto it = m_rank.find(Lower(modid)); + return (it == m_rank.end()) ? -1 : it->second; +} + +const char* ModIndex::Owner(const std::string& fileName) const +{ + const auto it = m_owner.find(Lower(BareName(fileName))); + return (it == m_owner.end()) ? nullptr : it->second.c_str(); +} + +} // namespace ptf diff --git a/Projects/PTFextender/src/mod_index.h b/Projects/PTFextender/src/mod_index.h new file mode 100644 index 00000000..00ac4704 --- /dev/null +++ b/Projects/PTFextender/src/mod_index.h @@ -0,0 +1,80 @@ +// ModIndex -- which MOD, if any, ships a given file. +// +// WHY A PART'S NAME IS NOT ENOUGH TO TRUST IT. +// +// `ApplyPtfPatches` merges every `__*` the pak filesystem returns. The pak filesystem is +// the BASE GAME plus every mod, flattened, and the base game ships a great many files that already +// carry a `__`: build-time table parts, per-quest dialogue variants, per-role barks. +// Counted from a retail 1.5.6 install, restricted to the directories a content mod patches: +// +// 5,902 files carrying a __, across 3,245 distinct suffixes +// muz 411 files zena 371 kuman 200 bandita 96 straz 68 autotests 20 +// +// Warhorse publish these as RESERVED MODIDS for exactly this reason. But "reject a list of names" +// is the wrong shape of fix: the list goes stale at every game patch, it cannot be derived by a +// plugin at runtime, and it answers a question about NAMES when the real question is about ORIGIN. +// A player who installs a mod called `muz` should not thereby merge 411 of Warhorse's own fixtures +// into their tables. +// +// PROVENANCE ANSWERS IT EXACTLY: a part is eligible when the file came from a MOD pak. That needs +// no list, cannot go stale, and is what a reserved-name list is a proxy for. +// +// WHY THIS CLASS EXISTS RATHER THAN A FLAG ON THE FIND RECORD. Measured in game, 2026-09-05: +// `ICryPak::FindFirst` fills `attrib / times / size / name` and no pak field, and `attrib` is +// `0x80000001` (in-pak | readonly) for all 184 hits tested across the base game and three separate +// mods. The enumeration separates pak from loose and nothing finer, so the plugin has to build the +// map itself. +// +// HOW: read `Mods\mod_order.txt`, then walk each ENABLED mod's own `.pak` files and record which +// names each one contains. A pak is a zip; only its central directory is read, which is a few +// hundred KB even for a large mod and needs no external library. +// +// COST, measured on an 89-mod install: 1.4 GB of mod paks, of which the central directories are a +// small fraction, read once at startup. + +#pragma once + +#include +#include +#include +#include + +namespace ptf { + +class ModIndex { +public: + // `gameRoot` is the install directory -- the parent of `Mods\`. Safe to call twice; the second + // call rebuilds. Returns false only when `Mods\mod_order.txt` cannot be read, which means no + // mod is enabled and no part can be eligible. + bool Build(const std::string& gameRoot); + + // Load order position of a modid, or -1 when it is not enabled. Comparison is case-insensitive + // because mod_order.txt is hand-edited as often as it is generated. + int Rank(const std::string& modid) const; + + // The modid that ships `fileName` (a BARE name, as the find record carries), or nullptr when no + // enabled mod does -- which for a file the pak filesystem just returned means the BASE GAME. + // + // Last enabled mod wins, matching the load order: if two mods ship the same name, the one the + // engine actually reads is the later one, and that is the one whose provenance applies. + const char* Owner(const std::string& fileName) const; + + std::size_t Mods() const { return m_order.size(); } + std::size_t Files() const { return m_owner.size(); } + std::size_t Paks() const { return m_paks; } + + // The whole map, so an independent implementation can be diffed against it. `kcdlib\vfs.py` + // builds the same relation in Python; two implementations agreeing on ~1,900 names is a far + // stronger claim than either one passing its own spot checks. + const std::unordered_map& Dump() const { return m_owner; } + +private: + void IndexPak(const std::string& pakPath, const std::string& modid); + + std::vector m_order; // as listed, original spelling + std::unordered_map m_rank; // lowercased modid -> position + std::unordered_map m_owner; // lowercased bare name -> modid + std::size_t m_paks = 0; +}; + +} // namespace ptf diff --git a/Projects/PTFextender/src/plugin.cpp b/Projects/PTFextender/src/plugin.cpp index 58fbe130..53ac84ea 100644 --- a/Projects/PTFextender/src/plugin.cpp +++ b/Projects/PTFextender/src/plugin.cpp @@ -24,9 +24,15 @@ #include "Offsets/vtables/IXmlNode.h" #include "playermodule/C_ActionSets.h" #include "playermodule/C_Keybinds.h" +#include "mod_index.h" +#include "ptf_families.h" +#include + +#include #include #include +#include using Offsets::IXmlNode; @@ -53,24 +59,125 @@ void CopyAttributes(IXmlNode* dst, IXmlNode* src) } } -// Row identity = tag + the identifying attributes, absent attrs compare as "". -// The config XML families use different id attrs: , -// , , and in -// keybindSuperactions / -// — so equality spans all of them instead of hardcoding one key per file. -bool SameIdentity(IXmlNode* a, IXmlNode* b) +// The text of a direct child element -- `x`, which an attribute-only +// identity cannot express. [49] getContent, VERIFIED: it is the trivial getter of the content +// member, and returns the shared static empty string rather than NULL when a node has no text. +const char* ChildText(IXmlNode* node, const char* tag) +{ + IXmlNode* c = nullptr; + node->findChild(&c, tag); + if (!c) + return ""; + const char* t = c->getContent(); + // the caller compares immediately; the string is interned in the node's own table and outlives + // this reference for as long as the tree does + c->Release(); + return t ? t : ""; +} + +// Row identity, from the FAMILY DESCRIPTOR rather than a fixed attribute list. +// +// WHY A DESCRIPTOR. The shipped rule compares seven hardcoded attribute names, which happens to +// span the three keybind files it was written for. It cannot express the families a content mod +// actually claims: `whdata_1` keys on a CHILD ELEMENT, `waitinglinks` on a PAIR of attributes plus +// a child, and an arbitrary `Libs\Tables` row on that table's primary key. Worse, a fixed list is +// silently wrong rather than absent -- measured, `skill_check_difficulty.xml` carries none of the +// seven, so every row keys identically and rows pair by POSITION. +// +// `spec` is the identity string from the generated table: `attr:X`, `attr:X+attr:Y`, `child:X`, or +// any combination. Absent parts compare as "" on both sides, so a row missing the key still matches +// only another row missing it. +bool IdentityMatches(IXmlNode* a, IXmlNode* b, const char* spec) { if (std::strcmp(a->getTag(), b->getTag()) != 0) return false; - static const char* const kIdAttrs[] = { "name", "actionmap", "action", "alternative", - "map", "input", "controller" }; - for (const char* key : kIdAttrs) - if (std::strcmp(AttrOrEmpty(a, key), AttrOrEmpty(b, key)) != 0) - return false; + if (!spec || !*spec) { + // No descriptor: fall back to the shipped seven-attribute key. That is the behaviour the + // three config files have always had, and it stays exactly as it was for them. + static const char* const kIdAttrs[] = { "name", "actionmap", "action", "alternative", + "map", "input", "controller" }; + for (const char* key : kIdAttrs) + if (std::strcmp(AttrOrEmpty(a, key), AttrOrEmpty(b, key)) != 0) + return false; + return true; + } + const char* p = spec; + while (*p) { + const char* plus = std::strchr(p, '+'); + const std::string term(p, plus ? static_cast(plus - p) : std::strlen(p)); + const size_t colon = term.find(':'); + if (colon != std::string::npos) { + const std::string kind = term.substr(0, colon); + const std::string key = term.substr(colon + 1); + if (kind == "attr") { + if (std::strcmp(AttrOrEmpty(a, key.c_str()), AttrOrEmpty(b, key.c_str())) != 0) + return false; + } else if (kind == "child") { + if (std::strcmp(ChildText(a, key.c_str()), ChildText(b, key.c_str())) != 0) + return false; + } + } + if (!plus) + break; + p = plus + 1; + } return true; } -int FindChildIndex(IXmlNode* base, IXmlNode* like) +// Glob match where `*` NEVER CROSSES A SEPARATOR. +// +// `fnmatch`-style matching lets `*` swallow `/`, and that is not a nicety here: the rule written for +// a graph NODE then also catches every EDGE inside it, the edges key on an attribute they do not +// carry, and a contested rewiring comes out silently applied. Measured on the Python side before +// this was written. +bool GlobMatch(const char* pat, const char* str) +{ + if (*pat == '\0') + return *str == '\0'; + if (*pat == '*') { + for (const char* s = str;; ++s) { + if (GlobMatch(pat + 1, s)) + return true; + if (*s == '\0' || *s == '/' || *s == '\\') + return false; // the wildcard stops at a separator + } + } + if (*str == '\0') + return false; + const char a = static_cast(::tolower(static_cast(*pat))); + const char b = static_cast(::tolower(static_cast(*str))); + const bool same = (a == b) || ((a == '/' || a == '\\') && (b == '/' || b == '\\')); + return same && GlobMatch(pat + 1, str + 1); +} + +// Which family, if any, describes this virtual path. +const ptf::Family* FamilyFor(const char* vpath) +{ + if (!vpath) + return nullptr; + for (std::size_t i = 0; i < ptf::kFamilyCount; ++i) + if (GlobMatch(ptf::kFamilies[i].glob, vpath)) + return &ptf::kFamilies[i]; + return nullptr; +} + +// The identity rule in force for the children of `parentPath`, or nullptr when no family describes +// it -- in which case `IdentityMatches` uses the shipped seven-attribute key, unchanged. +const char* IdentityFor(const ptf::Family* fam, const std::string& parentPath) +{ + if (!fam) + return nullptr; + for (std::size_t i = 0; i < fam->ruleCount; ++i) { + const std::string rule(fam->rules[i].path); + // the rule names the ROW path; we are asked about its parent + const size_t slash = rule.find_last_of('/'); + if (slash != std::string::npos && rule.substr(0, slash) == parentPath) + return fam->rules[i].identity; + } + return nullptr; +} + +int FindChildIndex(IXmlNode* base, IXmlNode* like, const char* idSpec) { const int n = base->getChildCount(); for (int i = 0; i < n; ++i) { @@ -78,7 +185,7 @@ int FindChildIndex(IXmlNode* base, IXmlNode* like) base->getChild(&c, i); if (!c) continue; - const bool match = SameIdentity(c, like); + const bool match = IdentityMatches(c, like, idSpec); c->Release(); if (match) return i; @@ -95,8 +202,14 @@ struct S_MergeStats { // containers so a mod can patch a single without restating the whole // actionmap. Patch subtrees are shared into the base document — the same // technique the stock XMLPatcher uses (insertChild/replaceChild AddRef the node). -void MergeChildren(IXmlNode* base, IXmlNode* patch, S_MergeStats& stats) +void MergeChildren(IXmlNode* base, IXmlNode* patch, S_MergeStats& stats, + const ptf::Family* fam = nullptr, const std::string& path = std::string()) { + // The element path is carried down so each level of the recursion can ask the descriptor which + // rule governs ITS children. Without it a family could only describe rows at one depth. + const std::string here = path + "/" + base->getTag(); + const char* idSpec = IdentityFor(fam, here); + const int n = patch->getChildCount(); for (int i = 0; i < n; ++i) { IXmlNode* p = nullptr; @@ -104,7 +217,7 @@ void MergeChildren(IXmlNode* base, IXmlNode* patch, S_MergeStats& stats) if (!p) continue; - const int at = FindChildIndex(base, p); + const int at = FindChildIndex(base, p, idSpec); if (at < 0) { IXmlNode* add = nullptr; // clone: p may be read-only p->clone(&add, false); @@ -132,7 +245,7 @@ void MergeChildren(IXmlNode* base, IXmlNode* patch, S_MergeStats& stats) CopyAttributes(b, p); // modified (attrs) ++stats.modified; } - MergeChildren(b, p, stats); // recurse + MergeChildren(b, p, stats, fam, here); // recurse, carrying the path } b->Release(); } @@ -141,6 +254,63 @@ void MergeChildren(IXmlNode* base, IXmlNode* patch, S_MergeStats& stats) } } +// The provenance map, built once. See mod_index.h: the find record carries no pak field, so a +// patch's ORIGIN cannot be read off the enumeration and has to be indexed from the mod paks. +ptf::ModIndex g_mods; +bool g_modsBuilt = false; + +const ptf::ModIndex& Mods() +{ + if (!g_modsBuilt) { + g_modsBuilt = true; + char exe[MAX_PATH] = {0}; + ::GetModuleFileNameA(nullptr, exe, MAX_PATH); + // \Bin\Win64MasterMasterSteamPGO\KingdomCome.exe -> + std::string root(exe); + for (int up = 0; up < 3; ++up) { + const size_t slash = root.find_last_of("/\\"); + if (slash == std::string::npos) + break; + root.erase(slash); + } + g_mods.Build(root); + auto* env = SSystemGlobalEnvironment::GetInstance(); + if (env && env->pLog) + env->pLog->LogAlways("[PTFextender] mod index: %zu enabled mod(s), %zu pak(s), " + "%zu file name(s) -- a part is eligible only if a MOD ships it", + g_mods.Mods(), g_mods.Paks(), g_mods.Files()); + } + return g_mods; +} + +// One candidate patch: the file the enumeration returned, and the position that decides when it +// applies relative to the others. +struct Candidate { + int rank; + std::string name; + std::string modid; + bool operator<(const Candidate& o) const { return rank < o.rank; } +}; + +// The suffix after the LAST `__`, which is the modid by convention. +// +// SPLIT FROM THE RIGHT, NEVER THE LEFT. Base files already contain `__` -- e.g. +// `konec_sluzby__komplet_vsechno.xml` -- so a part of one is +// `konec_sluzby__komplet_vsechno__.xml`, and a first-`__` split reads the modid as +// `komplet_vsechno__`, matches no enabled mod, and drops the part silently. +std::string SuffixOf(const std::string& fileName) +{ + std::string stem(fileName); + const size_t slash = stem.find_last_of("/\\"); + if (slash != std::string::npos) + stem = stem.substr(slash + 1); + const size_t dot = stem.rfind('.'); + if (dot != std::string::npos) + stem = stem.substr(0, dot); + const size_t sep = stem.rfind("__"); + return (sep == std::string::npos) ? std::string() : stem.substr(sep + 2); +} + // Merge every __ patch the pak FS has for basePath into the // tree at *ppBaseRoot. The base tree is a read-only pooled one, so patches land // in a mutable deep clone and *ppBaseRoot is swapped to it (old ref released). @@ -150,37 +320,85 @@ int ApplyPtfPatches(const char* basePath, IXmlNode** ppBaseRoot) if (!basePath || !ppBaseRoot || !*ppBaseRoot) return 0; + // The descriptor for this file, or nullptr -- in which case every rule below behaves exactly as + // the shipped plugin did. Adding a family cannot change how an undescribed file merges. + const ptf::Family* fam = FamilyFor(basePath); + auto* env = SSystemGlobalEnvironment::GetInstance(); if (!env || !env->pCryPak || !env->pSystem) return 0; // "libs/config/foo.xml" -> "libs/config/foo__*.xml" + // + // AND AN EXTENSION-LESS BASE PATH GETS `__*` APPENDED. `rfind('.')` returning npos used to mean + // "give up", which quietly excluded every file without a dot -- `levels\\whdata_1` among + // them, the registry that says which profile streams a layer. A file cannot be unpatchable + // because of how it is spelled. std::string wildcard(basePath); const size_t dot = wildcard.rfind('.'); if (dot == std::string::npos) - return 0; - wildcard.insert(dot, "__*"); + wildcard += "__*"; + else + wildcard.insert(dot, "__*"); // find records carry bare file names; keep the directory for reloading const size_t slash = wildcard.find_last_of("/\\"); const std::string dir = (slash == std::string::npos) ? std::string() : wildcard.substr(0, slash + 1); + // ---- COLLECT FIRST, then decide. Two rules cannot be applied while enumerating: eligibility + // needs the mod index, and ORDER needs every candidate in hand. Offsets::SCryPakFindData fd{}; const intptr_t h = env->pCryPak->FindFirst(wildcard.c_str(), &fd); if (h == -1) return 0; + const ptf::ModIndex& mods = Mods(); + std::vector queue; + int rejectedBase = 0, rejectedUnlisted = 0; + do { + const std::string name(fd.name); + // ELIGIBILITY IS PROVENANCE **AND** NAME. + // provenance: the file must come from a mod pak. The base game ships 5,902 files carrying + // a `__` in the directories mods patch, so a name-only rule merges Warhorse's + // own build fixtures the moment a player installs a mod called `muz` or `autotests`. + // name: the suffix must be an ENABLED mod, so a disabled mod's leftovers stay inert. + const char* owner = mods.Owner(name); + if (!owner) { + ++rejectedBase; + continue; + } + const std::string suffix = SuffixOf(name); + const int rank = suffix.empty() ? -1 : mods.Rank(suffix); + if (rank < 0) { + ++rejectedUnlisted; + continue; + } + queue.push_back(Candidate{rank, name, suffix}); + } while (env->pCryPak->FindNext(h, &fd) >= 0); + env->pCryPak->FindClose(h); + + if (queue.empty()) { + if ((rejectedBase || rejectedUnlisted) && env->pLog) + env->pLog->LogAlways("[PTFextender] '%s': no eligible patch (%d base-game, %d not in " + "mod_order)", basePath, rejectedBase, rejectedUnlisted); + return 0; + } + + // ORDER IS mod_order.txt, NOT ENUMERATION ORDER. `FindFirst` returns names in whatever order the + // pak filesystem holds them, so two mods touching one row would resolve differently on two + // machines -- and differently from what the player's load order says. Stable sort, so two parts + // from one mod keep their enumeration order relative to each other. + std::stable_sort(queue.begin(), queue.end()); + IXmlNode* merged = nullptr; (*ppBaseRoot)->clone(&merged, false); - if (!merged) { - env->pCryPak->FindClose(h); + if (!merged) return 0; - } int applied = 0; - do { - const std::string patchPath = dir + fd.name; + for (const Candidate& c : queue) { + const std::string patchPath = dir + c.name; IXmlNode* patchRoot = nullptr; // same call shape as the three stock config loaders: (&out, path, 0, 1, 1) env->pSystem->LoadXmlFromFile(&patchRoot, patchPath.c_str(), 0, 1, 1); @@ -190,14 +408,18 @@ int ApplyPtfPatches(const char* basePath, IXmlNode** ppBaseRoot) continue; } S_MergeStats stats; - MergeChildren(merged, patchRoot, stats); + MergeChildren(merged, patchRoot, stats, fam); patchRoot->Release(); ++applied; if (env->pLog) - env->pLog->LogAlways("[PTFextender] '%s' is patched by '%s', nodes added: %d, modified: %d", - basePath, fd.name, stats.added, stats.modified); - } while (env->pCryPak->FindNext(h, &fd) >= 0); - env->pCryPak->FindClose(h); + env->pLog->LogAlways("[PTFextender] '%s' is patched by '%s' (mod '%s', #%d of %zu), " + "nodes added: %d, modified: %d", + basePath, c.name.c_str(), c.modid.c_str(), c.rank + 1, + mods.Mods(), stats.added, stats.modified); + } + if (rejectedBase && env->pLog) + env->pLog->LogAlways("[PTFextender] '%s': %d base-game part(s) rejected -- not shipped by " + "any enabled mod", basePath, rejectedBase); if (applied > 0) { (*ppBaseRoot)->Release(); diff --git a/Projects/PTFextender/src/ptf_families.h b/Projects/PTFextender/src/ptf_families.h new file mode 100644 index 00000000..30a5bdd9 --- /dev/null +++ b/Projects/PTFextender/src/ptf_families.h @@ -0,0 +1,93 @@ +// GENERATED by mesh_bridge/kcdlib/ptf.py --emit-cpp= -- DO NOT EDIT BY HAND. +// +// The PTF merge table: for each file family, which element path carries rows, what makes +// two rows THE SAME row, where a new row sorts, and which number a second mod would +// collide on. Every rule here is checked against a shipped artifact by `ptf_verify.py`: +// merge(vanilla, diff(vanilla, override)) must reproduce that override byte for byte. +// +// Regenerate with: python mesh_bridge/kcdlib/ptf.py --emit-cpp= + +#pragma once + +#include + +namespace ptf { + +struct FamilyRule { + const char* path; // element path whose children are rows + const char* identity; // attr:X[+attr:Y][+child:Z] -- what makes two rows the same + const char* order; // append | sorted:attr:X[|suffix_] | sorted:child:X | clone + const char* allocate; // attr:X / child:X, or nullptr -- a number two mods would pick + const char* conflict; // last-wins | error | chain +}; + +struct Family { + const char* name; + const char* glob; // levels/*/leveldata.xml -- `*` never crosses a separator + const char* expect; + const FamilyRule* rules; + std::size_t ruleCount; +}; + +inline const FamilyRule kLeveldataRules[] = { + {"/LevelData/Layers/Layer", + "attr:Name", + "sorted:attr:Name|suffix_", "attr:Id", "last-wins"}, +}; + +inline const FamilyRule kWhdataRules[] = { + {"/Root/GameProfileManager/GameProfiles/GameProfile", + "child:Name", + "sorted:child:Name", "child:Id", "last-wins"}, +}; + +inline const FamilyRule kObjectsMissionRules[] = { + {"/Objects/Entity", + "attr:EntityGuid", + "sorted:attr:EntityGuid", "attr:EntityId", "last-wins"}, +}; + +inline const FamilyRule kWaitinglinksRules[] = { + {"/StaticLinksInfo/WaitingLinks/WaitingLink", + "attr:SourceId+attr:TargetId+child:LinkDefinition", + "append", nullptr, "last-wins"}, +}; + +inline const FamilyRule kUiTableRules[] = { + {"/database/Cutscenes/*", + "attr:Name", + "append", nullptr, "last-wins"}, +}; + +inline const FamilyRule kSkaldRules[] = { + {"**/Edge", + "attr:From+attr:To", + "append", nullptr, "chain"}, + {"/Database/Skald/*/EdgeVertices/EdgeVertex", + "attr:Name", + "append", nullptr, "last-wins"}, + {"/Database/Skald/*/Nodes/*", + "attr:Name", + "append", nullptr, "last-wins"}, + {"/Database/Skald/*/Ports/Port", + "attr:Name", + "append", nullptr, "last-wins"}, +}; + +inline const Family kFamilies[] = { + {"leveldata", "levels/*/leveldata.xml", "byte", + kLeveldataRules, sizeof(kLeveldataRules) / sizeof(kLeveldataRules[0])}, + {"whdata", "levels/*/whdata_1", "eol", + kWhdataRules, sizeof(kWhdataRules) / sizeof(kWhdataRules[0])}, + {"objects_mission", "levels/*/objects_mission0.xml", "byte", + kObjectsMissionRules, sizeof(kObjectsMissionRules) / sizeof(kObjectsMissionRules[0])}, + {"waitinglinks", "levels/*/waitinglinks.xml", "rowset", + kWaitinglinksRules, sizeof(kWaitinglinksRules) / sizeof(kWaitinglinksRules[0])}, + {"ui_table", "libs/tables/*/*.xml", "rowset", + kUiTableRules, sizeof(kUiTableRules) / sizeof(kUiTableRules[0])}, + {"skald", "quests/*", "report", + kSkaldRules, sizeof(kSkaldRules) / sizeof(kSkaldRules[0])}, +}; +inline const std::size_t kFamilyCount = sizeof(kFamilies) / sizeof(kFamilies[0]); + +} // namespace ptf diff --git a/include/Offsets/vtables/IXmlNode.h b/include/Offsets/vtables/IXmlNode.h index 151dca5c..619d2666 100644 --- a/include/Offsets/vtables/IXmlNode.h +++ b/include/Offsets/vtables/IXmlNode.h @@ -101,9 +101,25 @@ struct IXmlNode { // [46] 0x18045840C -- first child with the given tag, NEW reference to *out (NULL *out // when absent). virtual IXmlNode** findChild(IXmlNode** out, const char* tag) const = 0; - virtual void _vf47() = 0; // [47] 0x180459EDC + // [47] 0x180459EDC -- the PARENT node. MEASURED 2026-09-05: the only slot that touches +0x40, + // a load at byte 6 (it needs a null test -- a root's parent IS null). On a leaf +0x40 + // pointed at an object whose first qword is the CXmlNode vtable, i.e. another node. + virtual void _vf47() = 0; // [47] 0x180459EDC getParent (not declared: return type is a + // hidden-pointer XmlNodeRef and no caller here needs it) virtual void _vf48() = 0; // [48] 0x181A73BE0 - virtual void _vf49() = 0; // [49] 0x1819A2A90 + // [49] 0x1819A2A90 -- the element's own TEXT. MEASURED IN GAME 2026-09-05: the trivial getter + // `mov rax,[rcx+0x48]; ret` of the content member, found by dumping the node's 0x58-byte header + // and scanning every vtable slot for one that reads that offset. Confirmed by READING rather + // than by elimination -- the leaf under + // /Root/GameProfileManager/GameProfiles/GameProfile returned "apolena_abandonedCampsEnviro". + // + // NEVER NULL: a node with no text carries the shared static empty string (0x183A3D1E0), the + // same convention [33] getAttr uses -- which is why this one CAN be a trivial getter while + // [47] getParent needs a branch. + // + // The header is 0x58, not the 0x48 the clone impl's allocation suggests: on every node the tag + // pointer is exactly node+0x58, where the inline string pool begins. + virtual const char* getContent() const = 0; virtual void _vf50() = 0; // [50] 0x182488BA0 // [51] 0x18045A3F4 (shared by both vtables) -- internal deep-clone taking the string- // intern-table context as arg (impl 0x18045A7A0: allocs 0x48, plants the CXmlNode From e7c8e2d06d83f0f4f3034a078c51c0ee935c50c7 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 5 Sep 2026 08:36:16 -0500 Subject: [PATCH 2/2] Apply PTF parts to level data, not only the config files ApplyPtfPatches runs from the table loader, so the only files that could ever be patched were the ones that loader reads. Level data comes in through IXmlUtils::LoadXmlFromFile and never passed through it -- which is why a content mod has no choice but to ship a whole-file override of leveldata.xml, whdata_1, waitinglinks.xml and objects_mission0.xml, and why two such mods cannot coexist: the last one in mod_order wins the file outright and the other mod's rows simply cease to exist. The hook is slot [1] of the IXmlUtils vtable read off gEnv->pSystem->GetXmlUtils(). A load whose normalised path matches a family glob gets the same collect-filter-order-merge the config files get; everything else returns the original pointer untouched. A thread_local depth counter guards re-entrancy, because the merge loads the part through the very function being hooked. Path spellings vary by caller -- `levels/kutnohorsko/LevelData.xml`, `levels/kutnohorsko/ whdata_1`, `data/levels/kutnohorsko/WaitingLinks.xml` were all measured from one launch-load-quit -- so matching is done on a normalised form: lowercased, separators folded, a leading data/ stripped. Co-Authored-By: Claude Opus 5 (1M context) --- Projects/PTFextender/src/plugin.cpp | 118 +++++++++++++++++++++++++++- 1 file changed, 115 insertions(+), 3 deletions(-) diff --git a/Projects/PTFextender/src/plugin.cpp b/Projects/PTFextender/src/plugin.cpp index 53ac84ea..19d4efb7 100644 --- a/Projects/PTFextender/src/plugin.cpp +++ b/Projects/PTFextender/src/plugin.cpp @@ -22,6 +22,7 @@ #include "Offsets/vtables/ILog.h" #include "Offsets/vtables/ISystem.h" #include "Offsets/vtables/IXmlNode.h" +#include "Offsets/vtables/IXmlUtils.h" #include "playermodule/C_ActionSets.h" #include "playermodule/C_Keybinds.h" #include "mod_index.h" @@ -150,13 +151,41 @@ bool GlobMatch(const char* pat, const char* str) return same && GlobMatch(pat + 1, str + 1); } +// The path a family glob is matched against. +// +// SPELLING VARIES BY CALLER, measured from a launch-load-quit that logged all 28,019 paths the +// engine asked for: +// +// levels/kutnohorsko/LevelData.xml mixed case, no prefix +// levels/kutnohorsko/Objects_Mission0.xml +// levels/kutnohorsko/whdata_1 no extension at all +// data/levels/kutnohorsko/WaitingLinks.xml ...and this one carries `data/` +// +// A glob written one way would silently miss three of the four. Lowercase, forward slashes, and a +// leading `data/` removed -- CryPak is rooted there, so the two spellings name one file. The +// ORIGINAL string is still what FindFirst is given, because that is what the engine resolved. +std::string NormalisePath(const char* p) +{ + std::string v(p ? p : ""); + for (char& c : v) { + if (c == '\\') + c = '/'; + else + c = static_cast(::tolower(static_cast(c))); + } + if (v.compare(0, 5, "data/") == 0) + v.erase(0, 5); + return v; +} + // Which family, if any, describes this virtual path. const ptf::Family* FamilyFor(const char* vpath) { if (!vpath) return nullptr; + const std::string v = NormalisePath(vpath); for (std::size_t i = 0; i < ptf::kFamilyCount; ++i) - if (GlobMatch(ptf::kFamilies[i].glob, vpath)) + if (GlobMatch(ptf::kFamilies[i].glob, v.c_str())) return &ptf::kFamilies[i]; return nullptr; } @@ -500,6 +529,85 @@ class { static inline REL::Relocation orig; } hkLoadFromXML; +// ------------------------------------------------------ hook: LEVEL FILES --- +// +// ONE HOOK REACHES ALL OF THEM. Measured: 37,768 calls over 28,019 distinct paths in a single +// launch-load-quit, and every file family a level-content mod claims comes through +// `IXmlUtils::LoadXmlFromFile` -- leveldata, objects_mission0, waitinglinks and whdata_1 alike. +// So this is a hook plus a path lookup, and every future family is a descriptor rather than a +// reverse-engineering job. +// +// IXmlUtils SLOT [1], NOT ISystem [131]. The latter is a thin forwarder into the former, so hooking +// IXmlUtils catches both routes and no direct caller slips past. The target is read out of the live +// vtable via `gEnv->pSystem->GetXmlUtils()`, which is why this one installs on a KCSE message +// rather than at plugin load: the pointer does not exist yet when the DLL is loaded. +// +// `patching=1` on all 37,768 calls -- the stock XML patcher already runs inside this function, on +// every file. A merge placed here sits exactly where the engine already expects one. +namespace { + +using LoadXmlFn = IXmlNode** (*)(Offsets::IXmlUtils*, IXmlNode**, const char*, bool, bool, bool, + bool); +LoadXmlFn g_origLoadXml = nullptr; + +// RE-ENTRANCY. ApplyPtfPatches loads each part through this very function, so without a guard the +// first patched file recurses until the stack ends. Thread-local because XML loading is not +// single-threaded here -- two loader threads were observed interleaving in the same session. +thread_local int t_inPatchLoad = 0; + +struct PatchLoadGuard { + PatchLoadGuard() { ++t_inPatchLoad; } + ~PatchLoadGuard() { --t_inPatchLoad; } +}; + +IXmlNode** LoadXmlDetour(Offsets::IXmlUtils* pThis, IXmlNode** out, const char* sFilename, + bool bEnablePatching, bool b5, bool b6, bool b7) +{ + IXmlNode** r = g_origLoadXml(pThis, out, sFilename, bEnablePatching, b5, b6, b7); + if (t_inPatchLoad || !out || !*out || !sFilename) + return r; + if (!FamilyFor(sFilename)) + return r; // not a described file: the engine's own result, untouched + PatchLoadGuard guard; + ApplyPtfPatches(sFilename, out); + return r; +} + +bool InstallLevelHook() +{ + auto* env = SSystemGlobalEnvironment::GetInstance(); + if (!env || !env->pSystem) + return false; + auto* utils = reinterpret_cast(env->pSystem->GetXmlUtils()); + if (!utils) + return false; + void** vtbl = *reinterpret_cast(utils); + void* target = vtbl[1]; + if (MH_CreateHook(target, reinterpret_cast(&LoadXmlDetour), + reinterpret_cast(&g_origLoadXml)) != MH_OK) + return false; + if (MH_EnableHook(target) != MH_OK) + return false; + if (env->pLog) + env->pLog->LogAlways("[PTFextender] level families armed on IXmlUtils::LoadXmlFromFile " + "at %p -- %zu family descriptor(s)", target, ptf::kFamilyCount); + return true; +} + +bool g_levelHookArmed = false; + +void OnKcseMessage(KCSE::Message* msg) +{ + if (g_levelHookArmed || !msg) + return; + // PreDataLoaded is the earliest point gEnv->pSystem is usable; DataLoaded is the fallback. + if (msg->type == KCSE::IMessagingInterface::kMessage_PreDataLoaded || + msg->type == KCSE::IMessagingInterface::kMessage_DataLoaded) + g_levelHookArmed = InstallLevelHook(); +} + +} // namespace + // ---------------------------------------------------------------- install --- static bool InstallHooks() @@ -519,6 +627,10 @@ KCSE_PLUGIN_INFO("PTF Extender", "JerryYOJ", 1); KCSE_PLUGIN_LOAD(kcse) { KCSE::AllocTrampoline(1 << 10); - - return InstallHooks(); + + if (!InstallHooks()) + return false; + // The level hook needs gEnv->pSystem, which does not exist yet -- arm it on the first message. + auto* msg = kcse->GetMessagingInterface(); + return msg && msg->RegisterListener(&OnKcseMessage); }