diff --git a/Projects/PTFextender/src/mod_index.cpp b/Projects/PTFextender/src/mod_index.cpp new file mode 100644 index 00000000..7408f1fd --- /dev/null +++ b/Projects/PTFextender/src/mod_index.cpp @@ -0,0 +1,245 @@ +#include "mod_index.h" + +#include + +#include +#include +#include +#include + +namespace ptf { +namespace { + +std::string Lower(const std::string& s) +{ + std::string out(s); + std::transform(out.begin(), out.end(), out.begin(), + [](unsigned char c) { return static_cast(::tolower(c)); }); + return out; +} + +std::string BareName(const std::string& path) +{ + const size_t slash = path.find_last_of("/\\"); + return (slash == std::string::npos) ? path : path.substr(slash + 1); +} + +// ---------------------------------------------------------------- zip reading +// +// Only the central directory, and only the names. A pak is a plain zip: the End Of Central +// Directory record is the last thing in the file, and it points at a run of 0x02014b50 headers each +// carrying a name length and a name. Nothing here decompresses anything. +// +// The EOCD sits at the very end unless the archive has a trailing comment, so it is found by +// scanning backwards over the last 64 KB -- the largest a comment may be. + +#pragma pack(push, 1) +struct EOCD { + uint32_t sig; // 0x06054b50 + uint16_t disk, cdDisk; + uint16_t entriesHere, entriesTotal; + uint32_t cdSize, cdOffset; + uint16_t commentLen; +}; +struct CDHeader { + uint32_t sig; // 0x02014b50 + uint16_t madeBy, needed, flags, method; + uint16_t modTime, modDate; + uint32_t crc, compSize, uncompSize; + uint16_t nameLen, extraLen, commentLen; + uint16_t diskStart, internalAttr; + uint32_t externalAttr, localOffset; +}; +#pragma pack(pop) + +bool ReadAt(HANDLE h, long long offset, void* buf, DWORD n) +{ + LARGE_INTEGER li; + li.QuadPart = offset; + if (!::SetFilePointerEx(h, li, nullptr, FILE_BEGIN)) { + return false; + } + DWORD got = 0; + return ::ReadFile(h, buf, n, &got, nullptr) && got == n; +} + +} // namespace + +void ModIndex::IndexPak(const std::string& pakPath, const std::string& modid) +{ + HANDLE h = ::CreateFileA(pakPath.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (h == INVALID_HANDLE_VALUE) { + return; + } + LARGE_INTEGER size{}; + if (!::GetFileSizeEx(h, &size) || size.QuadPart < static_cast(sizeof(EOCD))) { + ::CloseHandle(h); + return; + } + + // scan back over at most 64 KB + the EOCD itself for the signature + const long long tailLen = (std::min)(static_cast(64 * 1024 + sizeof(EOCD)), + size.QuadPart); + std::vector tail(static_cast(tailLen)); + if (!ReadAt(h, size.QuadPart - tailLen, tail.data(), static_cast(tailLen))) { + ::CloseHandle(h); + return; + } + long long eocdAt = -1; + for (long long i = tailLen - static_cast(sizeof(EOCD)); i >= 0; --i) { + uint32_t sig = 0; + std::memcpy(&sig, tail.data() + i, sizeof(sig)); + if (sig == 0x06054b50u) { + eocdAt = i; + break; + } + } + if (eocdAt < 0) { + ::CloseHandle(h); + return; + } + EOCD eocd{}; + std::memcpy(&eocd, tail.data() + eocdAt, sizeof(eocd)); + + // ZIP64 archives put 0xFFFFFFFF here. No mod pak measured is near 4 GB, and reading a truncated + // directory would silently under-report provenance -- which fails OPEN, letting base files + // through. Skip the pak instead, and let the caller see it in the pak count. + if (eocd.cdOffset == 0xFFFFFFFFu || eocd.cdSize == 0xFFFFFFFFu || + eocd.entriesTotal == 0xFFFFu) { + ::CloseHandle(h); + return; + } + + std::vector cd(eocd.cdSize); + if (eocd.cdSize == 0 || !ReadAt(h, eocd.cdOffset, cd.data(), eocd.cdSize)) { + ::CloseHandle(h); + return; + } + ::CloseHandle(h); + + size_t at = 0; + for (uint16_t i = 0; i < eocd.entriesTotal; ++i) { + if (at + sizeof(CDHeader) > cd.size()) { + break; + } + CDHeader hdr{}; + std::memcpy(&hdr, cd.data() + at, sizeof(hdr)); + if (hdr.sig != 0x02014b50u) { + break; + } + const size_t nameAt = at + sizeof(CDHeader); + if (nameAt + hdr.nameLen > cd.size()) { + break; + } + std::string name(reinterpret_cast(cd.data() + nameAt), hdr.nameLen); + if (!name.empty() && name.back() != '/' && name.back() != '\\') { + // BARE NAME, because that is what the find record carries. Two mods shipping the same + // bare name in different directories is not a case this has to separate: eligibility + // asks "did a mod ship a file called this", and the merge then loads it by full path. + m_owner[Lower(BareName(name))] = modid; + } + at = nameAt + hdr.nameLen + hdr.extraLen + hdr.commentLen; + } + ++m_paks; +} + +bool ModIndex::Build(const std::string& gameRoot) +{ + m_order.clear(); + m_rank.clear(); + m_owner.clear(); + m_paks = 0; + + const std::string modsDir = gameRoot + "\\Mods"; + const std::string orderPath = modsDir + "\\mod_order.txt"; + + HANDLE h = ::CreateFileA(orderPath.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (h == INVALID_HANDLE_VALUE) { + return false; + } + LARGE_INTEGER size{}; + ::GetFileSizeEx(h, &size); + std::string text; + if (size.QuadPart > 0 && size.QuadPart < (1 << 20)) { + text.resize(static_cast(size.QuadPart)); + DWORD got = 0; + ::ReadFile(h, &text[0], static_cast(text.size()), &got, nullptr); + text.resize(got); + } + ::CloseHandle(h); + + // A BOM lands on the FIRST modid and nowhere else, so the list reads as correct while exactly + // one mod fails to match its folder. Windows PowerShell writes one for `-Encoding utf8`, and + // this file is hand-edited as often as it is generated. + if (text.size() >= 3 && static_cast(text[0]) == 0xEF && + static_cast(text[1]) == 0xBB && static_cast(text[2]) == 0xBF) { + text.erase(0, 3); + } + + size_t pos = 0; + while (pos <= text.size()) { + const size_t nl = text.find('\n', pos); + std::string line = text.substr(pos, (nl == std::string::npos ? text.size() : nl) - pos); + pos = (nl == std::string::npos) ? text.size() + 1 : nl + 1; + while (!line.empty() && (line.back() == '\r' || line.back() == ' ' || line.back() == '\t')) { + line.pop_back(); + } + size_t b = 0; + while (b < line.size() && (line[b] == ' ' || line[b] == '\t')) { + ++b; + } + line = line.substr(b); + if (line.empty() || line[0] == '#') { + continue; + } + m_rank[Lower(line)] = static_cast(m_order.size()); + m_order.push_back(line); + } + + // Walk each enabled mod for its paks. A mod listed with no folder on disk simply contributes + // nothing -- that is a real state (`temptation` is listed on this install and has no folder), + // not an error. + for (const std::string& modid : m_order) { + std::vector dirs{modsDir + "\\" + modid}; + while (!dirs.empty()) { + const std::string dir = dirs.back(); + dirs.pop_back(); + WIN32_FIND_DATAA fd{}; + HANDLE fh = ::FindFirstFileA((dir + "\\*").c_str(), &fd); + if (fh == INVALID_HANDLE_VALUE) { + continue; + } + do { + const std::string name = fd.cFileName; + if (name == "." || name == "..") { + continue; + } + const std::string full = dir + "\\" + name; + if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + dirs.push_back(full); + } else if (name.size() > 4 && + Lower(name.substr(name.size() - 4)) == ".pak") { + IndexPak(full, modid); + } + } while (::FindNextFileA(fh, &fd)); + ::FindClose(fh); + } + } + return true; +} + +int ModIndex::Rank(const std::string& modid) const +{ + const auto it = m_rank.find(Lower(modid)); + return (it == m_rank.end()) ? -1 : it->second; +} + +const char* ModIndex::Owner(const std::string& fileName) const +{ + const auto it = m_owner.find(Lower(BareName(fileName))); + return (it == m_owner.end()) ? nullptr : it->second.c_str(); +} + +} // namespace ptf diff --git a/Projects/PTFextender/src/mod_index.h b/Projects/PTFextender/src/mod_index.h new file mode 100644 index 00000000..00ac4704 --- /dev/null +++ b/Projects/PTFextender/src/mod_index.h @@ -0,0 +1,80 @@ +// ModIndex -- which MOD, if any, ships a given file. +// +// WHY A PART'S NAME IS NOT ENOUGH TO TRUST IT. +// +// `ApplyPtfPatches` merges every `__*` the pak filesystem returns. The pak filesystem is +// the BASE GAME plus every mod, flattened, and the base game ships a great many files that already +// carry a `__`: build-time table parts, per-quest dialogue variants, per-role barks. +// Counted from a retail 1.5.6 install, restricted to the directories a content mod patches: +// +// 5,902 files carrying a __, across 3,245 distinct suffixes +// muz 411 files zena 371 kuman 200 bandita 96 straz 68 autotests 20 +// +// Warhorse publish these as RESERVED MODIDS for exactly this reason. But "reject a list of names" +// is the wrong shape of fix: the list goes stale at every game patch, it cannot be derived by a +// plugin at runtime, and it answers a question about NAMES when the real question is about ORIGIN. +// A player who installs a mod called `muz` should not thereby merge 411 of Warhorse's own fixtures +// into their tables. +// +// PROVENANCE ANSWERS IT EXACTLY: a part is eligible when the file came from a MOD pak. That needs +// no list, cannot go stale, and is what a reserved-name list is a proxy for. +// +// WHY THIS CLASS EXISTS RATHER THAN A FLAG ON THE FIND RECORD. Measured in game, 2026-09-05: +// `ICryPak::FindFirst` fills `attrib / times / size / name` and no pak field, and `attrib` is +// `0x80000001` (in-pak | readonly) for all 184 hits tested across the base game and three separate +// mods. The enumeration separates pak from loose and nothing finer, so the plugin has to build the +// map itself. +// +// HOW: read `Mods\mod_order.txt`, then walk each ENABLED mod's own `.pak` files and record which +// names each one contains. A pak is a zip; only its central directory is read, which is a few +// hundred KB even for a large mod and needs no external library. +// +// COST, measured on an 89-mod install: 1.4 GB of mod paks, of which the central directories are a +// small fraction, read once at startup. + +#pragma once + +#include +#include +#include +#include + +namespace ptf { + +class ModIndex { +public: + // `gameRoot` is the install directory -- the parent of `Mods\`. Safe to call twice; the second + // call rebuilds. Returns false only when `Mods\mod_order.txt` cannot be read, which means no + // mod is enabled and no part can be eligible. + bool Build(const std::string& gameRoot); + + // Load order position of a modid, or -1 when it is not enabled. Comparison is case-insensitive + // because mod_order.txt is hand-edited as often as it is generated. + int Rank(const std::string& modid) const; + + // The modid that ships `fileName` (a BARE name, as the find record carries), or nullptr when no + // enabled mod does -- which for a file the pak filesystem just returned means the BASE GAME. + // + // Last enabled mod wins, matching the load order: if two mods ship the same name, the one the + // engine actually reads is the later one, and that is the one whose provenance applies. + const char* Owner(const std::string& fileName) const; + + std::size_t Mods() const { return m_order.size(); } + std::size_t Files() const { return m_owner.size(); } + std::size_t Paks() const { return m_paks; } + + // The whole map, so an independent implementation can be diffed against it. `kcdlib\vfs.py` + // builds the same relation in Python; two implementations agreeing on ~1,900 names is a far + // stronger claim than either one passing its own spot checks. + const std::unordered_map& Dump() const { return m_owner; } + +private: + void IndexPak(const std::string& pakPath, const std::string& modid); + + std::vector m_order; // as listed, original spelling + std::unordered_map m_rank; // lowercased modid -> position + std::unordered_map m_owner; // lowercased bare name -> modid + std::size_t m_paks = 0; +}; + +} // namespace ptf diff --git a/Projects/PTFextender/src/plugin.cpp b/Projects/PTFextender/src/plugin.cpp index 58fbe130..19d4efb7 100644 --- a/Projects/PTFextender/src/plugin.cpp +++ b/Projects/PTFextender/src/plugin.cpp @@ -22,11 +22,18 @@ #include "Offsets/vtables/ILog.h" #include "Offsets/vtables/ISystem.h" #include "Offsets/vtables/IXmlNode.h" +#include "Offsets/vtables/IXmlUtils.h" #include "playermodule/C_ActionSets.h" #include "playermodule/C_Keybinds.h" +#include "mod_index.h" +#include "ptf_families.h" +#include + +#include #include #include +#include using Offsets::IXmlNode; @@ -53,24 +60,153 @@ void CopyAttributes(IXmlNode* dst, IXmlNode* src) } } -// Row identity = tag + the identifying attributes, absent attrs compare as "". -// The config XML families use different id attrs: , -// , , and in -// keybindSuperactions / -// — so equality spans all of them instead of hardcoding one key per file. -bool SameIdentity(IXmlNode* a, IXmlNode* b) +// The text of a direct child element -- `x`, which an attribute-only +// identity cannot express. [49] getContent, VERIFIED: it is the trivial getter of the content +// member, and returns the shared static empty string rather than NULL when a node has no text. +const char* ChildText(IXmlNode* node, const char* tag) +{ + IXmlNode* c = nullptr; + node->findChild(&c, tag); + if (!c) + return ""; + const char* t = c->getContent(); + // the caller compares immediately; the string is interned in the node's own table and outlives + // this reference for as long as the tree does + c->Release(); + return t ? t : ""; +} + +// Row identity, from the FAMILY DESCRIPTOR rather than a fixed attribute list. +// +// WHY A DESCRIPTOR. The shipped rule compares seven hardcoded attribute names, which happens to +// span the three keybind files it was written for. It cannot express the families a content mod +// actually claims: `whdata_1` keys on a CHILD ELEMENT, `waitinglinks` on a PAIR of attributes plus +// a child, and an arbitrary `Libs\Tables` row on that table's primary key. Worse, a fixed list is +// silently wrong rather than absent -- measured, `skill_check_difficulty.xml` carries none of the +// seven, so every row keys identically and rows pair by POSITION. +// +// `spec` is the identity string from the generated table: `attr:X`, `attr:X+attr:Y`, `child:X`, or +// any combination. Absent parts compare as "" on both sides, so a row missing the key still matches +// only another row missing it. +bool IdentityMatches(IXmlNode* a, IXmlNode* b, const char* spec) { if (std::strcmp(a->getTag(), b->getTag()) != 0) return false; - static const char* const kIdAttrs[] = { "name", "actionmap", "action", "alternative", - "map", "input", "controller" }; - for (const char* key : kIdAttrs) - if (std::strcmp(AttrOrEmpty(a, key), AttrOrEmpty(b, key)) != 0) - return false; + if (!spec || !*spec) { + // No descriptor: fall back to the shipped seven-attribute key. That is the behaviour the + // three config files have always had, and it stays exactly as it was for them. + static const char* const kIdAttrs[] = { "name", "actionmap", "action", "alternative", + "map", "input", "controller" }; + for (const char* key : kIdAttrs) + if (std::strcmp(AttrOrEmpty(a, key), AttrOrEmpty(b, key)) != 0) + return false; + return true; + } + const char* p = spec; + while (*p) { + const char* plus = std::strchr(p, '+'); + const std::string term(p, plus ? static_cast(plus - p) : std::strlen(p)); + const size_t colon = term.find(':'); + if (colon != std::string::npos) { + const std::string kind = term.substr(0, colon); + const std::string key = term.substr(colon + 1); + if (kind == "attr") { + if (std::strcmp(AttrOrEmpty(a, key.c_str()), AttrOrEmpty(b, key.c_str())) != 0) + return false; + } else if (kind == "child") { + if (std::strcmp(ChildText(a, key.c_str()), ChildText(b, key.c_str())) != 0) + return false; + } + } + if (!plus) + break; + p = plus + 1; + } return true; } -int FindChildIndex(IXmlNode* base, IXmlNode* like) +// Glob match where `*` NEVER CROSSES A SEPARATOR. +// +// `fnmatch`-style matching lets `*` swallow `/`, and that is not a nicety here: the rule written for +// a graph NODE then also catches every EDGE inside it, the edges key on an attribute they do not +// carry, and a contested rewiring comes out silently applied. Measured on the Python side before +// this was written. +bool GlobMatch(const char* pat, const char* str) +{ + if (*pat == '\0') + return *str == '\0'; + if (*pat == '*') { + for (const char* s = str;; ++s) { + if (GlobMatch(pat + 1, s)) + return true; + if (*s == '\0' || *s == '/' || *s == '\\') + return false; // the wildcard stops at a separator + } + } + if (*str == '\0') + return false; + const char a = static_cast(::tolower(static_cast(*pat))); + const char b = static_cast(::tolower(static_cast(*str))); + const bool same = (a == b) || ((a == '/' || a == '\\') && (b == '/' || b == '\\')); + return same && GlobMatch(pat + 1, str + 1); +} + +// The path a family glob is matched against. +// +// SPELLING VARIES BY CALLER, measured from a launch-load-quit that logged all 28,019 paths the +// engine asked for: +// +// levels/kutnohorsko/LevelData.xml mixed case, no prefix +// levels/kutnohorsko/Objects_Mission0.xml +// levels/kutnohorsko/whdata_1 no extension at all +// data/levels/kutnohorsko/WaitingLinks.xml ...and this one carries `data/` +// +// A glob written one way would silently miss three of the four. Lowercase, forward slashes, and a +// leading `data/` removed -- CryPak is rooted there, so the two spellings name one file. The +// ORIGINAL string is still what FindFirst is given, because that is what the engine resolved. +std::string NormalisePath(const char* p) +{ + std::string v(p ? p : ""); + for (char& c : v) { + if (c == '\\') + c = '/'; + else + c = static_cast(::tolower(static_cast(c))); + } + if (v.compare(0, 5, "data/") == 0) + v.erase(0, 5); + return v; +} + +// Which family, if any, describes this virtual path. +const ptf::Family* FamilyFor(const char* vpath) +{ + if (!vpath) + return nullptr; + const std::string v = NormalisePath(vpath); + for (std::size_t i = 0; i < ptf::kFamilyCount; ++i) + if (GlobMatch(ptf::kFamilies[i].glob, v.c_str())) + return &ptf::kFamilies[i]; + return nullptr; +} + +// The identity rule in force for the children of `parentPath`, or nullptr when no family describes +// it -- in which case `IdentityMatches` uses the shipped seven-attribute key, unchanged. +const char* IdentityFor(const ptf::Family* fam, const std::string& parentPath) +{ + if (!fam) + return nullptr; + for (std::size_t i = 0; i < fam->ruleCount; ++i) { + const std::string rule(fam->rules[i].path); + // the rule names the ROW path; we are asked about its parent + const size_t slash = rule.find_last_of('/'); + if (slash != std::string::npos && rule.substr(0, slash) == parentPath) + return fam->rules[i].identity; + } + return nullptr; +} + +int FindChildIndex(IXmlNode* base, IXmlNode* like, const char* idSpec) { const int n = base->getChildCount(); for (int i = 0; i < n; ++i) { @@ -78,7 +214,7 @@ int FindChildIndex(IXmlNode* base, IXmlNode* like) base->getChild(&c, i); if (!c) continue; - const bool match = SameIdentity(c, like); + const bool match = IdentityMatches(c, like, idSpec); c->Release(); if (match) return i; @@ -95,8 +231,14 @@ struct S_MergeStats { // containers so a mod can patch a single without restating the whole // actionmap. Patch subtrees are shared into the base document — the same // technique the stock XMLPatcher uses (insertChild/replaceChild AddRef the node). -void MergeChildren(IXmlNode* base, IXmlNode* patch, S_MergeStats& stats) +void MergeChildren(IXmlNode* base, IXmlNode* patch, S_MergeStats& stats, + const ptf::Family* fam = nullptr, const std::string& path = std::string()) { + // The element path is carried down so each level of the recursion can ask the descriptor which + // rule governs ITS children. Without it a family could only describe rows at one depth. + const std::string here = path + "/" + base->getTag(); + const char* idSpec = IdentityFor(fam, here); + const int n = patch->getChildCount(); for (int i = 0; i < n; ++i) { IXmlNode* p = nullptr; @@ -104,7 +246,7 @@ void MergeChildren(IXmlNode* base, IXmlNode* patch, S_MergeStats& stats) if (!p) continue; - const int at = FindChildIndex(base, p); + const int at = FindChildIndex(base, p, idSpec); if (at < 0) { IXmlNode* add = nullptr; // clone: p may be read-only p->clone(&add, false); @@ -132,7 +274,7 @@ void MergeChildren(IXmlNode* base, IXmlNode* patch, S_MergeStats& stats) CopyAttributes(b, p); // modified (attrs) ++stats.modified; } - MergeChildren(b, p, stats); // recurse + MergeChildren(b, p, stats, fam, here); // recurse, carrying the path } b->Release(); } @@ -141,6 +283,63 @@ void MergeChildren(IXmlNode* base, IXmlNode* patch, S_MergeStats& stats) } } +// The provenance map, built once. See mod_index.h: the find record carries no pak field, so a +// patch's ORIGIN cannot be read off the enumeration and has to be indexed from the mod paks. +ptf::ModIndex g_mods; +bool g_modsBuilt = false; + +const ptf::ModIndex& Mods() +{ + if (!g_modsBuilt) { + g_modsBuilt = true; + char exe[MAX_PATH] = {0}; + ::GetModuleFileNameA(nullptr, exe, MAX_PATH); + // \Bin\Win64MasterMasterSteamPGO\KingdomCome.exe -> + std::string root(exe); + for (int up = 0; up < 3; ++up) { + const size_t slash = root.find_last_of("/\\"); + if (slash == std::string::npos) + break; + root.erase(slash); + } + g_mods.Build(root); + auto* env = SSystemGlobalEnvironment::GetInstance(); + if (env && env->pLog) + env->pLog->LogAlways("[PTFextender] mod index: %zu enabled mod(s), %zu pak(s), " + "%zu file name(s) -- a part is eligible only if a MOD ships it", + g_mods.Mods(), g_mods.Paks(), g_mods.Files()); + } + return g_mods; +} + +// One candidate patch: the file the enumeration returned, and the position that decides when it +// applies relative to the others. +struct Candidate { + int rank; + std::string name; + std::string modid; + bool operator<(const Candidate& o) const { return rank < o.rank; } +}; + +// The suffix after the LAST `__`, which is the modid by convention. +// +// SPLIT FROM THE RIGHT, NEVER THE LEFT. Base files already contain `__` -- e.g. +// `konec_sluzby__komplet_vsechno.xml` -- so a part of one is +// `konec_sluzby__komplet_vsechno__.xml`, and a first-`__` split reads the modid as +// `komplet_vsechno__`, matches no enabled mod, and drops the part silently. +std::string SuffixOf(const std::string& fileName) +{ + std::string stem(fileName); + const size_t slash = stem.find_last_of("/\\"); + if (slash != std::string::npos) + stem = stem.substr(slash + 1); + const size_t dot = stem.rfind('.'); + if (dot != std::string::npos) + stem = stem.substr(0, dot); + const size_t sep = stem.rfind("__"); + return (sep == std::string::npos) ? std::string() : stem.substr(sep + 2); +} + // Merge every __ patch the pak FS has for basePath into the // tree at *ppBaseRoot. The base tree is a read-only pooled one, so patches land // in a mutable deep clone and *ppBaseRoot is swapped to it (old ref released). @@ -150,37 +349,85 @@ int ApplyPtfPatches(const char* basePath, IXmlNode** ppBaseRoot) if (!basePath || !ppBaseRoot || !*ppBaseRoot) return 0; + // The descriptor for this file, or nullptr -- in which case every rule below behaves exactly as + // the shipped plugin did. Adding a family cannot change how an undescribed file merges. + const ptf::Family* fam = FamilyFor(basePath); + auto* env = SSystemGlobalEnvironment::GetInstance(); if (!env || !env->pCryPak || !env->pSystem) return 0; // "libs/config/foo.xml" -> "libs/config/foo__*.xml" + // + // AND AN EXTENSION-LESS BASE PATH GETS `__*` APPENDED. `rfind('.')` returning npos used to mean + // "give up", which quietly excluded every file without a dot -- `levels\\whdata_1` among + // them, the registry that says which profile streams a layer. A file cannot be unpatchable + // because of how it is spelled. std::string wildcard(basePath); const size_t dot = wildcard.rfind('.'); if (dot == std::string::npos) - return 0; - wildcard.insert(dot, "__*"); + wildcard += "__*"; + else + wildcard.insert(dot, "__*"); // find records carry bare file names; keep the directory for reloading const size_t slash = wildcard.find_last_of("/\\"); const std::string dir = (slash == std::string::npos) ? std::string() : wildcard.substr(0, slash + 1); + // ---- COLLECT FIRST, then decide. Two rules cannot be applied while enumerating: eligibility + // needs the mod index, and ORDER needs every candidate in hand. Offsets::SCryPakFindData fd{}; const intptr_t h = env->pCryPak->FindFirst(wildcard.c_str(), &fd); if (h == -1) return 0; + const ptf::ModIndex& mods = Mods(); + std::vector queue; + int rejectedBase = 0, rejectedUnlisted = 0; + do { + const std::string name(fd.name); + // ELIGIBILITY IS PROVENANCE **AND** NAME. + // provenance: the file must come from a mod pak. The base game ships 5,902 files carrying + // a `__` in the directories mods patch, so a name-only rule merges Warhorse's + // own build fixtures the moment a player installs a mod called `muz` or `autotests`. + // name: the suffix must be an ENABLED mod, so a disabled mod's leftovers stay inert. + const char* owner = mods.Owner(name); + if (!owner) { + ++rejectedBase; + continue; + } + const std::string suffix = SuffixOf(name); + const int rank = suffix.empty() ? -1 : mods.Rank(suffix); + if (rank < 0) { + ++rejectedUnlisted; + continue; + } + queue.push_back(Candidate{rank, name, suffix}); + } while (env->pCryPak->FindNext(h, &fd) >= 0); + env->pCryPak->FindClose(h); + + if (queue.empty()) { + if ((rejectedBase || rejectedUnlisted) && env->pLog) + env->pLog->LogAlways("[PTFextender] '%s': no eligible patch (%d base-game, %d not in " + "mod_order)", basePath, rejectedBase, rejectedUnlisted); + return 0; + } + + // ORDER IS mod_order.txt, NOT ENUMERATION ORDER. `FindFirst` returns names in whatever order the + // pak filesystem holds them, so two mods touching one row would resolve differently on two + // machines -- and differently from what the player's load order says. Stable sort, so two parts + // from one mod keep their enumeration order relative to each other. + std::stable_sort(queue.begin(), queue.end()); + IXmlNode* merged = nullptr; (*ppBaseRoot)->clone(&merged, false); - if (!merged) { - env->pCryPak->FindClose(h); + if (!merged) return 0; - } int applied = 0; - do { - const std::string patchPath = dir + fd.name; + for (const Candidate& c : queue) { + const std::string patchPath = dir + c.name; IXmlNode* patchRoot = nullptr; // same call shape as the three stock config loaders: (&out, path, 0, 1, 1) env->pSystem->LoadXmlFromFile(&patchRoot, patchPath.c_str(), 0, 1, 1); @@ -190,14 +437,18 @@ int ApplyPtfPatches(const char* basePath, IXmlNode** ppBaseRoot) continue; } S_MergeStats stats; - MergeChildren(merged, patchRoot, stats); + MergeChildren(merged, patchRoot, stats, fam); patchRoot->Release(); ++applied; if (env->pLog) - env->pLog->LogAlways("[PTFextender] '%s' is patched by '%s', nodes added: %d, modified: %d", - basePath, fd.name, stats.added, stats.modified); - } while (env->pCryPak->FindNext(h, &fd) >= 0); - env->pCryPak->FindClose(h); + env->pLog->LogAlways("[PTFextender] '%s' is patched by '%s' (mod '%s', #%d of %zu), " + "nodes added: %d, modified: %d", + basePath, c.name.c_str(), c.modid.c_str(), c.rank + 1, + mods.Mods(), stats.added, stats.modified); + } + if (rejectedBase && env->pLog) + env->pLog->LogAlways("[PTFextender] '%s': %d base-game part(s) rejected -- not shipped by " + "any enabled mod", basePath, rejectedBase); if (applied > 0) { (*ppBaseRoot)->Release(); @@ -278,6 +529,85 @@ class { static inline REL::Relocation orig; } hkLoadFromXML; +// ------------------------------------------------------ hook: LEVEL FILES --- +// +// ONE HOOK REACHES ALL OF THEM. Measured: 37,768 calls over 28,019 distinct paths in a single +// launch-load-quit, and every file family a level-content mod claims comes through +// `IXmlUtils::LoadXmlFromFile` -- leveldata, objects_mission0, waitinglinks and whdata_1 alike. +// So this is a hook plus a path lookup, and every future family is a descriptor rather than a +// reverse-engineering job. +// +// IXmlUtils SLOT [1], NOT ISystem [131]. The latter is a thin forwarder into the former, so hooking +// IXmlUtils catches both routes and no direct caller slips past. The target is read out of the live +// vtable via `gEnv->pSystem->GetXmlUtils()`, which is why this one installs on a KCSE message +// rather than at plugin load: the pointer does not exist yet when the DLL is loaded. +// +// `patching=1` on all 37,768 calls -- the stock XML patcher already runs inside this function, on +// every file. A merge placed here sits exactly where the engine already expects one. +namespace { + +using LoadXmlFn = IXmlNode** (*)(Offsets::IXmlUtils*, IXmlNode**, const char*, bool, bool, bool, + bool); +LoadXmlFn g_origLoadXml = nullptr; + +// RE-ENTRANCY. ApplyPtfPatches loads each part through this very function, so without a guard the +// first patched file recurses until the stack ends. Thread-local because XML loading is not +// single-threaded here -- two loader threads were observed interleaving in the same session. +thread_local int t_inPatchLoad = 0; + +struct PatchLoadGuard { + PatchLoadGuard() { ++t_inPatchLoad; } + ~PatchLoadGuard() { --t_inPatchLoad; } +}; + +IXmlNode** LoadXmlDetour(Offsets::IXmlUtils* pThis, IXmlNode** out, const char* sFilename, + bool bEnablePatching, bool b5, bool b6, bool b7) +{ + IXmlNode** r = g_origLoadXml(pThis, out, sFilename, bEnablePatching, b5, b6, b7); + if (t_inPatchLoad || !out || !*out || !sFilename) + return r; + if (!FamilyFor(sFilename)) + return r; // not a described file: the engine's own result, untouched + PatchLoadGuard guard; + ApplyPtfPatches(sFilename, out); + return r; +} + +bool InstallLevelHook() +{ + auto* env = SSystemGlobalEnvironment::GetInstance(); + if (!env || !env->pSystem) + return false; + auto* utils = reinterpret_cast(env->pSystem->GetXmlUtils()); + if (!utils) + return false; + void** vtbl = *reinterpret_cast(utils); + void* target = vtbl[1]; + if (MH_CreateHook(target, reinterpret_cast(&LoadXmlDetour), + reinterpret_cast(&g_origLoadXml)) != MH_OK) + return false; + if (MH_EnableHook(target) != MH_OK) + return false; + if (env->pLog) + env->pLog->LogAlways("[PTFextender] level families armed on IXmlUtils::LoadXmlFromFile " + "at %p -- %zu family descriptor(s)", target, ptf::kFamilyCount); + return true; +} + +bool g_levelHookArmed = false; + +void OnKcseMessage(KCSE::Message* msg) +{ + if (g_levelHookArmed || !msg) + return; + // PreDataLoaded is the earliest point gEnv->pSystem is usable; DataLoaded is the fallback. + if (msg->type == KCSE::IMessagingInterface::kMessage_PreDataLoaded || + msg->type == KCSE::IMessagingInterface::kMessage_DataLoaded) + g_levelHookArmed = InstallLevelHook(); +} + +} // namespace + // ---------------------------------------------------------------- install --- static bool InstallHooks() @@ -297,6 +627,10 @@ KCSE_PLUGIN_INFO("PTF Extender", "JerryYOJ", 1); KCSE_PLUGIN_LOAD(kcse) { KCSE::AllocTrampoline(1 << 10); - - return InstallHooks(); + + if (!InstallHooks()) + return false; + // The level hook needs gEnv->pSystem, which does not exist yet -- arm it on the first message. + auto* msg = kcse->GetMessagingInterface(); + return msg && msg->RegisterListener(&OnKcseMessage); } diff --git a/Projects/PTFextender/src/ptf_families.h b/Projects/PTFextender/src/ptf_families.h new file mode 100644 index 00000000..30a5bdd9 --- /dev/null +++ b/Projects/PTFextender/src/ptf_families.h @@ -0,0 +1,93 @@ +// GENERATED by mesh_bridge/kcdlib/ptf.py --emit-cpp= -- DO NOT EDIT BY HAND. +// +// The PTF merge table: for each file family, which element path carries rows, what makes +// two rows THE SAME row, where a new row sorts, and which number a second mod would +// collide on. Every rule here is checked against a shipped artifact by `ptf_verify.py`: +// merge(vanilla, diff(vanilla, override)) must reproduce that override byte for byte. +// +// Regenerate with: python mesh_bridge/kcdlib/ptf.py --emit-cpp= + +#pragma once + +#include + +namespace ptf { + +struct FamilyRule { + const char* path; // element path whose children are rows + const char* identity; // attr:X[+attr:Y][+child:Z] -- what makes two rows the same + const char* order; // append | sorted:attr:X[|suffix_] | sorted:child:X | clone + const char* allocate; // attr:X / child:X, or nullptr -- a number two mods would pick + const char* conflict; // last-wins | error | chain +}; + +struct Family { + const char* name; + const char* glob; // levels/*/leveldata.xml -- `*` never crosses a separator + const char* expect; + const FamilyRule* rules; + std::size_t ruleCount; +}; + +inline const FamilyRule kLeveldataRules[] = { + {"/LevelData/Layers/Layer", + "attr:Name", + "sorted:attr:Name|suffix_", "attr:Id", "last-wins"}, +}; + +inline const FamilyRule kWhdataRules[] = { + {"/Root/GameProfileManager/GameProfiles/GameProfile", + "child:Name", + "sorted:child:Name", "child:Id", "last-wins"}, +}; + +inline const FamilyRule kObjectsMissionRules[] = { + {"/Objects/Entity", + "attr:EntityGuid", + "sorted:attr:EntityGuid", "attr:EntityId", "last-wins"}, +}; + +inline const FamilyRule kWaitinglinksRules[] = { + {"/StaticLinksInfo/WaitingLinks/WaitingLink", + "attr:SourceId+attr:TargetId+child:LinkDefinition", + "append", nullptr, "last-wins"}, +}; + +inline const FamilyRule kUiTableRules[] = { + {"/database/Cutscenes/*", + "attr:Name", + "append", nullptr, "last-wins"}, +}; + +inline const FamilyRule kSkaldRules[] = { + {"**/Edge", + "attr:From+attr:To", + "append", nullptr, "chain"}, + {"/Database/Skald/*/EdgeVertices/EdgeVertex", + "attr:Name", + "append", nullptr, "last-wins"}, + {"/Database/Skald/*/Nodes/*", + "attr:Name", + "append", nullptr, "last-wins"}, + {"/Database/Skald/*/Ports/Port", + "attr:Name", + "append", nullptr, "last-wins"}, +}; + +inline const Family kFamilies[] = { + {"leveldata", "levels/*/leveldata.xml", "byte", + kLeveldataRules, sizeof(kLeveldataRules) / sizeof(kLeveldataRules[0])}, + {"whdata", "levels/*/whdata_1", "eol", + kWhdataRules, sizeof(kWhdataRules) / sizeof(kWhdataRules[0])}, + {"objects_mission", "levels/*/objects_mission0.xml", "byte", + kObjectsMissionRules, sizeof(kObjectsMissionRules) / sizeof(kObjectsMissionRules[0])}, + {"waitinglinks", "levels/*/waitinglinks.xml", "rowset", + kWaitinglinksRules, sizeof(kWaitinglinksRules) / sizeof(kWaitinglinksRules[0])}, + {"ui_table", "libs/tables/*/*.xml", "rowset", + kUiTableRules, sizeof(kUiTableRules) / sizeof(kUiTableRules[0])}, + {"skald", "quests/*", "report", + kSkaldRules, sizeof(kSkaldRules) / sizeof(kSkaldRules[0])}, +}; +inline const std::size_t kFamilyCount = sizeof(kFamilies) / sizeof(kFamilies[0]); + +} // namespace ptf diff --git a/include/Offsets/vtables/IXmlNode.h b/include/Offsets/vtables/IXmlNode.h index 151dca5c..619d2666 100644 --- a/include/Offsets/vtables/IXmlNode.h +++ b/include/Offsets/vtables/IXmlNode.h @@ -101,9 +101,25 @@ struct IXmlNode { // [46] 0x18045840C -- first child with the given tag, NEW reference to *out (NULL *out // when absent). virtual IXmlNode** findChild(IXmlNode** out, const char* tag) const = 0; - virtual void _vf47() = 0; // [47] 0x180459EDC + // [47] 0x180459EDC -- the PARENT node. MEASURED 2026-09-05: the only slot that touches +0x40, + // a load at byte 6 (it needs a null test -- a root's parent IS null). On a leaf +0x40 + // pointed at an object whose first qword is the CXmlNode vtable, i.e. another node. + virtual void _vf47() = 0; // [47] 0x180459EDC getParent (not declared: return type is a + // hidden-pointer XmlNodeRef and no caller here needs it) virtual void _vf48() = 0; // [48] 0x181A73BE0 - virtual void _vf49() = 0; // [49] 0x1819A2A90 + // [49] 0x1819A2A90 -- the element's own TEXT. MEASURED IN GAME 2026-09-05: the trivial getter + // `mov rax,[rcx+0x48]; ret` of the content member, found by dumping the node's 0x58-byte header + // and scanning every vtable slot for one that reads that offset. Confirmed by READING rather + // than by elimination -- the leaf under + // /Root/GameProfileManager/GameProfiles/GameProfile returned "apolena_abandonedCampsEnviro". + // + // NEVER NULL: a node with no text carries the shared static empty string (0x183A3D1E0), the + // same convention [33] getAttr uses -- which is why this one CAN be a trivial getter while + // [47] getParent needs a branch. + // + // The header is 0x58, not the 0x48 the clone impl's allocation suggests: on every node the tag + // pointer is exactly node+0x58, where the inline string pool begins. + virtual const char* getContent() const = 0; virtual void _vf50() = 0; // [50] 0x182488BA0 // [51] 0x18045A3F4 (shared by both vtables) -- internal deep-clone taking the string- // intern-table context as arg (impl 0x18045A7A0: allocs 0x48, plants the CXmlNode