From 025aaee1967aa919d8efa617bcacf0427822991d Mon Sep 17 00:00:00 2001 From: Tehsapper Date: Wed, 19 Aug 2026 12:57:44 +0200 Subject: [PATCH] allow instrumenting asset file reads --- include/limitless/assets.hpp | 14 ++ include/limitless/core/shader/shader.hpp | 8 +- .../limitless/core/shader/shader_compiler.hpp | 7 +- include/limitless/shader_storage.hpp | 3 +- include/limitless/text/font_atlas.hpp | 10 ++ src/limitless/assets.cpp | 24 ++- src/limitless/core/shader/shader.cpp | 27 +++- src/limitless/core/shader/shader_compiler.cpp | 12 +- src/limitless/loaders/gltf_model_loader.cpp | 141 ++++++++++++------ src/limitless/loaders/texture_loader.cpp | 26 +++- src/limitless/ms/material_compiler.cpp | 2 +- src/limitless/shader_storage.cpp | 5 +- src/limitless/text/font_atlas.cpp | 58 +++++-- 13 files changed, 260 insertions(+), 77 deletions(-) diff --git a/include/limitless/assets.hpp b/include/limitless/assets.hpp index 5e5020b9..688a3945 100644 --- a/include/limitless/assets.hpp +++ b/include/limitless/assets.hpp @@ -5,6 +5,9 @@ #include #include +#include +#include + namespace Limitless::ms { class Material; } @@ -129,6 +132,17 @@ namespace Limitless { [[nodiscard]] const auto& getBaseDir() const noexcept { return base_dir; } [[nodiscard]] const auto& getShaderDir() const noexcept { return shader_dir; } + /** + * Read file bytes from the given path. Default implementation uses the filesystem. + * Paths may be absolute or relative. + */ + [[nodiscard]] virtual std::vector readFile(const fs::path& path) const; + + /** + * Whether a regular file exists at the given path. Default implementation uses the filesystem. + */ + [[nodiscard]] virtual bool fileExists(const fs::path& path) const; + void reloadTextures(const TextureLoaderFlags& settings); }; } diff --git a/include/limitless/core/shader/shader.hpp b/include/limitless/core/shader/shader.hpp index e261328c..5c7d5ae8 100644 --- a/include/limitless/core/shader/shader.hpp +++ b/include/limitless/core/shader/shader.hpp @@ -24,6 +24,8 @@ namespace Limitless { using std::runtime_error::runtime_error; }; + class Assets; + /** * Shader class describes shader that is used to compile shader program by ShaderCompiler * @@ -68,6 +70,8 @@ namespace Limitless { */ std::set include_entries; + Assets* assets {nullptr}; + /** * Self-implemented "#include" directive for GLSL * @@ -92,7 +96,7 @@ namespace Limitless { * @return - file source code * @throw shader_file_not_found - if file at @param filepath does not exist */ - static std::string getSource(const fs::path& filepath); + std::string getSource(const fs::path& filepath); /** * Swap function used by move-semantics @@ -119,7 +123,7 @@ namespace Limitless { * @throw shader_file_not_found - if file at @param path does not exist * @throw shader_include_not_found - if file specified at include directive does not exist */ - Shader(fs::path path, Type type, const ShaderAction& action = {}); + Shader(fs::path path, Type type, const ShaderAction& action = {}, Assets* assets = nullptr); /** * Shader destructor diff --git a/include/limitless/core/shader/shader_compiler.hpp b/include/limitless/core/shader/shader_compiler.hpp index 894a1e6a..f30ea331 100644 --- a/include/limitless/core/shader/shader_compiler.hpp +++ b/include/limitless/core/shader/shader_compiler.hpp @@ -11,6 +11,7 @@ namespace Limitless { class ShaderProgram; class RendererSettings; class Context; + class Assets; class shader_linking_error : public std::runtime_error { public: @@ -46,6 +47,8 @@ namespace Limitless { */ std::optional render_settings; + Assets* assets {nullptr}; + void replaceCommonDefines(Shader& shader); public: /** @@ -58,8 +61,8 @@ namespace Limitless { * @param ctx * @param settings */ - ShaderCompiler(Context& ctx, const RendererSettings& settings); - explicit ShaderCompiler(Context& ctx); + ShaderCompiler(Context& ctx, const RendererSettings& settings, Assets* assets = nullptr); + explicit ShaderCompiler(Context& ctx, Assets* assets = nullptr); virtual ~ShaderCompiler() = default; ShaderCompiler(const ShaderCompiler&) noexcept = delete; diff --git a/include/limitless/shader_storage.hpp b/include/limitless/shader_storage.hpp index e24890ac..74023668 100644 --- a/include/limitless/shader_storage.hpp +++ b/include/limitless/shader_storage.hpp @@ -12,6 +12,7 @@ namespace Limitless { class RendererSettings; class ShaderProgram; class Context; + class Assets; class shader_storage_error : public std::runtime_error { public: @@ -27,7 +28,7 @@ namespace Limitless { std::mutex mutex; public: - void initialize(Context& ctx, const RendererSettings& settings, const fs::path& shader_dir); + void initialize(Context& ctx, const RendererSettings& settings, const fs::path& shader_dir, Assets& assets); ShaderProgram& get(const std::string& name) const; ShaderProgram& get(ShaderType material_type, InstanceType model_type, uint64_t material_index) const; diff --git a/include/limitless/text/font_atlas.hpp b/include/limitless/text/font_atlas.hpp index 89dbe791..3c7a9f4d 100644 --- a/include/limitless/text/font_atlas.hpp +++ b/include/limitless/text/font_atlas.hpp @@ -55,6 +55,8 @@ namespace Limitless { explicit font_error(const std::string& error) : runtime_error{error} {} }; + class Assets; + class FontAtlas { public: static std::shared_ptr load( @@ -64,6 +66,14 @@ namespace Limitless { std::optional _cjk_variant = std::nullopt ); + static std::shared_ptr load( + Assets& assets, + const fs::path& path, + uint32_t pixel_size, + std::vector> codepoint_ranges = {}, + std::optional _cjk_variant = std::nullopt + ); + static std::shared_ptr make( uint32_t font_size_in_pixels, std::unordered_map> icons diff --git a/src/limitless/assets.cpp b/src/limitless/assets.cpp index ba1f1653..0d6a2d1b 100644 --- a/src/limitless/assets.cpp +++ b/src/limitless/assets.cpp @@ -14,9 +14,31 @@ #include #include +#include using namespace Limitless; +std::vector Assets::readFile(const fs::path& path) const { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) { + throw std::runtime_error("Failed to read file: " + path.string()); + } + const auto size = file.tellg(); + file.seekg(0, std::ios::beg); + std::vector data(static_cast(std::max(size, 0))); + if (size > 0) { + file.read(reinterpret_cast(data.data()), size); + if (file.gcount() != size) { + throw std::runtime_error("Truncated file: " + path.string()); + } + } + return data; +} + +bool Assets::fileExists(const fs::path& path) const { + return fs::is_regular_file(path); +} + Assets::Assets(const fs::path& _base_dir) noexcept : base_dir {_base_dir} , shader_dir {_base_dir / "../shaders"} { @@ -105,7 +127,7 @@ void Assets::load([[maybe_unused]] Context& context) { } void Assets::initialize(Context& ctx, const RendererSettings& settings) { - shaders.initialize(ctx, settings, shader_dir); + shaders.initialize(ctx, settings, shader_dir, *this); } void Assets::add(const Assets& other) { diff --git a/src/limitless/core/shader/shader.cpp b/src/limitless/core/shader/shader.cpp index e4f42554..511f3dff 100644 --- a/src/limitless/core/shader/shader.cpp +++ b/src/limitless/core/shader/shader.cpp @@ -7,9 +7,20 @@ using namespace Limitless; -Shader::Shader(fs::path _path, Type _type, const ShaderAction& action) +#include +#include +#include +#include +#include +#include +#include + +using namespace Limitless; + +Shader::Shader(fs::path _path, Type _type, const ShaderAction& action, Assets* _assets) : path {std::move(_path)} - , type {_type} { + , type {_type} + , assets {_assets} { source = getSource(path); replaceIncludes(path.parent_path()); @@ -86,6 +97,14 @@ void Shader::replaceKey(const std::string& key, const std::string& value) noexce std::string Shader::getSource(const fs::path& filepath) { try { + if (assets) { + if (!assets->fileExists(filepath)) { + throw shader_file_not_found(filepath.string()); + } + const auto bytes = assets->readFile(filepath); + return std::string(reinterpret_cast(bytes.data()), bytes.size()); + } + std::ifstream file(filepath); std::string file_source; @@ -99,6 +118,8 @@ std::string Shader::getSource(const fs::path& filepath) { } return file_source; + } catch (const shader_file_not_found&) { + throw; } catch (...) { throw shader_file_not_found(filepath.string()); } @@ -157,6 +178,8 @@ void Limitless::swap(Shader &lhs, Shader &rhs) noexcept { swap(lhs.path, rhs.path); swap(lhs.type, rhs.type); swap(lhs.id, rhs.id); + swap(lhs.assets, rhs.assets); + swap(lhs.include_entries, rhs.include_entries); } Shader::Shader(Shader&& rhs) noexcept : Shader() { diff --git a/src/limitless/core/shader/shader_compiler.cpp b/src/limitless/core/shader/shader_compiler.cpp index 1bfe9ed8..9a59217d 100644 --- a/src/limitless/core/shader/shader_compiler.cpp +++ b/src/limitless/core/shader/shader_compiler.cpp @@ -10,13 +10,15 @@ using namespace Limitless; -ShaderCompiler::ShaderCompiler(Context& _context) - : context {_context} { +ShaderCompiler::ShaderCompiler(Context& _context, Assets* _assets) + : context {_context} + , assets {_assets} { } -ShaderCompiler::ShaderCompiler(Context& _context, const RendererSettings& _settings) +ShaderCompiler::ShaderCompiler(Context& _context, const RendererSettings& _settings, Assets* _assets) : context {_context} - , render_settings {_settings} { + , render_settings {_settings} + , assets {_assets} { } void ShaderCompiler::checkStatus(const GLuint program_id) { @@ -77,7 +79,7 @@ std::shared_ptr ShaderCompiler::compile(const fs::path& path, con uint8_t shader_count {}; for (const auto& [extension, type] : shader_file_extensions) { try { - Shader shader { path.string() + extension.data(), type, action }; + Shader shader { path.string() + extension.data(), type, action, assets }; replaceCommonDefines(shader); diff --git a/src/limitless/loaders/gltf_model_loader.cpp b/src/limitless/loaders/gltf_model_loader.cpp index 92583624..e158c54b 100644 --- a/src/limitless/loaders/gltf_model_loader.cpp +++ b/src/limitless/loaders/gltf_model_loader.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,18 @@ using namespace Limitless; +struct CgltfDataGuard { + cgltf_data** data; + + ~CgltfDataGuard() { + if (*data) { + cgltf_free(*data); + } + } +}; + +static cgltf_data* parseGltfFromAssets(Assets& assets, const fs::path& path, bool load_buffers); + static std::string toString(cgltf_type type) { switch (type) { case cgltf_type_invalid: @@ -1309,26 +1322,8 @@ std::vector> GltfModelLoader::loadModel const auto base_model_name = path.stem().string(); const auto variant_model_name = base_model_name + "_" + std::move(variant_name); - cgltf_options opts = cgltf_options { - cgltf_file_type_invalid, // autodetect - 0, // auto json token count - cgltf_memory_options {nullptr, nullptr, nullptr}, - cgltf_file_options {nullptr, nullptr, nullptr} - }; - cgltf_data* out_data = nullptr; - - const auto path_str = path.string(); - - cgltf_result gltf = cgltf_parse_file(&opts, path_str.c_str(), &out_data); - if (gltf != cgltf_result_success) { - throw ModelLoadError { - "failed to parse GLTF model file " + path.string() + ": " - + std::to_string(static_cast(gltf))}; - } - - if (out_data->scenes == nullptr) { - throw ModelLoadError {"no scene"}; - } + cgltf_data* out_data = parseGltfFromAssets(assets, path, false); + CgltfDataGuard data_guard(&out_data); std::vector> mesh_materials; InstanceTypes instance_types = flags.additional_instance_types; @@ -1465,45 +1460,103 @@ std::shared_ptr GltfModelLoader::simplifyMesh( throw ModelLoadError {"unsupported vertex stream type for mesh simplification"}; } -struct CgltfDataGuard { - cgltf_data** data; +struct CgltfAssetsUserData { + Assets* assets; +}; - ~CgltfDataGuard() { - if (*data) { - cgltf_free(*data); +static void* cgltfAlloc(void* /*user*/, cgltf_size size) { + return std::malloc(size); +} + +static void cgltfFree(void* /*user*/, void* ptr) { + std::free(ptr); +} + +static cgltf_result cgltfReadFile( + const struct cgltf_memory_options* memory_options, + const struct cgltf_file_options* file_options, + const char* path, + cgltf_size* size, + void** data +) { + auto* user = static_cast(file_options->user_data); + try { + auto bytes = user->assets->readFile(path); + if (bytes.empty()) { + return cgltf_result_file_not_found; + } + void* (*memory_alloc)(void*, cgltf_size) = + memory_options->alloc_func ? memory_options->alloc_func : cgltfAlloc; + void* memory = memory_alloc(memory_options->user_data, bytes.size()); + if (!memory) { + return cgltf_result_out_of_memory; } + std::memcpy(memory, bytes.data(), bytes.size()); + *size = bytes.size(); + *data = memory; + return cgltf_result_success; + } catch (...) { + return cgltf_result_file_not_found; } -}; +} -std::shared_ptr -GltfModelLoader::loadModel(Assets& assets, const fs::path& path, const ModelLoaderFlags& flags) { - cgltf_options opts = cgltf_options { - cgltf_file_type_invalid, // autodetect - 0, // auto json token count - cgltf_memory_options {nullptr, nullptr, nullptr}, - cgltf_file_options {nullptr, nullptr, nullptr} - }; - cgltf_data* out_data = nullptr; - CgltfDataGuard data_guard(&out_data); +static void cgltfReleaseFile( + const struct cgltf_memory_options* memory_options, + const struct cgltf_file_options* /*file_options*/, + void* data +) { + void (*memory_free)(void*, void*) = + memory_options->free_func ? memory_options->free_func : cgltfFree; + memory_free(memory_options->user_data, data); +} + +static cgltf_options makeCgltfOptions(CgltfAssetsUserData& user_data) { + cgltf_options opts {}; + opts.file.read = cgltfReadFile; + opts.file.release = cgltfReleaseFile; + opts.file.user_data = &user_data; + return opts; +} - const auto path_str = path.string(); +static cgltf_data* parseGltfFromAssets(Assets& assets, const fs::path& path, bool load_buffers) { + CgltfAssetsUserData user_data {&assets}; + auto opts = makeCgltfOptions(user_data); + const auto path_str = path.string(); - cgltf_result gltf = cgltf_parse_file(&opts, path_str.c_str(), &out_data); - if (gltf != cgltf_result_success) { + cgltf_data* out_data = nullptr; + // cgltf_parse_file keeps the file bytes in data->file_data. For GLB, the BIN + // chunk (embedded meshes and images) is a pointer into those bytes, so they + // must outlive cgltf_data — a stack vector passed to cgltf_parse does not. + const auto parse_result = cgltf_parse_file(&opts, path_str.c_str(), &out_data); + if (parse_result != cgltf_result_success) { + if (out_data) { + cgltf_free(out_data); + } throw ModelLoadError { "failed to parse GLTF model file " + path.string() + ": " - + std::to_string(static_cast(gltf))}; + + std::to_string(static_cast(parse_result))}; } - auto result = cgltf_load_buffers(&opts, out_data, path_str.c_str()); - if (result != cgltf_result_success) { - throw ModelLoadError { - "failed to load buffers: " + std::to_string(static_cast(result))}; + if (load_buffers) { + const auto buffer_result = cgltf_load_buffers(&opts, out_data, path_str.c_str()); + if (buffer_result != cgltf_result_success) { + cgltf_free(out_data); + throw ModelLoadError { + "failed to load buffers: " + std::to_string(static_cast(buffer_result))}; + } } if (out_data->scenes == nullptr) { + cgltf_free(out_data); throw ModelLoadError {"no scene"}; } + return out_data; +} + +std::shared_ptr +GltfModelLoader::loadModel(Assets& assets, const fs::path& path, const ModelLoaderFlags& flags) { + cgltf_data* out_data = parseGltfFromAssets(assets, path, true); + CgltfDataGuard data_guard(&out_data); return ::loadModel(assets, path, *out_data, flags); } diff --git a/src/limitless/loaders/texture_loader.cpp b/src/limitless/loaders/texture_loader.cpp index 3eb425d1..0c7f8183 100644 --- a/src/limitless/loaders/texture_loader.cpp +++ b/src/limitless/loaders/texture_loader.cpp @@ -151,7 +151,10 @@ std::shared_ptr TextureLoader::load(Assets& assets, const fs::path& _pa stbi_set_flip_vertically_on_load(static_cast((int)flags.origin)); int width = 0, height = 0, channels = 0; - unsigned char* data = stbi_load(path.string().c_str(), &width, &height, &channels, 0); + const auto file_bytes = assets.readFile(path); + unsigned char* data = stbi_load_from_memory( + file_bytes.data(), static_cast(file_bytes.size()), &width, &height, &channels, 0 + ); if (!data) { throw std::runtime_error("Failed to load texture: " + path.string() + " " + stbi_failure_reason()); @@ -228,7 +231,7 @@ std::shared_ptr TextureLoader::load(Assets& assets, const std::string& return texture; } -std::shared_ptr TextureLoader::loadCubemap([[maybe_unused]] Assets& assets, const fs::path& path, const TextureLoaderFlags& flags) { +std::shared_ptr TextureLoader::loadCubemap(Assets& assets, const fs::path& path, const TextureLoaderFlags& flags) { constexpr std::array ext = { "_right", "_left", "_top", "_bottom", "_front", "_back" }; size_t i = 0; @@ -251,7 +254,10 @@ std::shared_ptr TextureLoader::loadCubemap(Assets& assets, const std::a for (size_t i = 0; i < data.size(); ++i) { std::string p = convertPathSeparators(paths[i]).string(); - data[i] = stbi_load(p.c_str(), &width, &height, &channels, 0); + const auto file_bytes = assets.readFile(paths[i]); + data[i] = stbi_load_from_memory( + file_bytes.data(), static_cast(file_bytes.size()), &width, &height, &channels, 0 + ); if (!data[i]) { throw texture_loader_exception("Failed to load texture: " + p + " " + stbi_failure_reason()); @@ -280,13 +286,16 @@ std::shared_ptr TextureLoader::loadCubemap(Assets& assets, const std::a return texture; } -GLFWimage TextureLoader::loadGLFWImage([[maybe_unused]] Assets& assets, const fs::path& _path, const TextureLoaderFlags& flags) { +GLFWimage TextureLoader::loadGLFWImage(Assets& assets, const fs::path& _path, const TextureLoaderFlags& flags) { auto path = convertPathSeparators(_path); stbi_set_flip_vertically_on_load(static_cast(flags.origin)); int width = 0, height = 0, channels = 0; - unsigned char* data = stbi_load(path.string().c_str(), &width, &height, &channels, 0); + const auto file_bytes = assets.readFile(path); + unsigned char* data = stbi_load_from_memory( + file_bytes.data(), static_cast(file_bytes.size()), &width, &height, &channels, 0 + ); if (data) { return GLFWimage{ width, height, data }; @@ -348,7 +357,7 @@ bool TextureLoader::isPowerOfTwo(int width, int height) { return ((width != 0) && !(width & (width - 1))) && ((height != 0) && !(height & (height - 1))); } -std::shared_ptr TextureLoader::load([[maybe_unused]] Assets &assets, const std::vector &paths, const TextureLoaderFlags &flags) { +std::shared_ptr TextureLoader::load(Assets &assets, const std::vector &paths, const TextureLoaderFlags &flags) { //TODO: size equality check Texture::Builder builder = Texture::builder(); @@ -368,7 +377,10 @@ std::shared_ptr TextureLoader::load([[maybe_unused]] Assets &assets, co stbi_set_flip_vertically_on_load(static_cast((int)flags.origin)); int width = 0, height = 0, channels = 0; - unsigned char* data = stbi_load(path.string().c_str(), &width, &height, &channels, 0); + const auto file_bytes = assets.readFile(path); + unsigned char* data = stbi_load_from_memory( + file_bytes.data(), static_cast(file_bytes.size()), &width, &height, &channels, 0 + ); if (!data) { throw std::runtime_error("Failed to load texture: " + path.string() + " " + stbi_failure_reason()); diff --git a/src/limitless/ms/material_compiler.cpp b/src/limitless/ms/material_compiler.cpp index a87ba0f2..6f052859 100644 --- a/src/limitless/ms/material_compiler.cpp +++ b/src/limitless/ms/material_compiler.cpp @@ -8,7 +8,7 @@ using namespace Limitless::ms; MaterialCompiler::MaterialCompiler(Context& context, Assets& _assets, const RendererSettings& settings) noexcept - : ShaderCompiler {context, settings} + : ShaderCompiler {context, settings, &_assets} , assets {_assets} { } diff --git a/src/limitless/shader_storage.cpp b/src/limitless/shader_storage.cpp index 277677de..db25a894 100644 --- a/src/limitless/shader_storage.cpp +++ b/src/limitless/shader_storage.cpp @@ -1,6 +1,7 @@ #include #include #include +#include using namespace Limitless; @@ -102,8 +103,8 @@ void ShaderStorage::add(const fx::UniqueEmitterShaderKey& emitter_type, std::sha } } -void ShaderStorage::initialize(Context& ctx, const RendererSettings& settings, const fs::path& shader_dir) { - ShaderCompiler compiler {ctx, settings}; +void ShaderStorage::initialize(Context& ctx, const RendererSettings& settings, const fs::path& shader_dir, Assets& assets) { + ShaderCompiler compiler {ctx, settings, &assets}; if (settings.bloom) { add("blur_downsample", compiler.compile(shader_dir / "postprocessing/bloom/blur_downsample")); diff --git a/src/limitless/text/font_atlas.cpp b/src/limitless/text/font_atlas.cpp index 8555607b..f13a5dd0 100644 --- a/src/limitless/text/font_atlas.cpp +++ b/src/limitless/text/font_atlas.cpp @@ -1,8 +1,10 @@ #include +#include #define STB_RECT_PACK_IMPLEMENTATION #include +#include #include using namespace Limitless; @@ -268,12 +270,7 @@ std::shared_ptr FontAtlas::make( ); } -std::shared_ptr FontAtlas::load( - const fs::path& path, - uint32_t pixel_size, - std::vector> codepoint_ranges, - std::optional cjk_variant -) { +static FT_Library getFreeType() { static FT_Library ft {nullptr}; if (!ft) { @@ -282,11 +279,15 @@ std::shared_ptr FontAtlas::load( } } - FT_Face face; - if (FT_New_Face(ft, path.string().c_str(), 0, &face)) { - throw font_error{"Failed to load the font at path"s + path.string()}; - } + return ft; +} +static std::shared_ptr loadFontAtlasFromFace( + FT_Face face, + uint32_t pixel_size, + std::vector> codepoint_ranges, + std::optional cjk_variant +) { FT_Set_Pixel_Sizes(face, 0, pixel_size); std::unordered_map glyph_for_char; @@ -358,6 +359,43 @@ std::shared_ptr FontAtlas::load( ); } +std::shared_ptr FontAtlas::load( + const fs::path& path, + uint32_t pixel_size, + std::vector> codepoint_ranges, + std::optional cjk_variant +) { + FT_Face face; + if (FT_New_Face(getFreeType(), path.string().c_str(), 0, &face)) { + throw font_error{"Failed to load the font at path"s + path.string()}; + } + + return loadFontAtlasFromFace(face, pixel_size, std::move(codepoint_ranges), cjk_variant); +} + +std::shared_ptr FontAtlas::load( + Assets& assets, + const fs::path& path, + uint32_t pixel_size, + std::vector> codepoint_ranges, + std::optional cjk_variant +) { + const auto bytes = assets.readFile(path); + + FT_Face face; + if (FT_New_Memory_Face( + getFreeType(), + reinterpret_cast(bytes.data()), + static_cast(bytes.size()), + 0, + &face + )) { + throw font_error{"Failed to load the font at path"s + path.string()}; + } + + return loadFontAtlasFromFace(face, pixel_size, std::move(codepoint_ranges), cjk_variant); +} + FontAtlas::~FontAtlas() = default; static size_t utf8CharLength(char c) {