From 962486435ae7c38ab4b0b83269b2f6d72acc9e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Tue, 21 Apr 2026 20:08:38 +0200 Subject: [PATCH 01/21] WIP add data to primitive --- source/GltfState/gltf_state.js | 6 +- source/Renderer/renderer.js | 45 ++-- source/gltf/primitive.js | 451 +++++++++++++++++++++++++++------ 3 files changed, 408 insertions(+), 94 deletions(-) diff --git a/source/GltfState/gltf_state.js b/source/GltfState/gltf_state.js index d4b76038..1452660c 100644 --- a/source/GltfState/gltf_state.js +++ b/source/GltfState/gltf_state.js @@ -55,12 +55,16 @@ class GltfState { KHR_materials_specular: true, /** KHR_materials_iridescence adds a thin-film iridescence effect */ KHR_materials_iridescence: true, + /** KHR_materials_diffuse_transmission */ KHR_materials_diffuse_transmission: true, /** KHR_materials_anisotropy defines microfacet grooves in the surface, stretching the specular reflection on the surface */ KHR_materials_anisotropy: true, /** KHR_materials_dispersion defines configuring the strength of the angular separation of colors (chromatic abberation)*/ KHR_materials_dispersion: true, - KHR_materials_emissive_strength: true + /** KHR_materials_emissive_strength allows configuring the strength of the emissive component */ + KHR_materials_emissive_strength: true, + /** KHR_gaussian_splatting */ + KHR_gaussian_splatting: true }, /** clear color expressed as list of ints in the range [0, 255] */ clearColor: [58, 64, 74, 255], diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index ca0cbc51..2a6b8df1 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -298,7 +298,8 @@ class gltfRenderer { state.gltf.materials[primitive.material].alphaMode !== "BLEND" && (state.gltf.materials[primitive.material].extensions === undefined || state.gltf.materials[primitive.material].extensions - .KHR_materials_transmission === undefined) + .KHR_materials_transmission === undefined) && + primitive.extensions?.KHR_gaussian_splatting === undefined ); let counter = 0; @@ -330,17 +331,19 @@ class gltfRenderer { // transparent drawables need sorting before they can be drawn this.transparentDrawables = drawables.filter( ({ primitive }) => - state.gltf.materials[primitive.material].alphaMode === "BLEND" && - (state.gltf.materials[primitive.material].extensions === undefined || - state.gltf.materials[primitive.material].extensions - .KHR_materials_transmission === undefined) + (state.gltf.materials[primitive.material].alphaMode === "BLEND" && + (state.gltf.materials[primitive.material].extensions === undefined || + state.gltf.materials[primitive.material].extensions + .KHR_materials_transmission === undefined)) || + primitive.extensions?.KHR_gaussian_splatting !== undefined ); this.transmissionDrawables = drawables.filter( ({ primitive }) => state.gltf.materials[primitive.material].extensions !== undefined && state.gltf.materials[primitive.material].extensions.KHR_materials_transmission !== - undefined + undefined && + primitive.extensions?.KHR_gaussian_splatting === undefined ); this.scatterDrawables = drawables.filter( @@ -349,7 +352,8 @@ class gltfRenderer { state.gltf.materials[primitive.material].extensions.KHR_materials_volume_scatter !== undefined && state.gltf.materials[primitive.material].extensions.KHR_materials_volume !== - undefined + undefined && + primitive.extensions?.KHR_gaussian_splatting === undefined ); } @@ -651,16 +655,23 @@ class gltfRenderer { this.transparentDrawables ); for (const drawable of this.transparentDrawables.filter((a) => a.depth <= 0)) { - let renderpassConfiguration = {}; - renderpassConfiguration.linearOutput = false; - renderpassConfiguration.frameBufferSize = [this.currentWidth, this.currentHeight]; - this.drawPrimitive( - state, - renderpassConfiguration, - drawable.primitive, - drawable.node, - this.viewProjectionMatrix - ); + if ( + drawable.primitive.extensions?.KHR_gaussian_splatting !== undefined && + state.renderingParameters.enabledExtensions.KHR_gaussian_splatting + ) { + //TODO + } else { + let renderpassConfiguration = {}; + renderpassConfiguration.linearOutput = false; + renderpassConfiguration.frameBufferSize = [this.currentWidth, this.currentHeight]; + this.drawPrimitive( + state, + renderpassConfiguration, + drawable.primitive, + drawable.node, + this.viewProjectionMatrix + ); + } } } diff --git a/source/gltf/primitive.js b/source/gltf/primitive.js index 4ea1937e..d3eaddfd 100644 --- a/source/gltf/primitive.js +++ b/source/gltf/primitive.js @@ -34,10 +34,228 @@ class gltfPrimitive extends GltfObject { this.hasTexcoord = false; this.hasColor = false; + // Gaussian Splatting + this.hasDegree1 = false; + this.hasDegree2 = false; + this.hasDegree3 = false; + this.linear = true; + this.positionTextureInfo = undefined; + this.rotationTextureInfo = undefined; + this.scaleTextureInfo = undefined; + this.opacityTextureInfo = undefined; + this.sphericalHarmonicsTextureInfo = undefined; + this.sortOrder = undefined; + // The primitive centroid is used for depth sorting. this.centroid = undefined; } + //Currently only support types relevant for gaussian splatting + _getInternalTextureFormat(componentType, componentCount, normalized = false) { + if (componentType === GL.FLOAT) { + switch (componentCount) { + case 1: // OPACITIY + return { internalFormat: GL.R32F, format: GL.RED }; + case 3: // POSITION, SCALE, Spherical Harmonics + return { internalFormat: GL.RGB32F, format: GL.RGB }; + case 4: // ROTATION + return { internalFormat: GL.RGBA32F, format: GL.RGBA }; + } + } + if (componentType === GL.UNSIGNED_BYTE) { + switch (componentCount) { + case 1: // OPACITY + return { internalFormat: GL.R8, format: GL.RED }; // Opacity is always normalized + case 3: // POSITION, SCALE + return { + internalFormat: normalized ? GL.RGB8 : GL.RGB8UI, + format: normalized ? GL.RGB : GL.RGB_INTEGER + }; + } + } + if (componentType === GL.BYTE) { + switch (componentCount) { + case 3: // POSITION + return { + internalFormat: normalized ? GL.RGB8_SNORM : GL.RGB8I, + format: normalized ? GL.RGB : GL.RGB_INTEGER + }; + case 4: // ROTATION + return { internalFormat: GL.RGBA8_SNORM, format: GL.RGBA }; // Rotation is always normalized + } + } + + // There is no normalized format for unsigned short and short. Needs to be resolved in the shader + if (componentType === GL.UNSIGNED_SHORT) { + switch (componentCount) { + case 1: // OPACITY + return { internalFormat: GL.R16UI, format: GL.RED_INTEGER }; + case 3: // POSITION, SCALE + return { internalFormat: GL.RGB16UI, format: GL.RGB_INTEGER }; + } + } + if (componentType === GL.SHORT) { + switch (componentCount) { + case 3: // POSITION + return { internalFormat: GL.RGB16I, format: GL.RGB_INTEGER }; + case 4: // ROTATION + return { internalFormat: GL.RGBA16I, format: GL.RGBA_INTEGER }; + } + } + console.error( + "Unsupported texture format for componentType:", + componentType, + "and componentCount:", + componentCount + ); + return undefined; + } + + _createDataTexture(gltf, webGlContext, attributeName, accessor) { + let texture = webGlContext.createTexture(); + webGlContext.bindTexture(webGlContext.TEXTURE_2D, texture); + // Set texture format and upload data. + const componentType = accessor.componentType; + const componentCount = accessor.getComponentCount(accessor.type); + const formats = this._getInternalTextureFormat( + componentType, + componentCount, + accessor.normalized + ); + if ( + formats.format === GL.RED_INTEGER || + formats.format === GL.RGB_INTEGER || + formats.format === GL.RGBA_INTEGER + ) { + this.defines.push(`${attributeName}_IS_INTEGER 1`); + if (accessor.normalized) { + this.defines.push(`${attributeName}_NEEDS_NORMALIZATION 1`); + } + } else { + this.defines.push(`${attributeName}_IS_FLOAT 1`); + } + const size = Math.ceil(Math.sqrt(accessor.count)); + const data = accessor.getDeinterlacedView(gltf); + const paddedData = new data.constructor(size * size * componentCount); + paddedData.set(data); + webGlContext.texImage2D( + webGlContext.TEXTURE_2D, + 0, //level + formats.internalFormat, + size, + size, + 0, //border + formats.format, + accessor.componentType, + paddedData + ); + // Ensure mipmapping is disabled and the sampler is configured correctly. + webGlContext.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_WRAP_S, GL.CLAMP_TO_EDGE); + webGlContext.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_WRAP_T, GL.CLAMP_TO_EDGE); + webGlContext.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_MIN_FILTER, GL.NEAREST); + webGlContext.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_MAG_FILTER, GL.NEAREST); + + // Now we add the morph target texture as a gltf texture info resource, so that + // we can just call webGl.setTexture(..., gltfTextureInfo, ...) in the renderer. + const image = new gltfImage( + undefined, // uri + GL.TEXTURE_2D, // type + 0, // mip level + undefined, // buffer view + undefined, // name + ImageMimeType.GLTEXTURE, // mimeType + texture // image + ); + gltf.images.push(image); + + gltf.samplers.push( + new gltfSampler(GL.NEAREST, GL.NEAREST, GL.CLAMP_TO_EDGE, GL.CLAMP_TO_EDGE, undefined) + ); + + const tex = new gltfTexture( + gltf.samplers.length - 1, + gltf.images.length - 1, + GL.TEXTURE_2D + ); + // The webgl texture is already initialized -> this flag informs + // webgl.setTexture about this. + tex.initialized = true; + + gltf.textures.push(tex); + + const textureInfo = new gltfTextureInfo(gltf.textures.length - 1, 0, true); + textureInfo.samplerName = `u_${attributeName}Sampler`; //TODO Check if this works + textureInfo.generateMips = false; + return textureInfo; + } + + _createDataTextureArray( + gltf, + webGlContext, + data, + width, + textureCount, + componentCount, + samplerName + ) { + let texture = webGlContext.createTexture(); + webGlContext.bindTexture(webGlContext.TEXTURE_2D_ARRAY, texture); + // Set texture format and upload data. + let internalFormat = componentCount === 4 ? webGlContext.RGBA32F : webGlContext.RGB32F; + let format = componentCount === 4 ? webGlContext.RGBA : webGlContext.RGB; + let type = webGlContext.FLOAT; + webGlContext.texImage3D( + webGlContext.TEXTURE_2D_ARRAY, + 0, //level + internalFormat, + width, + width, + textureCount, //Layer count + 0, //border + format, + type, + data + ); + // Ensure mipmapping is disabled and the sampler is configured correctly. + webGlContext.texParameteri(GL.TEXTURE_2D_ARRAY, GL.TEXTURE_WRAP_S, GL.CLAMP_TO_EDGE); + webGlContext.texParameteri(GL.TEXTURE_2D_ARRAY, GL.TEXTURE_WRAP_T, GL.CLAMP_TO_EDGE); + webGlContext.texParameteri(GL.TEXTURE_2D_ARRAY, GL.TEXTURE_MIN_FILTER, GL.NEAREST); + webGlContext.texParameteri(GL.TEXTURE_2D_ARRAY, GL.TEXTURE_MAG_FILTER, GL.NEAREST); + + // Now we add the morph target texture as a gltf texture info resource, so that + // we can just call webGl.setTexture(..., gltfTextureInfo, ...) in the renderer. + const image = new gltfImage( + undefined, // uri + GL.TEXTURE_2D_ARRAY, // type + 0, // mip level + undefined, // buffer view + undefined, // name + ImageMimeType.GLTEXTURE, // mimeType + texture // image + ); + gltf.images.push(image); + + gltf.samplers.push( + new gltfSampler(GL.NEAREST, GL.NEAREST, GL.CLAMP_TO_EDGE, GL.CLAMP_TO_EDGE, undefined) + ); + + const tex = new gltfTexture( + gltf.samplers.length - 1, + gltf.images.length - 1, + GL.TEXTURE_2D_ARRAY + ); + // The webgl texture is already initialized -> this flag informs + // webgl.setTexture about this. + tex.initialized = true; + + gltf.textures.push(tex); + + const textureInfo = new gltfTextureInfo(gltf.textures.length - 1, 0, true); + textureInfo.generateMips = false; + textureInfo.samplerName = samplerName; + return textureInfo; + } + initGl(gltf, webGlContext) { // Use the default glTF material. if (this.material === undefined) { @@ -92,6 +310,7 @@ class gltfPrimitive extends GltfObject { break; } let knownAttribute = true; + let isTexture = false; switch (attribute) { case "POSITION": this.skip = false; @@ -123,11 +342,43 @@ class gltfPrimitive extends GltfObject { case "WEIGHTS_1": this.hasWeights = true; break; + case "KHR_gaussian_splatting:ROTATION": + case "KHR_gaussian_splatting:SCALE": + case "KHR_gaussian_splatting:OPACITY": + isTexture = true; + break; + case "KHR_gaussian_splatting:SH_DEGREE_0_COEF_0": + isTexture = true; + break; + case "KHR_gaussian_splatting:SH_DEGREE_1_COEF_0": + case "KHR_gaussian_splatting:SH_DEGREE_1_COEF_1": + case "KHR_gaussian_splatting:SH_DEGREE_1_COEF_2": + isTexture = true; + this.hasDegree1 = true; + break; + case "KHR_gaussian_splatting:SH_DEGREE_2_COEF_0": + case "KHR_gaussian_splatting:SH_DEGREE_2_COEF_1": + case "KHR_gaussian_splatting:SH_DEGREE_2_COEF_2": + case "KHR_gaussian_splatting:SH_DEGREE_2_COEF_3": + case "KHR_gaussian_splatting:SH_DEGREE_2_COEF_4": + isTexture = true; + this.hasDegree2 = true; + break; + case "KHR_gaussian_splatting:SH_DEGREE_3_COEF_0": + case "KHR_gaussian_splatting:SH_DEGREE_3_COEF_1": + case "KHR_gaussian_splatting:SH_DEGREE_3_COEF_2": + case "KHR_gaussian_splatting:SH_DEGREE_3_COEF_3": + case "KHR_gaussian_splatting:SH_DEGREE_3_COEF_4": + case "KHR_gaussian_splatting:SH_DEGREE_3_COEF_5": + case "KHR_gaussian_splatting:SH_DEGREE_3_COEF_6": + isTexture = true; + this.hasDegree3 = true; + break; default: knownAttribute = false; console.log("Unknown attribute: " + attribute); } - if (knownAttribute) { + if (knownAttribute && !isTexture) { const idx = this.attributes[attribute]; this.glAttributes.push({ attribute: attribute, @@ -138,6 +389,122 @@ class gltfPrimitive extends GltfObject { } } + // Gaussian Splatting + if (this.extensions?.KHR_gaussian_splatting !== undefined) { + const extension = this.extensions.KHR_gaussian_splatting; + if (extension.kernel !== "ellipse") { + console.warn( + `Unsupported kernel type for Gaussian Splatting: ${extension.kernel}. Using ellipse kernel.` + ); + } + if (extension.colorSpace === "srgb_rec_709_display") { + this.linear = false; + } else if (extension.colorSpace !== "lin_rec709_display") { + console.warn( + `Unsupported color space for Gaussian Splatting: ${extension.colorSpace}. Using linear Rec. 709 display.` + ); + } + if (extension.projection !== undefined && extension.projection !== "perspective") { + console.warn( + `Unsupported projection type for Gaussian Splatting: ${extension.projection}. Using perspective projection.` + ); + } + if ( + extension.sortingMethod !== undefined && + extension.sortingMethod !== "cameraDistance" + ) { + console.warn( + `Unsupported sorting method for Gaussian Splatting: ${extension.sortingMethod}. Using camera distance.` + ); + } + if (this.hasDegree2 && !this.hasDegree1) { + console.warn( + "Degree 2 SH Coefficients provided without Degree 1. This is not supported and Degree 2 coefficients will be ignored." + ); + this.hasDegree2 = false; + } + if (this.hasDegree3 && (!this.hasDegree1 || !this.hasDegree2)) { + console.warn( + "Degree 3 SH Coefficients provided without Degree 1 or Degree 2. This is not supported and Degree 3 coefficients will be ignored." + ); + this.hasDegree3 = false; + } + + const max2DTextureSize = Math.pow(webGlContext.getParameter(GL.MAX_TEXTURE_SIZE), 2); + const vertexCount = + gltf.accessors[this.attributes["KHR_gaussian_splatting:SH_DEGREE_0_COEF_0"]].count; + const singleTextureWidth = Math.ceil(Math.sqrt(vertexCount)); + const singleTextureSize = Math.pow(singleTextureWidth, 2); + + if (vertexCount > max2DTextureSize) { + console.error("Vertex count exceeds maximum 2D texture size."); + return; + } + + this.positionTextureInfo = this._createDataTexture( + gltf, + webGlContext, + "POSITION", + gltf.accessors[this.attributes.POSITION] + ); + + this.rotationTextureInfo = this._createDataTexture( + gltf, + webGlContext, + "KHR_gaussian_splatting_ROTATION", + gltf.accessors[this.attributes["KHR_gaussian_splatting:ROTATION"]] + ); + + this.scaleTextureInfo = this._createDataTexture( + gltf, + webGlContext, + "KHR_gaussian_splatting_SCALE", + gltf.accessors[this.attributes["KHR_gaussian_splatting:SCALE"]] + ); + + this.opacityTextureInfo = this._createDataTexture( + gltf, + webGlContext, + "KHR_gaussian_splatting_OPACITY", + gltf.accessors[this.attributes["KHR_gaussian_splatting:OPACITY"]] + ); + + let textureAtlasSize = 1; + if (this.hasDegree1) { + this.defines.push("HAS_GAUSSIAN_SPLATTING_DEGREE_1 1"); + textureAtlasSize += 3; + if (this.hasDegree2) { + this.defines.push("HAS_GAUSSIAN_SPLATTING_DEGREE_2 1"); + textureAtlasSize += 5; + if (this.hasDegree3) { + this.defines.push("HAS_GAUSSIAN_SPLATTING_DEGREE_3 1"); + textureAtlasSize += 7; + } + } + } + const shData = new Float32Array(singleTextureSize * textureAtlasSize * 3); + const shAttributes = Object.keys(this.attributes) + .filter((attr) => attr.startsWith("KHR_gaussian_splatting:SH_DEGREE_")) + .sort(); + for (let i = 0; i < shAttributes.length; i++) { + const accessor = gltf.accessors[this.attributes[shAttributes[i]]]; + const data = accessor.getDeinterlacedView(gltf); + shData.set(data, i * singleTextureSize * 3); + } + this._createDataTextureArray( + gltf, + webGlContext, + shData, + singleTextureWidth, + textureAtlasSize, + 3, + "u_SHCoefficients" + ); + + this.sortOrder = new Uint32Array(vertexCount); + for (let i = 0; i < vertexCount; i++) this.sortOrder[i] = i; + } + // MORPH TARGETS if (this.targets !== undefined && this.targets.length > 0) { const max2DTextureSize = Math.pow(webGlContext.getParameter(GL.MAX_TEXTURE_SIZE), 2); @@ -237,83 +604,15 @@ class gltfPrimitive extends GltfObject { } // Add the morph target texture. - // We have to create a WebGL2 texture as the format of the - // morph target texture has to be explicitly specified - // (gltf image would assume uint8). - let texture = webGlContext.createTexture(); - webGlContext.bindTexture(webGlContext.TEXTURE_2D_ARRAY, texture); - // Set texture format and upload data. - let internalFormat = webGlContext.RGBA32F; - let format = webGlContext.RGBA; - let type = webGlContext.FLOAT; - let data = morphTargetTextureArray; - webGlContext.texImage3D( - webGlContext.TEXTURE_2D_ARRAY, - 0, //level - internalFormat, - width, + this._createDataTextureArray( + gltf, + webGlContext, + morphTargetTextureArray, width, - targetCount * attributes.length, //Layer count - 0, //border - format, - type, - data - ); - // Ensure mipmapping is disabled and the sampler is configured correctly. - webGlContext.texParameteri( - GL.TEXTURE_2D_ARRAY, - GL.TEXTURE_WRAP_S, - GL.CLAMP_TO_EDGE - ); - webGlContext.texParameteri( - GL.TEXTURE_2D_ARRAY, - GL.TEXTURE_WRAP_T, - GL.CLAMP_TO_EDGE - ); - webGlContext.texParameteri(GL.TEXTURE_2D_ARRAY, GL.TEXTURE_MIN_FILTER, GL.NEAREST); - webGlContext.texParameteri(GL.TEXTURE_2D_ARRAY, GL.TEXTURE_MAG_FILTER, GL.NEAREST); - - // Now we add the morph target texture as a gltf texture info resource, so that - // we can just call webGl.setTexture(..., gltfTextureInfo, ...) in the renderer. - const morphTargetImage = new gltfImage( - undefined, // uri - GL.TEXTURE_2D_ARRAY, // type - 0, // mip level - undefined, // buffer view - undefined, // name - ImageMimeType.GLTEXTURE, // mimeType - texture // image - ); - gltf.images.push(morphTargetImage); - - gltf.samplers.push( - new gltfSampler( - GL.NEAREST, - GL.NEAREST, - GL.CLAMP_TO_EDGE, - GL.CLAMP_TO_EDGE, - undefined - ) - ); - - const morphTargetTexture = new gltfTexture( - gltf.samplers.length - 1, - gltf.images.length - 1, - GL.TEXTURE_2D_ARRAY - ); - // The webgl texture is already initialized -> this flag informs - // webgl.setTexture about this. - morphTargetTexture.initialized = true; - - gltf.textures.push(morphTargetTexture); - - this.morphTargetTextureInfo = new gltfTextureInfo( - gltf.textures.length - 1, - 0, - true + targetCount * attributes.length, + 4, + "u_MorphTargetsSampler" ); - this.morphTargetTextureInfo.samplerName = "u_MorphTargetsSampler"; - this.morphTargetTextureInfo.generateMips = false; } else { console.warn("Mesh of Morph targets too big. Cannot apply morphing."); } From 0fe18370b2bad78aaf4c66e1c09482e0df23bc61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Wed, 22 Apr 2026 15:23:15 +0200 Subject: [PATCH 02/21] Improve error handling --- source/gltf/primitive.js | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/source/gltf/primitive.js b/source/gltf/primitive.js index d207d041..825fbeec 100644 --- a/source/gltf/primitive.js +++ b/source/gltf/primitive.js @@ -112,6 +112,9 @@ class gltfPrimitive extends GltfObject { } _createDataTexture(gltf, webGlContext, attributeName, accessor) { + if (accessor === undefined) { + return undefined; + } let texture = webGlContext.createTexture(); webGlContext.bindTexture(webGlContext.TEXTURE_2D, texture); // Set texture format and upload data. @@ -438,6 +441,7 @@ class gltfPrimitive extends GltfObject { if (vertexCount > max2DTextureSize) { console.error("Vertex count exceeds maximum 2D texture size."); + this.skip = true; return; } @@ -455,6 +459,14 @@ class gltfPrimitive extends GltfObject { gltf.accessors[this.attributes["KHR_gaussian_splatting:ROTATION"]] ); + if (this.rotationTextureInfo === undefined) { + console.error( + "Rotation attribute is required for Gaussian Splatting but not found. Skipping primitive." + ); + this.skip = true; + return; + } + this.scaleTextureInfo = this._createDataTexture( gltf, webGlContext, @@ -462,6 +474,14 @@ class gltfPrimitive extends GltfObject { gltf.accessors[this.attributes["KHR_gaussian_splatting:SCALE"]] ); + if (this.scaleTextureInfo === undefined) { + console.error( + "Scale attribute is required for Gaussian Splatting but not found. Skipping primitive." + ); + this.skip = true; + return; + } + this.opacityTextureInfo = this._createDataTexture( gltf, webGlContext, @@ -469,6 +489,22 @@ class gltfPrimitive extends GltfObject { gltf.accessors[this.attributes["KHR_gaussian_splatting:OPACITY"]] ); + if (this.opacityTextureInfo === undefined) { + console.error( + "Opacity attribute is required for Gaussian Splatting but not found. Skipping primitive." + ); + this.skip = true; + return; + } + + if (this.attributes["KHR_gaussian_splatting:SH_DEGREE_0_COEF_0"] === undefined) { + console.error( + "SH Degree 0 Coefficient 0 attribute is required for Gaussian Splatting but not found. Skipping primitive." + ); + this.skip = true; + return; + } + let textureAtlasSize = 1; if (this.hasDegree1) { this.defines.push("HAS_GAUSSIAN_SPLATTING_DEGREE_1 1"); From a0abde8b9f93f96d59950b9cf24e1c5c66570e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Mon, 27 Apr 2026 15:48:50 +0200 Subject: [PATCH 03/21] WIP add shaders --- source/Renderer/renderer.js | 121 +++++++++++++++++- source/Renderer/shader.js | 12 ++ source/Renderer/shaders/splat.frag | 23 ++++ source/Renderer/shaders/splat.vert | 191 +++++++++++++++++++++++++++++ source/gltf/primitive.js | 29 +++-- 5 files changed, 364 insertions(+), 12 deletions(-) create mode 100644 source/Renderer/shaders/splat.frag create mode 100644 source/Renderer/shaders/splat.vert diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index 2a6b8df1..6c25d02b 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -19,6 +19,8 @@ import cubemapVertShader from "./shaders/cubemap.vert"; import cubemapFragShader from "./shaders/cubemap.frag"; import scatterShader from "./shaders/scatter.frag"; import specularGlossinesShader from "./shaders/specular_glossiness.frag"; +import splatVertexShader from "./shaders/splat.vert"; +import splatFragmentShader from "./shaders/splat.frag"; import { gltfLight } from "../gltf/light.js"; import { jsToGl } from "../gltf/utils.js"; import { gltfMaterial } from "../gltf/material.js"; @@ -61,6 +63,8 @@ class gltfRenderer { shaderSources.set("cubemap.vert", cubemapVertShader); shaderSources.set("cubemap.frag", cubemapFragShader); shaderSources.set("specular_glossiness.frag", specularGlossinesShader); + shaderSources.set("splat.vert", splatVertexShader); + shaderSources.set("splat.frag", splatFragmentShader); this.shaderCache = new ShaderCache(shaderSources, this.webGl); @@ -86,6 +90,9 @@ class gltfRenderer { this.maxVertAttributes = undefined; this.instanceBuffer = undefined; + + this.splatVBO = undefined; + this.currentSortBuffer = undefined; } ///////////////////////////////////////////////////////////////////// @@ -181,6 +188,14 @@ class gltfRenderer { context.viewport(0, 0, this.opaqueFramebufferWidth, this.opaqueFramebufferHeight); context.bindFramebuffer(context.FRAMEBUFFER, null); + const quatVertices = new Float32Array([-1, -1, 1, -1, -1, 1, 1, 1]); + this.splatVBO = context.createBuffer(); + context.bindBuffer(context.ARRAY_BUFFER, this.splatVBO); + context.bufferData(context.ARRAY_BUFFER, quatVertices, context.STATIC_DRAW); + context.bindBuffer(context.ARRAY_BUFFER, null); + + this.currentSortBuffer = context.createBuffer(); + this.maxVertAttributes = context.getParameter(context.MAX_VERTEX_ATTRIBS); this.initialized = true; @@ -659,7 +674,13 @@ class gltfRenderer { drawable.primitive.extensions?.KHR_gaussian_splatting !== undefined && state.renderingParameters.enabledExtensions.KHR_gaussian_splatting ) { - //TODO + this.drawSplat( + state, + drawable.primitive, + drawable.node, + this.projMatrix, + this.viewMatrix + ); } else { let renderpassConfiguration = {}; renderpassConfiguration.linearOutput = false; @@ -675,6 +696,104 @@ class gltfRenderer { } } + drawSplat(state, primitive, node, projectionMatrix, viewMatrix) { + if (primitive.skip) return; + let vertDefines = primitive.defines.slice(); + if (primitive.linear === true) { + vertDefines.push("LINEAR_OUTPUT 1"); + } + + const fragmentHash = this.shaderCache.selectShader("splat.frag", vertDefines); + const vertexHash = this.shaderCache.selectShader("splat.vert", vertDefines); + if (fragmentHash && vertexHash) { + this.shader = this.shaderCache.getShaderProgram(fragmentHash, vertexHash); + } + + if (this.shader === undefined) { + return; + } + + this.webGl.context.useProgram(this.shader.program); + + this.webGl.context.uniformMatrix4fv( + this.shader.getUniformLocation("u_ModelMatrix"), + false, + node.worldTransform + ); + this.webGl.context.uniformMatrix4fv( + this.shader.getUniformLocation("u_ViewMatrix"), + false, + this.viewMatrix + ); + this.webGl.context.uniformMatrix4fv( + this.shader.getUniformLocation("u_ProjectionMatrix"), + false, + this.projMatrix + ); + this.webGl.context.uniform2i( + this.shader.getUniformLocation("u_FramebufferSize"), + this.currentWidth, + this.currentHeight + ); + // The projection matrix stores the focal length in the first and second element of the diagonal. + // We need to convert from NDC space to screen space, which is done by multiplying with the framebuffer dimensions and dividing by 2, since NDC goes from -1 to 1. + this.webGl.context.uniform2f( + this.shader.getUniformLocation("u_FocalLength"), + this.projMatrix[0] * this.currentWidth * 0.5, + this.projMatrix[5] * this.currentHeight * 0.5 + ); + this.shader.updateUniform("u_TextureWidth", primitive.splatTextureWidth); + + this.webGl.context.enable(GL.BLEND); + this.webGl.context.blendFuncSeparate( + GL.ONE, + GL.ONE_MINUS_SRC_ALPHA, + GL.ONE, + GL.ONE_MINUS_SRC_ALPHA + ); + this.webGl.context.blendEquation(GL.FUNC_ADD); + + let textureIndex = 0; + + let location = this.shader.getUniformLocation(primitive.positionTextureInfo.samplerName); + this.webGl.setTexture(location, state.gltf, primitive.positionTextureInfo, textureIndex++); + + location = this.shader.getUniformLocation(primitive.rotationTextureInfo.samplerName); + this.webGl.setTexture(location, state.gltf, primitive.rotationTextureInfo, textureIndex++); + + location = this.shader.getUniformLocation(primitive.scaleTextureInfo.samplerName); + this.webGl.setTexture(location, state.gltf, primitive.scaleTextureInfo, textureIndex++); + + location = this.shader.getUniformLocation(primitive.opacityTextureInfo.samplerName); + this.webGl.setTexture(location, state.gltf, primitive.opacityTextureInfo, textureIndex++); + + location = this.shader.getUniformLocation(primitive.shArray.samplerName); + this.webGl.setTexture(location, state.gltf, primitive.shArray, textureIndex++); + + this.webGl.context.bindBuffer(this.webGl.context.ARRAY_BUFFER, this.splatVBO); + location = this.shader.getAttributeLocation("a_position"); + this.webGl.context.enableVertexAttribArray(location); + this.webGl.context.vertexAttribPointer(location, 2, GL.FLOAT, false, 0, 0); + this.webGl.context.bindBuffer(this.webGl.context.ARRAY_BUFFER, null); + + this.webGl.context.bindBuffer(this.webGl.context.ARRAY_BUFFER, this.currentSortBuffer); + this.webGl.context.bufferData( + this.webGl.context.ARRAY_BUFFER, + primitive.sortOrder, + this.webGl.context.DYNAMIC_DRAW + ); + location = this.shader.getAttributeLocation("a_instance_sort_index"); + this.webGl.context.enableVertexAttribArray(location); + this.webGl.context.vertexAttribIPointer(location, 1, GL.UNSIGNED_INT, 0, 0); + this.webGl.context.vertexAttribDivisor(location, 1); + this.webGl.context.bindBuffer(this.webGl.context.ARRAY_BUFFER, null); + + this.webGl.context.drawArraysInstanced(GL.TRIANGLE_STRIP, 0, 4, primitive.sortOrder.length); + + this.webGl.context.vertexAttribDivisor(location, 0); + this.webGl.context.disableVertexAttribArray(location); + } + // vertices with given material // prettier-ignore drawPrimitive(state, renderpassConfiguration, primitive, node, viewProjectionMatrix, sampledTextures, instanceOffset = undefined) diff --git a/source/Renderer/shader.js b/source/Renderer/shader.js index f33ff1a7..958f957f 100644 --- a/source/Renderer/shader.js +++ b/source/Renderer/shader.js @@ -154,6 +154,18 @@ class gltfShader { } break; } + case GL.UNSIGNED_INT: { + if ( + Array.isArray(value) || + value instanceof Uint32Array || + value instanceof Int32Array + ) { + this.gl.context.uniform1uiv(uniform.loc, value); + } else { + this.gl.context.uniform1ui(uniform.loc, value); + } + break; + } case GL.INT_VEC2: this.gl.context.uniform2iv(uniform.loc, value); break; diff --git a/source/Renderer/shaders/splat.frag b/source/Renderer/shaders/splat.frag new file mode 100644 index 00000000..164a6b82 --- /dev/null +++ b/source/Renderer/shaders/splat.frag @@ -0,0 +1,23 @@ +precision highp float; + +in vec4 v_color; // (r, g, b, opacity) +in vec2 v_uv; // pixel-space offset from splat centre +in vec3 v_conic; // (A, B, C) coefficients of the conic section defining the splat's shape + +out vec4 g_finalColor; + +void main() { + + // Equation 4 + float exponent = -0.5 * (v_conic.x * v_uv.x * v_uv.x + v_conic.z * v_uv.y * v_uv.y) - v_conic.y * v_uv.x * v_uv.y; + + if (exponent > 0.0) { + discard; + } + + float alpha = min(0.99, exp(exponent) * v_color.a); // opacity modulated by Gaussian falloff + if (alpha < 1.0 / 255.0) { + discard; + } + g_finalColor = vec4(v_color.rgb * alpha, alpha); // premultiplied-alpha output +} diff --git a/source/Renderer/shaders/splat.vert b/source/Renderer/shaders/splat.vert new file mode 100644 index 00000000..26cedfa8 --- /dev/null +++ b/source/Renderer/shaders/splat.vert @@ -0,0 +1,191 @@ +// Original CUDA kernel https://github.com/graphdeco-inria/diff-gaussian-rasterization/blob/9c5c2028f6fbee2be239bc4c9421ff894fe4fbe0/cuda_rasterizer/forward.cu + +precision highp float; +precision highp int; + +in vec2 a_position; +in uint a_instance_sort_index; + +out vec4 v_color; +out vec2 v_uv; +out vec3 v_conic; + +uniform mat4 u_ProjectionMatrix; +uniform mat4 u_ViewMatrix; +uniform mat4 u_ModelMatrix; +uniform uint u_TextureWidth; +uniform ivec2 u_FramebufferSize; +uniform vec2 u_FocalLength; + +#ifdef POSITION_IS_INTEGER +uniform isampler2D u_POSITIONSampler; +#elif defined(POSITION_IS_UINTEGER) +uniform usampler2D u_POSITIONSampler; +#else +uniform sampler2D u_POSITIONSampler; +#endif + +#ifdef ROTATION_IS_INTEGER +uniform isampler2D u_ROTATIONSampler; +#else +uniform sampler2D u_ROTATIONSampler; +#endif + +#ifdef SCALE_IS_UINTEGER +uniform usampler2D u_SCALESampler; +#else +uniform sampler2D u_SCALESampler; +#endif + +#ifdef OPACITY_IS_UINTEGER +uniform usampler2D u_OPACITYSampler; +#else +uniform sampler2D u_OPACITYSampler; +#endif + +uniform highp sampler2DArray u_SHCoefficientsSampler; + +mat3 computeC(vec3 scale, vec4 rotation) +{ + + float qx = rotation.x; + float qy = rotation.y; + float qz = rotation.z; + float qw = rotation.w; + float sx = scale.x; + float sy = scale.y; + float sz = scale.z; + + mat3 C = mat3( + sx * (1.0 - 2.0*(qy*qy + qz*qz)), sx * (2.0*(qx*qy + qw*qz)), sx * (2.0*(qx*qz - qw*qy)), + sy * (2.0*(qx*qy - qw*qz)), sy * (1.0 - 2.0*(qx*qx + qz*qz)), sy * (2.0*(qy*qz + qw*qx)), + sz * (2.0*(qx*qz + qw*qy)), sz * (2.0*(qy*qz - qw*qx)), sz * (1.0 - 2.0*(qx*qx + qy*qy)) + ); + + return C; +} + +vec3 computeCameraCovariance(mat3 world_covariance, vec3 view_splat_center) +{ + float x = view_splat_center.x; + float y = view_splat_center.y; + float z = view_splat_center.z; + + mat3 J = mat3( + u_FocalLength.x / z, 0.0, -(u_FocalLength.x * x) / (z * z), + 0.0, u_FocalLength.y / z, -(u_FocalLength.y * y) / (z * z), + 0, 0, 0 + ); + + mat3 W = transpose(mat3(u_ViewMatrix)); + + // Equation 5 + mat3 T = W * J; + + mat3 cov = transpose(T) * transpose(world_covariance) * T; + + // Low-pass filter (EWA Splatting) + cov[0][0] += 0.3; + cov[1][1] += 0.3; + + return vec3(cov[0][0], cov[0][1], cov[1][1]); +} + +vec3 calculateSphericalHarmonics(ivec2 texelCoord) +{ + ivec3 coord = ivec3(texelCoord.x, texelCoord.y, 0); + // Degree 0 + vec3 sh0 = texelFetch(u_SHCoefficientsSampler, coord, 0).rgb; + + vec3 result = sh0 * 0.2820947917738781; + + //TODO + + result += 0.5; + return result; + +} + + +void main() +{ + vec3 splat_center; + ivec2 texelCoord = ivec2(a_instance_sort_index % u_TextureWidth, a_instance_sort_index / u_TextureWidth); +#if defined(POSITION_IS_INTEGER) || defined(POSITION_IS_UINTEGER) + splat_center = vec3(texelFetch(u_POSITIONSampler, texelCoord, 0).xyz); +#ifdef POSITION_NEEDS_NORMALIZATION +#ifdef POSITION_IS_UINTEGER + splat_center = splat_center / 65535.0; // normalize unsigned short +#else + splat_center = max(splat_center / 32767.0, -1.0); // normalize signed short +#endif +#endif +#else + splat_center = texelFetch(u_POSITIONSampler, texelCoord, 0).xyz; +#endif + + vec4 rotation; +#if defined(ROTATION_IS_INTEGER) + rotation = vec4(texelFetch(u_ROTATIONSampler, texelCoord, 0).xyzw); +#ifdef ROTATION_NEEDS_NORMALIZATION + rotation = max(rotation / 32767.0, -1.0); // normalize signed short +#endif +#else + rotation = texelFetch(u_ROTATIONSampler, texelCoord, 0); +#endif + + vec3 scale; +#if defined(SCALE_IS_UINTEGER) + scale = vec3(texelFetch(u_SCALESampler, texelCoord, 0).xyz); +#ifdef SCALE_NEEDS_NORMALIZATION + scale = scale / 65535.0; // normalize unsigned short +#endif +#else + scale = texelFetch(u_SCALESampler, texelCoord, 0).xyz; +#endif + + float opacity; +#if defined(OPACITY_IS_UINTEGER) + opacity = float(texelFetch(u_OPACITYSampler, texelCoord, 0).x); +#ifdef OPACITY_NEEDS_NORMALIZATION + opacity = opacity / 65535.0; // normalize unsigned short +#endif +#else + opacity = texelFetch(u_OPACITYSampler, texelCoord, 0).x; +#endif + + splat_center = (u_ModelMatrix * vec4(splat_center, 1.0)).xyz; + + vec4 view_splat_center = u_ViewMatrix * vec4(splat_center, 1.0); + vec4 clip_splat_center = u_ProjectionMatrix * view_splat_center; + clip_splat_center /= clip_splat_center.w; // perspective division + + mat3 C = computeC(scale, rotation); + mat3 M = mat3(u_ModelMatrix); + mat3 world_covariance = M * C * transpose(C) * transpose(M); + vec3 camera_covariance = computeCameraCovariance(world_covariance, view_splat_center.xyz); + + float a = camera_covariance.x; // Variance x + float b = camera_covariance.y; // Covariance xy + float c = camera_covariance.z; // Variance y + + float det = (a * c - b * b); + if (det == 0.0) { + gl_Position = vec4(0.0); + return; + } + + float det_inv = 1.0 / det; + // Calculate the inverse of the covariance matrix + v_conic = vec3(c * det_inv, -b * det_inv, a * det_inv); + + // pow(e, pow(-3.4, 2) * -0.5) = 1/255, so 3.4 is the standard deviation in terms of the Gaussian falloff that results in a radius of 1 pixel when the variance is 1. + // sqrt(a) and sqrt(c) are the standard deviations in x and y direction, so multiplying them with 3.4 gives us the radius in pixels where the Gaussian falloff results in 1/255 opacity. + vec2 quad_pixel_size = vec2(3.4 * sqrt(a), 3.4 * sqrt(c)); // screen space half quad height and width + vec2 quad_ndc_size = quad_pixel_size / vec2(u_FramebufferSize) * 2.0; // in ndc space + clip_splat_center.xy = clip_splat_center.xy + a_position * quad_ndc_size; + v_uv = a_position * quad_pixel_size; + gl_Position = clip_splat_center; + + v_color = vec4(calculateSphericalHarmonics(texelCoord), opacity); +} diff --git a/source/gltf/primitive.js b/source/gltf/primitive.js index 825fbeec..a8f3d7fe 100644 --- a/source/gltf/primitive.js +++ b/source/gltf/primitive.js @@ -45,6 +45,7 @@ class gltfPrimitive extends GltfObject { this.opacityTextureInfo = undefined; this.sphericalHarmonicsTextureInfo = undefined; this.sortOrder = undefined; + this.splatTextureWidth = undefined; // The primitive centroid is used for depth sorting. this.centroid = undefined; @@ -130,8 +131,13 @@ class gltfPrimitive extends GltfObject { formats.format === GL.RGB_INTEGER || formats.format === GL.RGBA_INTEGER ) { - this.defines.push(`${attributeName}_IS_INTEGER 1`); + if (componentType === GL.UNSIGNED_BYTE || componentType === GL.UNSIGNED_SHORT) { + this.defines.push(`${attributeName}_IS_UINTEGER 1`); + } else { + this.defines.push(`${attributeName}_IS_INTEGER 1`); + } if (accessor.normalized) { + // Only shorts do not support normalized integer formats, so we need to normalize them manually in the shader. this.defines.push(`${attributeName}_NEEDS_NORMALIZATION 1`); } } else { @@ -400,7 +406,7 @@ class gltfPrimitive extends GltfObject { `Unsupported kernel type for Gaussian Splatting: ${extension.kernel}. Using ellipse kernel.` ); } - if (extension.colorSpace === "srgb_rec_709_display") { + if (extension.colorSpace === "srgb_rec709_display") { this.linear = false; } else if (extension.colorSpace !== "lin_rec709_display") { console.warn( @@ -436,8 +442,8 @@ class gltfPrimitive extends GltfObject { const max2DTextureSize = Math.pow(webGlContext.getParameter(GL.MAX_TEXTURE_SIZE), 2); const vertexCount = gltf.accessors[this.attributes["KHR_gaussian_splatting:SH_DEGREE_0_COEF_0"]].count; - const singleTextureWidth = Math.ceil(Math.sqrt(vertexCount)); - const singleTextureSize = Math.pow(singleTextureWidth, 2); + this.splatTextureWidth = Math.ceil(Math.sqrt(vertexCount)); + const singleTextureSize = Math.pow(this.splatTextureWidth, 2); if (vertexCount > max2DTextureSize) { console.error("Vertex count exceeds maximum 2D texture size."); @@ -455,7 +461,7 @@ class gltfPrimitive extends GltfObject { this.rotationTextureInfo = this._createDataTexture( gltf, webGlContext, - "KHR_gaussian_splatting_ROTATION", + "ROTATION", gltf.accessors[this.attributes["KHR_gaussian_splatting:ROTATION"]] ); @@ -470,7 +476,7 @@ class gltfPrimitive extends GltfObject { this.scaleTextureInfo = this._createDataTexture( gltf, webGlContext, - "KHR_gaussian_splatting_SCALE", + "SCALE", gltf.accessors[this.attributes["KHR_gaussian_splatting:SCALE"]] ); @@ -485,7 +491,7 @@ class gltfPrimitive extends GltfObject { this.opacityTextureInfo = this._createDataTexture( gltf, webGlContext, - "KHR_gaussian_splatting_OPACITY", + "OPACITY", gltf.accessors[this.attributes["KHR_gaussian_splatting:OPACITY"]] ); @@ -527,14 +533,15 @@ class gltfPrimitive extends GltfObject { const data = accessor.getDeinterlacedView(gltf); shData.set(data, i * singleTextureSize * 3); } - this._createDataTextureArray( + + this.shArray = this._createDataTextureArray( gltf, webGlContext, shData, - singleTextureWidth, + this.splatTextureWidth, textureAtlasSize, 3, - "u_SHCoefficients" + "u_SHCoefficientsSampler" ); this.sortOrder = new Uint32Array(vertexCount); @@ -640,7 +647,7 @@ class gltfPrimitive extends GltfObject { } // Add the morph target texture. - this._createDataTextureArray( + this.morphTargetTextureInfo = this._createDataTextureArray( gltf, webGlContext, morphTargetTextureArray, From aa1c057e1f30fb653033a15f2c8ed55729b582e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Thu, 7 May 2026 18:11:18 +0200 Subject: [PATCH 04/21] Add WASM sorting --- source/GltfState/gltf_state.js | 3 + source/Renderer/renderer.js | 12 +- source/gltf/primitive.js | 124 +++++++++++++ source/libs/mkkellogg-sort.worker.js | 164 ++++++++++++++++++ source/libs/sorter_no_simd_non_shared.wasm | Bin 0 -> 2093 bytes thirdparty/GaussianSplats3D/BUILD.md | 4 + thirdparty/GaussianSplats3D/LICENSE | 21 +++ .../compile_wasm_no_simd_non_shared.sh | 1 + .../GaussianSplats3D/sorter_no_simd.cpp | 156 +++++++++++++++++ thirdparty/Thirdparty.md | 13 ++ 10 files changed, 496 insertions(+), 2 deletions(-) create mode 100644 source/libs/mkkellogg-sort.worker.js create mode 100644 source/libs/sorter_no_simd_non_shared.wasm create mode 100644 thirdparty/GaussianSplats3D/BUILD.md create mode 100644 thirdparty/GaussianSplats3D/LICENSE create mode 100644 thirdparty/GaussianSplats3D/compile_wasm_no_simd_non_shared.sh create mode 100644 thirdparty/GaussianSplats3D/sorter_no_simd.cpp create mode 100644 thirdparty/Thirdparty.md diff --git a/source/GltfState/gltf_state.js b/source/GltfState/gltf_state.js index 1452660c..52ddc452 100644 --- a/source/GltfState/gltf_state.js +++ b/source/GltfState/gltf_state.js @@ -31,6 +31,9 @@ class GltfState { /** KHR_materials_variants */ this.variant = undefined; + /** Indicates whether the view needs to be redrawn, currently used to indicate new sorting orders for gaussian splatting */ + this.needsRedraw = false; + /** parameters used to configure the rendering */ this.renderingParameters = { /** morphing between vertices */ diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index 6c25d02b..ec9195bc 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -669,6 +669,7 @@ class gltfRenderer { state.gltf, this.transparentDrawables ); + this.needsRedraw = false; for (const drawable of this.transparentDrawables.filter((a) => a.depth <= 0)) { if ( drawable.primitive.extensions?.KHR_gaussian_splatting !== undefined && @@ -694,10 +695,17 @@ class gltfRenderer { ); } } + state.needsRedraw = this.needsRedraw; } drawSplat(state, primitive, node, projectionMatrix, viewMatrix) { if (primitive.skip) return; + // Request an async worker sort each frame (no-op if the previous sort + // has not yet finished or no worker is available). + primitive.requestSort(viewMatrix); + if (primitive.sortPending) { + this.needsRedraw = true; + } let vertDefines = primitive.defines.slice(); if (primitive.linear === true) { vertDefines.push("LINEAR_OUTPUT 1"); @@ -723,12 +731,12 @@ class gltfRenderer { this.webGl.context.uniformMatrix4fv( this.shader.getUniformLocation("u_ViewMatrix"), false, - this.viewMatrix + viewMatrix ); this.webGl.context.uniformMatrix4fv( this.shader.getUniformLocation("u_ProjectionMatrix"), false, - this.projMatrix + projectionMatrix ); this.webGl.context.uniform2i( this.shader.getUniformLocation("u_FramebufferSize"), diff --git a/source/gltf/primitive.js b/source/gltf/primitive.js index a8f3d7fe..f6244b2a 100644 --- a/source/gltf/primitive.js +++ b/source/gltf/primitive.js @@ -47,6 +47,16 @@ class gltfPrimitive extends GltfObject { this.sortOrder = undefined; this.splatTextureWidth = undefined; + // Worker-based sort state + this.sortWorker = undefined; + this.sortWorkerReady = false; + this.sortPending = false; + + // Store the view matrix if sort is currently running + this.queuedViewMatrix = undefined; + // Compare against a stored value to detect whether a redraw is needed. + this.lastSortViewMatrix = undefined; + // The primitive centroid is used for depth sorting. this.centroid = undefined; } @@ -442,6 +452,7 @@ class gltfPrimitive extends GltfObject { const max2DTextureSize = Math.pow(webGlContext.getParameter(GL.MAX_TEXTURE_SIZE), 2); const vertexCount = gltf.accessors[this.attributes["KHR_gaussian_splatting:SH_DEGREE_0_COEF_0"]].count; + this.initSortWorker(gltf, vertexCount); this.splatTextureWidth = Math.ceil(Math.sqrt(vertexCount)); const singleTextureSize = Math.pow(this.splatTextureWidth, 2); @@ -664,6 +675,119 @@ class gltfPrimitive extends GltfObject { this.computeCentroid(gltf); } + /** + * Spawn a mkkellogg WASM sort worker and hand it the splat centre positions. + * Called once during initGl for Gaussian Splatting primitives. + * @param {object} gltf + * @param {number} vertexCount + */ + initSortWorker(gltf, vertexCount) { + try { + this.sortWorker = new Worker( + new URL("./libs/mkkellogg-sort.worker.js", import.meta.url), //URL needs to be relative to rollup build + { type: "module" } + ); + } catch (err) { + console.warn("Failed to spawn sort worker:", err); + return; + } + + // Build stride-4 Float32Array (x, y, z, 1) from the POSITION accessor. + const posAccessor = gltf.accessors[this.attributes.POSITION]; + const rawPositions = posAccessor.getDeinterlacedView(gltf); + const posOp = new Float32Array(vertexCount * 4); + for (let i = 0; i < vertexCount; i++) { + posOp[i * 4 + 0] = rawPositions[i * 3 + 0]; + posOp[i * 4 + 1] = rawPositions[i * 3 + 1]; + posOp[i * 4 + 2] = rawPositions[i * 3 + 2]; + posOp[i * 4 + 3] = 1.0; + } + + this.sortWorker.onmessage = (e) => { + const { type } = e.data; + if (type === "ready") { + this.sortWorkerReady = true; + this.sortPending = false; + // Fire any sort that was queued while the worker was initialising. + if (this.queuedViewMatrix !== undefined) { + // Skip if the view matrix hasn't changed since the last dispatched sort. + if (this.lastSortViewMatrix !== undefined) { + let same = true; + for (let i = 0; i < 16; i++) { + if (this.lastSortViewMatrix[i] !== this.queuedViewMatrix[i]) { + same = false; + break; + } + } + if (same) return; + } + this.sortPending = true; + this.sortWorker.postMessage({ + type: "sort", + viewMatrix: this.queuedViewMatrix + }); + this.queuedViewMatrix = undefined; + } + } else if (type === "sorted") { + this.sortOrder = e.data.indices; + this.sortPending = false; + } else if (type === "error") { + console.error("Sort worker error:", e.data.message); + this.sortPending = false; + } + }; + + this.sortWorker.onerror = (err) => { + console.error( + "Sort worker uncaught error:", + err.message, + err.filename, + "line", + err.lineno, + err + ); + this.sortPending = false; + }; + + // Transfer positions buffer to the worker (zero-copy). + this.sortWorker.postMessage({ type: "init", posOp: posOp, splatCount: vertexCount }, [ + posOp.buffer + ]); + this.sortPending = true; + } + + /** + * Request an asynchronous back-to-front sort of the splat indices. + * Safe to call every frame — the request is dropped while a previous sort + * is still in flight. + * @param {Float32Array} viewMatrix Column-major 4×4 view matrix. + */ + requestSort(viewMatrix) { + if (this.sortWorker === undefined) { + return; + } + if (!this.sortWorkerReady || this.sortPending) { + // Worker is busy — keep the latest matrix so it sorts immediately once ready + this.queuedViewMatrix = new Float32Array(viewMatrix); + return; + } + // Skip if the view matrix hasn't changed since the last dispatched sort. + if (this.lastSortViewMatrix !== undefined) { + let same = true; + for (let i = 0; i < 16; i++) { + if (this.lastSortViewMatrix[i] !== viewMatrix[i]) { + same = false; + break; + } + } + if (same) return; + } + const vm = new Float32Array(viewMatrix); + this.lastSortViewMatrix = vm; + this.sortPending = true; + this.sortWorker.postMessage({ type: "sort", viewMatrix: vm }); + } + computeCentroid(gltf) { const positionsAccessor = gltf.accessors[this.attributes.POSITION]; const positions = positionsAccessor.getNormalizedDeinterlacedView(gltf); diff --git a/source/libs/mkkellogg-sort.worker.js b/source/libs/mkkellogg-sort.worker.js new file mode 100644 index 00000000..f98d9ccc --- /dev/null +++ b/source/libs/mkkellogg-sort.worker.js @@ -0,0 +1,164 @@ +/** + * mkkellogg-sort.worker.js + * + * Web Worker that runs the mkkellogg counting-sort WASM extracted from + * @mkkellogg/gaussian-splats-3d. + * + * Exported function (from sorter.cpp): + * sortIndexes( + * indexes*, // i32 – input/output index array + * centers*, // i32 – float positions stride-4 (x,y,z,w) + * precomputed*, // i32 – pass 0 (unused when usePrecomputedDistances=false) + * mappedDistances*, // i32 – scratch int array, size=renderCount + * frequencies*, // i32 – scratch uint array, size=distanceMapRange + * modelViewProj*, // i32 – view matrix (4×4 column-major float) + * indexesOut*, // i32 – output sorted index array + * sceneIndexes*, // i32 – pass 0 (unused when dynamicMode=false) + * transforms*, // i32 – pass 0 (unused when dynamicMode=false) + * distanceMapRange, // i32 – histogram bin count (32768) + * sortCount, // i32 – splats to sort (= splatCount) + * renderCount, // i32 – rendered splat count (= splatCount) + * splatCount, // i32 – total splat count + * usePrecomputedDistances, // i32 – 0 (false) + * useIntegerSort, // i32 – 0 (false → float path) + * dynamicMode // i32 – 0 (false → single scene) + * ) + * + * Messages in: + * { type:'init', posOp: Float32Array, splatCount: number } + * { type:'sort', viewMatrix: Float32Array[16] } + * + * Messages out: + * { type:'ready' } + * { type:'sorted', indices: Uint32Array } (buffer transferred) + * { type:'error', message: string } + */ + +// Resolve the WASM URL relative to this worker file, compatible with both +// Rollup (import.meta.url is the worker script URL at runtime) and dev servers. +const wasmUrl = new URL('./sorter_no_simd_non_shared.wasm', import.meta.url).href; + +// ── Constants ────────────────────────────────────────────────────────────────── + +const DIST_MAP_RANGE = 32768; +const HEADER_PAGES = 1; // reserve 1 WASM page (64 KiB) for internal use +const PAGE_BYTES = 65536; + +// ── Module-level state ──────────────────────────────────────────────────────── + +let sortFn = null; // wasm export function +let memBuf = null; // WebAssembly.Memory + +// Byte offsets (set during init) +let pIndexes = 0, pCenters = 0, pMappedDist = 0, + pFrequencies = 0, pMVP = 0, pIndexesOut = 0; +let N = 0; + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +/** Align a byte offset up to 16-byte boundary. */ +const align16 = (b) => (b + 15) & ~15; + +// ── Initialisation ──────────────────────────────────────────────────────────── + +async function init({ posOp, splatCount }) { + N = splatCount; + + // Layout each buffer (byte offsets in WebAssembly.Memory) + let off = HEADER_PAGES * PAGE_BYTES; + + pIndexes = off; off = align16(off + N * 4); // Uint32[N] + pCenters = off; off = align16(off + N * 4 * 4); // Float32[N*4] + pMappedDist = off; off = align16(off + N * 4); // Int32[N] + pFrequencies = off; off = align16(off + DIST_MAP_RANGE * 4); // Uint32[32768] + pMVP = off; off = align16(off + 16 * 4); // Float32[16] + pIndexesOut = off; off = align16(off + N * 4); // Uint32[N] + + const totalBytes = off; + const pages = Math.ceil(totalBytes / PAGE_BYTES) + 2; // +2 for safety + + memBuf = new WebAssembly.Memory({ initial: pages, maximum: pages + 4 }); + + let instance; + try { + const response = await fetch(wasmUrl); + ({ instance } = await WebAssembly.instantiateStreaming(response, { + env: { memory: memBuf }, + })); + } catch (err) { + // Fallback: fetch as ArrayBuffer (needed when streaming isn't available). + const ab = await (await fetch(wasmUrl)).arrayBuffer(); + ({ instance } = await WebAssembly.instantiate(ab, { + env: { memory: memBuf }, + })); + } + + sortFn = instance.exports.sortIndexes; + if (typeof instance.exports.__wasm_call_ctors === 'function') { + instance.exports.__wasm_call_ctors(); + } + + // Populate centres (stride-4): Float32 (x, y, z, opacity) — raw positions. + const fv = new Float32Array(memBuf.buffer); + fv.set(new Float32Array(posOp, 0, N * 4), pCenters >> 2); + + // Initialise indexes 0..N-1 + const uv = new Uint32Array(memBuf.buffer); + for (let i = 0; i < N; i++) uv[(pIndexes >> 2) + i] = i; + + self.postMessage({ type: 'ready' }); +} + +// ── Sort ────────────────────────────────────────────────────────────────────── + +function sort({ viewMatrix }) { + const uv = new Uint32Array(memBuf.buffer); + const fv = new Float32Array(memBuf.buffer); + + // Reset frequencies histogram to zero + uv.fill(0, pFrequencies >> 2, (pFrequencies >> 2) + DIST_MAP_RANGE); + + // Write view matrix into WASM memory, with the Z row negated. + // The standard lookAt puts -forward in row 2, so visible splats have negative + // distances (near ≈ −ε, far ≈ −∞). The WASM counting sort maps low values to + // low bins and places them at the END of indexesOut, giving front-to-back order. + // By negating row 2 (columns 2, 6, 10, 14) we flip all distances to positive + // (near ≈ +ε, far ≈ +∞), so the sort naturally produces back-to-front order + // with no O(N) reversal pass needed. + fv.set(viewMatrix, pMVP >> 2); + const base = pMVP >> 2; + fv[base + 2] = -fv[base + 2]; + fv[base + 6] = -fv[base + 6]; + fv[base + 10] = -fv[base + 10]; + fv[base + 14] = -fv[base + 14]; + + // Call the counting sort + // sortIndexes(indexes, centers, precomputed, mappedDist, frequencies, + // mvp, indexesOut, sceneIdxs, transforms, + // distMapRange, sortCount, renderCount, splatCount, + // usePrecomputed, useIntegerSort, dynamicMode) + sortFn( + pIndexes, pCenters, 0, + pMappedDist, pFrequencies, pMVP, + pIndexesOut, 0, 0, + DIST_MAP_RANGE, N, N, N, + 0, 0, 0, + ); + + const out = new Uint32Array(uv.subarray(pIndexesOut >> 2, (pIndexesOut >> 2) + N)); + self.postMessage({ type: 'sorted', indices: out }, [out.buffer]); +} + +// ── Message dispatcher ──────────────────────────────────────────────────────── + +self.onmessage = async (e) => { + try { + if (e.data.type === 'init') { + await init(e.data); + } else if (e.data.type === 'sort') { + sort(e.data); + } + } catch (err) { + self.postMessage({ type: 'error', message: String(err) }); + } +}; diff --git a/source/libs/sorter_no_simd_non_shared.wasm b/source/libs/sorter_no_simd_non_shared.wasm new file mode 100644 index 0000000000000000000000000000000000000000..e96cbdf038042521508fafb9061e0f74bbb027b8 GIT binary patch literal 2093 zcmbtVO=}cE5bf@n%}(ZXca6!!M5rD-EI}j(5f36es6->;FObcW?7`W|2g#xdD9KUq zAjI_`c*;pUNj&;z{2AWjtDf1A7!4aT(9_*B{a$s~tEv{AtyL-TM9#`i{&KB%YhKzs zms4szg}D2+LAJR_|N&cKbEh+RL3hZ!h=PH@Ad{%Ws{H zjeMuQ(&=~Fo85eU8Na-(_09g1wUzF6cS}h3AW}+rUm{DsslFZCC6C;m_Ek%@D70G? z*sb*U(a}*=yXm{Vy}fN6V2S@Mu*laHt3?)7Qq@mnMv4TU%+$Hjk!WVM6J)Y6r z2iz8CKvGF%mQ=`F2!!Wb9z`%_h*i3)qcb*rtzayoDvT&Lc&)UQ%otnYg56TPRGCwE zlR7yRrepmqiRRS30e@^J4Z`HEnK;Q6p3Iv`oJ{cK_F#X1|K~%&PwPYLAgO&WCgNAR zjIGta`O-__fW2)`$FNjq)Xf1I-sXS|B@M{1-M9$b^pI_~WQ$9-TFEwX5w_VOn_sdy zC7USO?2E8nr%*>8M81|mxfxbA37g8M*__Y|x}Ik|aA>+1_!Q?@!petG-lY&~;|=8l z3K0bXaPUry8UHeW;oHQgqu^MJe~M{e)tyZuqJ3`M^J{< z`R9(JN5jOFCXsZVyI424bM%#eWGc!6c2_qZGR2{K>?Q>a$0K^gV}pB+#}Rj~LquXV zBNJYu9>5e8FK~h{vh;6yY?U%H9tO4#_?I_z7F~bx?+k zMS@y7Dm@+#Z%!std^PqAEtr_n`vVi(h5-No literal 0 HcmV?d00001 diff --git a/thirdparty/GaussianSplats3D/BUILD.md b/thirdparty/GaussianSplats3D/BUILD.md new file mode 100644 index 00000000..fd68a429 --- /dev/null +++ b/thirdparty/GaussianSplats3D/BUILD.md @@ -0,0 +1,4 @@ +To build `sorter_no_simd_non_shared.wasm` + +1. [Install emscripten SDK](https://emscripten.org/docs/getting_started/downloads.html#) +2. Run `compile_wasm_no_simd_non_shared.sh` or copy its content into your shell to build the WASM file diff --git a/thirdparty/GaussianSplats3D/LICENSE b/thirdparty/GaussianSplats3D/LICENSE new file mode 100644 index 00000000..6370ba2d --- /dev/null +++ b/thirdparty/GaussianSplats3D/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2023 Mark Kellogg + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/thirdparty/GaussianSplats3D/compile_wasm_no_simd_non_shared.sh b/thirdparty/GaussianSplats3D/compile_wasm_no_simd_non_shared.sh new file mode 100644 index 00000000..cdabf646 --- /dev/null +++ b/thirdparty/GaussianSplats3D/compile_wasm_no_simd_non_shared.sh @@ -0,0 +1 @@ +em++ -std=c++11 sorter_no_simd.cpp -Os -s WASM=1 -s SIDE_MODULE=2 -o sorter_no_simd_non_shared.wasm -s IMPORTED_MEMORY=1 \ No newline at end of file diff --git a/thirdparty/GaussianSplats3D/sorter_no_simd.cpp b/thirdparty/GaussianSplats3D/sorter_no_simd.cpp new file mode 100644 index 00000000..c07fc71a --- /dev/null +++ b/thirdparty/GaussianSplats3D/sorter_no_simd.cpp @@ -0,0 +1,156 @@ +#include +#include + +#ifdef __cplusplus +#define EXTERN extern "C" +#else +#define EXTERN +#endif + +#define computeMatMul4x4ThirdRow(a, b, out) \ + out[0] = a[2] * b[0] + a[6] * b[1] + a[10] * b[2] + a[14] * b[3]; \ + out[1] = a[2] * b[4] + a[6] * b[5] + a[10] * b[6] + a[14] * b[7]; \ + out[2] = a[2] * b[8] + a[6] * b[9] + a[10] * b[10] + a[14] * b[11]; \ + out[3] = a[2] * b[12] + a[6] * b[13] + a[10] * b[14] + a[14] * b[15]; + +EXTERN EMSCRIPTEN_KEEPALIVE void sortIndexes(unsigned int* indexes, void* centers, void* precomputedDistances, + int* mappedDistances, unsigned int * frequencies, float* modelViewProj, + unsigned int* indexesOut, unsigned int* sceneIndexes, float* transforms, + unsigned int distanceMapRange, unsigned int sortCount, unsigned int renderCount, + unsigned int splatCount, bool usePrecomputedDistances, bool useIntegerSort, + bool dynamicMode) { + + int maxDistance = -2147483640; + int minDistance = 2147483640; + + float fMVPTRow3[4]; + int iMVPTRow3[4]; + unsigned int sortStart = renderCount - sortCount; + if (useIntegerSort) { + int* intCenters = (int*)centers; + if (usePrecomputedDistances) { + int* intPrecomputedDistances = (int*)precomputedDistances; + for (unsigned int i = sortStart; i < renderCount; i++) { + int distance = intPrecomputedDistances[indexes[i]]; + mappedDistances[i] = distance; + if (distance > maxDistance) maxDistance = distance; + if (distance < minDistance) minDistance = distance; + } + } else { + if (dynamicMode) { + int lastTransformIndex = -1; + for (unsigned int i = sortStart; i < renderCount; i++) { + unsigned int realIndex = indexes[i]; + unsigned int indexOffset = 4 * realIndex; + unsigned int sceneIndex = sceneIndexes[realIndex]; + if ((int)sceneIndex != lastTransformIndex) { + float* transform = &transforms[sceneIndex * 16]; + computeMatMul4x4ThirdRow(modelViewProj, transform, fMVPTRow3); + iMVPTRow3[0] = (int)(fMVPTRow3[0] * 1000.0); + iMVPTRow3[1] = (int)(fMVPTRow3[1] * 1000.0); + iMVPTRow3[2] = (int)(fMVPTRow3[2] * 1000.0); + iMVPTRow3[3] = (int)(fMVPTRow3[3] * 1000.0); + lastTransformIndex = (int)sceneIndex; + } + int distance = + (int)((iMVPTRow3[0] * intCenters[indexOffset] + + iMVPTRow3[1] * intCenters[indexOffset + 1] + + iMVPTRow3[2] * intCenters[indexOffset + 2] + + iMVPTRow3[3] * intCenters[indexOffset + 3])); + mappedDistances[i] = distance; + if (distance > maxDistance) maxDistance = distance; + if (distance < minDistance) minDistance = distance; + } + } else { + iMVPTRow3[0] = (int)(modelViewProj[2] * 1000.0); + iMVPTRow3[1] = (int)(modelViewProj[6] * 1000.0); + iMVPTRow3[2] = (int)(modelViewProj[10] * 1000.0); + iMVPTRow3[3] = 1; + for (unsigned int i = sortStart; i < renderCount; i++) { + unsigned int indexOffset = 4 * (unsigned int)indexes[i]; + int distance = + (int)((iMVPTRow3[0] * intCenters[indexOffset] + + iMVPTRow3[1] * intCenters[indexOffset + 1] + + iMVPTRow3[2] * intCenters[indexOffset + 2])); + mappedDistances[i] = distance; + if (distance > maxDistance) maxDistance = distance; + if (distance < minDistance) minDistance = distance; + } + } + } + } else { + float* floatCenters = (float*)centers; + if (usePrecomputedDistances) { + float* floatPrecomputedDistances = (float*)precomputedDistances; + for (unsigned int i = sortStart; i < renderCount; i++) { + int distance = (int)(floatPrecomputedDistances[indexes[i]] * 4096.0); + mappedDistances[i] = distance; + if (distance > maxDistance) maxDistance = distance; + if (distance < minDistance) minDistance = distance; + } + } else { + float* fMVP = (float*)modelViewProj; + float* floatTransforms = (float *)transforms; + + if (dynamicMode) { + int lastTransformIndex = -1; + for (unsigned int i = sortStart; i < renderCount; i++) { + unsigned int realIndex = indexes[i]; + unsigned int indexOffset = 4 * realIndex; + unsigned int sceneIndex = sceneIndexes[realIndex]; + if ((int)sceneIndex != lastTransformIndex) { + float* transform = &transforms[sceneIndex * 16]; + computeMatMul4x4ThirdRow(modelViewProj, transform, fMVPTRow3); + lastTransformIndex = (int)sceneIndex; + } + int distance = + (int)((fMVPTRow3[0] * floatCenters[indexOffset] + + fMVPTRow3[1] * floatCenters[indexOffset + 1] + + fMVPTRow3[2] * floatCenters[indexOffset + 2] + + fMVPTRow3[3] * floatCenters[indexOffset + 3]) * 4096.0); + mappedDistances[i] = distance; + if (distance > maxDistance) maxDistance = distance; + if (distance < minDistance) minDistance = distance; + } + } else { + for (unsigned int i = sortStart; i < renderCount; i++) { + unsigned int indexOffset = 4 * (unsigned int)indexes[i]; + int distance = + (int)((fMVP[2] * floatCenters[indexOffset] + + fMVP[6] * floatCenters[indexOffset + 1] + + fMVP[10] * floatCenters[indexOffset + 2]) * 4096.0); + mappedDistances[i] = distance; + if (distance > maxDistance) maxDistance = distance; + if (distance < minDistance) minDistance = distance; + } + } + } + } + + float distancesRange = (float)maxDistance - (float)minDistance; + float rangeMap = (float)(distanceMapRange - 1) / distancesRange; + + for (unsigned int i = sortStart; i < renderCount; i++) { + unsigned int frequenciesIndex = (int)((float)(mappedDistances[i] - minDistance) * rangeMap); + mappedDistances[i] = frequenciesIndex; + frequencies[frequenciesIndex] = frequencies[frequenciesIndex] + 1; + } + + unsigned int cumulativeFreq = frequencies[0]; + for (unsigned int i = 1; i < distanceMapRange; i++) { + unsigned int freq = frequencies[i]; + cumulativeFreq += freq; + frequencies[i] = cumulativeFreq; + } + + for (int i = (int)sortStart - 1; i >= 0; i--) { + indexesOut[i] = indexes[i]; + } + + for (int i = (int)renderCount - 1; i >= (int)sortStart; i--) { + unsigned int frequenciesIndex = mappedDistances[i]; + unsigned int freq = frequencies[frequenciesIndex]; + indexesOut[renderCount - freq] = indexes[i]; + frequencies[frequenciesIndex] = freq - 1; + } +} diff --git a/thirdparty/Thirdparty.md b/thirdparty/Thirdparty.md new file mode 100644 index 00000000..e7926356 --- /dev/null +++ b/thirdparty/Thirdparty.md @@ -0,0 +1,13 @@ +# Thirdparty + +This folder contains all third-party code that is not directly used in the distribution package. + +Libraries used directly are located in the `source/libs` folder or are included as npm packages. + +The following libraries are used: + +### GaussianSplats3D WASM sorter + +The `GaussianSplats3D` folder contains the C++ code to compile the WASM file `/source/libs/sorter_no_simd_non_shared.wasm`. + +The files were copied from the [original repository](https://github.com/mkkellogg/GaussianSplats3D) for reference. From 7c0beafd9b8b241788cfd473040fbcf400100d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Fri, 8 May 2026 13:15:41 +0200 Subject: [PATCH 05/21] Add all spherical harmoic degrees --- source/Renderer/renderer.js | 1 + source/Renderer/shaders/splat.vert | 109 ++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index ec9195bc..1cdfaa2e 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -751,6 +751,7 @@ class gltfRenderer { this.projMatrix[5] * this.currentHeight * 0.5 ); this.shader.updateUniform("u_TextureWidth", primitive.splatTextureWidth); + this.shader.updateUniform("u_Camera", this.currentCameraPosition); this.webGl.context.enable(GL.BLEND); this.webGl.context.blendFuncSeparate( diff --git a/source/Renderer/shaders/splat.vert b/source/Renderer/shaders/splat.vert index 26cedfa8..f4793fef 100644 --- a/source/Renderer/shaders/splat.vert +++ b/source/Renderer/shaders/splat.vert @@ -16,6 +16,7 @@ uniform mat4 u_ModelMatrix; uniform uint u_TextureWidth; uniform ivec2 u_FramebufferSize; uniform vec2 u_FocalLength; +uniform vec3 u_Camera; #ifdef POSITION_IS_INTEGER uniform isampler2D u_POSITIONSampler; @@ -45,6 +46,35 @@ uniform sampler2D u_OPACITYSampler; uniform highp sampler2DArray u_SHCoefficientsSampler; +#define SH_C0 0.28209479177387814 + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_1 +#define SH_C1_0 -0.4886025119029199 +#define SH_C1_1 0.4886025119029199 +#define SH_C1_2 -0.4886025119029199 + + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_2 +#define SH_C2_0 1.0925484305920792 +#define SH_C2_1 -1.0925484305920792 +#define SH_C2_2 0.31539156525252005 +#define SH_C2_3 -1.0925484305920792 +#define SH_C2_4 0.5462742152960396 + + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_3 +#define SH_C3_0 -0.5900435899266435 +#define SH_C3_1 2.890611442640554 +#define SH_C3_2 -0.4570457994644658 +#define SH_C3_3 0.3731763325901154 +#define SH_C3_4 -0.4570457994644658 +#define SH_C3_5 1.445305721320277 +#define SH_C3_6 -0.5900435899266435 +#endif + +#endif +#endif + mat3 computeC(vec3 scale, vec4 rotation) { @@ -154,6 +184,40 @@ void main() opacity = texelFetch(u_OPACITYSampler, texelCoord, 0).x; #endif + // Fetch SH coefficients early to avoid GPU stall + // Degree 0 + vec3 sh0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 0), 0).rgb; + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_1 + // Degree 1 + vec3 sh1_0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 1), 0).rgb; + vec3 sh1_1 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 2), 0).rgb; + vec3 sh1_2 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 3), 0).rgb; + + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_2 + // Degree 2 + vec3 sh2_0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 4), 0).rgb; + vec3 sh2_1 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 5), 0).rgb; + vec3 sh2_2 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 6), 0).rgb; + vec3 sh2_3 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 7), 0).rgb; + vec3 sh2_4 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 8), 0).rgb; + + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_3 + // Degree 3 + vec3 sh3_0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 9), 0).rgb; + vec3 sh3_1 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 10), 0).rgb; + vec3 sh3_2 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 11), 0).rgb; + vec3 sh3_3 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 12), 0).rgb; + vec3 sh3_4 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 13), 0).rgb; + vec3 sh3_5 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 14), 0).rgb; + vec3 sh3_6 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 15), 0).rgb; +#endif + +#endif +#endif + splat_center = (u_ModelMatrix * vec4(splat_center, 1.0)).xyz; vec4 view_splat_center = u_ViewMatrix * vec4(splat_center, 1.0); @@ -187,5 +251,48 @@ void main() v_uv = a_position * quad_pixel_size; gl_Position = clip_splat_center; - v_color = vec4(calculateSphericalHarmonics(texelCoord), opacity); + vec3 color = sh0 * SH_C0; + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_1 + + vec3 view_dir = normalize(splat_center - u_Camera); + float x = view_dir.x; + float y = view_dir.y; + float z = view_dir.z; + + color += SH_C1_0 * y * sh1_0 + + SH_C1_1 * z * sh1_1 + + SH_C1_2 * x * sh1_2; + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_2 + float xx = x * x; + float yy = y * y; + float zz = z * z; + float xy = x * y; + float yz = y * z; + float xz = x * z; + + color += SH_C2_0 * xy * sh2_0 + + SH_C2_1 * yz * sh2_1 + + SH_C2_2 * (2.0 * zz - xx - yy) * sh2_2 + + SH_C2_3 * xz * sh2_3 + + SH_C2_4 * (xx - yy) * sh2_4; + + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_3 + color += SH_C3_0 * y * (3.0 * xx - yy) * sh3_0 + + SH_C3_1 * xy * z * sh3_1 + + SH_C3_2 * y * (4.0 * zz - xx - yy) * sh3_2 + + SH_C3_3 * z * (2.0 * zz - 3.0 * xx - 3.0 * yy) * sh3_3 + + SH_C3_4 * x * (4.0 * zz - xx - yy) * sh3_4 + + SH_C3_5 * z * (xx - yy) * sh3_5 + + SH_C3_6 * x * (xx - 3.0 * yy) * sh3_6; +#endif + +#endif +#endif + + color += 0.5; + + v_color = vec4(color, opacity); } From 9793afcc5d351e5af7516c88ebe9b8cfb4994aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Fri, 8 May 2026 15:23:51 +0200 Subject: [PATCH 06/21] Removed unused code --- source/Renderer/shaders/splat.vert | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/source/Renderer/shaders/splat.vert b/source/Renderer/shaders/splat.vert index f4793fef..74d5fc9f 100644 --- a/source/Renderer/shaders/splat.vert +++ b/source/Renderer/shaders/splat.vert @@ -121,22 +121,6 @@ vec3 computeCameraCovariance(mat3 world_covariance, vec3 view_splat_center) return vec3(cov[0][0], cov[0][1], cov[1][1]); } -vec3 calculateSphericalHarmonics(ivec2 texelCoord) -{ - ivec3 coord = ivec3(texelCoord.x, texelCoord.y, 0); - // Degree 0 - vec3 sh0 = texelFetch(u_SHCoefficientsSampler, coord, 0).rgb; - - vec3 result = sh0 * 0.2820947917738781; - - //TODO - - result += 0.5; - return result; - -} - - void main() { vec3 splat_center; @@ -278,7 +262,6 @@ void main() SH_C2_3 * xz * sh2_3 + SH_C2_4 * (xx - yy) * sh2_4; - #ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_3 color += SH_C3_0 * y * (3.0 * xx - yy) * sh3_0 + SH_C3_1 * xy * z * sh3_1 + From 5b86ca7f102665a05f3cf1fe7b729a20a0f996ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Fri, 8 May 2026 15:24:09 +0200 Subject: [PATCH 07/21] Add DEBUG channels --- source/GltfState/gltf_state.js | 11 +++++++++++ source/Renderer/renderer.js | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/source/GltfState/gltf_state.js b/source/GltfState/gltf_state.js index 52ddc452..928d1d41 100644 --- a/source/GltfState/gltf_state.js +++ b/source/GltfState/gltf_state.js @@ -233,6 +233,17 @@ GltfState.DebugOutput = { SINGLE_SCATTER_COLOR: "Single-Scatter Color", /** output for the pre scatter pass, which collects all lighting contribution for scattering */ PRE_SCATTER_PASS: "Pre-Scatter Pass" + }, + + gaussianSplatting: { + /** output the spherical harmonics degree 0 */ + SH_DEGREE_0: "SH Degree 0", + /** output the spherical harmonics degree 0-1 */ + SH_DEGREE_1: "SH Degree 1", + /** output the spherical harmonics degree 0-2 */ + SH_DEGREE_2: "SH Degree 2", + /** output the spherical harmonics degree 0-3 */ + SH_DEGREE_3: "SH Degree 3" } }; diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index 1cdfaa2e..d908e238 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -711,6 +711,34 @@ class gltfRenderer { vertDefines.push("LINEAR_OUTPUT 1"); } + // Debug views + if (state.renderingParameters.debugOutput !== GltfState.DebugOutput.NONE) { + if ( + state.renderingParameters.debugOutput === + GltfState.DebugOutput.gaussianSplatting.SH_DEGREE_0 + ) { + vertDefines = vertDefines.filter( + (define) => !define.startsWith("HAS_GAUSSIAN_SPLATTING_DEGREE") + ); + } else if ( + state.renderingParameters.debugOutput === + GltfState.DebugOutput.gaussianSplatting.SH_DEGREE_1 + ) { + vertDefines = vertDefines.filter( + (define) => + define !== "HAS_GAUSSIAN_SPLATTING_DEGREE_2 1" && + define !== "HAS_GAUSSIAN_SPLATTING_DEGREE_3 1" + ); + } else if ( + state.renderingParameters.debugOutput === + GltfState.DebugOutput.gaussianSplatting.SH_DEGREE_2 + ) { + vertDefines = vertDefines.filter( + (define) => define !== "HAS_GAUSSIAN_SPLATTING_DEGREE_3 1" + ); + } + } + const fragmentHash = this.shaderCache.selectShader("splat.frag", vertDefines); const vertexHash = this.shaderCache.selectShader("splat.vert", vertDefines); if (fragmentHash && vertexHash) { From bcf3c0eccbd12b9fa1bd02ec240993c9a3a3577c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Thu, 21 May 2026 19:11:13 +0200 Subject: [PATCH 08/21] Apply tone-mapping in postprocess. Add possibility of half-float framebuffer --- source/GltfState/gltf_state.js | 4 +- source/Renderer/renderer.js | 259 ++++++++++++++++++++-- source/Renderer/shaders/cubemap.frag | 9 +- source/Renderer/shaders/fullscreen.vert | 13 ++ source/Renderer/shaders/pbr.frag | 4 +- source/Renderer/shaders/splat.frag | 10 +- source/Renderer/shaders/tonemap_main.frag | 30 +++ 7 files changed, 304 insertions(+), 25 deletions(-) create mode 100644 source/Renderer/shaders/fullscreen.vert create mode 100644 source/Renderer/shaders/tonemap_main.frag diff --git a/source/GltfState/gltf_state.js b/source/GltfState/gltf_state.js index 928d1d41..729d408c 100644 --- a/source/GltfState/gltf_state.js +++ b/source/GltfState/gltf_state.js @@ -99,7 +99,9 @@ class GltfState { /** If this is set to true, directional lights will be generated if IBL is disabled */ useDirectionalLightsWithDisabledIBL: false, /** MSAA used for cases which are not handled by the browser (e.g. Transmission)*/ - internalMSAA: 4 + internalMSAA: 4, + /** Use RGBA16F floating-point main framebuffer instead of RGBA8 */ + floatingPointFramebuffer: true }; // retain a reference to the view with which the state was created, so that it can be validated diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index d908e238..a9524f5d 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -21,6 +21,8 @@ import scatterShader from "./shaders/scatter.frag"; import specularGlossinesShader from "./shaders/specular_glossiness.frag"; import splatVertexShader from "./shaders/splat.vert"; import splatFragmentShader from "./shaders/splat.frag"; +import fullscreenVertShader from "./shaders/fullscreen.vert"; +import tonemapMainFragShader from "./shaders/tonemap_main.frag"; import { gltfLight } from "../gltf/light.js"; import { jsToGl } from "../gltf/utils.js"; import { gltfMaterial } from "../gltf/material.js"; @@ -65,6 +67,8 @@ class gltfRenderer { shaderSources.set("specular_glossiness.frag", specularGlossinesShader); shaderSources.set("splat.vert", splatVertexShader); shaderSources.set("splat.frag", splatFragmentShader); + shaderSources.set("fullscreen.vert", fullscreenVertShader); + shaderSources.set("tonemap_main.frag", tonemapMainFragShader); this.shaderCache = new ShaderCache(shaderSources, this.webGl); @@ -93,6 +97,19 @@ class gltfRenderer { this.splatVBO = undefined; this.currentSortBuffer = undefined; + + this.mainTexture = undefined; + this.mainTextureHDR = undefined; + this.mainDepthTexture = undefined; + this.mainTonemapTexture = undefined; + this.mainFramebuffer = undefined; + + this.framebufferFormat = undefined; + this.framebufferType = undefined; + + // Tracks last value of state.renderingParameters.floatingPointFramebuffer so we + // only re-attach when the setting actually changes. + this._floatingPointFramebuffer = undefined; } ///////////////////////////////////////////////////////////////////// @@ -196,6 +213,72 @@ class gltfRenderer { this.currentSortBuffer = context.createBuffer(); + this.framebufferFormat = context.supports_EXT_color_buffer_half_float + ? context.RGBA16F : context.RGBA; + this.framebufferType = context.supports_EXT_color_buffer_half_float + ? context.HALF_FLOAT : context.UNSIGNED_BYTE; + + const hdrW = Math.max(this.currentWidth, 1); + const hdrH = Math.max(this.currentHeight, 1); + + // Main color texture + this.mainTexture = context.createTexture(); + context.bindTexture(context.TEXTURE_2D, this.mainTexture); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MIN_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MAG_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_S, context.CLAMP_TO_EDGE); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_T, context.CLAMP_TO_EDGE); + context.texImage2D(context.TEXTURE_2D, 0, context.RGBA, + hdrW, hdrH, 0, context.RGBA, context.UNSIGNED_BYTE, null); + context.bindTexture(context.TEXTURE_2D, null); + + if (context.supports_EXT_color_buffer_half_float) { + this.mainTextureHDR = context.createTexture(); + context.bindTexture(context.TEXTURE_2D, this.mainTextureHDR); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MIN_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MAG_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_S, context.CLAMP_TO_EDGE); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_T, context.CLAMP_TO_EDGE); + context.texImage2D(context.TEXTURE_2D, 0, context.RGBA16F, + hdrW, hdrH, 0, context.RGBA, context.HALF_FLOAT, null); + context.bindTexture(context.TEXTURE_2D, null); + } + + // Main depth texture + this.mainDepthTexture = context.createTexture(); + context.bindTexture(context.TEXTURE_2D, this.mainDepthTexture); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MIN_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MAG_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_S, context.CLAMP_TO_EDGE); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_T, context.CLAMP_TO_EDGE); + context.texImage2D(context.TEXTURE_2D, 0, context.DEPTH_COMPONENT24, + hdrW, hdrH, 0, context.DEPTH_COMPONENT, context.UNSIGNED_INT, null); + context.bindTexture(context.TEXTURE_2D, null); + + this.mainFramebuffer = context.createFramebuffer(); + context.bindFramebuffer(context.FRAMEBUFFER, this.mainFramebuffer); + context.framebufferTexture2D(context.FRAMEBUFFER, context.COLOR_ATTACHMENT0, + context.TEXTURE_2D, this.mainTexture, 0); + context.framebufferTexture2D(context.FRAMEBUFFER, context.DEPTH_ATTACHMENT, + context.TEXTURE_2D, this.mainDepthTexture, 0); + + // Tonemap flag texture: R8UI, one flag per pixel. + // 0 = linear only, 1 = sRGB gamma only, 2 = tonemap + sRGB. + this.mainTonemapTexture = context.createTexture(); + context.bindTexture(context.TEXTURE_2D, this.mainTonemapTexture); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MIN_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MAG_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_S, context.CLAMP_TO_EDGE); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_T, context.CLAMP_TO_EDGE); + context.texImage2D(context.TEXTURE_2D, 0, context.R8UI, + hdrW, hdrH, 0, context.RED_INTEGER, context.UNSIGNED_BYTE, null); + context.bindTexture(context.TEXTURE_2D, null); + + context.framebufferTexture2D(context.FRAMEBUFFER, context.COLOR_ATTACHMENT1, + context.TEXTURE_2D, this.mainTonemapTexture, 0); + context.drawBuffers([context.COLOR_ATTACHMENT0, context.COLOR_ATTACHMENT1]); + context.bindFramebuffer(context.FRAMEBUFFER, null); + this.maxVertAttributes = context.getParameter(context.MAX_VERTEX_ATTRIBS); this.initialized = true; @@ -265,6 +348,74 @@ class gltfRenderer { ); this.webGl.context.bindTexture(this.webGl.context.TEXTURE_2D, null); this.webGl.context.bindFramebuffer(this.webGl.context.FRAMEBUFFER, null); + + // Resize main color texture (always RGBA/UNSIGNED_BYTE). + this.webGl.context.bindTexture(this.webGl.context.TEXTURE_2D, this.mainTexture); + this.webGl.context.texImage2D( + this.webGl.context.TEXTURE_2D, + 0, + this.webGl.context.RGBA, + this.currentWidth, + this.currentHeight, + 0, + this.webGl.context.RGBA, + this.webGl.context.UNSIGNED_BYTE, + null + ); + + // Resize HDR color texture (RGBA16F/HALF_FLOAT, only when available). + if (this.mainTextureHDR) { + this.webGl.context.bindTexture( + this.webGl.context.TEXTURE_2D, + this.mainTextureHDR + ); + this.webGl.context.texImage2D( + this.webGl.context.TEXTURE_2D, + 0, + this.webGl.context.RGBA16F, + this.currentWidth, + this.currentHeight, + 0, + this.webGl.context.RGBA, + this.webGl.context.HALF_FLOAT, + null + ); + } + + this.webGl.context.bindTexture( + this.webGl.context.TEXTURE_2D, + this.mainDepthTexture + ); + this.webGl.context.texImage2D( + this.webGl.context.TEXTURE_2D, + 0, + this.webGl.context.DEPTH_COMPONENT24, + this.currentWidth, + this.currentHeight, + 0, + this.webGl.context.DEPTH_COMPONENT, + this.webGl.context.UNSIGNED_INT, + null + ); + this.webGl.context.bindTexture(this.webGl.context.TEXTURE_2D, null); + + // Resize tonemap flag texture (R8UI). + this.webGl.context.bindTexture( + this.webGl.context.TEXTURE_2D, + this.mainTonemapTexture + ); + this.webGl.context.texImage2D( + this.webGl.context.TEXTURE_2D, + 0, + this.webGl.context.R8UI, + this.currentWidth, + this.currentHeight, + 0, + this.webGl.context.RED_INTEGER, + this.webGl.context.UNSIGNED_BYTE, + null + ); + this.webGl.context.bindTexture(this.webGl.context.TEXTURE_2D, null); } } } @@ -283,6 +434,18 @@ class gltfRenderer { this.webGl.context.bindFramebuffer(this.webGl.context.FRAMEBUFFER, this.scatterFramebuffer); this.webGl.context.clearColor(0, 0, 0, 0); this.webGl.context.clear(GL.COLOR_BUFFER_BIT | GL.DEPTH_BUFFER_BIT); + // Main framebuffer: clear color (attachment 0) and depth separately, + // then zero the tonemap flag attachment (attachment 1) via clearBufferuiv. + if (this.mainFramebuffer) { + const gl = this.webGl.context; + gl.bindFramebuffer(gl.FRAMEBUFFER, this.mainFramebuffer); + // Clear color attachment 0 to the scene background colour. + gl.clearBufferfv(gl.COLOR, 0, clearColor); + // Clear tonemap flag attachment to 0 (linear only = no-op pass-through). + gl.clearBufferuiv(gl.COLOR, 1, new Uint32Array([0, 0, 0, 0])); + // Clear depth. + gl.clearBufferfv(gl.DEPTH, 0, new Float32Array([1.0])); + } this.webGl.context.bindFramebuffer(this.webGl.context.FRAMEBUFFER, null); this.webGl.context.clearColor(...clearColor); this.webGl.context.clear(GL.COLOR_BUFFER_BIT | GL.DEPTH_BUFFER_BIT); @@ -601,26 +764,41 @@ class gltfRenderer { this.webGl.context.generateMipmap(this.webGl.context.TEXTURE_2D); } - // Render to canvas - this.webGl.context.bindFramebuffer(this.webGl.context.FRAMEBUFFER, null); + // Re-attach mainFramebuffer COLOR_ATTACHMENT0 if the floating-point toggle changed. + const wantFP = + state.renderingParameters.floatingPointFramebuffer !== false && + this.mainTextureHDR !== undefined; + if (wantFP !== this._floatingPointFramebuffer) { + this._floatingPointFramebuffer = wantFP; + const gl = this.webGl.context; + gl.bindFramebuffer(gl.FRAMEBUFFER, this.mainFramebuffer); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + gl.COLOR_ATTACHMENT0, + gl.TEXTURE_2D, + wantFP ? this.mainTextureHDR : this.mainTexture, + 0 + ); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + } + + this.webGl.context.bindFramebuffer(this.webGl.context.FRAMEBUFFER, this.mainFramebuffer); this.webGl.context.viewport(aspectOffsetX, aspectOffsetY, aspectWidth, aspectHeight); - // Render environment - const fragDefines = []; - this.pushFragParameterDefines(fragDefines, state); + // Environment (always linear in this pass) this.environmentRenderer.drawEnvironmentMap( this.webGl, this.viewProjectionMatrix, state, this.shaderCache, - fragDefines + ["LINEAR_OUTPUT 1"] ); let drawableCounter = 0; for (const instance of Object.values(this.opaqueDrawables)) { const drawable = instance[0]; let renderpassConfiguration = {}; - renderpassConfiguration.linearOutput = false; + renderpassConfiguration.linearOutput = true; renderpassConfiguration.frameBufferSize = [this.currentWidth, this.currentHeight]; const instanceOffset = instanceWorldTransforms[drawableCounter]; drawableCounter++; @@ -647,7 +825,7 @@ class gltfRenderer { ); for (const drawable of this.transmissionDrawables.filter((a) => a.depth <= 0)) { let renderpassConfiguration = {}; - renderpassConfiguration.linearOutput = false; + renderpassConfiguration.linearOutput = true; renderpassConfiguration.frameBufferSize = [this.currentWidth, this.currentHeight]; let sampledTextures = {}; sampledTextures.transmissionSampleTexture = this.opaqueRenderTexture; @@ -684,7 +862,7 @@ class gltfRenderer { ); } else { let renderpassConfiguration = {}; - renderpassConfiguration.linearOutput = false; + renderpassConfiguration.linearOutput = true; renderpassConfiguration.frameBufferSize = [this.currentWidth, this.currentHeight]; this.drawPrimitive( state, @@ -695,6 +873,10 @@ class gltfRenderer { ); } } + + // ── Final tonemapping pass → canvas ─────────────────────────────────── + this.tonemapPass(state, aspectOffsetX, aspectOffsetY, aspectWidth, aspectHeight); + state.needsRedraw = this.needsRedraw; } @@ -706,9 +888,9 @@ class gltfRenderer { if (primitive.sortPending) { this.needsRedraw = true; } - let vertDefines = primitive.defines.slice(); + let defines = primitive.defines.slice(); if (primitive.linear === true) { - vertDefines.push("LINEAR_OUTPUT 1"); + defines.push("LINEAR_OUTPUT 1"); } // Debug views @@ -717,14 +899,14 @@ class gltfRenderer { state.renderingParameters.debugOutput === GltfState.DebugOutput.gaussianSplatting.SH_DEGREE_0 ) { - vertDefines = vertDefines.filter( + defines = defines.filter( (define) => !define.startsWith("HAS_GAUSSIAN_SPLATTING_DEGREE") ); } else if ( state.renderingParameters.debugOutput === GltfState.DebugOutput.gaussianSplatting.SH_DEGREE_1 ) { - vertDefines = vertDefines.filter( + defines = defines.filter( (define) => define !== "HAS_GAUSSIAN_SPLATTING_DEGREE_2 1" && define !== "HAS_GAUSSIAN_SPLATTING_DEGREE_3 1" @@ -733,14 +915,14 @@ class gltfRenderer { state.renderingParameters.debugOutput === GltfState.DebugOutput.gaussianSplatting.SH_DEGREE_2 ) { - vertDefines = vertDefines.filter( + defines = defines.filter( (define) => define !== "HAS_GAUSSIAN_SPLATTING_DEGREE_3 1" ); } } - const fragmentHash = this.shaderCache.selectShader("splat.frag", vertDefines); - const vertexHash = this.shaderCache.selectShader("splat.vert", vertDefines); + const fragmentHash = this.shaderCache.selectShader("splat.frag", defines); + const vertexHash = this.shaderCache.selectShader("splat.vert", defines); if (fragmentHash && vertexHash) { this.shader = this.shaderCache.getShaderProgram(fragmentHash, vertexHash); } @@ -831,6 +1013,51 @@ class gltfRenderer { this.webGl.context.disableVertexAttribArray(location); } + tonemapPass(state, aspectOffsetX, aspectOffsetY, aspectWidth, aspectHeight) { + const gl = this.webGl.context; + + const fragDefines = []; + this.pushFragParameterDefines(fragDefines, state); + + const fragHash = this.shaderCache.selectShader("tonemap_main.frag", fragDefines); + const vertHash = this.shaderCache.selectShader("fullscreen.vert", []); + if (!fragHash || !vertHash) return; + const shader = this.shaderCache.getShaderProgram(fragHash, vertHash); + if (!shader) return; + + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.viewport(aspectOffsetX, aspectOffsetY, aspectWidth, aspectHeight); + + gl.useProgram(shader.program); + shader.updateUniform("u_Exposure", state.renderingParameters.exposure, false); + + const hdrLoc = shader.getUniformLocation("u_MainSampler"); + gl.activeTexture(GL.TEXTURE0); + gl.bindTexture( + gl.TEXTURE_2D, + this._floatingPointFramebuffer ? this.mainTextureHDR : this.mainTexture + ); + gl.uniform1i(hdrLoc, 0); + + const tonemapLoc = shader.getUniformLocation("u_TonemapSampler"); + gl.activeTexture(GL.TEXTURE1); + gl.bindTexture(GL.TEXTURE_2D, this.mainTonemapTexture); + gl.uniform1i(tonemapLoc, 1); + + gl.bindBuffer(gl.ARRAY_BUFFER, this.splatVBO); + const posLoc = shader.getAttributeLocation("a_position"); + gl.enableVertexAttribArray(posLoc); + gl.vertexAttribPointer(posLoc, 2, GL.FLOAT, false, 0, 0); + gl.bindBuffer(gl.ARRAY_BUFFER, null); + + gl.disable(gl.DEPTH_TEST); + gl.disable(gl.BLEND); + gl.drawArrays(GL.TRIANGLE_STRIP, 0, 4); + + gl.enable(gl.DEPTH_TEST); + gl.disableVertexAttribArray(posLoc); + } + // vertices with given material // prettier-ignore drawPrimitive(state, renderpassConfiguration, primitive, node, viewProjectionMatrix, sampledTextures, instanceOffset = undefined) diff --git a/source/Renderer/shaders/cubemap.frag b/source/Renderer/shaders/cubemap.frag index 352da8b9..6bb4d6ad 100644 --- a/source/Renderer/shaders/cubemap.frag +++ b/source/Renderer/shaders/cubemap.frag @@ -9,7 +9,8 @@ uniform float u_EnvBlurNormalized; uniform int u_MipCount; uniform samplerCube u_GGXEnvSampler; -out vec4 FragColor; +layout(location = 0) out vec4 FragColor; +layout(location = 1) out uint toneMapFlag; in vec3 v_TexCoords; @@ -18,10 +19,6 @@ void main() vec4 color = textureLod(u_GGXEnvSampler, v_TexCoords, u_EnvBlurNormalized * float(u_MipCount - 1)); color.rgb *= u_EnvIntensity; color.a = 1.0; - -#ifdef LINEAR_OUTPUT FragColor = color.rgba; -#else - FragColor = vec4(toneMap(color.rgb), color.a); -#endif + toneMapFlag = 2u; } diff --git a/source/Renderer/shaders/fullscreen.vert b/source/Renderer/shaders/fullscreen.vert new file mode 100644 index 00000000..fd902190 --- /dev/null +++ b/source/Renderer/shaders/fullscreen.vert @@ -0,0 +1,13 @@ +precision highp float; + +// Fullscreen quad vertex shader. +// Expects the splatVBO quad: positions in [-1, 1] x [-1, 1]. +in vec2 a_position; + +out vec2 v_uv; + +void main() +{ + v_uv = a_position * 0.5 + 0.5; + gl_Position = vec4(a_position, 0.0, 1.0); +} diff --git a/source/Renderer/shaders/pbr.frag b/source/Renderer/shaders/pbr.frag index a9771938..0edd9812 100644 --- a/source/Renderer/shaders/pbr.frag +++ b/source/Renderer/shaders/pbr.frag @@ -30,7 +30,8 @@ precision highp float; #endif -out vec4 g_finalColor; +layout(location = 0) out vec4 g_finalColor; +layout(location = 1) out uint toneMapFlag; void main() @@ -386,6 +387,7 @@ void main() #ifdef LINEAR_OUTPUT g_finalColor = vec4(color.rgb, baseColor.a); + toneMapFlag = 2u; #else g_finalColor = vec4(toneMap(color), baseColor.a); #endif diff --git a/source/Renderer/shaders/splat.frag b/source/Renderer/shaders/splat.frag index 164a6b82..d2e0875e 100644 --- a/source/Renderer/shaders/splat.frag +++ b/source/Renderer/shaders/splat.frag @@ -4,7 +4,8 @@ in vec4 v_color; // (r, g, b, opacity) in vec2 v_uv; // pixel-space offset from splat centre in vec3 v_conic; // (A, B, C) coefficients of the conic section defining the splat's shape -out vec4 g_finalColor; +layout(location = 0) out vec4 g_finalColor; +layout(location = 1) out uint toneMapFlag; void main() { @@ -20,4 +21,11 @@ void main() { discard; } g_finalColor = vec4(v_color.rgb * alpha, alpha); // premultiplied-alpha output +#ifdef LINEAR_OUTPUT + // Convert linear to sRGB? + toneMapFlag = 1u; +#else + // Color is already in sRGB + toneMapFlag = 0u; +#endif } diff --git a/source/Renderer/shaders/tonemap_main.frag b/source/Renderer/shaders/tonemap_main.frag new file mode 100644 index 00000000..54dedd79 --- /dev/null +++ b/source/Renderer/shaders/tonemap_main.frag @@ -0,0 +1,30 @@ +precision highp float; + +// Final tonemapping pass. +// Applies tonemapping for mesh pixels, linearToSRGB for splat pixels, +// and mixes between them based on splatCoverage. +// +// The TONEMAP_* define is injected at compile time to match the selected tone mapper, +// mirroring what pbr.frag does. The u_Exposure uniform is provided by tonemapping.glsl. + +#include + +uniform sampler2D u_MainSampler; +uniform lowp usampler2D u_TonemapSampler; + +in vec2 v_uv; + +out vec4 g_finalColor; + +void main() +{ + vec4 color = texture(u_MainSampler, v_uv); + uint splatCoverage = texture(u_TonemapSampler, v_uv).r; + + if (splatCoverage == 2u) { + color.rgb = toneMap(color.rgb); + } else if (splatCoverage == 1u) { + color.rgb = linearTosRGB(color.rgb); + } + g_finalColor = vec4(color.rgb, color.a); +} From 44d340c2b8cf662e7230824763e4c6eff50569d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Fri, 22 May 2026 15:48:31 +0200 Subject: [PATCH 09/21] Fix depthMask and transmission --- source/Renderer/renderer.js | 12 +++++++++++- source/Renderer/shaders/pbr.frag | 6 ++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index a9524f5d..c5e89114 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -685,7 +685,7 @@ class gltfRenderer { this.viewProjectionMatrix, state, this.shaderCache, - ["LINEAR_OUTPUT 1"] + ["TRANSMISSION_PASS 1"] ); let drawableCounter = 0; @@ -693,6 +693,7 @@ class gltfRenderer { const drawable = instance[0]; let renderpassConfiguration = {}; renderpassConfiguration.linearOutput = true; + renderpassConfiguration.transmission = true; renderpassConfiguration.frameBufferSize = [ this.opaqueFramebufferWidth, this.opaqueFramebufferHeight @@ -723,6 +724,7 @@ class gltfRenderer { for (const drawable of this.transparentDrawables) { let renderpassConfiguration = {}; renderpassConfiguration.linearOutput = true; + renderpassConfiguration.transmission = true; renderpassConfiguration.frameBufferSize = [ this.opaqueFramebufferWidth, this.opaqueFramebufferHeight @@ -971,6 +973,7 @@ class gltfRenderer { GL.ONE_MINUS_SRC_ALPHA ); this.webGl.context.blendEquation(GL.FUNC_ADD); + this.webGl.context.depthMask(false); let textureIndex = 0; @@ -1011,6 +1014,7 @@ class gltfRenderer { this.webGl.context.vertexAttribDivisor(location, 0); this.webGl.context.disableVertexAttribArray(location); + this.webGl.context.depthMask(true); } tonemapPass(state, aspectOffsetX, aspectOffsetY, aspectWidth, aspectHeight) { @@ -1112,6 +1116,10 @@ class gltfRenderer { { fragDefines.push("LINEAR_OUTPUT 1"); } + if (renderpassConfiguration.transmission) + { + fragDefines.push("TRANSMISSION_PASS 1"); + } // POINTS, LINES, LINE_LOOP, LINE_STRIP if (primitive.mode < 4) { @@ -1188,10 +1196,12 @@ class gltfRenderer { this.webGl.context.enable(GL.BLEND); this.webGl.context.blendFuncSeparate(GL.SRC_ALPHA, GL.ONE_MINUS_SRC_ALPHA, GL.ONE, GL.ONE_MINUS_SRC_ALPHA); this.webGl.context.blendEquation(GL.FUNC_ADD); + this.webGl.context.depthMask(false); } else { this.webGl.context.disable(GL.BLEND); + this.webGl.context.depthMask(true); } diff --git a/source/Renderer/shaders/pbr.frag b/source/Renderer/shaders/pbr.frag index ae31f2dc..5f9b1cac 100644 --- a/source/Renderer/shaders/pbr.frag +++ b/source/Renderer/shaders/pbr.frag @@ -49,11 +49,13 @@ void main() } baseColor.a = 1.0; #endif -#ifdef LINEAR_OUTPUT +#ifdef TRANSMISSION_PASS // If used for transmission, we need to invert exposure and tone mapping, so the original color is computed in the general render pass g_finalColor = vec4(toneMapInverse(baseColor.rgb), baseColor.a); + toneMapFlag = 2u; #else - g_finalColor = vec4(linearTosRGB(baseColor.rgb), baseColor.a); + g_finalColor = baseColor; + toneMapFlag = 1u; #endif // PBR Flow. This else goes all the way to the end of the file #else From 5d25964432fbe8b2d8816ffb63afaa74dbd2fc9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Wed, 27 May 2026 18:20:38 +0200 Subject: [PATCH 10/21] Add extra framebuffer for splats --- source/GltfState/gltf_state.js | 4 +- source/Renderer/renderer.js | 200 ++++++++++++++++-- source/Renderer/shaders/pbr.frag | 7 +- .../Renderer/shaders/specular_glossiness.frag | 13 +- source/Renderer/shaders/splat.frag | 10 +- source/Renderer/shaders/splat_composite.frag | 39 ++++ source/Renderer/shaders/tonemap_main.frag | 2 +- 7 files changed, 236 insertions(+), 39 deletions(-) create mode 100644 source/Renderer/shaders/splat_composite.frag diff --git a/source/GltfState/gltf_state.js b/source/GltfState/gltf_state.js index 729d408c..bd279724 100644 --- a/source/GltfState/gltf_state.js +++ b/source/GltfState/gltf_state.js @@ -69,8 +69,8 @@ class GltfState { /** KHR_gaussian_splatting */ KHR_gaussian_splatting: true }, - /** clear color expressed as list of ints in the range [0, 255] */ - clearColor: [58, 64, 74, 255], + /** clear color expressed as list of floats in the range [0, 1] */ + clearColor: [0.22, 0.25, 0.29, 1], /** exposure factor */ exposure: 1.0, /** KHR_lights_punctual */ diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index c5e89114..b5176cf9 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -23,6 +23,7 @@ import splatVertexShader from "./shaders/splat.vert"; import splatFragmentShader from "./shaders/splat.frag"; import fullscreenVertShader from "./shaders/fullscreen.vert"; import tonemapMainFragShader from "./shaders/tonemap_main.frag"; +import splatCompositeFragShader from "./shaders/splat_composite.frag"; import { gltfLight } from "../gltf/light.js"; import { jsToGl } from "../gltf/utils.js"; import { gltfMaterial } from "../gltf/material.js"; @@ -69,6 +70,7 @@ class gltfRenderer { shaderSources.set("splat.frag", splatFragmentShader); shaderSources.set("fullscreen.vert", fullscreenVertShader); shaderSources.set("tonemap_main.frag", tonemapMainFragShader); + shaderSources.set("splat_composite.frag", splatCompositeFragShader); this.shaderCache = new ShaderCache(shaderSources, this.webGl); @@ -110,6 +112,12 @@ class gltfRenderer { // Tracks last value of state.renderingParameters.floatingPointFramebuffer so we // only re-attach when the setting actually changes. this._floatingPointFramebuffer = undefined; + + // Splat isolation framebuffer: same colour format as mainFramebuffer, + // shares mainDepthTexture for depth-test (no depth writes). + this.splatColorTexture = undefined; + this.splatColorTextureHDR = undefined; + this.splatFramebuffer = undefined; } ///////////////////////////////////////////////////////////////////// @@ -279,6 +287,39 @@ class gltfRenderer { context.drawBuffers([context.COLOR_ATTACHMENT0, context.COLOR_ATTACHMENT1]); context.bindFramebuffer(context.FRAMEBUFFER, null); + this.splatColorTexture = context.createTexture(); + context.bindTexture(context.TEXTURE_2D, this.splatColorTexture); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MIN_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MAG_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_S, context.CLAMP_TO_EDGE); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_T, context.CLAMP_TO_EDGE); + context.texImage2D(context.TEXTURE_2D, 0, context.RGBA, + hdrW, hdrH, 0, context.RGBA, context.UNSIGNED_BYTE, null); + context.bindTexture(context.TEXTURE_2D, null); + + if (context.supports_EXT_color_buffer_half_float) { + this.splatColorTextureHDR = context.createTexture(); + context.bindTexture(context.TEXTURE_2D, this.splatColorTextureHDR); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MIN_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_MAG_FILTER, context.NEAREST); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_S, context.CLAMP_TO_EDGE); + context.texParameteri(context.TEXTURE_2D, context.TEXTURE_WRAP_T, context.CLAMP_TO_EDGE); + context.texImage2D(context.TEXTURE_2D, 0, context.RGBA16F, + hdrW, hdrH, 0, context.RGBA, context.HALF_FLOAT, null); + context.bindTexture(context.TEXTURE_2D, null); + } + + // Initially attach the LDR colour texture; re-attached together with + // mainFramebuffer when _floatingPointFramebuffer is resolved. + this.splatFramebuffer = context.createFramebuffer(); + context.bindFramebuffer(context.FRAMEBUFFER, this.splatFramebuffer); + context.framebufferTexture2D(context.FRAMEBUFFER, context.COLOR_ATTACHMENT0, + context.TEXTURE_2D, this.splatColorTexture, 0); + context.framebufferTexture2D(context.FRAMEBUFFER, context.DEPTH_ATTACHMENT, + context.TEXTURE_2D, this.mainDepthTexture, 0); + context.drawBuffers([context.COLOR_ATTACHMENT0]); + context.bindFramebuffer(context.FRAMEBUFFER, null); + this.maxVertAttributes = context.getParameter(context.MAX_VERTEX_ATTRIBS); this.initialized = true; @@ -416,20 +457,57 @@ class gltfRenderer { null ); this.webGl.context.bindTexture(this.webGl.context.TEXTURE_2D, null); + + // Resize splat isolation colour textures. + this.webGl.context.bindTexture( + this.webGl.context.TEXTURE_2D, + this.splatColorTexture + ); + this.webGl.context.texImage2D( + this.webGl.context.TEXTURE_2D, + 0, + this.webGl.context.RGBA, + this.currentWidth, + this.currentHeight, + 0, + this.webGl.context.RGBA, + this.webGl.context.UNSIGNED_BYTE, + null + ); + if (this.splatColorTextureHDR) { + this.webGl.context.bindTexture( + this.webGl.context.TEXTURE_2D, + this.splatColorTextureHDR + ); + this.webGl.context.texImage2D( + this.webGl.context.TEXTURE_2D, + 0, + this.webGl.context.RGBA16F, + this.currentWidth, + this.currentHeight, + 0, + this.webGl.context.RGBA, + this.webGl.context.HALF_FLOAT, + null + ); + } + this.webGl.context.bindTexture(this.webGl.context.TEXTURE_2D, null); } } } // frame state clearFrame(clearColor) { + // Convert clear color from sRGB to linear since we will always transfer to sRGB by default + const linearClearColor = clearColor.map((c) => Math.pow(c, 2.2)); this.webGl.context.bindFramebuffer(this.webGl.context.FRAMEBUFFER, this.opaqueFramebuffer); - this.webGl.context.clearColor(...clearColor); + this.webGl.context.clearColor(...linearClearColor); this.webGl.context.clear(GL.COLOR_BUFFER_BIT | GL.DEPTH_BUFFER_BIT); this.webGl.context.bindFramebuffer( this.webGl.context.FRAMEBUFFER, this.opaqueFramebufferMSAA ); - this.webGl.context.clearColor(...clearColor); + this.webGl.context.clearColor(...linearClearColor); this.webGl.context.clear(GL.COLOR_BUFFER_BIT | GL.DEPTH_BUFFER_BIT); this.webGl.context.bindFramebuffer(this.webGl.context.FRAMEBUFFER, this.scatterFramebuffer); this.webGl.context.clearColor(0, 0, 0, 0); @@ -440,14 +518,14 @@ class gltfRenderer { const gl = this.webGl.context; gl.bindFramebuffer(gl.FRAMEBUFFER, this.mainFramebuffer); // Clear color attachment 0 to the scene background colour. - gl.clearBufferfv(gl.COLOR, 0, clearColor); - // Clear tonemap flag attachment to 0 (linear only = no-op pass-through). - gl.clearBufferuiv(gl.COLOR, 1, new Uint32Array([0, 0, 0, 0])); + gl.clearBufferfv(gl.COLOR, 0, linearClearColor); + // Clear tonemap flag attachment to 1 (linear to sRGB by default). + gl.clearBufferuiv(gl.COLOR, 1, new Uint32Array([1, 1, 1, 1])); // Clear depth. gl.clearBufferfv(gl.DEPTH, 0, new Float32Array([1.0])); } this.webGl.context.bindFramebuffer(this.webGl.context.FRAMEBUFFER, null); - this.webGl.context.clearColor(...clearColor); + this.webGl.context.clearColor(...linearClearColor); this.webGl.context.clear(GL.COLOR_BUFFER_BIT | GL.DEPTH_BUFFER_BIT); this.webGl.context.bindFramebuffer(this.webGl.context.FRAMEBUFFER, null); } @@ -766,21 +844,34 @@ class gltfRenderer { this.webGl.context.generateMipmap(this.webGl.context.TEXTURE_2D); } - // Re-attach mainFramebuffer COLOR_ATTACHMENT0 if the floating-point toggle changed. + // Re-attach mainFramebuffer and splatFramebuffer COLOR_ATTACHMENT0 if + // the floating-point toggle changed. const wantFP = state.renderingParameters.floatingPointFramebuffer !== false && this.mainTextureHDR !== undefined; if (wantFP !== this._floatingPointFramebuffer) { this._floatingPointFramebuffer = wantFP; const gl = this.webGl.context; + const colorTex = wantFP ? this.mainTextureHDR : this.mainTexture; + const splatColorTex = wantFP ? this.splatColorTextureHDR : this.splatColorTexture; gl.bindFramebuffer(gl.FRAMEBUFFER, this.mainFramebuffer); gl.framebufferTexture2D( gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, - wantFP ? this.mainTextureHDR : this.mainTexture, + colorTex, 0 ); + if (this.splatFramebuffer) { + gl.bindFramebuffer(gl.FRAMEBUFFER, this.splatFramebuffer); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + gl.COLOR_ATTACHMENT0, + gl.TEXTURE_2D, + splatColorTex ?? this.splatColorTexture, + 0 + ); + } gl.bindFramebuffer(gl.FRAMEBUFFER, null); } @@ -850,18 +941,41 @@ class gltfRenderer { this.transparentDrawables ); this.needsRedraw = false; + + // ── Transparent drawables (non-splat and splat) in depth-sorted order ─── + // Splats are rendered one at a time into the dedicated splatFramebuffer + // (which shares mainDepthTexture for depth-testing) and + // immediately composited back into mainFramebuffer before the next + // drawable is processed. This preserves correct depth-sorted blending + // order between splats and regular transparent geometry. for (const drawable of this.transparentDrawables.filter((a) => a.depth <= 0)) { if ( drawable.primitive.extensions?.KHR_gaussian_splatting !== undefined && state.renderingParameters.enabledExtensions.KHR_gaussian_splatting ) { - this.drawSplat( - state, - drawable.primitive, - drawable.node, - this.projMatrix, - this.viewMatrix - ); + // Isolate this splat in its own framebuffer pass. + if (this.splatFramebuffer) { + const gl = this.webGl.context; + + // Clear only the colour attachment – depth is shared with + // mainFramebuffer and must not be touched here. + gl.bindFramebuffer(gl.FRAMEBUFFER, this.splatFramebuffer); + gl.clearBufferfv(gl.COLOR, 0, [0, 0, 0, 0]); + gl.viewport(aspectOffsetX, aspectOffsetY, aspectWidth, aspectHeight); + + this.drawSplat( + state, + drawable.primitive, + drawable.node, + this.projMatrix, + this.viewMatrix + ); + + // Composite into mainFramebuffer. + gl.bindFramebuffer(gl.FRAMEBUFFER, this.mainFramebuffer); + gl.viewport(aspectOffsetX, aspectOffsetY, aspectWidth, aspectHeight); + this.splatCompositePass(state, !drawable.primitive.linear); + } } else { let renderpassConfiguration = {}; renderpassConfiguration.linearOutput = true; @@ -1062,6 +1176,61 @@ class gltfRenderer { gl.disableVertexAttribArray(posLoc); } + // Composites the splat isolation framebuffer onto the currently-bound mainFramebuffer. + splatCompositePass(state, srgbToLinear) { + const gl = this.webGl.context; + + const fragDefines = []; + this.pushFragParameterDefines(fragDefines, state); + const fragHash = this.shaderCache.selectShader("splat_composite.frag", fragDefines); + const vertHash = this.shaderCache.selectShader("fullscreen.vert", []); + if (!fragHash || !vertHash) return; + const shader = this.shaderCache.getShaderProgram(fragHash, vertHash); + if (!shader) return; + + gl.useProgram(shader.program); + + const splatTex = this._floatingPointFramebuffer + ? this.splatColorTextureHDR + : this.splatColorTexture; + const samplerLoc = shader.getUniformLocation("u_SplatSampler"); + gl.activeTexture(GL.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, splatTex ?? this.splatColorTexture); + gl.uniform1i(samplerLoc, 0); + + const srgbLoc = shader.getUniformLocation("u_SrgbToLinear"); + gl.uniform1i(srgbLoc, srgbToLinear ? 1 : 0); + shader.updateUniform("u_Exposure", state.renderingParameters.exposure, false); + + // Bind the fullscreen quad VBO (a_position attribute expected by fullscreen.vert). + gl.bindBuffer(gl.ARRAY_BUFFER, this.splatVBO); + const posLoc = shader.getAttributeLocation("a_position"); + gl.enableVertexAttribArray(posLoc); + gl.vertexAttribPointer(posLoc, 2, GL.FLOAT, false, 0, 0); + gl.bindBuffer(gl.ARRAY_BUFFER, null); + + gl.enable(gl.BLEND); + this.webGl.context.blendFuncSeparate( + GL.SRC_ALPHA, + GL.ONE_MINUS_SRC_ALPHA, + GL.ONE, + GL.ONE_MINUS_SRC_ALPHA + ); + this.webGl.context.blendEquation(GL.FUNC_ADD); + + // No depth test or depth write – this is a pure compositing pass. + gl.disable(gl.DEPTH_TEST); + gl.depthMask(false); + + gl.drawArrays(GL.TRIANGLE_STRIP, 0, 4); + + // Restore state. + gl.disable(gl.BLEND); + gl.enable(gl.DEPTH_TEST); + gl.depthMask(true); + gl.disableVertexAttribArray(posLoc); + } + // vertices with given material // prettier-ignore drawPrimitive(state, renderpassConfiguration, primitive, node, viewProjectionMatrix, sampledTextures, instanceOffset = undefined) @@ -1550,6 +1719,7 @@ class gltfRenderer { break; case GltfState.ToneMaps.NONE: default: + fragDefines.push("TONEMAP_NONE 1"); break; } diff --git a/source/Renderer/shaders/pbr.frag b/source/Renderer/shaders/pbr.frag index 5f9b1cac..690b9b2c 100644 --- a/source/Renderer/shaders/pbr.frag +++ b/source/Renderer/shaders/pbr.frag @@ -400,15 +400,14 @@ void main() baseColor.a = 1.0; #endif -#ifdef LINEAR_OUTPUT + g_finalColor = vec4(color.rgb, baseColor.a); toneMapFlag = 2u; -#else - g_finalColor = vec4(toneMap(color), baseColor.a); -#endif + #else // In case of missing data for a debug view, render a checkerboard. + toneMapFlag = 0u; g_finalColor = vec4(1.0); { float frequency = 0.02; diff --git a/source/Renderer/shaders/specular_glossiness.frag b/source/Renderer/shaders/specular_glossiness.frag index 5ba6572a..45038d7a 100644 --- a/source/Renderer/shaders/specular_glossiness.frag +++ b/source/Renderer/shaders/specular_glossiness.frag @@ -10,7 +10,8 @@ precision highp float; #include -out vec4 g_finalColor; +layout(location = 0) out vec4 g_finalColor; +layout(location = 1) out uint toneMapFlag; // Specular Glossiness uniform vec3 u_SpecularFactor; @@ -142,9 +143,7 @@ void main() #endif -#ifdef MATERIAL_UNLIT - color = baseColor.rgb; -#elif defined(NOT_TRIANGLE) && !defined(HAS_NORMAL_VEC3) +#if defined(NOT_TRIANGLE) && !defined(HAS_NORMAL_VEC3) //Points or Lines with no NORMAL attribute SHOULD be rendered without lighting and instead use the sum of the base color value and the emissive value. color = f_emissive + baseColor.rgb; #else @@ -162,14 +161,12 @@ void main() baseColor.a = 1.0; #endif -#ifdef LINEAR_OUTPUT g_finalColor = vec4(color.rgb, baseColor.a); -#else - g_finalColor = vec4(toneMap(color), baseColor.a); -#endif + toneMapFlag = 2u; #else // In case of missing data for a debug view, render a checkerboard. + toneMapFlag = 0u; g_finalColor = vec4(1.0); { float frequency = 0.02; diff --git a/source/Renderer/shaders/splat.frag b/source/Renderer/shaders/splat.frag index d2e0875e..76bf5a1a 100644 --- a/source/Renderer/shaders/splat.frag +++ b/source/Renderer/shaders/splat.frag @@ -5,7 +5,6 @@ in vec2 v_uv; // pixel-space offset from splat centre in vec3 v_conic; // (A, B, C) coefficients of the conic section defining the splat's shape layout(location = 0) out vec4 g_finalColor; -layout(location = 1) out uint toneMapFlag; void main() { @@ -20,12 +19,5 @@ void main() { if (alpha < 1.0 / 255.0) { discard; } - g_finalColor = vec4(v_color.rgb * alpha, alpha); // premultiplied-alpha output -#ifdef LINEAR_OUTPUT - // Convert linear to sRGB? - toneMapFlag = 1u; -#else - // Color is already in sRGB - toneMapFlag = 0u; -#endif + g_finalColor = max(vec4(v_color.rgb * alpha, alpha), vec4(0.0)); // premultiplied-alpha output } diff --git a/source/Renderer/shaders/splat_composite.frag b/source/Renderer/shaders/splat_composite.frag new file mode 100644 index 00000000..72ea01d2 --- /dev/null +++ b/source/Renderer/shaders/splat_composite.frag @@ -0,0 +1,39 @@ +precision highp float; + +#include + +// Composites the splat isolation framebuffer onto the main framebuffer. +// Outputs: +// location 0 – color (blended onto main) +// location 1 – toneMapFlag 1u = "linearToSRGB" for the tonemap_main pass + +uniform sampler2D u_SplatSampler; +uniform bool u_SrgbToLinear; + +in vec2 v_uv; + +layout(location = 0) out vec4 g_finalColor; +layout(location = 1) out uint g_toneMapFlag; + +void main() { + vec4 splat = texture(u_SplatSampler, v_uv); + + // Discard fully-transparent fragments so the toneMapFlag texture is not + // overwritten for pixels that were never touched by a splat. +#ifndef TONEMAP_NONE + vec3 multiplied = splat.rgb * splat.a; + float maxComponent = max(max(multiplied.r, multiplied.g), multiplied.b); + if (maxComponent < 1.0 / 255.0) { + discard; + } +#endif + splat.rgb = clamp(splat.rgb, vec3(0.0), vec3(1.0)); + if (u_SrgbToLinear) { + splat.rgb = sRGBToLinear(splat.rgb); + } + g_finalColor = splat; + // The composited content is linear; tonemap_main.frag should apply + // linearToSRGB (flag value 0) in its final step. + g_toneMapFlag = 1u; + +} diff --git a/source/Renderer/shaders/tonemap_main.frag b/source/Renderer/shaders/tonemap_main.frag index 54dedd79..c242c4b6 100644 --- a/source/Renderer/shaders/tonemap_main.frag +++ b/source/Renderer/shaders/tonemap_main.frag @@ -18,8 +18,8 @@ out vec4 g_finalColor; void main() { - vec4 color = texture(u_MainSampler, v_uv); uint splatCoverage = texture(u_TonemapSampler, v_uv).r; + vec4 color = texture(u_MainSampler, v_uv); if (splatCoverage == 2u) { color.rgb = toneMap(color.rgb); From bb6e6a22c1522e5199f97cd83e6785bf753d549a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Wed, 27 May 2026 18:21:16 +0200 Subject: [PATCH 11/21] Improve inverse tonemapping for KHR_PBR_NEUTRAL --- source/Renderer/shaders/tonemapping.glsl | 58 ++++++++++++++++-------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/source/Renderer/shaders/tonemapping.glsl b/source/Renderer/shaders/tonemapping.glsl index 1232f078..c270be86 100644 --- a/source/Renderer/shaders/tonemapping.glsl +++ b/source/Renderer/shaders/tonemapping.glsl @@ -212,28 +212,50 @@ vec3 toneMap_KhronosPbrNeutralInverse(vec3 toneMapped) { const float startCompression = 0.8 - 0.04; const float desaturation = 0.15; - - // This is an approximation - the forward function has complex conditional logic - // that makes perfect inversion difficult + const float d = 1.0 - startCompression; + vec3 color = toneMapped; - - // Try to undo the desaturation mix - float peak = max(color.r, max(color.g, color.b)); - - // Approximate inverse of the compression - if (peak >= startCompression) { - const float d = 1.0 - startCompression; - // Approximate inverse of: newPeak = 1. - d * d / (peak + d - startCompression) - // This is a rough approximation - float originalPeak = peak / (1.0 - peak + startCompression); - color *= originalPeak / peak; + + float newPeak = max(color.r, max(color.g, color.b)); + + if (newPeak >= startCompression) + { + // Exact inverse of: newPeak = 1 - d*d / (peak + d - startCompression) + // => peak = d*d / (1 - newPeak) - d + startCompression + float peak = d * d / (1.0 - newPeak) - d + startCompression; + + // Undo desaturation mix: + // forward: color = mix(color_scaled, newPeak * vec3(1), g) + // => color_scaled = (color - g * newPeak) / (1 - g) + float g = 1.0 - 1.0 / (desaturation * (peak - newPeak) + 1.0); + // Guard against divide-by-zero when g ~ 1 (extremely bright/saturated, info lost) + if (g < 0.9999) + { + color = (color - g * newPeak) / (1.0 - g); + } + + // Undo peak scaling: forward did color *= newPeak / peak + color *= peak / newPeak; } - - // Try to undo the offset + + // Undo offset: forward subtracted offset computed from original x_orig. + // For x_orig >= 0.08: offset = 0.04, so x_out = x_orig - 0.04 >= 0.04 + // For x_orig < 0.08: offset = x_orig - 6.25*x_orig^2, so x_out = 6.25*x_orig^2 + // => x_orig = sqrt(x_out / 6.25), threshold in output space: x_out < 0.04 float x = min(color.r, min(color.g, color.b)); - float offset = x < 0.08 ? x - 6.25 * x * x : 0.04; + float offset; + if (x < 0.04) + { + // Clamp to 0 before sqrt to avoid NaN from floating-point rounding going slightly negative + float x_orig = sqrt(max(x, 0.0) / 6.25); + offset = x_orig - x; + } + else + { + offset = 0.04; + } color += offset; - + return color; } From d561f5fd600c93c1e7d783197e5ba40e5b2ed922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Wed, 27 May 2026 18:34:09 +0200 Subject: [PATCH 12/21] Discard too large splats --- source/Renderer/shaders/splat.vert | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/Renderer/shaders/splat.vert b/source/Renderer/shaders/splat.vert index 74d5fc9f..d515db6b 100644 --- a/source/Renderer/shaders/splat.vert +++ b/source/Renderer/shaders/splat.vert @@ -232,6 +232,15 @@ void main() vec2 quad_pixel_size = vec2(3.4 * sqrt(a), 3.4 * sqrt(c)); // screen space half quad height and width vec2 quad_ndc_size = quad_pixel_size / vec2(u_FramebufferSize) * 2.0; // in ndc space clip_splat_center.xy = clip_splat_center.xy + a_position * quad_ndc_size; + + // Discard too large splats that would cover the entire screen + float min_screen = float(min(u_FramebufferSize.x, u_FramebufferSize.y)); + float max_quad_size = max(quad_pixel_size.x, quad_pixel_size.y); + if (max_quad_size > min_screen) + { + gl_Position = vec4(0.0, 0.0, 2.0, 1.0); + return; + } v_uv = a_position * quad_pixel_size; gl_Position = clip_splat_center; From cc83b3c759751c21fc6b2228f21ef4c14df16216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Thu, 28 May 2026 11:45:47 +0200 Subject: [PATCH 13/21] Apply inverse global rotation to spherical harmonics --- source/Renderer/renderer.js | 6 ++++++ source/Renderer/shaders/splat.vert | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index b5176cf9..34e6d4ff 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -1054,6 +1054,12 @@ class gltfRenderer { false, node.worldTransform ); + + const rotationMatrixInv = mat3.create(); + mat3.fromQuat(rotationMatrixInv, node.worldQuaternion); + mat3.transpose(rotationMatrixInv, rotationMatrixInv); + this.shader.updateUniform("u_ModelRotationInv", rotationMatrixInv); + this.webGl.context.uniformMatrix4fv( this.shader.getUniformLocation("u_ViewMatrix"), false, diff --git a/source/Renderer/shaders/splat.vert b/source/Renderer/shaders/splat.vert index d515db6b..a1199d5b 100644 --- a/source/Renderer/shaders/splat.vert +++ b/source/Renderer/shaders/splat.vert @@ -13,6 +13,7 @@ out vec3 v_conic; uniform mat4 u_ProjectionMatrix; uniform mat4 u_ViewMatrix; uniform mat4 u_ModelMatrix; +uniform mat3 u_ModelRotationInv; uniform uint u_TextureWidth; uniform ivec2 u_FramebufferSize; uniform vec2 u_FocalLength; @@ -248,7 +249,9 @@ void main() #ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_1 - vec3 view_dir = normalize(splat_center - u_Camera); + vec3 view_dir_world = normalize(splat_center - u_Camera); + vec3 view_dir = u_ModelRotationInv * view_dir_world; // local-frame direction + float x = view_dir.x; float y = view_dir.y; float z = view_dir.z; From 95b63f06e21eec89c14d0635bcb1a6cca21475b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Thu, 28 May 2026 18:30:06 +0200 Subject: [PATCH 14/21] Fix quantization --- source/Renderer/shaders/splat.vert | 10 +++++----- source/gltf/primitive.js | 18 ++++++++++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/source/Renderer/shaders/splat.vert b/source/Renderer/shaders/splat.vert index a1199d5b..b866aa55 100644 --- a/source/Renderer/shaders/splat.vert +++ b/source/Renderer/shaders/splat.vert @@ -20,27 +20,27 @@ uniform vec2 u_FocalLength; uniform vec3 u_Camera; #ifdef POSITION_IS_INTEGER -uniform isampler2D u_POSITIONSampler; +uniform mediump isampler2D u_POSITIONSampler; #elif defined(POSITION_IS_UINTEGER) -uniform usampler2D u_POSITIONSampler; +uniform mediump usampler2D u_POSITIONSampler; #else uniform sampler2D u_POSITIONSampler; #endif #ifdef ROTATION_IS_INTEGER -uniform isampler2D u_ROTATIONSampler; +uniform mediump isampler2D u_ROTATIONSampler; #else uniform sampler2D u_ROTATIONSampler; #endif #ifdef SCALE_IS_UINTEGER -uniform usampler2D u_SCALESampler; +uniform mediump usampler2D u_SCALESampler; #else uniform sampler2D u_SCALESampler; #endif #ifdef OPACITY_IS_UINTEGER -uniform usampler2D u_OPACITYSampler; +uniform mediump usampler2D u_OPACITYSampler; #else uniform sampler2D u_OPACITYSampler; #endif diff --git a/source/gltf/primitive.js b/source/gltf/primitive.js index 90737942..17a0e3ae 100644 --- a/source/gltf/primitive.js +++ b/source/gltf/primitive.js @@ -62,6 +62,7 @@ class gltfPrimitive extends GltfObject { } //Currently only support types relevant for gaussian splatting + // If an alignment of 4 bytes is not possible for a given format, the correct alignment is returned. _getInternalTextureFormat(componentType, componentCount, normalized = false) { if (componentType === GL.FLOAT) { switch (componentCount) { @@ -76,11 +77,12 @@ class gltfPrimitive extends GltfObject { if (componentType === GL.UNSIGNED_BYTE) { switch (componentCount) { case 1: // OPACITY - return { internalFormat: GL.R8, format: GL.RED }; // Opacity is always normalized + return { internalFormat: GL.R8, format: GL.RED, alignment: 1 }; // Opacity is always normalized case 3: // POSITION, SCALE return { internalFormat: normalized ? GL.RGB8 : GL.RGB8UI, - format: normalized ? GL.RGB : GL.RGB_INTEGER + format: normalized ? GL.RGB : GL.RGB_INTEGER, + alignment: 1 }; } } @@ -89,7 +91,8 @@ class gltfPrimitive extends GltfObject { case 3: // POSITION return { internalFormat: normalized ? GL.RGB8_SNORM : GL.RGB8I, - format: normalized ? GL.RGB : GL.RGB_INTEGER + format: normalized ? GL.RGB : GL.RGB_INTEGER, + alignment: 1 }; case 4: // ROTATION return { internalFormat: GL.RGBA8_SNORM, format: GL.RGBA }; // Rotation is always normalized @@ -100,15 +103,15 @@ class gltfPrimitive extends GltfObject { if (componentType === GL.UNSIGNED_SHORT) { switch (componentCount) { case 1: // OPACITY - return { internalFormat: GL.R16UI, format: GL.RED_INTEGER }; + return { internalFormat: GL.R16UI, format: GL.RED_INTEGER, alignment: 2 }; // Opacity is always normalized case 3: // POSITION, SCALE - return { internalFormat: GL.RGB16UI, format: GL.RGB_INTEGER }; + return { internalFormat: GL.RGB16UI, format: GL.RGB_INTEGER, alignment: 2 }; } } if (componentType === GL.SHORT) { switch (componentCount) { case 3: // POSITION - return { internalFormat: GL.RGB16I, format: GL.RGB_INTEGER }; + return { internalFormat: GL.RGB16I, format: GL.RGB_INTEGER, alignment: 2 }; case 4: // ROTATION return { internalFormat: GL.RGBA16I, format: GL.RGBA_INTEGER }; } @@ -157,6 +160,8 @@ class gltfPrimitive extends GltfObject { const data = accessor.getDeinterlacedView(gltf); const paddedData = new data.constructor(size * size * componentCount); paddedData.set(data); + + webGlContext.pixelStorei(webGlContext.UNPACK_ALIGNMENT, formats.alignment ?? 4); webGlContext.texImage2D( webGlContext.TEXTURE_2D, 0, //level @@ -168,6 +173,7 @@ class gltfPrimitive extends GltfObject { accessor.componentType, paddedData ); + webGlContext.pixelStorei(webGlContext.UNPACK_ALIGNMENT, 4); // restore default // Ensure mipmapping is disabled and the sampler is configured correctly. webGlContext.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_WRAP_S, GL.CLAMP_TO_EDGE); webGlContext.texParameteri(GL.TEXTURE_2D, GL.TEXTURE_WRAP_T, GL.CLAMP_TO_EDGE); From 245901f2c056486d19efde1f780805cc951d3786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Thu, 28 May 2026 20:11:35 +0200 Subject: [PATCH 15/21] Use define instead of uniform --- source/Renderer/renderer.js | 9 +++++---- source/Renderer/shaders/splat_composite.frag | 7 +++---- source/gltf/gltf.js | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index 34e6d4ff..b11d18d0 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -974,7 +974,7 @@ class gltfRenderer { // Composite into mainFramebuffer. gl.bindFramebuffer(gl.FRAMEBUFFER, this.mainFramebuffer); gl.viewport(aspectOffsetX, aspectOffsetY, aspectWidth, aspectHeight); - this.splatCompositePass(state, !drawable.primitive.linear); + this.splatCompositePass(state, drawable.primitive.linear); } } else { let renderpassConfiguration = {}; @@ -1183,11 +1183,14 @@ class gltfRenderer { } // Composites the splat isolation framebuffer onto the currently-bound mainFramebuffer. - splatCompositePass(state, srgbToLinear) { + splatCompositePass(state, isLinear) { const gl = this.webGl.context; const fragDefines = []; this.pushFragParameterDefines(fragDefines, state); + if (isLinear) { + fragDefines.push("LINEAR_OUTPUT 1"); + } const fragHash = this.shaderCache.selectShader("splat_composite.frag", fragDefines); const vertHash = this.shaderCache.selectShader("fullscreen.vert", []); if (!fragHash || !vertHash) return; @@ -1204,8 +1207,6 @@ class gltfRenderer { gl.bindTexture(gl.TEXTURE_2D, splatTex ?? this.splatColorTexture); gl.uniform1i(samplerLoc, 0); - const srgbLoc = shader.getUniformLocation("u_SrgbToLinear"); - gl.uniform1i(srgbLoc, srgbToLinear ? 1 : 0); shader.updateUniform("u_Exposure", state.renderingParameters.exposure, false); // Bind the fullscreen quad VBO (a_position attribute expected by fullscreen.vert). diff --git a/source/Renderer/shaders/splat_composite.frag b/source/Renderer/shaders/splat_composite.frag index 72ea01d2..ba9d0050 100644 --- a/source/Renderer/shaders/splat_composite.frag +++ b/source/Renderer/shaders/splat_composite.frag @@ -8,7 +8,6 @@ precision highp float; // location 1 – toneMapFlag 1u = "linearToSRGB" for the tonemap_main pass uniform sampler2D u_SplatSampler; -uniform bool u_SrgbToLinear; in vec2 v_uv; @@ -28,9 +27,9 @@ void main() { } #endif splat.rgb = clamp(splat.rgb, vec3(0.0), vec3(1.0)); - if (u_SrgbToLinear) { - splat.rgb = sRGBToLinear(splat.rgb); - } +#ifndef LINEAR_OUTPUT + splat.rgb = sRGBToLinear(splat.rgb); +#endif g_finalColor = splat; // The composited content is linear; tonemap_main.frag should apply // linearToSRGB (flag value 0) in its final step. diff --git a/source/gltf/gltf.js b/source/gltf/gltf.js index 8daf7545..1d13d776 100644 --- a/source/gltf/gltf.js +++ b/source/gltf/gltf.js @@ -21,6 +21,7 @@ import { gltfVariant } from "./variant.js"; const allowedExtensions = [ "KHR_animation_pointer", "KHR_draco_mesh_compression", + "KHR_gaussian_splatting", "KHR_lights_image_based", "KHR_lights_punctual", "KHR_materials_anisotropy", From ff1dd602dde021036ed87cad4d33e64755dd1a5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Thu, 28 May 2026 20:43:24 +0200 Subject: [PATCH 16/21] Improve performance by decreasing sigma and discard earlier --- source/Renderer/shaders/splat.vert | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/source/Renderer/shaders/splat.vert b/source/Renderer/shaders/splat.vert index b866aa55..40278844 100644 --- a/source/Renderer/shaders/splat.vert +++ b/source/Renderer/shaders/splat.vert @@ -49,6 +49,11 @@ uniform highp sampler2DArray u_SHCoefficientsSampler; #define SH_C0 0.28209479177387814 +// Gaussian radius multiplier: how many standard deviations to extend the screen-space quad. +// exp(-0.5 * SPLAT_SIGMA²) gives the minimum opacity at the quad boundary. +// 3.33 → 1/255 (mathematically exact), 3.0 → 1/90 (23% smaller quads, ~40% fewer fragments). +#define SPLAT_SIGMA 3.0 + #ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_1 #define SH_C1_0 -0.4886025119029199 #define SH_C1_1 0.4886025119029199 @@ -228,16 +233,19 @@ void main() // Calculate the inverse of the covariance matrix v_conic = vec3(c * det_inv, -b * det_inv, a * det_inv); - // pow(e, pow(-3.4, 2) * -0.5) = 1/255, so 3.4 is the standard deviation in terms of the Gaussian falloff that results in a radius of 1 pixel when the variance is 1. - // sqrt(a) and sqrt(c) are the standard deviations in x and y direction, so multiplying them with 3.4 gives us the radius in pixels where the Gaussian falloff results in 1/255 opacity. - vec2 quad_pixel_size = vec2(3.4 * sqrt(a), 3.4 * sqrt(c)); // screen space half quad height and width + // pow(e, pow(-SPLAT_SIGMA, 2) * -0.5) gives the boundary opacity. + // sqrt(a) and sqrt(c) are the standard deviations in x and y, so multiplying + // by SPLAT_SIGMA gives the quad half-size in pixels at that opacity threshold. + vec2 quad_pixel_size = vec2(SPLAT_SIGMA * sqrt(a), SPLAT_SIGMA * sqrt(c)); // screen space half quad height and width vec2 quad_ndc_size = quad_pixel_size / vec2(u_FramebufferSize) * 2.0; // in ndc space clip_splat_center.xy = clip_splat_center.xy + a_position * quad_ndc_size; - // Discard too large splats that would cover the entire screen + // Discard splats whose projected size exceeds half the screen — + // they are almost certainly behind or very close to the camera and + // would cause extreme overdraw with negligible visual contribution. float min_screen = float(min(u_FramebufferSize.x, u_FramebufferSize.y)); float max_quad_size = max(quad_pixel_size.x, quad_pixel_size.y); - if (max_quad_size > min_screen) + if (max_quad_size > min_screen * 0.5) { gl_Position = vec4(0.0, 0.0, 2.0, 1.0); return; From 7fe0f1c8bc1fa0bdfc1822bfeee5f41b54694877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Fri, 29 May 2026 14:25:17 +0200 Subject: [PATCH 17/21] Use half float texture and move texture fetch --- source/Renderer/shaders/splat.vert | 66 ++++++++++++++---------------- source/gltf/primitive.js | 5 ++- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/source/Renderer/shaders/splat.vert b/source/Renderer/shaders/splat.vert index 40278844..0a47514c 100644 --- a/source/Renderer/shaders/splat.vert +++ b/source/Renderer/shaders/splat.vert @@ -45,7 +45,7 @@ uniform mediump usampler2D u_OPACITYSampler; uniform sampler2D u_OPACITYSampler; #endif -uniform highp sampler2DArray u_SHCoefficientsSampler; +uniform mediump sampler2DArray u_SHCoefficientsSampler; #define SH_C0 0.28209479177387814 @@ -174,40 +174,6 @@ void main() opacity = texelFetch(u_OPACITYSampler, texelCoord, 0).x; #endif - // Fetch SH coefficients early to avoid GPU stall - // Degree 0 - vec3 sh0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 0), 0).rgb; - -#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_1 - // Degree 1 - vec3 sh1_0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 1), 0).rgb; - vec3 sh1_1 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 2), 0).rgb; - vec3 sh1_2 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 3), 0).rgb; - - -#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_2 - // Degree 2 - vec3 sh2_0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 4), 0).rgb; - vec3 sh2_1 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 5), 0).rgb; - vec3 sh2_2 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 6), 0).rgb; - vec3 sh2_3 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 7), 0).rgb; - vec3 sh2_4 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 8), 0).rgb; - - -#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_3 - // Degree 3 - vec3 sh3_0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 9), 0).rgb; - vec3 sh3_1 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 10), 0).rgb; - vec3 sh3_2 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 11), 0).rgb; - vec3 sh3_3 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 12), 0).rgb; - vec3 sh3_4 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 13), 0).rgb; - vec3 sh3_5 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 14), 0).rgb; - vec3 sh3_6 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 15), 0).rgb; -#endif - -#endif -#endif - splat_center = (u_ModelMatrix * vec4(splat_center, 1.0)).xyz; vec4 view_splat_center = u_ViewMatrix * vec4(splat_center, 1.0); @@ -253,6 +219,36 @@ void main() v_uv = a_position * quad_pixel_size; gl_Position = clip_splat_center; + // Degree 0 + vec3 sh0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 0), 0).rgb; + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_1 + // Degree 1 + vec3 sh1_0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 1), 0).rgb; + vec3 sh1_1 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 2), 0).rgb; + vec3 sh1_2 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 3), 0).rgb; + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_2 + // Degree 2 + vec3 sh2_0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 4), 0).rgb; + vec3 sh2_1 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 5), 0).rgb; + vec3 sh2_2 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 6), 0).rgb; + vec3 sh2_3 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 7), 0).rgb; + vec3 sh2_4 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 8), 0).rgb; + +#ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_3 + // Degree 3 + vec3 sh3_0 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 9), 0).rgb; + vec3 sh3_1 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 10), 0).rgb; + vec3 sh3_2 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 11), 0).rgb; + vec3 sh3_3 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 12), 0).rgb; + vec3 sh3_4 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 13), 0).rgb; + vec3 sh3_5 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 14), 0).rgb; + vec3 sh3_6 = texelFetch(u_SHCoefficientsSampler, ivec3(texelCoord, 15), 0).rgb; +#endif +#endif +#endif + vec3 color = sh0 * SH_C0; #ifdef HAS_GAUSSIAN_SPLATTING_DEGREE_1 diff --git a/source/gltf/primitive.js b/source/gltf/primitive.js index 17a0e3ae..fad3c828 100644 --- a/source/gltf/primitive.js +++ b/source/gltf/primitive.js @@ -226,7 +226,10 @@ class gltfPrimitive extends GltfObject { let texture = webGlContext.createTexture(); webGlContext.bindTexture(webGlContext.TEXTURE_2D_ARRAY, texture); // Set texture format and upload data. - let internalFormat = componentCount === 4 ? webGlContext.RGBA32F : webGlContext.RGB32F; + // Use 16-bit half-precision floats: half the bandwidth of RGB32F with negligible + // quality loss for SH coefficients. WebGL2 accepts Float32Array with FLOAT type + // when the internal format is a 16F format — the driver converts on upload. + let internalFormat = componentCount === 4 ? webGlContext.RGBA16F : webGlContext.RGB16F; let format = componentCount === 4 ? webGlContext.RGBA : webGlContext.RGB; let type = webGlContext.FLOAT; webGlContext.texImage3D( From 6ba0a38e8bb92613b3852b8d9fb63004d671f85d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Fri, 29 May 2026 15:33:23 +0200 Subject: [PATCH 18/21] Update docs --- API.md | 35 +++++++++++++++++++++++++++++------ README.md | 1 + 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/API.md b/API.md index 088c5613..99afe983 100644 --- a/API.md +++ b/API.md @@ -117,6 +117,7 @@ GltfState containing a state for visualization in GltfView * [.animationIndices](#GltfState+animationIndices) * [.animationTimer](#GltfState+animationTimer) * [.variant](#GltfState+variant) + * [.needsRedraw](#GltfState+needsRedraw) * [.renderingParameters](#GltfState+renderingParameters) * [.morphing](#GltfState+renderingParameters.morphing) * [.skinning](#GltfState+renderingParameters.skinning) @@ -132,6 +133,7 @@ GltfState containing a state for visualization in GltfView * [.environmentRotation](#GltfState+renderingParameters.environmentRotation) * [.useDirectionalLightsWithDisabledIBL](#GltfState+renderingParameters.useDirectionalLightsWithDisabledIBL) * [.internalMSAA](#GltfState+renderingParameters.internalMSAA) + * [.floatingPointFramebuffer](#GltfState+renderingParameters.floatingPointFramebuffer) * _static_ * [.ToneMaps](#GltfState.ToneMaps) * [.KHR_PBR_NEUTRAL](#GltfState.ToneMaps.KHR_PBR_NEUTRAL) @@ -239,6 +241,12 @@ animation timer allows to control the animation time ### gltfState.variant KHR_materials_variants +**Kind**: instance property of [GltfState](#GltfState) + + +### gltfState.needsRedraw +Indicates whether the view needs to be redrawn, currently used to indicate new sorting orders for gaussian splatting + **Kind**: instance property of [GltfState](#GltfState) @@ -262,6 +270,7 @@ parameters used to configure the rendering * [.environmentRotation](#GltfState+renderingParameters.environmentRotation) * [.useDirectionalLightsWithDisabledIBL](#GltfState+renderingParameters.useDirectionalLightsWithDisabledIBL) * [.internalMSAA](#GltfState+renderingParameters.internalMSAA) + * [.floatingPointFramebuffer](#GltfState+renderingParameters.floatingPointFramebuffer) @@ -278,7 +287,7 @@ skin / skeleton #### renderingParameters.clearColor -clear color expressed as list of ints in the range [0, 255] +clear color expressed as list of floats in the range [0, 1] **Kind**: static property of [renderingParameters](#GltfState+renderingParameters) @@ -351,6 +360,12 @@ If this is set to true, directional lights will be generated if IBL is disabled #### renderingParameters.internalMSAA MSAA used for cases which are not handled by the browser (e.g. Transmission) +**Kind**: static property of [renderingParameters](#GltfState+renderingParameters) + + +#### renderingParameters.floatingPointFramebuffer +Use RGBA16F floating-point main framebuffer instead of RGBA8 + **Kind**: static property of [renderingParameters](#GltfState+renderingParameters) @@ -1010,7 +1025,9 @@ Fit view to updated canvas size without changing rotation if distance is incorre ## ResourceLoaderUtils -Utility class providing static helper methods for resource loading operations, such as extracting file extensions, resolving folder paths, normalizing relative paths, and detecting absolute URLs. +Utility class providing static helper methods for resource loading operations, +such as extracting file extensions, resolving folder paths, normalizing relative +paths, and detecting absolute URLs. **Kind**: global class @@ -1026,7 +1043,8 @@ Utility class providing static helper methods for resource loading operations, s Extracts the file extension from a filename. **Kind**: static method of [ResourceLoaderUtils](#ResourceLoaderUtils) -**Returns**: string \| undefined - The lowercase file extension (without the leading dot), or `undefined` if the filename has no extension. +**Returns**: string \| undefined - The lowercase file extension (without the leading dot), + or `undefined` if the filename has no extension. | Param | Type | Description | | --- | --- | --- | @@ -1038,7 +1056,8 @@ Extracts the file extension from a filename. Returns the directory portion of a file path, including the trailing slash. **Kind**: static method of [ResourceLoaderUtils](#ResourceLoaderUtils) -**Returns**: string - The path up to and including the last `/`, or an empty string if no `/` is present. +**Returns**: string - The path up to and including the last `/`, or an empty string + if no `/` is present. | Param | Type | Description | | --- | --- | --- | @@ -1047,7 +1066,10 @@ Returns the directory portion of a file path, including the trailing slash. ### ResourceLoaderUtils.cleanRelativePath(relativePath) ⇒ string -Normalizes a relative URL path by resolving `.` and `..` segments. - Strips a leading `./` prefix. - Collapses `/./` sequences to `/`. - Resolves `/../` sequences by removing the preceding path segment. +Normalizes a relative URL path by resolving `.` and `..` segments. +- Strips a leading `./` prefix. +- Collapses `/./` sequences to `/`. +- Resolves `/../` sequences by removing the preceding path segment. **Kind**: static method of [ResourceLoaderUtils](#ResourceLoaderUtils) **Returns**: string - The normalized path with dot segments resolved. @@ -1059,7 +1081,8 @@ Normalizes a relative URL path by resolving `.` and `..` segments. - Strips a le ### ResourceLoaderUtils.isAbsoluteUrl(url) ⇒ boolean -Determines whether a URL is absolute (i.e. contains a scheme such as `http:` or `data:`). A URL is considered absolute when it contains a `:` that appears before any `/`. +Determines whether a URL is absolute (i.e. contains a scheme such as `http:` or `data:`). +A URL is considered absolute when it contains a `:` that appears before any `/`. **Kind**: static method of [ResourceLoaderUtils](#ResourceLoaderUtils) **Returns**: boolean - `true` if the URL is absolute, `false` otherwise. diff --git a/README.md b/README.md index ed089363..60f04a3c 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ Formerly hosted together with the example frontend at the [glTF Sample Viewer](h - [x] glTF 2.0 - [x] [KHR_animation_pointer](https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_animation_pointer) - [x] [KHR_draco_mesh_compression](https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_draco_mesh_compression) +- [x] [KHR_gaussian_splatting](https://github.com/KhronosGroup/glTF/blob/main/extensions/2.0/Khronos/KHR_gaussian_splatting/README.md) - [x] [KHR_lights_punctual](https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_lights_punctual) - [x] [KHR_materials_anisotropy](https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_materials_anisotropy) - [x] [KHR_materials_clearcoat](https://github.com/KhronosGroup/glTF/tree/main/extensions/2.0/Khronos/KHR_materials_clearcoat) From c681d2026f08ce3b3e5b994cf6e9fd6e90b606eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Tue, 2 Jun 2026 15:41:57 +0200 Subject: [PATCH 19/21] Revert "Improve inverse tonemapping for KHR_PBR_NEUTRAL" This reverts commit bb6e6a22c1522e5199f97cd83e6785bf753d549a. --- source/Renderer/shaders/tonemapping.glsl | 58 ++++++++---------------- 1 file changed, 18 insertions(+), 40 deletions(-) diff --git a/source/Renderer/shaders/tonemapping.glsl b/source/Renderer/shaders/tonemapping.glsl index c270be86..1232f078 100644 --- a/source/Renderer/shaders/tonemapping.glsl +++ b/source/Renderer/shaders/tonemapping.glsl @@ -212,50 +212,28 @@ vec3 toneMap_KhronosPbrNeutralInverse(vec3 toneMapped) { const float startCompression = 0.8 - 0.04; const float desaturation = 0.15; - const float d = 1.0 - startCompression; - + + // This is an approximation - the forward function has complex conditional logic + // that makes perfect inversion difficult vec3 color = toneMapped; - - float newPeak = max(color.r, max(color.g, color.b)); - - if (newPeak >= startCompression) - { - // Exact inverse of: newPeak = 1 - d*d / (peak + d - startCompression) - // => peak = d*d / (1 - newPeak) - d + startCompression - float peak = d * d / (1.0 - newPeak) - d + startCompression; - - // Undo desaturation mix: - // forward: color = mix(color_scaled, newPeak * vec3(1), g) - // => color_scaled = (color - g * newPeak) / (1 - g) - float g = 1.0 - 1.0 / (desaturation * (peak - newPeak) + 1.0); - // Guard against divide-by-zero when g ~ 1 (extremely bright/saturated, info lost) - if (g < 0.9999) - { - color = (color - g * newPeak) / (1.0 - g); - } - - // Undo peak scaling: forward did color *= newPeak / peak - color *= peak / newPeak; + + // Try to undo the desaturation mix + float peak = max(color.r, max(color.g, color.b)); + + // Approximate inverse of the compression + if (peak >= startCompression) { + const float d = 1.0 - startCompression; + // Approximate inverse of: newPeak = 1. - d * d / (peak + d - startCompression) + // This is a rough approximation + float originalPeak = peak / (1.0 - peak + startCompression); + color *= originalPeak / peak; } - - // Undo offset: forward subtracted offset computed from original x_orig. - // For x_orig >= 0.08: offset = 0.04, so x_out = x_orig - 0.04 >= 0.04 - // For x_orig < 0.08: offset = x_orig - 6.25*x_orig^2, so x_out = 6.25*x_orig^2 - // => x_orig = sqrt(x_out / 6.25), threshold in output space: x_out < 0.04 + + // Try to undo the offset float x = min(color.r, min(color.g, color.b)); - float offset; - if (x < 0.04) - { - // Clamp to 0 before sqrt to avoid NaN from floating-point rounding going slightly negative - float x_orig = sqrt(max(x, 0.0) / 6.25); - offset = x_orig - x; - } - else - { - offset = 0.04; - } + float offset = x < 0.08 ? x - 6.25 * x * x : 0.04; color += offset; - + return color; } From 83339904d875ff9fb571a7a4611af6cd5b957581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Wed, 3 Jun 2026 15:09:10 +0200 Subject: [PATCH 20/21] Add warning for orthographic cameras --- source/Renderer/renderer.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index b11d18d0..d2032bdd 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -953,6 +953,9 @@ class gltfRenderer { drawable.primitive.extensions?.KHR_gaussian_splatting !== undefined && state.renderingParameters.enabledExtensions.KHR_gaussian_splatting ) { + if (currentCamera.type !== "perspective") { + console.warn("Splat rendering with non-perspective projection is undefined"); + } // Isolate this splat in its own framebuffer pass. if (this.splatFramebuffer) { const gl = this.webGl.context; From d09f6de2d61cbf993583ad6d5d031b6335fcfd93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=A4rtl?= Date: Wed, 3 Jun 2026 18:55:21 +0200 Subject: [PATCH 21/21] Apply world transform to WASM sorter --- source/Renderer/renderer.js | 3 ++- source/gltf/primitive.js | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/source/Renderer/renderer.js b/source/Renderer/renderer.js index d2032bdd..ac2fe9c2 100644 --- a/source/Renderer/renderer.js +++ b/source/Renderer/renderer.js @@ -1003,7 +1003,8 @@ class gltfRenderer { if (primitive.skip) return; // Request an async worker sort each frame (no-op if the previous sort // has not yet finished or no worker is available). - primitive.requestSort(viewMatrix); + const modelViewMatrix = mat4.multiply(mat4.create(), viewMatrix, node.worldTransform); + primitive.requestSort(modelViewMatrix); if (primitive.sortPending) { this.needsRedraw = true; } diff --git a/source/gltf/primitive.js b/source/gltf/primitive.js index fad3c828..f6a18702 100644 --- a/source/gltf/primitive.js +++ b/source/gltf/primitive.js @@ -769,29 +769,29 @@ class gltfPrimitive extends GltfObject { * Request an asynchronous back-to-front sort of the splat indices. * Safe to call every frame — the request is dropped while a previous sort * is still in flight. - * @param {Float32Array} viewMatrix Column-major 4×4 view matrix. + * @param {Float32Array} modelViewMatrix Column-major 4×4 model-view matrix (view * world). */ - requestSort(viewMatrix) { + requestSort(modelViewMatrix) { if (this.sortWorker === undefined) { return; } if (!this.sortWorkerReady || this.sortPending) { // Worker is busy — keep the latest matrix so it sorts immediately once ready - this.queuedViewMatrix = new Float32Array(viewMatrix); + this.queuedViewMatrix = new Float32Array(modelViewMatrix); return; } // Skip if the view matrix hasn't changed since the last dispatched sort. if (this.lastSortViewMatrix !== undefined) { let same = true; for (let i = 0; i < 16; i++) { - if (this.lastSortViewMatrix[i] !== viewMatrix[i]) { + if (this.lastSortViewMatrix[i] !== modelViewMatrix[i]) { same = false; break; } } if (same) return; } - const vm = new Float32Array(viewMatrix); + const vm = new Float32Array(modelViewMatrix); this.lastSortViewMatrix = vm; this.sortPending = true; this.sortWorker.postMessage({ type: "sort", viewMatrix: vm });