Reuse unchanged region draw batches and merge adjacent commands

src/main/java/me/jellysquid/mods/sodium/client/render/chunk/DefaultChunkRenderer.java

@@ -26,6 +26,7 @@
 import me.jellysquid.mods.sodium.client.render.viewport.CameraTransform;
 import me.jellysquid.mods.sodium.client.util.BitwiseMath;
 import org.lwjgl.system.MemoryUtil;
+import org.embeddedt.embeddium.render.ShaderModBridge;
 import java.util.Iterator;
 
 public class DefaultChunkRenderer extends ShaderChunkRenderer {
@@ -102,6 +103,18 @@ private static void fillCommandBuffer(MultiDrawBatch batch,
                                           CameraTransform camera,
                                           TerrainRenderPass pass,
                                           boolean useBlockFaceCulling) {
+        boolean merged = !pass.isSorted() && !ShaderModBridge.areShadersEnabled();
+        int count = renderList.getSectionsWithGeometryCount();
+        long faceBits = useBlockFaceCulling && !pass.isSorted() && count > 0
+                ? DrawBatchCache.faceBits(renderRegion, camera) : 0L;
+        var cache = renderDataStorage.getDrawBatchCache();
+        var hit = cache.find(count, renderList, pass.isReverseOrder(), pass.isSorted(),
+                useBlockFaceCulling, merged, faceBits);
+        if (hit != null) {
+            hit.load(batch);
+            return;
+        }
+
         batch.clear();
 
         var iterator = renderList.sectionsWithGeometryIterator(pass.isReverseOrder());
@@ -123,7 +136,7 @@ private static void fillCommandBuffer(MultiDrawBatch batch,
             int chunkY = originY + LocalSectionIndex.unpackY(sectionIndex);
             int chunkZ = originZ + LocalSectionIndex.unpackZ(sectionIndex);
 
-            var pMeshData = renderDataStorage.getDataPointer(sectionIndex);
+            var pMeshData = renderDataStorage.getDataPointerForRead(sectionIndex);
 
             int slices;
 
@@ -139,6 +152,14 @@ private static void fillCommandBuffer(MultiDrawBatch batch,
                 addDrawCommands(batch, pMeshData, slices, indexPointerMask);
             }
         }
+
+        if (merged) {
+            MergedDraws.merge(batch);
+        }
+        if (cache.isEnabled()) {
+            cache.victim().store(batch, count, renderList, pass.isReverseOrder(), pass.isSorted(),
+                    useBlockFaceCulling, merged, faceBits);
+        }
     }
 
     @SuppressWarnings("IntegerMultiplicationImplicitCastToLong")

src/main/java/me/jellysquid/mods/sodium/client/render/chunk/DrawBatchCache.java

@@ -0,0 +1,153 @@
+package me.jellysquid.mods.sodium.client.render.chunk;
+
+import java.lang.reflect.Field;
+import org.lwjgl.system.Pointer;
+import me.jellysquid.mods.sodium.client.gl.device.MultiDrawBatch;
+import me.jellysquid.mods.sodium.client.render.chunk.lists.ChunkRenderList;
+import me.jellysquid.mods.sodium.client.render.chunk.region.RenderRegion;
+import me.jellysquid.mods.sodium.client.render.viewport.CameraTransform;
+import me.jellysquid.mods.sodium.client.util.BitwiseMath;
+
+/** The last two draw command lists for one region and terrain pass, invalidated whenever its mesh data changes. */
+public final class DrawBatchCache {
+    private static final sun.misc.Unsafe U;
+    private static final long INT_BASE, LONG_BASE;
+    static {
+        try {
+            Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe");
+            field.setAccessible(true);
+            U = (sun.misc.Unsafe) field.get(null);
+        } catch (ReflectiveOperationException e) {
+            throw new ExceptionInInitializerError(e);
+        }
+        INT_BASE = U.arrayBaseOffset(int[].class);
+        LONG_BASE = U.arrayBaseOffset(long[].class);
+    }
+
+    private DrawBatchCache() {}
+
+    /**
+     * The comparison bits Embeddium's getVisibleFaces computes for every section coordinate of the region: for each
+     * x (8), y (4) and z (8) the "greater than min - 3" and "less than max + 3" tests of the camera's integer position.
+     * Two cameras with equal bits give every section of the region the same visible-face mask.
+     */
+    public static long faceBits(RenderRegion region, CameraTransform camera) {
+        long bits = 0L;
+        int bit = 0;
+        int cx = camera.intX, cy = camera.intY, cz = camera.intZ;
+        int ox = region.getChunkX(), oy = region.getChunkY(), oz = region.getChunkZ();
+        for (int i = 0; i < RenderRegion.REGION_WIDTH; i++) {
+            int min = ox + i << 4;
+            bits |= (long) BitwiseMath.greaterThan(cx, min - 3) << bit++;
+            bits |= (long) BitwiseMath.lessThan(cx, min + 16 + 3) << bit++;
+        }
+        for (int i = 0; i < RenderRegion.REGION_HEIGHT; i++) {
+            int min = oy + i << 4;
+            bits |= (long) BitwiseMath.greaterThan(cy, min - 3) << bit++;
+            bits |= (long) BitwiseMath.lessThan(cy, min + 16 + 3) << bit++;
+        }
+        for (int i = 0; i < RenderRegion.REGION_LENGTH; i++) {
+            int min = oz + i << 4;
+            bits |= (long) BitwiseMath.greaterThan(cz, min - 3) << bit++;
+            bits |= (long) BitwiseMath.lessThan(cz, min + 16 + 3) << bit++;
+        }
+        return bits;
+    }
+
+    /** Per SectionRenderDataStorage: the commands of its last two fills. */
+    public static final class Slots {
+        private boolean enabled = Pointer.POINTER_SIZE == Long.BYTES;
+
+        public boolean isEnabled() {
+            return this.enabled;
+        }
+
+        public void disable() {
+            this.enabled = false;
+            this.invalidate();
+        }
+
+        public void invalidate() {
+            this.a.valid = this.b.valid = false;
+        }
+        final Entry a = new Entry(), b = new Entry();
+        long uses;
+
+        /** The entry whose inputs equal these, or null. */
+        public Entry find(int listCount, ChunkRenderList renderList, boolean reverse, boolean sorted, boolean culling, boolean merged, long faceBits) {
+            if (!this.enabled) return null;
+            if (this.a.same(listCount, renderList, reverse, sorted, culling, merged, faceBits)) return this.a.touch(++this.uses);
+            if (this.b.same(listCount, renderList, reverse, sorted, culling, merged, faceBits)) return this.b.touch(++this.uses);
+            return null;
+        }
+
+        /** The entry to overwrite with a fresh fill (the least recently used one). */
+        public Entry victim() {
+            return (this.a.lastUse <= this.b.lastUse ? this.a : this.b).touch(++this.uses);
+        }
+    }
+
+    /** One remembered fill: its inputs and the commands in [0, size). */
+    public static final class Entry {
+        boolean valid;
+        int listCount;
+        final byte[] list = new byte[RenderRegion.REGION_SIZE];
+        boolean reverse, sorted, culling, merged;
+        long faceBits;
+        int size;
+        int[] baseVertex = new int[16], elementCount = new int[16];
+        long[] elementPointer = new long[16];
+        long lastUse;
+
+        Entry touch(long stamp) {
+            this.lastUse = stamp;
+            return this;
+        }
+
+        boolean same(int listCount, ChunkRenderList renderList, boolean reverse, boolean sorted, boolean culling, boolean merged, long faceBits) {
+            return this.valid && this.listCount == listCount && this.reverse == reverse && this.sorted == sorted
+                    && this.culling == culling && this.merged == merged && this.faceBits == faceBits
+                    && renderList.matchesGeometry(this.list, listCount);
+        }
+
+        /**
+         * Copies the batch's commands [0, size) and the inputs they were filled from.
+         */
+        public void store(MultiDrawBatch batch, int listCount, ChunkRenderList renderList, boolean reverse, boolean sorted, boolean culling, boolean merged,
+                          long faceBits) {
+            int n = batch.size;
+            if (n > this.baseVertex.length) {
+                int cap = Math.max(n, this.baseVertex.length * 2);
+                this.baseVertex = new int[cap];
+                this.elementCount = new int[cap];
+                this.elementPointer = new long[cap];
+            }
+            if (n > 0) {
+                U.copyMemory(null, batch.pBaseVertex, this.baseVertex, INT_BASE, (long) n << 2);
+                U.copyMemory(null, batch.pElementCount, this.elementCount, INT_BASE, (long) n << 2);
+                U.copyMemory(null, batch.pElementPointer, this.elementPointer, LONG_BASE, (long) n << 3);
+            }
+            this.size = n;
+            this.listCount = listCount;
+            renderList.copyGeometry(this.list);
+            this.reverse = reverse;
+            this.sorted = sorted;
+            this.culling = culling;
+            this.merged = merged;
+            this.faceBits = faceBits;
+            this.valid = true;
+        }
+
+        /** Writes the remembered commands into the batch: batch.size and [0, size) of its three arrays. */
+        public void load(MultiDrawBatch batch) {
+            int n = this.size;
+            if (n > 0) {
+                U.copyMemory(this.baseVertex, INT_BASE, null, batch.pBaseVertex, (long) n << 2);
+                U.copyMemory(this.elementCount, INT_BASE, null, batch.pElementCount, (long) n << 2);
+                U.copyMemory(this.elementPointer, LONG_BASE, null, batch.pElementPointer, (long) n << 3);
+            }
+            batch.size = n;
+        }
+
+    }
+}

src/main/java/me/jellysquid/mods/sodium/client/render/chunk/MergedDraws.java

@@ -0,0 +1,40 @@
+package me.jellysquid.mods.sodium.client.render.chunk;
+
+import me.jellysquid.mods.sodium.client.gl.device.MultiDrawBatch;
+import org.lwjgl.system.MemoryUtil;
+import org.lwjgl.system.Pointer;
+
+/** Merges adjacent commands using the shared quad index buffer without changing their triangle order. */
+final class MergedDraws {
+    private MergedDraws() {}
+
+    public static void merge(MultiDrawBatch batch) {
+        int n = batch.size;
+        if (n < 2 || Pointer.POINTER_SIZE != Long.BYTES) return;
+        long pb = batch.pBaseVertex, pc = batch.pElementCount, pp = batch.pElementPointer;
+        int base = MemoryUtil.memGetInt(pb), count = MemoryUtil.memGetInt(pc);
+        long ptr = MemoryUtil.memGetAddress(pp);
+        int out = 0;
+        for (int i = 1; i < n; i++) {
+            int b = MemoryUtil.memGetInt(pb + ((long) i << 2)), c = MemoryUtil.memGetInt(pc + ((long) i << 2));
+            long p = MemoryUtil.memGetAddress(pp + ((long) i << 3));
+            if (ptr == 0L && p == 0L && count >= 0 && c >= 0 && count % 6 == 0 && c % 6 == 0 && (long) base + (long) (count / 6) * 4L == (long) b
+                    && (long) count + (long) c <= Integer.MAX_VALUE) {
+                count += c;
+            } else {
+                MemoryUtil.memPutInt(pb + ((long) out << 2), base);
+                MemoryUtil.memPutInt(pc + ((long) out << 2), count);
+                MemoryUtil.memPutAddress(pp + ((long) out << 3), ptr);
+                out++;
+                base = b;
+                count = c;
+                ptr = p;
+            }
+        }
+        MemoryUtil.memPutInt(pb + ((long) out << 2), base);
+        MemoryUtil.memPutInt(pc + ((long) out << 2), count);
+        MemoryUtil.memPutAddress(pp + ((long) out << 3), ptr);
+        batch.size = out + 1;
+    }
+
+}

src/main/java/me/jellysquid/mods/sodium/client/render/chunk/data/SectionRenderDataStorage.java

@@ -1,6 +1,7 @@
 package me.jellysquid.mods.sodium.client.render.chunk.data;
 
 import me.jellysquid.mods.sodium.client.gl.arena.GlBufferSegment;
+import me.jellysquid.mods.sodium.client.render.chunk.DrawBatchCache;
 import me.jellysquid.mods.sodium.client.gl.util.VertexRange;
 import me.jellysquid.mods.sodium.client.model.quad.properties.ModelQuadFacing;
 import me.jellysquid.mods.sodium.client.render.chunk.region.RenderRegion;
@@ -13,13 +14,19 @@ public class SectionRenderDataStorage {
     private final GlBufferSegment[] indexAllocations = new GlBufferSegment[RenderRegion.REGION_SIZE];
 
     private final long pMeshDataArray;
+    private final DrawBatchCache.Slots drawBatchCache = new DrawBatchCache.Slots();
+
+    public DrawBatchCache.Slots getDrawBatchCache() {
+        return this.drawBatchCache;
+    }
 
     public SectionRenderDataStorage() {
         this.pMeshDataArray = SectionRenderDataUnsafe.allocateHeap(RenderRegion.REGION_SIZE);
     }
 
     public void setMeshes(int localSectionIndex,
                           GlBufferSegment allocation, @Nullable GlBufferSegment indexAllocation, VertexRange[] ranges) {
+        this.drawBatchCache.invalidate();
         if (this.allocations[localSectionIndex] != null) {
             this.allocations[localSectionIndex].delete();
             this.allocations[localSectionIndex] = null;
@@ -33,7 +40,7 @@ public void setMeshes(int localSectionIndex,
         this.allocations[localSectionIndex] = allocation;
         this.indexAllocations[localSectionIndex] = indexAllocation;
 
-        var pMeshData = this.getDataPointer(localSectionIndex);
+        var pMeshData = this.getInternalDataPointer(localSectionIndex);
 
         int sliceMask = 0;
         int vertexOffset = allocation.getOffset();
@@ -67,29 +74,32 @@ public void setMeshes(int localSectionIndex,
     }
 
     public void removeMeshes(int localSectionIndex) {
+        this.drawBatchCache.invalidate();
         if (this.allocations[localSectionIndex] != null) {
             this.allocations[localSectionIndex].delete();
             this.allocations[localSectionIndex] = null;
 
-            SectionRenderDataUnsafe.clear(this.getDataPointer(localSectionIndex));
+            SectionRenderDataUnsafe.clear(this.getInternalDataPointer(localSectionIndex));
         }
 
         removeIndexBuffer(localSectionIndex);
     }
 
     public void removeIndexBuffer(int localSectionIndex) {
+        this.drawBatchCache.invalidate();
         if (this.indexAllocations[localSectionIndex] != null) {
             this.indexAllocations[localSectionIndex].delete();
             this.indexAllocations[localSectionIndex] = null;
         }
     }
 
     public void replaceIndexBuffer(int localSectionIndex, GlBufferSegment indexAllocation) {
+        this.drawBatchCache.invalidate();
         removeIndexBuffer(localSectionIndex);
 
         this.indexAllocations[localSectionIndex] = indexAllocation;
 
-        var pMeshData = this.getDataPointer(localSectionIndex);
+        var pMeshData = this.getInternalDataPointer(localSectionIndex);
 
         int indexOffset = indexAllocation != null ? indexAllocation.getOffset() * 4 : 0;
 
@@ -101,6 +111,7 @@ public void replaceIndexBuffer(int localSectionIndex, GlBufferSegment indexAlloc
     }
 
     public void onBufferResized() {
+        this.drawBatchCache.invalidate();
         for (int sectionIndex = 0; sectionIndex < RenderRegion.REGION_SIZE; sectionIndex++) {
             this.updateMeshes(sectionIndex);
         }
@@ -118,7 +129,7 @@ private void updateMeshes(int sectionIndex) {
         var vertexOffset = allocation.getOffset();
         var indexOffset = indexAllocation != null ? indexAllocation.getOffset() * 4 : 0;
 
-        var data = this.getDataPointer(sectionIndex);
+        var data = this.getInternalDataPointer(sectionIndex);
 
         for (int facing = 0; facing < ModelQuadFacing.COUNT; facing++) {
             SectionRenderDataUnsafe.setVertexOffset(data, facing, vertexOffset);
@@ -130,11 +141,28 @@ private void updateMeshes(int sectionIndex) {
         }
     }
 
+    /**
+     * Exposes a mutable pointer. The caller may retain it, so command caching stays disabled for this storage.
+     */
     public long getDataPointer(int sectionIndex) {
+        this.drawBatchCache.disable();
+        return this.getInternalDataPointer(sectionIndex);
+    }
+
+    /**
+     * Returns a pointer for reading render data only. Callers must not write through this pointer;
+     * use the storage mutation methods or {@link #getDataPointer(int)} for mutable access.
+     */
+    public long getDataPointerForRead(int sectionIndex) {
+        return this.getInternalDataPointer(sectionIndex);
+    }
+
+    private long getInternalDataPointer(int sectionIndex) {
         return SectionRenderDataUnsafe.heapPointer(this.pMeshDataArray, sectionIndex);
     }
 
     public void delete() {
+        this.drawBatchCache.invalidate();
         for (var allocation : this.allocations) {
             if (allocation != null) {
                 allocation.delete();

src/main/java/me/jellysquid/mods/sodium/client/render/chunk/lists/ChunkRenderList.java

@@ -81,6 +81,15 @@ public void add(RenderSection render) {
         return new ByteArrayIterator(this.sectionsWithEntities, this.sectionsWithEntitiesCount);
     }
 
+    public boolean matchesGeometry(byte[] sections, int count) {
+        return count == this.sectionsWithGeometryCount
+                && java.util.Arrays.equals(this.sectionsWithGeometry, 0, count, sections, 0, count);
+    }
+
+    public void copyGeometry(byte[] destination) {
+        System.arraycopy(this.sectionsWithGeometry, 0, destination, 0, this.sectionsWithGeometryCount);
+    }
+
     public int getSectionsWithGeometryCount() {
         return this.sectionsWithGeometryCount;
     }