Support PRAGMA mmap_size (unix)

ext/fts5/fts5.go

@@ -5,7 +5,7 @@ package fts5
 
 import (
 	"github.com/ncruces/go-sqlite3"
-	"github.com/ncruces/go-sqlite3-wasm/v3/fts5"
+	"github.com/ncruces/go-sqlite3-wasm/v4/fts5"
 )
 
 // Register registers the fts5 extension.

ext/rtree/rtree.go

@@ -6,7 +6,7 @@ package rtree
 
 import (
 	"github.com/ncruces/go-sqlite3"
-	"github.com/ncruces/go-sqlite3-wasm/v3/rtree"
+	"github.com/ncruces/go-sqlite3-wasm/v4/rtree"
 )
 
 // Register registers the rtree and geopoly virtual tables.

ext/spellfix1/spellfix.go

@@ -5,7 +5,7 @@ package spellfix1
 
 import (
 	"github.com/ncruces/go-sqlite3"
-	"github.com/ncruces/go-sqlite3-wasm/v3/spellfix"
+	"github.com/ncruces/go-sqlite3-wasm/v4/spellfix"
 )
 
 // Register registers the spellfix1 virtual table.

ext/vec1/vec1.go

@@ -5,7 +5,7 @@ package vec1
 
 import (
 	"github.com/ncruces/go-sqlite3"
-	"github.com/ncruces/go-sqlite3-wasm/v3/vec1"
+	"github.com/ncruces/go-sqlite3-wasm/v4/vec1"
 )
 
 // Register registers the vec1 vector extension.

go.mod

@@ -3,7 +3,7 @@ module github.com/ncruces/go-sqlite3
 go 1.26.0
 
 require (
-	github.com/ncruces/go-sqlite3-wasm/v3 v3.4.35304
+	github.com/ncruces/go-sqlite3-wasm/v4 v4.0.35304
 	github.com/ncruces/julianday v1.0.0
 	github.com/ncruces/sort v1.0.0
 	github.com/ncruces/wbt v1.0.0

go.sum

@@ -2,8 +2,8 @@ github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
 github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/ncruces/go-sqlite3-wasm/v3 v3.4.35304 h1:5i8tdrFKOaWR80ruV9nWuPvqSX3P+2zuB7Al8ym3/hY=
-github.com/ncruces/go-sqlite3-wasm/v3 v3.4.35304/go.mod h1:aIkZSy1oIDrgwKtlNzI/qBezIH/CKAL8fNjYw7SxnXk=
+github.com/ncruces/go-sqlite3-wasm/v4 v4.0.35304 h1:rqpCiJNqT21zczDJ9GtAQkc3rLDoeEyd+Sckxk/ogF4=
+github.com/ncruces/go-sqlite3-wasm/v4 v4.0.35304/go.mod h1:2jqV+aun+kgJl3uxD8w0v9SEX7Ifq/UNhxK/tJvDCYI=
 github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
 github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
 github.com/ncruces/sort v1.0.0 h1:tXeitUzE3B8I2z+rsqbKUBThBlJJ34glqCDP0XReJmk=

internal/sqlite3_wrap/wrap.go

@@ -3,7 +3,7 @@ package sqlite3_wrap
 import (
 	"io"
 
-	sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v3"
+	sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v4"
 	"github.com/ncruces/go-sqlite3/internal/errutil"
 )
 

util/sql3util/parse.go

@@ -6,7 +6,7 @@ import (
 	"encoding/binary"
 	"strings"
 
-	parser "github.com/ncruces/go-sqlite3-wasm/v3/parser"
+	parser "github.com/ncruces/go-sqlite3-wasm/v4/parser"
 	"github.com/ncruces/go-sqlite3/internal/errutil"
 )
 

util/vfsutil/wrap.go

@@ -183,3 +183,11 @@ func WrapSharedMemory(f vfs.File) vfs.SharedMemory {
 	}
 	return nil
 }
+
+// WrapMemoryMapper helps wrap [vfs.FileMemoryMapper].
+func WrapMemoryMapper(f vfs.File) vfs.MemoryMapper {
+	if f, ok := f.(vfs.FileMemoryMapper); ok {
+		return f.MemoryMapper()
+	}
+	return nil
+}

vfs/api.go

@@ -206,13 +206,31 @@ type blockingSharedMemory interface {
 	shmEnableBlocking(block bool)
 }
 
+// FileMemoryMapper extends File to possibly implement
+// memory mapped files.
+// The same memory mapper instance must be returned
+// for the entire life of the file.
+// It's OK for MemoryMapper to return nil.
+type FileMemoryMapper interface {
+	File
+	MemoryMapper() MemoryMapper
+}
+
+// MemoryMapper is a file mapper implementation.
+// Use [NewMemoryMapper] to create a mapper.
+type MemoryMapper interface {
+	mmapSize(*sqlite3_wrap.Wrapper, ptr_t)
+	fetch(*sqlite3_wrap.Wrapper, int64, int32, ptr_t) error
+	io.Closer
+}
+
 // FileControl makes it easy to forward all fileControl methods,
 // which we want to do for the checksum VFS.
 // However, this is not a safe default, and other VFSes
 // should explicitly wrap the methods they want to wrap.
 type fileControl interface {
 	File
-	fileControl(wrp *sqlite3_wrap.Wrapper, op _FcntlOpcode, pArg ptr_t) _ErrorCode
+	fileControl(*sqlite3_wrap.Wrapper, _FcntlOpcode, ptr_t) _ErrorCode
 }
 
 type filePDB interface {

vfs/cksm.go

@@ -109,6 +109,16 @@ func (c *cksmFile) SharedMemory() SharedMemory {
 	return nil
 }
 
+func (c *cksmFile) MemoryMapper() MemoryMapper {
+	if c.verifyCksm {
+		return nil
+	}
+	if f, ok := c.File.(FileMemoryMapper); ok {
+		return f.MemoryMapper()
+	}
+	return nil
+}
+
 func (c *cksmFile) Unwrap() File {
 	return c.File
 }

vfs/const.go

@@ -53,6 +53,7 @@ const (
 	_IOERR_SHMMAP            _ErrorCode = sqlite3_wrap.IOERR_SHMMAP
 	_IOERR_SEEK              _ErrorCode = sqlite3_wrap.IOERR_SEEK
 	_IOERR_DELETE_NOENT      _ErrorCode = sqlite3_wrap.IOERR_DELETE_NOENT
+	_IOERR_MMAP              _ErrorCode = sqlite3_wrap.IOERR_MMAP
 	_IOERR_GETTEMPPATH       _ErrorCode = sqlite3_wrap.IOERR_GETTEMPPATH
 	_IOERR_BEGIN_ATOMIC      _ErrorCode = sqlite3_wrap.IOERR_BEGIN_ATOMIC
 	_IOERR_COMMIT_ATOMIC     _ErrorCode = sqlite3_wrap.IOERR_COMMIT_ATOMIC

vfs/file.go

@@ -127,6 +127,7 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error
 	file := vfsFile{
 		File:  f,
 		flags: flags | _FLAG_PSOW,
+		mmap:  NewMemoryMapper(f, flags),
 	}
 	if osBatchAtomic(f) {
 		file.flags |= _FLAG_ATOMIC
@@ -143,6 +144,7 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error
 type vfsFile struct {
 	*os.File
 	shm   SharedMemory
+	mmap  MemoryMapper
 	lock  LockLevel
 	flags OpenFlag
 }

vfs/map.go

@@ -0,0 +1,63 @@
+//go:build linux || darwin || freebsd
+
+package vfs
+
+import (
+	"os"
+
+	"github.com/ncruces/go-sqlite3/internal/sqlite3_wrap"
+)
+
+func (f *vfsFile) MemoryMapper() MemoryMapper { return f.mmap }
+
+// NewMemoryMapper returns a memory mapper for the given file.
+// It will return nil if file mapping is not supported,
+// or not appropriate for the given flags.
+// Only databases use memory mapping.
+func NewMemoryMapper(file *os.File, flags OpenFlag) MemoryMapper {
+	if flags&(OPEN_MAIN_DB|OPEN_TEMP_DB|OPEN_TRANSIENT_DB) == 0 || flags&OPEN_MEMORY != 0 {
+		return nil
+	}
+	return &vfsMapper{File: file}
+}
+
+type vfsMapper struct {
+	*os.File
+	mmap *sqlite3_wrap.MappedRegion
+	size int32
+}
+
+func (m *vfsMapper) mmapSize(wrp *sqlite3_wrap.Wrapper, p ptr_t) {
+	size := int64(wrp.Read64(p))
+	wrp.Write64(p, uint64(m.size))
+	if size >= 0 && m.mmap == nil {
+		m.size = int32(min(size, 1024*1024*1024))
+	}
+}
+
+func (m *vfsMapper) fetch(wrp *sqlite3_wrap.Wrapper, iOfst int64, iAmt int32, pp ptr_t) error {
+	var ptr ptr_t
+	if iOfst+int64(iAmt)+256 <= int64(m.size) {
+		if m.mmap == nil {
+			var err error
+			m.mmap, err = wrp.MapRegion(m.File, 0, m.size, true)
+			if err != nil {
+				return err
+			}
+		}
+		if m.mmap != nil {
+			ptr = m.mmap.Ptr + ptr_t(iOfst)
+		}
+	}
+	wrp.Write32(pp, uint32(ptr))
+	return nil
+}
+
+func (m *vfsMapper) Close() error {
+	c := m.mmap
+	if c == nil {
+		return nil
+	}
+	m.mmap = nil
+	return c.Unmap()
+}

vfs/map_other.go

@@ -0,0 +1,13 @@
+//go:build !(linux || darwin || freebsd)
+
+package vfs
+
+import "os"
+
+// NewMemoryMapper returns a memory mapper for the given file.
+// It will return nil if file mapping is not supported,
+// or not appropriate for the given flags.
+// Only databases use memory mapping.
+func NewMemoryMapper(file *os.File, flags OpenFlag) MemoryMapper {
+	return nil
+}

vfs/tests/mptest/download.sh

@@ -3,12 +3,13 @@ set -euo pipefail
 
 cd -P -- "$(dirname -- "$0")"
 
-GITHUB_TAG="https://github.com/sqlite/sqlite/raw/version-3.53.3"
+GITHUB_TAG="https://github.com/sqlite/sqlite/raw/version-3.53.4"
 
 cd testdata/
 curl -#OL "$GITHUB_TAG/mptest/config01.test"
 curl -#OL "$GITHUB_TAG/mptest/config02.test"
 curl -#OL "$GITHUB_TAG/mptest/crash01.test"
 curl -#OL "$GITHUB_TAG/mptest/crash02.subtest"
 curl -#OL "$GITHUB_TAG/mptest/multiwrite01.test"
+sed -i "s/if vfsname() GLOB 'unix'/if vfsname() GLOB 'os'/" config01.test
 cd ~-

vfs/tests/mptest/testdata/config01.test

@@ -1,7 +1,7 @@
 /*
 ** Configure five tasks in different ways, then run tests.
 */
---if vfsname() GLOB 'unix'
+--if vfsname() GLOB 'os'
 PRAGMA page_size=8192;
 --task 1
   PRAGMA journal_mode=PERSIST;

vfs/tests/speedtest1/speedtest1_test.go

@@ -60,6 +60,12 @@ func runBenchmark(b *testing.B, args ...string) {
 	}
 
 	testenv.TB = b
+	testenv.Exit = func(c int32) {
+		if c != 0 {
+			b.Fatal("exit error:", c)
+		}
+		b.SkipNow()
+	}
 	wrp, err := createWrapper(b.Context())
 	if err != nil {
 		b.Fatal(err)

vfs/vfs.go

@@ -66,12 +66,17 @@ func vfsOpen(wrp *sqlite3_wrap.Wrapper, pVfs, zPath, pFile ptr_t, flags OpenFlag
 			file.SetPowersafeOverwrite(b)
 		}
 	}
-	if file, ok := file.(FileSharedMemory); ok && pOutVFS != 0 {
-		wrp.WriteBool(pOutVFS, file.SharedMemory() != nil)
-	}
 	if pOutFlags != 0 {
 		wrp.Write32(pOutFlags, uint32(flags))
 	}
+	var outVFS uint32
+	if file, ok := file.(FileSharedMemory); ok && file.SharedMemory() != nil {
+		outVFS |= 1
+	}
+	if file, ok := file.(FileMemoryMapper); ok && file.MemoryMapper() != nil {
+		outVFS |= 2
+	}
+	wrp.Write32(pOutVFS, outVFS)
 	file = cksmWrapFile(file, flags)
 	vfsFileRegister(wrp, pFile, file)
 	return _OK
@@ -302,6 +307,14 @@ func vfsFileControlImpl(wrp *sqlite3_wrap.Wrapper, file File, op _FcntlOpcode, p
 			}
 		}
 
+	case _FCNTL_MMAP_SIZE:
+		if file, ok := file.(FileMemoryMapper); ok {
+			if mmap := file.MemoryMapper(); mmap != nil {
+				mmap.mmapSize(wrp, pArg)
+				return _OK
+			}
+		}
+
 	case _FCNTL_PDB:
 		if file, ok := file.(filePDB); ok {
 			file.SetDB(wrp.DB)
@@ -356,6 +369,22 @@ func vfsShmUnmap(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, bDelete int32) _ErrorCo
 	return _OK
 }
 
+//go:linkname vfsFetch
+func vfsFetch(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, iOfst int64, iAmt int32, pp ptr_t) _ErrorCode {
+	mem := vfsFileGet(wrp, pFile).(FileMemoryMapper).MemoryMapper()
+	err := mem.fetch(wrp, iOfst, iAmt, pp)
+	return vfsErrorCode(wrp, err, _IOERR_MMAP)
+}
+
+//go:linkname vfsUnfetch
+func vfsUnfetch(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, _ int64, p ptr_t) _ErrorCode {
+	if p == 0 {
+		mmap := vfsFileGet(wrp, pFile).(FileMemoryMapper).MemoryMapper()
+		mmap.Close()
+	}
+	return _OK
+}
+
 func vfsGet(wrp *sqlite3_wrap.Wrapper, pVfs ptr_t) VFS {
 	var name string
 	if pVfs != 0 {

wrap.go

@@ -8,7 +8,7 @@ import (
 	"time"
 	_ "unsafe"
 
-	sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v3"
+	sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v4"
 	"github.com/ncruces/go-sqlite3/internal/errutil"
 	"github.com/ncruces/go-sqlite3/internal/sqlite3_wrap"
 	"github.com/ncruces/go-sqlite3/vfs"
@@ -214,3 +214,17 @@ func vfsShmUnmap(_ *sqlite3_wrap.Wrapper, v0, v1 int32) int32
 func (e *env) Xgo_shm_unmap(v0, v1 int32) int32 {
 	return vfsShmUnmap(e.Wrapper, v0, v1)
 }
+
+//go:linkname vfsFetch github.com/ncruces/go-sqlite3/vfs.vfsFetch
+func vfsFetch(_ *sqlite3_wrap.Wrapper, v0 int32, v1 int64, v2, v3 int32) int32
+
+func (e *env) Xgo_fetch(v0 int32, v1 int64, v2, v3 int32) int32 {
+	return vfsFetch(e.Wrapper, v0, v1, v2, v3)
+}
+
+//go:linkname vfsUnfetch github.com/ncruces/go-sqlite3/vfs.vfsUnfetch
+func vfsUnfetch(_ *sqlite3_wrap.Wrapper, v0 int32, v1 int64, v2 int32) int32
+
+func (e *env) Xgo_unfetch(v0 int32, v1 int64, v2 int32) int32 {
+	return vfsUnfetch(e.Wrapper, v0, v1, v2)
+}