From f303ec761eba931cdc5f9213df4d6b1849117211 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Tue, 18 Aug 2026 18:38:10 +0100 Subject: [PATCH 1/3] PRAGMA mmap_size. --- util/vfsutil/wrap.go | 8 ++++ vfs/api.go | 20 +++++++- vfs/cksm.go | 10 ++++ vfs/const.go | 1 + vfs/file.go | 2 + vfs/map_linux.go | 62 +++++++++++++++++++++++++ vfs/map_other.go | 13 ++++++ vfs/tests/mptest/download.sh | 3 +- vfs/tests/mptest/testdata/config01.test | 2 +- vfs/tests/speedtest1/speedtest1_test.go | 6 +++ vfs/vfs.go | 35 ++++++++++++-- wrap.go | 14 ++++++ 12 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 vfs/map_linux.go create mode 100644 vfs/map_other.go diff --git a/util/vfsutil/wrap.go b/util/vfsutil/wrap.go index ad96547f..6b0dc03b 100644 --- a/util/vfsutil/wrap.go +++ b/util/vfsutil/wrap.go @@ -183,3 +183,11 @@ func WrapSharedMemory(f vfs.File) vfs.SharedMemory { } return nil } + +// WrapMemoryMapper helps wrap [vfs.FileMemoryMapper]. +func WrapMemoryMapper(f vfs.File) vfs.MemoryMapper { + if f, ok := f.(vfs.FileMemoryMapper); ok { + return f.MemoryMapper() + } + return nil +} diff --git a/vfs/api.go b/vfs/api.go index 5f538ea3..beafc628 100644 --- a/vfs/api.go +++ b/vfs/api.go @@ -206,13 +206,31 @@ type blockingSharedMemory interface { shmEnableBlocking(block bool) } +// FileMemoryMapper extends File to possibly implement +// memory mapped files. +// The same memory mapper instance must be returned +// for the entire life of the file. +// It's OK for MemoryMapper to return nil. +type FileMemoryMapper interface { + File + MemoryMapper() MemoryMapper +} + +// MemoryMapper is a file mapper implementation. +// Use [NewMemoryMapper] to create a mapper. +type MemoryMapper interface { + mmapSize(*sqlite3_wrap.Wrapper, ptr_t) + fetch(*sqlite3_wrap.Wrapper, int64, int32, ptr_t) error + io.Closer +} + // FileControl makes it easy to forward all fileControl methods, // which we want to do for the checksum VFS. // However, this is not a safe default, and other VFSes // should explicitly wrap the methods they want to wrap. type fileControl interface { File - fileControl(wrp *sqlite3_wrap.Wrapper, op _FcntlOpcode, pArg ptr_t) _ErrorCode + fileControl(*sqlite3_wrap.Wrapper, _FcntlOpcode, ptr_t) _ErrorCode } type filePDB interface { diff --git a/vfs/cksm.go b/vfs/cksm.go index 48289ece..adbc6068 100644 --- a/vfs/cksm.go +++ b/vfs/cksm.go @@ -109,6 +109,16 @@ func (c *cksmFile) SharedMemory() SharedMemory { return nil } +func (c *cksmFile) MemoryMapper() MemoryMapper { + if c.verifyCksm { + return nil + } + if f, ok := c.File.(FileMemoryMapper); ok { + return f.MemoryMapper() + } + return nil +} + func (c *cksmFile) Unwrap() File { return c.File } diff --git a/vfs/const.go b/vfs/const.go index 7ca31350..1a8a37a7 100644 --- a/vfs/const.go +++ b/vfs/const.go @@ -53,6 +53,7 @@ const ( _IOERR_SHMMAP _ErrorCode = sqlite3_wrap.IOERR_SHMMAP _IOERR_SEEK _ErrorCode = sqlite3_wrap.IOERR_SEEK _IOERR_DELETE_NOENT _ErrorCode = sqlite3_wrap.IOERR_DELETE_NOENT + _IOERR_MMAP _ErrorCode = sqlite3_wrap.IOERR_MMAP _IOERR_GETTEMPPATH _ErrorCode = sqlite3_wrap.IOERR_GETTEMPPATH _IOERR_BEGIN_ATOMIC _ErrorCode = sqlite3_wrap.IOERR_BEGIN_ATOMIC _IOERR_COMMIT_ATOMIC _ErrorCode = sqlite3_wrap.IOERR_COMMIT_ATOMIC diff --git a/vfs/file.go b/vfs/file.go index d6f12ee3..eafe701a 100644 --- a/vfs/file.go +++ b/vfs/file.go @@ -127,6 +127,7 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error file := vfsFile{ File: f, flags: flags | _FLAG_PSOW, + mmap: NewMemoryMapper(f, flags), } if osBatchAtomic(f) { file.flags |= _FLAG_ATOMIC @@ -143,6 +144,7 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error type vfsFile struct { *os.File shm SharedMemory + mmap MemoryMapper lock LockLevel flags OpenFlag } diff --git a/vfs/map_linux.go b/vfs/map_linux.go new file mode 100644 index 00000000..d536416f --- /dev/null +++ b/vfs/map_linux.go @@ -0,0 +1,62 @@ +package vfs + +import ( + "os" + + "github.com/ncruces/go-sqlite3/internal/sqlite3_wrap" +) + +func (f *vfsFile) MemoryMapper() MemoryMapper { return f.mmap } + +// NewMemoryMapper returns a memory mapper for the given file. +// It will return nil if file mapping is not supported, +// or not appropriate for the given flags. +// Only databases use memory mapping. +func NewMemoryMapper(file *os.File, flags OpenFlag) MemoryMapper { + if flags&(OPEN_MAIN_DB|OPEN_TEMP_DB|OPEN_TRANSIENT_DB) == 0 || flags&OPEN_MEMORY != 0 { + return nil + } + return &vfsMapper{File: file} +} + +type vfsMapper struct { + *os.File + mmap *sqlite3_wrap.MappedRegion + size int32 +} + +func (m *vfsMapper) mmapSize(wrp *sqlite3_wrap.Wrapper, p ptr_t) { + size := int64(wrp.Read64(p)) + wrp.Write64(p, uint64(m.size)) + if size >= 0 && m.mmap == nil { + m.size = int32(min(size, 1024*1024*1024)) + } +} + +func (m *vfsMapper) fetch(wrp *sqlite3_wrap.Wrapper, iOfst int64, iAmt int32, pp ptr_t) error { + var ptr ptr_t + if iOfst+int64(iAmt)+256 <= int64(m.size) { + if m.mmap == nil { + var err error + m.mmap, err = wrp.MapRegion(m.File, 0, m.size, true) + if err != nil { + return err + } + if m.mmap == nil { + return _IOERR_NOMEM + } + } + ptr = m.mmap.Ptr + ptr_t(iOfst) + } + wrp.Write32(pp, uint32(ptr)) + return nil +} + +func (m *vfsMapper) Close() error { + c := m.mmap + if c == nil { + return nil + } + m.mmap = nil + return c.Unmap() +} diff --git a/vfs/map_other.go b/vfs/map_other.go new file mode 100644 index 00000000..a29ac6ad --- /dev/null +++ b/vfs/map_other.go @@ -0,0 +1,13 @@ +//go:build !linux + +package vfs + +import "os" + +// NewMemoryMapper returns a memory mapper for the given file. +// It will return nil if file mapping is not supported, +// or not appropriate for the given flags. +// Only databases use memory mapping. +func NewMemoryMapper(file *os.File, flags OpenFlag) MemoryMapper { + return nil +} diff --git a/vfs/tests/mptest/download.sh b/vfs/tests/mptest/download.sh index 1b594b85..b45d9734 100755 --- a/vfs/tests/mptest/download.sh +++ b/vfs/tests/mptest/download.sh @@ -3,7 +3,7 @@ set -euo pipefail cd -P -- "$(dirname -- "$0")" -GITHUB_TAG="https://github.com/sqlite/sqlite/raw/version-3.53.3" +GITHUB_TAG="https://github.com/sqlite/sqlite/raw/version-3.53.4" cd testdata/ curl -#OL "$GITHUB_TAG/mptest/config01.test" @@ -11,4 +11,5 @@ curl -#OL "$GITHUB_TAG/mptest/config02.test" curl -#OL "$GITHUB_TAG/mptest/crash01.test" curl -#OL "$GITHUB_TAG/mptest/crash02.subtest" curl -#OL "$GITHUB_TAG/mptest/multiwrite01.test" +sed -i "s/if vfsname() GLOB 'unix'/if vfsname() GLOB 'os'/" config01.test cd ~- diff --git a/vfs/tests/mptest/testdata/config01.test b/vfs/tests/mptest/testdata/config01.test index 683ee911..2edf2d7c 100644 --- a/vfs/tests/mptest/testdata/config01.test +++ b/vfs/tests/mptest/testdata/config01.test @@ -1,7 +1,7 @@ /* ** Configure five tasks in different ways, then run tests. */ ---if vfsname() GLOB 'unix' +--if vfsname() GLOB 'os' PRAGMA page_size=8192; --task 1 PRAGMA journal_mode=PERSIST; diff --git a/vfs/tests/speedtest1/speedtest1_test.go b/vfs/tests/speedtest1/speedtest1_test.go index ffc6d8ad..66675a80 100644 --- a/vfs/tests/speedtest1/speedtest1_test.go +++ b/vfs/tests/speedtest1/speedtest1_test.go @@ -60,6 +60,12 @@ func runBenchmark(b *testing.B, args ...string) { } testenv.TB = b + testenv.Exit = func(c int32) { + if c != 0 { + b.Fatal("exit error:", c) + } + b.SkipNow() + } wrp, err := createWrapper(b.Context()) if err != nil { b.Fatal(err) diff --git a/vfs/vfs.go b/vfs/vfs.go index f2a7104a..174cf8f3 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -66,12 +66,17 @@ func vfsOpen(wrp *sqlite3_wrap.Wrapper, pVfs, zPath, pFile ptr_t, flags OpenFlag file.SetPowersafeOverwrite(b) } } - if file, ok := file.(FileSharedMemory); ok && pOutVFS != 0 { - wrp.WriteBool(pOutVFS, file.SharedMemory() != nil) - } if pOutFlags != 0 { wrp.Write32(pOutFlags, uint32(flags)) } + var outVFS uint32 + if file, ok := file.(FileSharedMemory); ok && file.SharedMemory() != nil { + outVFS |= 1 + } + if file, ok := file.(FileMemoryMapper); ok && file.MemoryMapper() != nil { + outVFS |= 2 + } + wrp.Write32(pOutVFS, outVFS) file = cksmWrapFile(file, flags) vfsFileRegister(wrp, pFile, file) return _OK @@ -302,6 +307,14 @@ func vfsFileControlImpl(wrp *sqlite3_wrap.Wrapper, file File, op _FcntlOpcode, p } } + case _FCNTL_MMAP_SIZE: + if file, ok := file.(FileMemoryMapper); ok { + if mmap := file.MemoryMapper(); mmap != nil { + mmap.mmapSize(wrp, pArg) + return _OK + } + } + case _FCNTL_PDB: if file, ok := file.(filePDB); ok { file.SetDB(wrp.DB) @@ -356,6 +369,22 @@ func vfsShmUnmap(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, bDelete int32) _ErrorCo return _OK } +//go:linkname vfsFetch +func vfsFetch(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, iOfst int64, iAmt int32, pp ptr_t) _ErrorCode { + mmap := vfsFileGet(wrp, pFile).(FileMemoryMapper).MemoryMapper() + err := mmap.fetch(wrp, iOfst, iAmt, pp) + return vfsErrorCode(wrp, err, _IOERR_MMAP) +} + +//go:linkname vfsUnfetch +func vfsUnfetch(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, _ int64, p ptr_t) _ErrorCode { + if p == 0 { + mmap := vfsFileGet(wrp, pFile).(FileMemoryMapper).MemoryMapper() + mmap.Close() + } + return _OK +} + func vfsGet(wrp *sqlite3_wrap.Wrapper, pVfs ptr_t) VFS { var name string if pVfs != 0 { diff --git a/wrap.go b/wrap.go index a18aa6c4..b00e9636 100644 --- a/wrap.go +++ b/wrap.go @@ -214,3 +214,17 @@ func vfsShmUnmap(_ *sqlite3_wrap.Wrapper, v0, v1 int32) int32 func (e *env) Xgo_shm_unmap(v0, v1 int32) int32 { return vfsShmUnmap(e.Wrapper, v0, v1) } + +//go:linkname vfsFetch github.com/ncruces/go-sqlite3/vfs.vfsFetch +func vfsFetch(_ *sqlite3_wrap.Wrapper, v0 int32, v1 int64, v2, v3 int32) int32 + +func (e *env) Xgo_fetch(v0 int32, v1 int64, v2, v3 int32) int32 { + return vfsFetch(e.Wrapper, v0, v1, v2, v3) +} + +//go:linkname vfsUnfetch github.com/ncruces/go-sqlite3/vfs.vfsUnfetch +func vfsUnfetch(_ *sqlite3_wrap.Wrapper, v0 int32, v1 int64, v2 int32) int32 + +func (e *env) Xgo_unfetch(v0 int32, v1 int64, v2 int32) int32 { + return vfsUnfetch(e.Wrapper, v0, v1, v2) +} From 06ef01b540120b994947ad794d6b2f033ef2fd22 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Wed, 19 Aug 2026 13:43:04 +0100 Subject: [PATCH 2/3] Fix. --- vfs/map_linux.go | 7 +++---- vfs/vfs.go | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/vfs/map_linux.go b/vfs/map_linux.go index d536416f..5df05dfe 100644 --- a/vfs/map_linux.go +++ b/vfs/map_linux.go @@ -42,11 +42,10 @@ func (m *vfsMapper) fetch(wrp *sqlite3_wrap.Wrapper, iOfst int64, iAmt int32, pp if err != nil { return err } - if m.mmap == nil { - return _IOERR_NOMEM - } } - ptr = m.mmap.Ptr + ptr_t(iOfst) + if m.mmap != nil { + ptr = m.mmap.Ptr + ptr_t(iOfst) + } } wrp.Write32(pp, uint32(ptr)) return nil diff --git a/vfs/vfs.go b/vfs/vfs.go index 174cf8f3..f71cb2ba 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -371,8 +371,8 @@ func vfsShmUnmap(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, bDelete int32) _ErrorCo //go:linkname vfsFetch func vfsFetch(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, iOfst int64, iAmt int32, pp ptr_t) _ErrorCode { - mmap := vfsFileGet(wrp, pFile).(FileMemoryMapper).MemoryMapper() - err := mmap.fetch(wrp, iOfst, iAmt, pp) + mem := vfsFileGet(wrp, pFile).(FileMemoryMapper).MemoryMapper() + err := mem.fetch(wrp, iOfst, iAmt, pp) return vfsErrorCode(wrp, err, _IOERR_MMAP) } From 095d317ed55e4a74009058580d4f32a5154fee58 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Thu, 20 Aug 2026 15:09:20 +0100 Subject: [PATCH 3/3] BSD. --- ext/fts5/fts5.go | 2 +- ext/rtree/rtree.go | 2 +- ext/spellfix1/spellfix.go | 2 +- ext/vec1/vec1.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- internal/sqlite3_wrap/wrap.go | 2 +- util/sql3util/parse.go | 2 +- vfs/{map_linux.go => map.go} | 2 ++ vfs/map_other.go | 2 +- wrap.go | 2 +- 11 files changed, 13 insertions(+), 11 deletions(-) rename vfs/{map_linux.go => map.go} (97%) diff --git a/ext/fts5/fts5.go b/ext/fts5/fts5.go index 9e0103ce..b7af8d44 100644 --- a/ext/fts5/fts5.go +++ b/ext/fts5/fts5.go @@ -5,7 +5,7 @@ package fts5 import ( "github.com/ncruces/go-sqlite3" - "github.com/ncruces/go-sqlite3-wasm/v3/fts5" + "github.com/ncruces/go-sqlite3-wasm/v4/fts5" ) // Register registers the fts5 extension. diff --git a/ext/rtree/rtree.go b/ext/rtree/rtree.go index ab0c25cc..91451cf2 100644 --- a/ext/rtree/rtree.go +++ b/ext/rtree/rtree.go @@ -6,7 +6,7 @@ package rtree import ( "github.com/ncruces/go-sqlite3" - "github.com/ncruces/go-sqlite3-wasm/v3/rtree" + "github.com/ncruces/go-sqlite3-wasm/v4/rtree" ) // Register registers the rtree and geopoly virtual tables. diff --git a/ext/spellfix1/spellfix.go b/ext/spellfix1/spellfix.go index 050de9eb..cba67cb9 100644 --- a/ext/spellfix1/spellfix.go +++ b/ext/spellfix1/spellfix.go @@ -5,7 +5,7 @@ package spellfix1 import ( "github.com/ncruces/go-sqlite3" - "github.com/ncruces/go-sqlite3-wasm/v3/spellfix" + "github.com/ncruces/go-sqlite3-wasm/v4/spellfix" ) // Register registers the spellfix1 virtual table. diff --git a/ext/vec1/vec1.go b/ext/vec1/vec1.go index 307019f9..5ee2894a 100644 --- a/ext/vec1/vec1.go +++ b/ext/vec1/vec1.go @@ -5,7 +5,7 @@ package vec1 import ( "github.com/ncruces/go-sqlite3" - "github.com/ncruces/go-sqlite3-wasm/v3/vec1" + "github.com/ncruces/go-sqlite3-wasm/v4/vec1" ) // Register registers the vec1 vector extension. diff --git a/go.mod b/go.mod index 61b11381..c1c4d310 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ncruces/go-sqlite3 go 1.26.0 require ( - github.com/ncruces/go-sqlite3-wasm/v3 v3.4.35304 + github.com/ncruces/go-sqlite3-wasm/v4 v4.0.35304 github.com/ncruces/julianday v1.0.0 github.com/ncruces/sort v1.0.0 github.com/ncruces/wbt v1.0.0 diff --git a/go.sum b/go.sum index f3b20c27..81063ea5 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/ncruces/go-sqlite3-wasm/v3 v3.4.35304 h1:5i8tdrFKOaWR80ruV9nWuPvqSX3P+2zuB7Al8ym3/hY= -github.com/ncruces/go-sqlite3-wasm/v3 v3.4.35304/go.mod h1:aIkZSy1oIDrgwKtlNzI/qBezIH/CKAL8fNjYw7SxnXk= +github.com/ncruces/go-sqlite3-wasm/v4 v4.0.35304 h1:rqpCiJNqT21zczDJ9GtAQkc3rLDoeEyd+Sckxk/ogF4= +github.com/ncruces/go-sqlite3-wasm/v4 v4.0.35304/go.mod h1:2jqV+aun+kgJl3uxD8w0v9SEX7Ifq/UNhxK/tJvDCYI= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/ncruces/sort v1.0.0 h1:tXeitUzE3B8I2z+rsqbKUBThBlJJ34glqCDP0XReJmk= diff --git a/internal/sqlite3_wrap/wrap.go b/internal/sqlite3_wrap/wrap.go index 885f5f8b..05a60978 100644 --- a/internal/sqlite3_wrap/wrap.go +++ b/internal/sqlite3_wrap/wrap.go @@ -3,7 +3,7 @@ package sqlite3_wrap import ( "io" - sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v3" + sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v4" "github.com/ncruces/go-sqlite3/internal/errutil" ) diff --git a/util/sql3util/parse.go b/util/sql3util/parse.go index 979a8fc1..fe179579 100644 --- a/util/sql3util/parse.go +++ b/util/sql3util/parse.go @@ -6,7 +6,7 @@ import ( "encoding/binary" "strings" - parser "github.com/ncruces/go-sqlite3-wasm/v3/parser" + parser "github.com/ncruces/go-sqlite3-wasm/v4/parser" "github.com/ncruces/go-sqlite3/internal/errutil" ) diff --git a/vfs/map_linux.go b/vfs/map.go similarity index 97% rename from vfs/map_linux.go rename to vfs/map.go index 5df05dfe..79eac486 100644 --- a/vfs/map_linux.go +++ b/vfs/map.go @@ -1,3 +1,5 @@ +//go:build linux || darwin || freebsd + package vfs import ( diff --git a/vfs/map_other.go b/vfs/map_other.go index a29ac6ad..a8b376c0 100644 --- a/vfs/map_other.go +++ b/vfs/map_other.go @@ -1,4 +1,4 @@ -//go:build !linux +//go:build !(linux || darwin || freebsd) package vfs diff --git a/wrap.go b/wrap.go index b00e9636..828133e3 100644 --- a/wrap.go +++ b/wrap.go @@ -8,7 +8,7 @@ import ( "time" _ "unsafe" - sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v3" + sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v4" "github.com/ncruces/go-sqlite3/internal/errutil" "github.com/ncruces/go-sqlite3/internal/sqlite3_wrap" "github.com/ncruces/go-sqlite3/vfs"