diff --git a/ext/fts5/fts5.go b/ext/fts5/fts5.go index 9e0103ce..b7af8d44 100644 --- a/ext/fts5/fts5.go +++ b/ext/fts5/fts5.go @@ -5,7 +5,7 @@ package fts5 import ( "github.com/ncruces/go-sqlite3" - "github.com/ncruces/go-sqlite3-wasm/v3/fts5" + "github.com/ncruces/go-sqlite3-wasm/v4/fts5" ) // Register registers the fts5 extension. diff --git a/ext/rtree/rtree.go b/ext/rtree/rtree.go index ab0c25cc..91451cf2 100644 --- a/ext/rtree/rtree.go +++ b/ext/rtree/rtree.go @@ -6,7 +6,7 @@ package rtree import ( "github.com/ncruces/go-sqlite3" - "github.com/ncruces/go-sqlite3-wasm/v3/rtree" + "github.com/ncruces/go-sqlite3-wasm/v4/rtree" ) // Register registers the rtree and geopoly virtual tables. diff --git a/ext/spellfix1/spellfix.go b/ext/spellfix1/spellfix.go index 050de9eb..cba67cb9 100644 --- a/ext/spellfix1/spellfix.go +++ b/ext/spellfix1/spellfix.go @@ -5,7 +5,7 @@ package spellfix1 import ( "github.com/ncruces/go-sqlite3" - "github.com/ncruces/go-sqlite3-wasm/v3/spellfix" + "github.com/ncruces/go-sqlite3-wasm/v4/spellfix" ) // Register registers the spellfix1 virtual table. diff --git a/ext/vec1/vec1.go b/ext/vec1/vec1.go index 307019f9..5ee2894a 100644 --- a/ext/vec1/vec1.go +++ b/ext/vec1/vec1.go @@ -5,7 +5,7 @@ package vec1 import ( "github.com/ncruces/go-sqlite3" - "github.com/ncruces/go-sqlite3-wasm/v3/vec1" + "github.com/ncruces/go-sqlite3-wasm/v4/vec1" ) // Register registers the vec1 vector extension. diff --git a/go.mod b/go.mod index 61b11381..c1c4d310 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ncruces/go-sqlite3 go 1.26.0 require ( - github.com/ncruces/go-sqlite3-wasm/v3 v3.4.35304 + github.com/ncruces/go-sqlite3-wasm/v4 v4.0.35304 github.com/ncruces/julianday v1.0.0 github.com/ncruces/sort v1.0.0 github.com/ncruces/wbt v1.0.0 diff --git a/go.sum b/go.sum index f3b20c27..81063ea5 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/ncruces/go-sqlite3-wasm/v3 v3.4.35304 h1:5i8tdrFKOaWR80ruV9nWuPvqSX3P+2zuB7Al8ym3/hY= -github.com/ncruces/go-sqlite3-wasm/v3 v3.4.35304/go.mod h1:aIkZSy1oIDrgwKtlNzI/qBezIH/CKAL8fNjYw7SxnXk= +github.com/ncruces/go-sqlite3-wasm/v4 v4.0.35304 h1:rqpCiJNqT21zczDJ9GtAQkc3rLDoeEyd+Sckxk/ogF4= +github.com/ncruces/go-sqlite3-wasm/v4 v4.0.35304/go.mod h1:2jqV+aun+kgJl3uxD8w0v9SEX7Ifq/UNhxK/tJvDCYI= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/ncruces/sort v1.0.0 h1:tXeitUzE3B8I2z+rsqbKUBThBlJJ34glqCDP0XReJmk= diff --git a/internal/sqlite3_wrap/wrap.go b/internal/sqlite3_wrap/wrap.go index 885f5f8b..05a60978 100644 --- a/internal/sqlite3_wrap/wrap.go +++ b/internal/sqlite3_wrap/wrap.go @@ -3,7 +3,7 @@ package sqlite3_wrap import ( "io" - sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v3" + sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v4" "github.com/ncruces/go-sqlite3/internal/errutil" ) diff --git a/util/sql3util/parse.go b/util/sql3util/parse.go index 979a8fc1..fe179579 100644 --- a/util/sql3util/parse.go +++ b/util/sql3util/parse.go @@ -6,7 +6,7 @@ import ( "encoding/binary" "strings" - parser "github.com/ncruces/go-sqlite3-wasm/v3/parser" + parser "github.com/ncruces/go-sqlite3-wasm/v4/parser" "github.com/ncruces/go-sqlite3/internal/errutil" ) diff --git a/util/vfsutil/wrap.go b/util/vfsutil/wrap.go index ad96547f..6b0dc03b 100644 --- a/util/vfsutil/wrap.go +++ b/util/vfsutil/wrap.go @@ -183,3 +183,11 @@ func WrapSharedMemory(f vfs.File) vfs.SharedMemory { } return nil } + +// WrapMemoryMapper helps wrap [vfs.FileMemoryMapper]. +func WrapMemoryMapper(f vfs.File) vfs.MemoryMapper { + if f, ok := f.(vfs.FileMemoryMapper); ok { + return f.MemoryMapper() + } + return nil +} diff --git a/vfs/api.go b/vfs/api.go index 5f538ea3..beafc628 100644 --- a/vfs/api.go +++ b/vfs/api.go @@ -206,13 +206,31 @@ type blockingSharedMemory interface { shmEnableBlocking(block bool) } +// FileMemoryMapper extends File to possibly implement +// memory mapped files. +// The same memory mapper instance must be returned +// for the entire life of the file. +// It's OK for MemoryMapper to return nil. +type FileMemoryMapper interface { + File + MemoryMapper() MemoryMapper +} + +// MemoryMapper is a file mapper implementation. +// Use [NewMemoryMapper] to create a mapper. +type MemoryMapper interface { + mmapSize(*sqlite3_wrap.Wrapper, ptr_t) + fetch(*sqlite3_wrap.Wrapper, int64, int32, ptr_t) error + io.Closer +} + // FileControl makes it easy to forward all fileControl methods, // which we want to do for the checksum VFS. // However, this is not a safe default, and other VFSes // should explicitly wrap the methods they want to wrap. type fileControl interface { File - fileControl(wrp *sqlite3_wrap.Wrapper, op _FcntlOpcode, pArg ptr_t) _ErrorCode + fileControl(*sqlite3_wrap.Wrapper, _FcntlOpcode, ptr_t) _ErrorCode } type filePDB interface { diff --git a/vfs/cksm.go b/vfs/cksm.go index 48289ece..adbc6068 100644 --- a/vfs/cksm.go +++ b/vfs/cksm.go @@ -109,6 +109,16 @@ func (c *cksmFile) SharedMemory() SharedMemory { return nil } +func (c *cksmFile) MemoryMapper() MemoryMapper { + if c.verifyCksm { + return nil + } + if f, ok := c.File.(FileMemoryMapper); ok { + return f.MemoryMapper() + } + return nil +} + func (c *cksmFile) Unwrap() File { return c.File } diff --git a/vfs/const.go b/vfs/const.go index 7ca31350..1a8a37a7 100644 --- a/vfs/const.go +++ b/vfs/const.go @@ -53,6 +53,7 @@ const ( _IOERR_SHMMAP _ErrorCode = sqlite3_wrap.IOERR_SHMMAP _IOERR_SEEK _ErrorCode = sqlite3_wrap.IOERR_SEEK _IOERR_DELETE_NOENT _ErrorCode = sqlite3_wrap.IOERR_DELETE_NOENT + _IOERR_MMAP _ErrorCode = sqlite3_wrap.IOERR_MMAP _IOERR_GETTEMPPATH _ErrorCode = sqlite3_wrap.IOERR_GETTEMPPATH _IOERR_BEGIN_ATOMIC _ErrorCode = sqlite3_wrap.IOERR_BEGIN_ATOMIC _IOERR_COMMIT_ATOMIC _ErrorCode = sqlite3_wrap.IOERR_COMMIT_ATOMIC diff --git a/vfs/file.go b/vfs/file.go index d6f12ee3..eafe701a 100644 --- a/vfs/file.go +++ b/vfs/file.go @@ -127,6 +127,7 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error file := vfsFile{ File: f, flags: flags | _FLAG_PSOW, + mmap: NewMemoryMapper(f, flags), } if osBatchAtomic(f) { file.flags |= _FLAG_ATOMIC @@ -143,6 +144,7 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error type vfsFile struct { *os.File shm SharedMemory + mmap MemoryMapper lock LockLevel flags OpenFlag } diff --git a/vfs/map.go b/vfs/map.go new file mode 100644 index 00000000..79eac486 --- /dev/null +++ b/vfs/map.go @@ -0,0 +1,63 @@ +//go:build linux || darwin || freebsd + +package vfs + +import ( + "os" + + "github.com/ncruces/go-sqlite3/internal/sqlite3_wrap" +) + +func (f *vfsFile) MemoryMapper() MemoryMapper { return f.mmap } + +// NewMemoryMapper returns a memory mapper for the given file. +// It will return nil if file mapping is not supported, +// or not appropriate for the given flags. +// Only databases use memory mapping. +func NewMemoryMapper(file *os.File, flags OpenFlag) MemoryMapper { + if flags&(OPEN_MAIN_DB|OPEN_TEMP_DB|OPEN_TRANSIENT_DB) == 0 || flags&OPEN_MEMORY != 0 { + return nil + } + return &vfsMapper{File: file} +} + +type vfsMapper struct { + *os.File + mmap *sqlite3_wrap.MappedRegion + size int32 +} + +func (m *vfsMapper) mmapSize(wrp *sqlite3_wrap.Wrapper, p ptr_t) { + size := int64(wrp.Read64(p)) + wrp.Write64(p, uint64(m.size)) + if size >= 0 && m.mmap == nil { + m.size = int32(min(size, 1024*1024*1024)) + } +} + +func (m *vfsMapper) fetch(wrp *sqlite3_wrap.Wrapper, iOfst int64, iAmt int32, pp ptr_t) error { + var ptr ptr_t + if iOfst+int64(iAmt)+256 <= int64(m.size) { + if m.mmap == nil { + var err error + m.mmap, err = wrp.MapRegion(m.File, 0, m.size, true) + if err != nil { + return err + } + } + if m.mmap != nil { + ptr = m.mmap.Ptr + ptr_t(iOfst) + } + } + wrp.Write32(pp, uint32(ptr)) + return nil +} + +func (m *vfsMapper) Close() error { + c := m.mmap + if c == nil { + return nil + } + m.mmap = nil + return c.Unmap() +} diff --git a/vfs/map_other.go b/vfs/map_other.go new file mode 100644 index 00000000..a8b376c0 --- /dev/null +++ b/vfs/map_other.go @@ -0,0 +1,13 @@ +//go:build !(linux || darwin || freebsd) + +package vfs + +import "os" + +// NewMemoryMapper returns a memory mapper for the given file. +// It will return nil if file mapping is not supported, +// or not appropriate for the given flags. +// Only databases use memory mapping. +func NewMemoryMapper(file *os.File, flags OpenFlag) MemoryMapper { + return nil +} diff --git a/vfs/tests/mptest/download.sh b/vfs/tests/mptest/download.sh index 1b594b85..b45d9734 100755 --- a/vfs/tests/mptest/download.sh +++ b/vfs/tests/mptest/download.sh @@ -3,7 +3,7 @@ set -euo pipefail cd -P -- "$(dirname -- "$0")" -GITHUB_TAG="https://github.com/sqlite/sqlite/raw/version-3.53.3" +GITHUB_TAG="https://github.com/sqlite/sqlite/raw/version-3.53.4" cd testdata/ curl -#OL "$GITHUB_TAG/mptest/config01.test" @@ -11,4 +11,5 @@ curl -#OL "$GITHUB_TAG/mptest/config02.test" curl -#OL "$GITHUB_TAG/mptest/crash01.test" curl -#OL "$GITHUB_TAG/mptest/crash02.subtest" curl -#OL "$GITHUB_TAG/mptest/multiwrite01.test" +sed -i "s/if vfsname() GLOB 'unix'/if vfsname() GLOB 'os'/" config01.test cd ~- diff --git a/vfs/tests/mptest/testdata/config01.test b/vfs/tests/mptest/testdata/config01.test index 683ee911..2edf2d7c 100644 --- a/vfs/tests/mptest/testdata/config01.test +++ b/vfs/tests/mptest/testdata/config01.test @@ -1,7 +1,7 @@ /* ** Configure five tasks in different ways, then run tests. */ ---if vfsname() GLOB 'unix' +--if vfsname() GLOB 'os' PRAGMA page_size=8192; --task 1 PRAGMA journal_mode=PERSIST; diff --git a/vfs/tests/speedtest1/speedtest1_test.go b/vfs/tests/speedtest1/speedtest1_test.go index ffc6d8ad..66675a80 100644 --- a/vfs/tests/speedtest1/speedtest1_test.go +++ b/vfs/tests/speedtest1/speedtest1_test.go @@ -60,6 +60,12 @@ func runBenchmark(b *testing.B, args ...string) { } testenv.TB = b + testenv.Exit = func(c int32) { + if c != 0 { + b.Fatal("exit error:", c) + } + b.SkipNow() + } wrp, err := createWrapper(b.Context()) if err != nil { b.Fatal(err) diff --git a/vfs/vfs.go b/vfs/vfs.go index f2a7104a..f71cb2ba 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -66,12 +66,17 @@ func vfsOpen(wrp *sqlite3_wrap.Wrapper, pVfs, zPath, pFile ptr_t, flags OpenFlag file.SetPowersafeOverwrite(b) } } - if file, ok := file.(FileSharedMemory); ok && pOutVFS != 0 { - wrp.WriteBool(pOutVFS, file.SharedMemory() != nil) - } if pOutFlags != 0 { wrp.Write32(pOutFlags, uint32(flags)) } + var outVFS uint32 + if file, ok := file.(FileSharedMemory); ok && file.SharedMemory() != nil { + outVFS |= 1 + } + if file, ok := file.(FileMemoryMapper); ok && file.MemoryMapper() != nil { + outVFS |= 2 + } + wrp.Write32(pOutVFS, outVFS) file = cksmWrapFile(file, flags) vfsFileRegister(wrp, pFile, file) return _OK @@ -302,6 +307,14 @@ func vfsFileControlImpl(wrp *sqlite3_wrap.Wrapper, file File, op _FcntlOpcode, p } } + case _FCNTL_MMAP_SIZE: + if file, ok := file.(FileMemoryMapper); ok { + if mmap := file.MemoryMapper(); mmap != nil { + mmap.mmapSize(wrp, pArg) + return _OK + } + } + case _FCNTL_PDB: if file, ok := file.(filePDB); ok { file.SetDB(wrp.DB) @@ -356,6 +369,22 @@ func vfsShmUnmap(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, bDelete int32) _ErrorCo return _OK } +//go:linkname vfsFetch +func vfsFetch(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, iOfst int64, iAmt int32, pp ptr_t) _ErrorCode { + mem := vfsFileGet(wrp, pFile).(FileMemoryMapper).MemoryMapper() + err := mem.fetch(wrp, iOfst, iAmt, pp) + return vfsErrorCode(wrp, err, _IOERR_MMAP) +} + +//go:linkname vfsUnfetch +func vfsUnfetch(wrp *sqlite3_wrap.Wrapper, pFile ptr_t, _ int64, p ptr_t) _ErrorCode { + if p == 0 { + mmap := vfsFileGet(wrp, pFile).(FileMemoryMapper).MemoryMapper() + mmap.Close() + } + return _OK +} + func vfsGet(wrp *sqlite3_wrap.Wrapper, pVfs ptr_t) VFS { var name string if pVfs != 0 { diff --git a/wrap.go b/wrap.go index a18aa6c4..828133e3 100644 --- a/wrap.go +++ b/wrap.go @@ -8,7 +8,7 @@ import ( "time" _ "unsafe" - sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v3" + sqlite3_wasm "github.com/ncruces/go-sqlite3-wasm/v4" "github.com/ncruces/go-sqlite3/internal/errutil" "github.com/ncruces/go-sqlite3/internal/sqlite3_wrap" "github.com/ncruces/go-sqlite3/vfs" @@ -214,3 +214,17 @@ func vfsShmUnmap(_ *sqlite3_wrap.Wrapper, v0, v1 int32) int32 func (e *env) Xgo_shm_unmap(v0, v1 int32) int32 { return vfsShmUnmap(e.Wrapper, v0, v1) } + +//go:linkname vfsFetch github.com/ncruces/go-sqlite3/vfs.vfsFetch +func vfsFetch(_ *sqlite3_wrap.Wrapper, v0 int32, v1 int64, v2, v3 int32) int32 + +func (e *env) Xgo_fetch(v0 int32, v1 int64, v2, v3 int32) int32 { + return vfsFetch(e.Wrapper, v0, v1, v2, v3) +} + +//go:linkname vfsUnfetch github.com/ncruces/go-sqlite3/vfs.vfsUnfetch +func vfsUnfetch(_ *sqlite3_wrap.Wrapper, v0 int32, v1 int64, v2 int32) int32 + +func (e *env) Xgo_unfetch(v0 int32, v1 int64, v2 int32) int32 { + return vfsUnfetch(e.Wrapper, v0, v1, v2) +}