diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4c8a47d..63d1035 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ jobs: test: strategy: matrix: - go-version: [1.19.x, 1.20.x, 1.21.x, 1.22.x] + go-version: [1.24.x, 1.25.x] os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: @@ -23,7 +23,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: 1.22.x + go-version: 1.25.x - name: Checkout code uses: actions/checkout@v4 - name: Install musl diff --git a/Makefile b/Makefile index e3907ea..f335783 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help all test format fmtcheck vet lint qa deps clean nuke +.PHONY: help all test format fmtcheck vet lint qa deps clean nuke @@ -27,8 +27,8 @@ help: # Alias for help target all: help -test: - go test +test: + go test ./... # Format the source code format: @find ./ -type f -name "*.go" -exec gofmt -w {} \; @@ -41,32 +41,30 @@ fmtcheck: # Check for syntax errors vet: - GOPATH=$(GOPATH) go vet ./... + go vet ./... # Check for style errors lint: - GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint ./... + go vet ./... # Alias to run all quality-assurance checks -qa: fmtcheck test vet lint +qa: fmtcheck test vet # --- INSTALL --- # Get the dependencies deps: - GOPATH=$(GOPATH) go get github.com/smartystreets/goconvey/convey - GOPATH=$(GOPATH) go get github.com/willf/bitset - GOPATH=$(GOPATH) go get github.com/golang/lint/golint + go mod download # Remove any build artifact clean: - GOPATH=$(GOPATH) go clean ./... + go clean ./... # Deletes any intermediate file nuke: rm -rf ./target - GOPATH=$(GOPATH) go clean -i ./... + go clean -i ./... diff --git a/README.md b/README.md index b06b452..29ecbde 100644 --- a/README.md +++ b/README.md @@ -45,13 +45,51 @@ http://arxiv.org/abs/1402.6407 This paper used data from http://lemire.me/data/r -### Dependencies +### Requirements -None in particular. +Go 1.24 or better, and a C compiler (cgo). The CRoaring sources are bundled +with the package, so there is nothing else to install: -Naturally, you also need to grab the roaring code itself: - go get github.com/RoaringBitmap/gocroaring +The bundled CRoaring version is reported by `gocroaring.CRoaringVersion`. + +### 32-bit and 64-bit bitmaps + +The package wraps both C APIs. `gocroaring.Bitmap` stores 32-bit integers and +wraps `roaring_bitmap_t`; `gocroaring.Bitmap64` stores 64-bit integers and +wraps `roaring64_bitmap_t`. The two types offer the same operations, with the +free functions of the 64-bit type carrying a `64` suffix (`Or64`, `And64`, +`Read64`, and so on). + +```go +rb := gocroaring.New64() +rb.AddRange(1<<40, 1<<40+1000) +rb.RunOptimize() +fmt.Println(rb.Cardinality(), rb.Contains(1<<40+5)) +``` + +### Memory management + +Bitmaps and iterators hold memory allocated by C. That memory is released +automatically once the Go value becomes unreachable: we register a cleanup with +`runtime.AddCleanup` rather than a finalizer, which lets the garbage collector +reclaim a bitmap in a single cycle instead of two, and makes creating and +discarding bitmaps about 20% cheaper than the finalizer-based approach we used +previously. You may still call `Free` to release the C memory eagerly; doing so +cancels the cleanup, and calling `Free` twice is harmless. + +Iteration reads values from C in blocks rather than one at a time, so walking a +bitmap through `Iterator()` is more than an order of magnitude faster than +paying for a crossing of the Go/C boundary per value. Use `NewIterator()` when +you need to move backwards or seek. + +The CRoaring entry points are declared `#cgo nocallback` (none of them calls +back into Go) and, where the function does not keep the buffer it is given, +`#cgo noescape` (so the Go buffers we hand to C are not forced onto the heap). +The frozen views are deliberately left out of the `noescape` list, since they +do retain their buffer. + ### Example diff --git a/api_test.go b/api_test.go new file mode 100644 index 0000000..24ac6c4 --- /dev/null +++ b/api_test.go @@ -0,0 +1,583 @@ +package gocroaring + +import ( + "math" + "reflect" + "testing" +) + +func TestVersion(t *testing.T) { + if CRoaringMajor < 5 { + t.Errorf("expected CRoaring 5 or better, got %d", CRoaringMajor) + } + expected := "5.1.0" + if CRoaringVersion != expected { + t.Errorf("expected version %s, got %s", expected, CRoaringVersion) + } +} + +func TestNewWithCapacity(t *testing.T) { + rb := NewWithCapacity(16) + if !rb.IsEmpty() { + t.Error("expected an empty bitmap") + } + rb.Add(1, 2, 3) + if rb.Cardinality() != 3 { + t.Errorf("expected 3, got %d", rb.Cardinality()) + } +} + +func TestFromRange(t *testing.T) { + rb := FromRange(0, 10, 3) + if got, want := rb.ToArray(), []uint32{0, 3, 6, 9}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + defer func() { + if recover() == nil { + t.Error("expected a panic on a zero step") + } + }() + FromRange(0, 10, 0) +} + +func TestAddRemoveChecked(t *testing.T) { + rb := New() + if !rb.AddChecked(7) { + t.Error("expected 7 to be new") + } + if rb.AddChecked(7) { + t.Error("expected 7 to already be present") + } + if !rb.RemoveChecked(7) { + t.Error("expected 7 to be removed") + } + if rb.RemoveChecked(7) { + t.Error("expected 7 to already be gone") + } +} + +func TestRemoveMany(t *testing.T) { + rb := New(1, 2, 3, 4, 5) + rb.RemoveMany(2, 4) + if got, want := rb.ToArray(), []uint32{1, 3, 5}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } +} + +func TestClosedRanges(t *testing.T) { + rb := New() + rb.AddRangeClosed(1, 5) + if got, want := rb.ToArray(), []uint32{1, 2, 3, 4, 5}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if !rb.ContainsRangeClosed(1, 5) { + t.Error("expected to contain [1, 5]") + } + if rb.ContainsRangeClosed(1, 6) { + t.Error("did not expect to contain [1, 6]") + } + if got := rb.RangeCardinalityClosed(2, 4); got != 3 { + t.Errorf("expected 3, got %d", got) + } + if got := rb.RangeCardinality(2, 4); got != 2 { + t.Errorf("expected 2, got %d", got) + } + rb.RemoveRangeClosed(2, 4) + if got, want := rb.ToArray(), []uint32{1, 5}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } +} + +func TestBulkContext(t *testing.T) { + rb := New() + ctx := NewBulkContext() + for i := uint32(0); i < 100000; i += 3 { + rb.AddBulk(ctx, i) + } + if rb.Cardinality() != 33334 { + t.Errorf("expected 33334, got %d", rb.Cardinality()) + } + ctx = NewBulkContext() + for i := uint32(0); i < 100000; i += 3 { + if !rb.ContainsBulk(ctx, i) { + t.Fatalf("expected to contain %d", i) + } + } +} + +func TestRankSelectIndex(t *testing.T) { + rb := New(2, 4, 8, 16) + if got := rb.Rank(8); got != 3 { + t.Errorf("expected 3, got %d", got) + } + if got := rb.GetIndex(8); got != 2 { + t.Errorf("expected 2, got %d", got) + } + if got := rb.GetIndex(9); got != -1 { + t.Errorf("expected -1, got %d", got) + } + if got, err := rb.Select(2); err != nil || got != 8 { + t.Errorf("expected 8, got %d (%v)", got, err) + } + if _, err := rb.Select(4); err != ErrNoSuchElement { + t.Errorf("expected ErrNoSuchElement, got %v", err) + } + if got, want := rb.RankMany([]uint32{2, 4, 8, 16}), []uint64{1, 2, 3, 4}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if got := rb.RankMany(nil); len(got) != 0 { + t.Errorf("expected an empty result, got %v", got) + } +} + +func TestSubsets(t *testing.T) { + rb := New(1, 2, 3) + sub := New(1, 3) + if !sub.IsSubset(rb) || !sub.IsStrictSubset(rb) { + t.Error("expected sub to be a strict subset of rb") + } + if !rb.IsSubset(rb) { + t.Error("expected rb to be a subset of itself") + } + if rb.IsStrictSubset(rb) { + t.Error("did not expect rb to be a strict subset of itself") + } +} + +func TestIntersectWithRange(t *testing.T) { + rb := New(10, 20) + if !rb.IntersectWithRange(5, 15) { + t.Error("expected an intersection with [5, 15)") + } + if rb.IntersectWithRange(11, 20) { + t.Error("did not expect an intersection with [11, 20)") + } +} + +func TestCopyOnWrite(t *testing.T) { + rb := New(1, 2, 3) + if rb.GetCopyOnWrite() { + t.Error("expected copy-on-write to be off by default") + } + rb.SetCopyOnWrite(true) + if !rb.GetCopyOnWrite() { + t.Error("expected copy-on-write to be on") + } + clone := rb.Clone() + clone.Add(4) + if rb.Contains(4) { + t.Error("the clone should not have modified the original") + } + rb.SetCopyOnWrite(false) + if rb.GetCopyOnWrite() { + t.Error("expected copy-on-write to be off") + } +} + +func TestLazyOperations(t *testing.T) { + rb1 := New(1, 2, 3) + rb2 := New(3, 4, 5) + + lazy := LazyOr(rb1, rb2, false) + lazy.RepairAfterLazy() + if got, want := lazy.ToArray(), []uint32{1, 2, 3, 4, 5}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + + lazyXor := LazyXor(rb1, rb2) + lazyXor.RepairAfterLazy() + if got, want := lazyXor.ToArray(), []uint32{1, 2, 4, 5}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + + inplace := rb1.Clone() + inplace.LazyOrInplace(rb2, true) + inplace.RepairAfterLazy() + if got, want := inplace.ToArray(), []uint32{1, 2, 3, 4, 5}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + + inplaceXor := rb1.Clone() + inplaceXor.LazyXorInplace(rb2) + inplaceXor.RepairAfterLazy() + if got, want := inplaceXor.ToArray(), []uint32{1, 2, 4, 5}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } +} + +func TestFastOperations(t *testing.T) { + rb1 := New(1, 2) + rb2 := New(2, 3) + rb3 := New(3, 4) + + if got, want := FastOr(rb1, rb2, rb3).ToArray(), []uint32{1, 2, 3, 4}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if got, want := FastOrHeap(rb1, rb2, rb3).ToArray(), []uint32{1, 2, 3, 4}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if got, want := FastXor(rb1, rb2, rb3).ToArray(), []uint32{1, 4}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if !FastOr().IsEmpty() || !FastOrHeap().IsEmpty() || !FastXor().IsEmpty() { + t.Error("expected empty bitmaps for empty inputs") + } +} + +func TestFlipClosed(t *testing.T) { + rb := New(1, 2, 3) + flipped := FlipClosed(rb, 2, 4) + if got, want := flipped.ToArray(), []uint32{1, 4}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + rb.FlipClosed(2, 4) + if got, want := rb.ToArray(), []uint32{1, 4}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } +} + +func TestAddOffset(t *testing.T) { + rb := New(1, 2, 3) + if got, want := rb.AddOffset(10).ToArray(), []uint32{11, 12, 13}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if got, want := rb.AddOffset(-2).ToArray(), []uint32{0, 1}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } +} + +func TestShrinkToFitAndValidate(t *testing.T) { + rb := NewWithCapacity(1024) + rb.Add(1, 2, 3) + rb.ShrinkToFit() + if err := rb.InternalValidate(); err != nil { + t.Errorf("expected a valid bitmap, got %v", err) + } +} + +func TestNativeSerialization(t *testing.T) { + rb := New(1, 2, 3, 400000) + buf := make([]byte, rb.NativeSerializedSizeInBytes()) + if err := rb.WriteNative(buf); err != nil { + t.Fatal(err) + } + back, err := ReadNative(buf) + if err != nil { + t.Fatal(err) + } + if !rb.Equals(back) { + t.Error("round trip through the native format failed") + } + if err := rb.WriteNative(buf[:1]); err != ErrNotEnoughSpace { + t.Errorf("expected ErrNotEnoughSpace, got %v", err) + } + if _, err := ReadNative(nil); err != ErrEmptyBuffer { + t.Errorf("expected ErrEmptyBuffer, got %v", err) + } +} + +func TestPortableSerialization(t *testing.T) { + rb := New(1, 2, 3, 400000) + buf := rb.ToBytes() + if got := PortableDeserializeSize(buf); got != len(buf) { + t.Errorf("expected %d, got %d", len(buf), got) + } + back, err := Read(buf) + if err != nil { + t.Fatal(err) + } + if !rb.Equals(back) { + t.Error("round trip through the portable format failed") + } + validated, err := ReadValidated(buf) + if err != nil { + t.Fatal(err) + } + if !rb.Equals(validated) { + t.Error("round trip through ReadValidated failed") + } + if _, err := ReadValidated([]byte{1, 2, 3, 4, 5, 6, 7, 8}); err == nil { + t.Error("expected garbage to be rejected") + } + if _, err := Read(nil); err != ErrEmptyBuffer { + t.Errorf("expected ErrEmptyBuffer, got %v", err) + } +} + +func TestFrozenViews(t *testing.T) { + rb := New() + rb.AddRange(0, 100000) + rb.RunOptimize() + + frozen := AlignedBuffer(rb.FrozenSizeInBytes()) + if err := rb.WriteFrozen(frozen); err != nil { + t.Fatal(err) + } + view, err := ReadFrozenView(frozen) + if err != nil { + t.Fatal(err) + } + if !rb.Equals(view) { + t.Error("the frozen view differs from the original") + } + + // A misaligned buffer must be reported as such rather than crashing. + misaligned := make([]byte, len(frozen)+1) + copy(misaligned[1:], frozen) + if _, err := ReadFrozenView(misaligned[1:]); err == nil { + t.Error("expected a misaligned buffer to be rejected") + } + + portable := rb.ToBytes() + pview, err := ReadPortableFrozenView(portable) + if err != nil { + t.Fatal(err) + } + if !rb.Equals(pview) { + t.Error("the portable frozen view differs from the original") + } +} + +func TestAlignedBuffer(t *testing.T) { + if AlignedBuffer(0) != nil { + t.Error("expected nil for a zero-sized buffer") + } + for size := 1; size < 200; size++ { + b := AlignedBuffer(size) + if len(b) != size { + t.Fatalf("expected a buffer of size %d, got %d", size, len(b)) + } + if !isAligned(b) { + t.Fatalf("buffer of size %d is not aligned", size) + } + } +} + +func TestRangeToArray(t *testing.T) { + rb := New(1, 2, 3, 4, 5) + if got, want := rb.RangeToArray(1, 3), []uint32{2, 3, 4}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if got := rb.RangeToArray(3, 100); len(got) != 2 { + t.Errorf("expected 2 values, got %v", got) + } + if got := rb.RangeToArray(10, 1); len(got) != 0 { + t.Errorf("expected no values, got %v", got) + } +} + +func TestToDenseBitset(t *testing.T) { + rb := New(0, 1, 64, 130) + words, err := rb.ToDenseBitset() + if err != nil { + t.Fatal(err) + } + if len(words) < 3 { + t.Fatalf("expected at least 3 words, got %d", len(words)) + } + for _, v := range []uint32{0, 1, 64, 130} { + if words[v/64]&(1<<(v%64)) == 0 { + t.Errorf("expected bit %d to be set", v) + } + } + if words[0]&(1<<2) != 0 { + t.Error("did not expect bit 2 to be set") + } +} + +func TestAssignAndClear(t *testing.T) { + rb := New(1, 2, 3) + other := New(9) + if !other.Assign(rb) { + t.Error("Assign failed") + } + if !other.Equals(rb) { + t.Error("expected the two bitmaps to be equal") + } + other.Clear() + if !other.IsEmpty() { + t.Error("expected an empty bitmap") + } +} + +func TestEqualsWithOtherType(t *testing.T) { + if New(1).Equals("not a bitmap") { + t.Error("expected a bitmap not to equal a string") + } +} + +func TestMinimumMaximumEmpty(t *testing.T) { + rb := New() + if rb.Minimum() != math.MaxUint32 { + t.Errorf("expected MaxUint32, got %d", rb.Minimum()) + } + if rb.Maximum() != 0 { + t.Errorf("expected 0, got %d", rb.Maximum()) + } +} + +func TestFreeIsIdempotent(t *testing.T) { + rb := New(1, 2, 3) + rb.Free() + rb.Free() +} + +func TestIterate(t *testing.T) { + rb := New() + rb.AddRange(0, 5000) + var collected []uint32 + rb.Iterate(func(x uint32) bool { + collected = append(collected, x) + return true + }) + if len(collected) != 5000 { + t.Fatalf("expected 5000 values, got %d", len(collected)) + } + if !reflect.DeepEqual(collected, rb.ToArray()) { + t.Error("Iterate disagrees with ToArray") + } + + count := 0 + rb.Iterate(func(x uint32) bool { + count++ + return count < 10 + }) + if count != 10 { + t.Errorf("expected the iteration to stop after 10 values, got %d", count) + } +} + +func TestBufferedIteratorCrossesBlocks(t *testing.T) { + rb := New() + // More values than the internal buffer holds, so that refills are covered. + rb.AddRange(0, uint64(iterBufferSize)*3+7) + expected := rb.ToArray() + var got []uint32 + it := rb.Iterator() + for it.HasNext() { + got = append(got, it.Next()) + } + if !reflect.DeepEqual(got, expected) { + t.Errorf("expected %d values, got %d", len(expected), len(got)) + } + if it.HasNext() { + t.Error("expected the iterator to be exhausted") + } +} + +func TestIteratorNavigation(t *testing.T) { + rb := New(1, 2, 3, 100, 1000) + + it := rb.NewIterator() + if !it.HasValue() || it.Value() != 1 { + t.Fatalf("expected to start at 1, got %d", it.Value()) + } + if !it.Next() || it.Value() != 2 { + t.Fatalf("expected 2, got %d", it.Value()) + } + if !it.AdvanceIfNeeded(100) || it.Value() != 100 { + t.Fatalf("expected 100, got %d", it.Value()) + } + if !it.Previous() || it.Value() != 3 { + t.Fatalf("expected 3, got %d", it.Value()) + } + if n := it.Skip(2); n != 2 || it.Value() != 1000 { + t.Fatalf("expected to skip to 1000, got %d after %d skips", it.Value(), n) + } + if n := it.SkipBackward(1); n != 1 || it.Value() != 100 { + t.Fatalf("expected to move back to 100, got %d after %d skips", it.Value(), n) + } + + clone := it.Clone() + if clone.Value() != it.Value() { + t.Error("the clone should point at the same value") + } + clone.Free() + clone.Free() + + it.Reset() + if it.Value() != 1 { + t.Errorf("expected 1 after Reset, got %d", it.Value()) + } + it.ResetToLast() + if it.Value() != 1000 { + t.Errorf("expected 1000 after ResetToLast, got %d", it.Value()) + } + + rev := rb.ReverseIterator() + if !rev.HasValue() || rev.Value() != 1000 { + t.Fatalf("expected the reverse iterator to start at 1000, got %d", rev.Value()) + } + var backwards []uint32 + for rev.HasValue() { + backwards = append(backwards, rev.Value()) + rev.Previous() + } + if !reflect.DeepEqual(backwards, []uint32{1000, 100, 3, 2, 1}) { + t.Errorf("unexpected reverse iteration: %v", backwards) + } +} + +func TestIteratorRead(t *testing.T) { + rb := New() + rb.AddRange(0, 1000) + + it := rb.NewIterator() + buf := make([]uint32, 300) + total := 0 + for { + n := it.Read(buf) + if n == 0 { + break + } + total += n + } + if total != 1000 { + t.Errorf("expected 1000 values, got %d", total) + } + if it.Read(nil) != 0 { + t.Error("expected zero values for an empty buffer") + } + + rev := rb.ReverseIterator() + if n := rev.ReadBackward(buf); n != 300 || buf[0] != 999 { + t.Errorf("expected to read 300 values starting at 999, got %d starting at %d", n, buf[0]) + } + if rev.ReadBackward(nil) != 0 { + t.Error("expected zero values for an empty buffer") + } +} + +func TestIteratorReadRanges(t *testing.T) { + rb := New() + rb.AddRange(0, 10) + rb.AddRange(100, 110) + rb.RunOptimize() + + it := rb.NewIterator() + ranges := make([]Range, 4) + n := it.ReadRanges(ranges) + if n != 2 { + t.Fatalf("expected 2 ranges, got %d", n) + } + want := []Range{{0, 9}, {100, 109}} + if !reflect.DeepEqual(ranges[:2], want) { + t.Errorf("expected %v, got %v", want, ranges[:2]) + } + if it.ReadRanges(nil) != 0 { + t.Error("expected zero ranges for an empty buffer") + } + + rev := rb.ReverseIterator() + n = rev.ReadPreviousRanges(ranges) + if n != 2 { + t.Fatalf("expected 2 ranges, got %d", n) + } + want = []Range{{100, 109}, {0, 9}} + if !reflect.DeepEqual(ranges[:2], want) { + t.Errorf("expected %v, got %v", want, ranges[:2]) + } + if rev.ReadPreviousRanges(nil) != 0 { + t.Error("expected zero ranges for an empty buffer") + } +} diff --git a/bench_test.go b/bench_test.go index cf0965d..cdcb174 100644 --- a/bench_test.go +++ b/bench_test.go @@ -50,3 +50,106 @@ func BenchmarkAddOrderedArity(b *testing.B) { benchmarkAddMany(b, ordered) } func BenchmarkRandomNewFromPtr(b *testing.B) { benchmarkNewFromPtr(b, random) } func BenchmarkOrderedNewFromPtr(b *testing.B) { benchmarkNewFromPtr(b, ordered) } + +func benchmarkBitmap() *gocroaring.Bitmap { + rb := gocroaring.New() + rb.AddRange(0, 1000000) + rb.RunOptimize() + return rb +} + +// BenchmarkIterateBuffered walks the bitmap through the buffered iterator, +// which pulls values from C in blocks. +func BenchmarkIterateBuffered(b *testing.B) { + rb := benchmarkBitmap() + b.ResetTimer() + for n := 0; n < b.N; n++ { + var sum uint64 + it := rb.Iterator() + for it.HasNext() { + sum += uint64(it.Next()) + } + _ = sum + } +} + +// BenchmarkIterateOneByOne walks the bitmap one value at a time, paying for a +// crossing of the Go/C boundary per value. +func BenchmarkIterateOneByOne(b *testing.B) { + rb := benchmarkBitmap() + b.ResetTimer() + for n := 0; n < b.N; n++ { + var sum uint64 + it := rb.NewIterator() + for it.HasValue() { + sum += uint64(it.Value()) + it.Next() + } + _ = sum + } +} + +func BenchmarkToArray(b *testing.B) { + rb := benchmarkBitmap() + b.ResetTimer() + for n := 0; n < b.N; n++ { + _ = rb.ToArray() + } +} + +// BenchmarkCreateAndDiscard measures the cost of handing bitmaps to the +// garbage collector, which is what the cleanup machinery pays for. +func BenchmarkCreateAndDiscard(b *testing.B) { + for n := 0; n < b.N; n++ { + rb := gocroaring.New() + rb.Add(1, 2, 3) + } +} + +func BenchmarkCreateAndFree(b *testing.B) { + for n := 0; n < b.N; n++ { + rb := gocroaring.New() + rb.Add(1, 2, 3) + rb.Free() + } +} + +func BenchmarkAdd64(b *testing.B) { + for n := 0; n < b.N; n++ { + rb := gocroaring.New64() + for _, i := range ordered { + rb.Add(uint64(i)) + } + } +} + +func BenchmarkIterateBuffered64(b *testing.B) { + rb := gocroaring.New64() + rb.AddRange(0, 1000000) + rb.RunOptimize() + b.ResetTimer() + for n := 0; n < b.N; n++ { + var sum uint64 + it := rb.Iterator() + for it.HasNext() { + sum += it.Next() + } + _ = sum + } +} + +func BenchmarkIterateOneByOne64(b *testing.B) { + rb := gocroaring.New64() + rb.AddRange(0, 1000000) + rb.RunOptimize() + b.ResetTimer() + for n := 0; n < b.N; n++ { + var sum uint64 + it := rb.NewIterator() + for it.HasValue() { + sum += it.Value() + it.Next() + } + _ = sum + } +} diff --git a/free_test/explicit_free.go b/free_test/explicit_free.go index 71b11ea..db5e0ef 100644 --- a/free_test/explicit_free.go +++ b/free_test/explicit_free.go @@ -31,7 +31,7 @@ func main() { bitmap := gocroaring.New() for i := uint32(0); i < M; i++ { - bitmap.Add(i*10) + bitmap.Add(i * 10) } fmt.Printf("Created reference bitmap... ") if err := syscall.Getrusage(syscall.RUSAGE_SELF, &rusage); err != nil { diff --git a/go.mod b/go.mod index af7516c..0ff0bf0 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/RoaringBitmap/gocroaring -go 1.19 +go 1.24 diff --git a/gocroaring.go b/gocroaring.go index 9607090..ca25cb4 100644 --- a/gocroaring.go +++ b/gocroaring.go @@ -1,12 +1,220 @@ -// Package gocroaring is an wrapper for CRoaring in go +// Package gocroaring is a wrapper for CRoaring in go. // It provides a fast compressed bitmap data structure. // See http://roaringbitmap.org for details. +// +// The package exposes two types: Bitmap, which stores 32-bit integers and +// wraps the roaring_bitmap_t C type, and Bitmap64, which stores 64-bit +// integers and wraps the roaring64_bitmap_t C type. +// +// Bitmaps hold memory allocated by C. That memory is released automatically +// once the Go value becomes unreachable (we rely on runtime.AddCleanup), but +// you may call Free explicitly to release it eagerly. package gocroaring /* -#cgo CFLAGS: -O3 -std=c11 +#cgo CFLAGS: -O3 -std=c11 + +// None of the CRoaring entry points below calls back into Go, and none of them +// retains a pointer to the memory it is handed. Saying so lets cgo use the +// cheaper calling convention and keeps the Go buffers we pass from escaping to +// the heap. +// +// The frozen views are the exception: they keep the buffer they are given, so +// they are deliberately absent from the noescape list. +#cgo noescape bitset_create +#cgo noescape bitset_free +#cgo noescape bitset_size_in_words +#cgo noescape gocroaring_deserialize_validate +#cgo noescape roaring_bitmap_add +#cgo noescape roaring_bitmap_add_bulk +#cgo noescape roaring_bitmap_add_checked +#cgo noescape roaring_bitmap_add_many +#cgo noescape roaring_bitmap_add_offset +#cgo noescape roaring_bitmap_add_range +#cgo noescape roaring_bitmap_add_range_closed +#cgo noescape roaring_bitmap_and +#cgo noescape roaring_bitmap_and_cardinality +#cgo noescape roaring_bitmap_and_inplace +#cgo noescape roaring_bitmap_andnot +#cgo noescape roaring_bitmap_andnot_cardinality +#cgo noescape roaring_bitmap_andnot_inplace +#cgo noescape roaring_bitmap_clear +#cgo noescape roaring_bitmap_contains +#cgo noescape roaring_bitmap_contains_bulk +#cgo noescape roaring_bitmap_contains_range +#cgo noescape roaring_bitmap_contains_range_closed +#cgo noescape roaring_bitmap_copy +#cgo noescape roaring_bitmap_create +#cgo noescape roaring_bitmap_create_with_capacity +#cgo noescape roaring_bitmap_deserialize_safe +#cgo noescape roaring_bitmap_equals +#cgo noescape roaring_bitmap_flip +#cgo noescape roaring_bitmap_flip_closed +#cgo noescape roaring_bitmap_flip_inplace +#cgo noescape roaring_bitmap_flip_inplace_closed +#cgo noescape roaring_bitmap_free +#cgo noescape roaring_bitmap_from_range +#cgo noescape roaring_bitmap_frozen_serialize +#cgo noescape roaring_bitmap_frozen_size_in_bytes +#cgo noescape roaring_bitmap_get_cardinality +#cgo noescape roaring_bitmap_get_copy_on_write +#cgo noescape roaring_bitmap_get_index +#cgo noescape roaring_bitmap_internal_validate +#cgo noescape roaring_bitmap_intersect +#cgo noescape roaring_bitmap_intersect_with_range +#cgo noescape roaring_bitmap_is_empty +#cgo noescape roaring_bitmap_is_strict_subset +#cgo noescape roaring_bitmap_is_subset +#cgo noescape roaring_bitmap_jaccard_index +#cgo noescape roaring_bitmap_lazy_or +#cgo noescape roaring_bitmap_lazy_or_inplace +#cgo noescape roaring_bitmap_lazy_xor +#cgo noescape roaring_bitmap_lazy_xor_inplace +#cgo noescape roaring_bitmap_maximum +#cgo noescape roaring_bitmap_minimum +#cgo noescape roaring_bitmap_of_ptr +#cgo noescape roaring_bitmap_or +#cgo noescape roaring_bitmap_or_cardinality +#cgo noescape roaring_bitmap_or_inplace +#cgo noescape roaring_bitmap_or_many +#cgo noescape roaring_bitmap_or_many_heap +#cgo noescape roaring_bitmap_overwrite +#cgo noescape roaring_bitmap_portable_deserialize_safe +#cgo noescape roaring_bitmap_portable_deserialize_size +#cgo noescape roaring_bitmap_portable_serialize +#cgo noescape roaring_bitmap_portable_size_in_bytes +#cgo noescape roaring_bitmap_range_cardinality +#cgo noescape roaring_bitmap_range_cardinality_closed +#cgo noescape roaring_bitmap_range_uint32_array +#cgo noescape roaring_bitmap_rank +#cgo noescape roaring_bitmap_rank_many +#cgo noescape roaring_bitmap_remove +#cgo noescape roaring_bitmap_remove_checked +#cgo noescape roaring_bitmap_remove_many +#cgo noescape roaring_bitmap_remove_range +#cgo noescape roaring_bitmap_remove_range_closed +#cgo noescape roaring_bitmap_remove_run_compression +#cgo noescape roaring_bitmap_repair_after_lazy +#cgo noescape roaring_bitmap_run_optimize +#cgo noescape roaring_bitmap_select +#cgo noescape roaring_bitmap_serialize +#cgo noescape roaring_bitmap_set_copy_on_write +#cgo noescape roaring_bitmap_shrink_to_fit +#cgo noescape roaring_bitmap_size_in_bytes +#cgo noescape roaring_bitmap_statistics +#cgo noescape roaring_bitmap_to_bitset +#cgo noescape roaring_bitmap_to_uint32_array +#cgo noescape roaring_bitmap_xor +#cgo noescape roaring_bitmap_xor_cardinality +#cgo noescape roaring_bitmap_xor_inplace +#cgo noescape roaring_bitmap_xor_many + +#cgo nocallback bitset_create +#cgo nocallback bitset_free +#cgo nocallback bitset_size_in_words +#cgo nocallback gocroaring_deserialize_validate +#cgo nocallback roaring_bitmap_add +#cgo nocallback roaring_bitmap_add_bulk +#cgo nocallback roaring_bitmap_add_checked +#cgo nocallback roaring_bitmap_add_many +#cgo nocallback roaring_bitmap_add_offset +#cgo nocallback roaring_bitmap_add_range +#cgo nocallback roaring_bitmap_add_range_closed +#cgo nocallback roaring_bitmap_and +#cgo nocallback roaring_bitmap_and_cardinality +#cgo nocallback roaring_bitmap_and_inplace +#cgo nocallback roaring_bitmap_andnot +#cgo nocallback roaring_bitmap_andnot_cardinality +#cgo nocallback roaring_bitmap_andnot_inplace +#cgo nocallback roaring_bitmap_clear +#cgo nocallback roaring_bitmap_contains +#cgo nocallback roaring_bitmap_contains_bulk +#cgo nocallback roaring_bitmap_contains_range +#cgo nocallback roaring_bitmap_contains_range_closed +#cgo nocallback roaring_bitmap_copy +#cgo nocallback roaring_bitmap_create +#cgo nocallback roaring_bitmap_create_with_capacity +#cgo nocallback roaring_bitmap_deserialize_safe +#cgo nocallback roaring_bitmap_equals +#cgo nocallback roaring_bitmap_flip +#cgo nocallback roaring_bitmap_flip_closed +#cgo nocallback roaring_bitmap_flip_inplace +#cgo nocallback roaring_bitmap_flip_inplace_closed +#cgo nocallback roaring_bitmap_free +#cgo nocallback roaring_bitmap_from_range +#cgo nocallback roaring_bitmap_frozen_serialize +#cgo nocallback roaring_bitmap_frozen_size_in_bytes +#cgo nocallback roaring_bitmap_frozen_view +#cgo nocallback roaring_bitmap_get_cardinality +#cgo nocallback roaring_bitmap_get_copy_on_write +#cgo nocallback roaring_bitmap_get_index +#cgo nocallback roaring_bitmap_internal_validate +#cgo nocallback roaring_bitmap_intersect +#cgo nocallback roaring_bitmap_intersect_with_range +#cgo nocallback roaring_bitmap_is_empty +#cgo nocallback roaring_bitmap_is_strict_subset +#cgo nocallback roaring_bitmap_is_subset +#cgo nocallback roaring_bitmap_jaccard_index +#cgo nocallback roaring_bitmap_lazy_or +#cgo nocallback roaring_bitmap_lazy_or_inplace +#cgo nocallback roaring_bitmap_lazy_xor +#cgo nocallback roaring_bitmap_lazy_xor_inplace +#cgo nocallback roaring_bitmap_maximum +#cgo nocallback roaring_bitmap_minimum +#cgo nocallback roaring_bitmap_of_ptr +#cgo nocallback roaring_bitmap_or +#cgo nocallback roaring_bitmap_or_cardinality +#cgo nocallback roaring_bitmap_or_inplace +#cgo nocallback roaring_bitmap_or_many +#cgo nocallback roaring_bitmap_or_many_heap +#cgo nocallback roaring_bitmap_overwrite +#cgo nocallback roaring_bitmap_portable_deserialize_frozen +#cgo nocallback roaring_bitmap_portable_deserialize_safe +#cgo nocallback roaring_bitmap_portable_deserialize_size +#cgo nocallback roaring_bitmap_portable_serialize +#cgo nocallback roaring_bitmap_portable_size_in_bytes +#cgo nocallback roaring_bitmap_range_cardinality +#cgo nocallback roaring_bitmap_range_cardinality_closed +#cgo nocallback roaring_bitmap_range_uint32_array +#cgo nocallback roaring_bitmap_rank +#cgo nocallback roaring_bitmap_rank_many +#cgo nocallback roaring_bitmap_remove +#cgo nocallback roaring_bitmap_remove_checked +#cgo nocallback roaring_bitmap_remove_many +#cgo nocallback roaring_bitmap_remove_range +#cgo nocallback roaring_bitmap_remove_range_closed +#cgo nocallback roaring_bitmap_remove_run_compression +#cgo nocallback roaring_bitmap_repair_after_lazy +#cgo nocallback roaring_bitmap_run_optimize +#cgo nocallback roaring_bitmap_select +#cgo nocallback roaring_bitmap_serialize +#cgo nocallback roaring_bitmap_set_copy_on_write +#cgo nocallback roaring_bitmap_shrink_to_fit +#cgo nocallback roaring_bitmap_size_in_bytes +#cgo nocallback roaring_bitmap_statistics +#cgo nocallback roaring_bitmap_to_bitset +#cgo nocallback roaring_bitmap_to_uint32_array +#cgo nocallback roaring_bitmap_xor +#cgo nocallback roaring_bitmap_xor_cardinality +#cgo nocallback roaring_bitmap_xor_inplace +#cgo nocallback roaring_bitmap_xor_many #include "roaring.h" +// Deserialize and validate in a single cgo crossing: two calls across the +// Go/C boundary would cost roughly twice as much as one. +static inline roaring_bitmap_t *gocroaring_deserialize_validate( + const char *buf, size_t maxbytes, const char **reason) { + roaring_bitmap_t *r = roaring_bitmap_portable_deserialize_safe(buf, maxbytes); + if (r == NULL) { + *reason = "deserialization failed"; + return NULL; + } + if (!roaring_bitmap_internal_validate(r, reason)) { + roaring_bitmap_free(r); + return NULL; + } + return r; +} */ import "C" import ( @@ -18,49 +226,120 @@ import ( "unsafe" ) +// CRoaringMajor, CRoaringMinor and CRoaringRevision report the version of the +// bundled CRoaring library. const CRoaringMajor = C.ROARING_VERSION_MAJOR const CRoaringMinor = C.ROARING_VERSION_MINOR const CRoaringRevision = C.ROARING_VERSION_REVISION -func free(a *Bitmap) { - C.roaring_bitmap_free(a.cpointer) -} +// CRoaringVersion is the version of the bundled CRoaring library, as a string. +const CRoaringVersion = C.ROARING_VERSION + +// FrozenAlignment is the alignment that a buffer must have before it can back +// a frozen view. See AlignedBuffer and ReadFrozenView. +const FrozenAlignment = 32 + +var ( + // ErrNotEnoughSpace is returned by the serialization routines when the + // provided buffer is too small. + ErrNotEnoughSpace = errors.New("not enough space") + // ErrEmptyBuffer is returned when a deserialization routine is handed an + // empty buffer. + ErrEmptyBuffer = errors.New("empty buffer") + // ErrDeserialize is returned when a buffer does not contain a valid bitmap. + ErrDeserialize = errors.New("failed to read roaring bitmap") + // ErrMisaligned is returned by the frozen view routines when the buffer is + // not aligned on a FrozenAlignment boundary. + ErrMisaligned = fmt.Errorf("buffer is not aligned on a %d-byte boundary", FrozenAlignment) + // ErrNoSuchElement is returned by Select when the rank is out of range. + ErrNoSuchElement = errors.New("no such element") +) -// Bitmap is the roaring bitmap +// Bitmap is a compressed bitmap of 32-bit integers. +// +// A Bitmap is not safe for concurrent modification. type Bitmap struct { - cpointer *C.struct_roaring_bitmap_s + cpointer *C.roaring_bitmap_t + cleanup runtime.Cleanup + // pinned keeps the buffer backing a frozen view alive for as long as the + // bitmap is alive. It is nil for ordinary bitmaps. + pinned *byte +} + +// wrap takes ownership of a C bitmap and arranges for it to be freed once the +// returned Bitmap becomes unreachable. It panics if p is nil. +func wrap(p *C.roaring_bitmap_t) *Bitmap { + if p == nil { + panic("C code returned a null pointer.") + } + rb := &Bitmap{cpointer: p} + // runtime.AddCleanup is cheaper than runtime.SetFinalizer: the bitmap is + // reclaimed in a single garbage collection cycle and it is never + // resurrected. The C pointer is passed as the cleanup argument so that the + // closure does not capture rb, which would keep it alive forever. + rb.cleanup = runtime.AddCleanup(rb, func(p *C.roaring_bitmap_t) { + C.roaring_bitmap_free(p) + }, p) + return rb } -type frozenBitmap struct { - Bitmap - buffer *byte +// Free releases the memory held by the bitmap. Using the bitmap afterwards is +// a mistake. Calling Free more than once is harmless. +func (rb *Bitmap) Free() { + if rb.cpointer == nil { + return + } + rb.cleanup.Stop() + C.roaring_bitmap_free(rb.cpointer) + rb.cpointer = nil + rb.pinned = nil } // New creates a new Bitmap with any number of initial values. // This function may panic if the allocation failed. func New(x ...uint32) *Bitmap { - var answer *Bitmap - if len(x) > 0 { - ptr := unsafe.Pointer(&x[0]) - answer = &Bitmap{C.roaring_bitmap_of_ptr(C.size_t(len(x)), (*C.uint32_t)(ptr))} - runtime.KeepAlive(x) - } else { - answer = &Bitmap{C.roaring_bitmap_create()} + if len(x) == 0 { + return wrap(C.roaring_bitmap_create()) } - if answer.cpointer == nil { - panic("C code returned a null pointer.") + rb := wrap(C.roaring_bitmap_of_ptr(C.size_t(len(x)), (*C.uint32_t)(unsafe.Pointer(&x[0])))) + runtime.KeepAlive(x) + return rb +} + +// NewWithCapacity creates a new Bitmap with room for the given number of +// containers, avoiding some reallocations when the bitmap is populated. +// This function may panic if the allocation failed. +func NewWithCapacity(capacity uint32) *Bitmap { + return wrap(C.roaring_bitmap_create_with_capacity(C.uint32_t(capacity))) +} + +// FromRange creates a bitmap containing min, min+step, min+2*step... up to but +// not including max. The step must be strictly positive. +// This function may panic if the allocation failed. +func FromRange(min, max uint64, step uint32) *Bitmap { + if step == 0 { + panic("gocroaring: FromRange requires a strictly positive step") } - runtime.SetFinalizer(answer, free) - return answer + return wrap(C.roaring_bitmap_from_range(C.uint64_t(min), C.uint64_t(max), C.uint32_t(step))) } -func (rb *Bitmap) Free() { - // Clear the finalizer to avoid double frees - runtime.SetFinalizer(rb, nil) - free(rb) +// Clone creates a copy of the Bitmap. +// This function may panic if the allocation failed. +func (rb *Bitmap) Clone() *Bitmap { + b := wrap(C.roaring_bitmap_copy(rb.cpointer)) + runtime.KeepAlive(rb) + return b +} + +// Assign copies x2 over rb, returning false if the copy failed. +func (rb *Bitmap) Assign(x2 *Bitmap) bool { + answer := bool(C.roaring_bitmap_overwrite(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer } -// Printf writes a description of the bitmap to stdout +// Printf writes a description of the bitmap to stdout. func (rb *Bitmap) Printf() { fmt.Print("{") i := rb.Iterator() @@ -78,475 +357,880 @@ func (rb *Bitmap) Printf() { fmt.Print("}") } -// Add the integer(s) x to the bitmap +// String creates a string representation of the Bitmap. +func (rb *Bitmap) String() string { + var buffer bytes.Buffer + buffer.WriteString("{") + i := rb.Iterator() + counter := 0 + for i.HasNext() { + // to avoid exhausting the memory + if counter > 0x40000 { + buffer.WriteString("...") + break + } + if counter > 0 { + buffer.WriteString(",") + } + buffer.WriteString(strconv.FormatUint(uint64(i.Next()), 10)) + counter++ + } + buffer.WriteString("}") + return buffer.String() +} + +//////////////////////////////////////////////////////////////////////////////// +// Adding and removing values +//////////////////////////////////////////////////////////////////////////////// + +// Add the integer(s) x to the bitmap. func (rb *Bitmap) Add(x ...uint32) { - if len(x) == 1 { + switch len(x) { + case 0: + return + case 1: C.roaring_bitmap_add(rb.cpointer, C.uint32_t(x[0])) - } else { - ptr := unsafe.Pointer(&x[0]) - C.roaring_bitmap_add_many(rb.cpointer, C.size_t(len(x)), (*C.uint32_t)(ptr)) + default: + C.roaring_bitmap_add_many(rb.cpointer, C.size_t(len(x)), (*C.uint32_t)(unsafe.Pointer(&x[0]))) runtime.KeepAlive(x) } runtime.KeepAlive(rb) } -// AddRange - add all values in range [min, max) +// AddChecked adds the integer x to the bitmap and reports whether a new value +// was actually added. +func (rb *Bitmap) AddChecked(x uint32) bool { + answer := bool(C.roaring_bitmap_add_checked(rb.cpointer, C.uint32_t(x))) + runtime.KeepAlive(rb) + return answer +} + +// AddRange adds all values in the range [min, max). func (rb *Bitmap) AddRange(min, max uint64) { C.roaring_bitmap_add_range(rb.cpointer, C.uint64_t(min), C.uint64_t(max)) runtime.KeepAlive(rb) } -// RemoveRange - remove all values in range [min, max) -func (rb *Bitmap) RemoveRange(min, max uint64) { - C.roaring_bitmap_remove_range(rb.cpointer, C.uint64_t(min), C.uint64_t(max)) +// AddRangeClosed adds all values in the range [min, max]. +func (rb *Bitmap) AddRangeClosed(min, max uint32) { + C.roaring_bitmap_add_range_closed(rb.cpointer, C.uint32_t(min), C.uint32_t(max)) runtime.KeepAlive(rb) } -// RunOptimize the compression of the bitmap (call this after populating a new bitmap), return true if the bitmap was modified -func (rb *Bitmap) RunOptimize() bool { - answer := bool(C.roaring_bitmap_run_optimize(rb.cpointer)) +// Remove the integer x from the bitmap. +func (rb *Bitmap) Remove(x uint32) { + C.roaring_bitmap_remove(rb.cpointer, C.uint32_t(x)) runtime.KeepAlive(rb) - return answer } -// RemoveRunCompression Remove run-length encoding even when it is more space efficient return whether a change was applied -func (rb *Bitmap) RemoveRunCompression() bool { - answer := bool(C.roaring_bitmap_remove_run_compression(rb.cpointer)) +// RemoveChecked removes the integer x from the bitmap and reports whether a +// value was actually removed. +func (rb *Bitmap) RemoveChecked(x uint32) bool { + answer := bool(C.roaring_bitmap_remove_checked(rb.cpointer, C.uint32_t(x))) runtime.KeepAlive(rb) return answer } -// FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly. -// It might also be faster than calling Or repeatedly. -// This function may panic if the allocation failed. -func FastOr(bitmaps ...*Bitmap) *Bitmap { - number := len(bitmaps) - po := make([]*C.struct_roaring_bitmap_s, number) - for i, v := range bitmaps { - po[i] = v.cpointer - } - b := &Bitmap{C.roaring_bitmap_or_many(C.size_t(number), (**C.struct_roaring_bitmap_s)(unsafe.Pointer(&po[0])))} - runtime.KeepAlive(bitmaps) - if b.cpointer == nil { - panic("C code returned a null pointer.") +// RemoveMany removes the integer(s) x from the bitmap. +func (rb *Bitmap) RemoveMany(x ...uint32) { + if len(x) == 0 { + return } - runtime.SetFinalizer(b, free) - runtime.KeepAlive(po) - return b + C.roaring_bitmap_remove_many(rb.cpointer, C.size_t(len(x)), (*C.uint32_t)(unsafe.Pointer(&x[0]))) + runtime.KeepAlive(x) + runtime.KeepAlive(rb) } -// Contains returns true if the integer is contained in the bitmap +// RemoveRange removes all values in the range [min, max). +func (rb *Bitmap) RemoveRange(min, max uint64) { + C.roaring_bitmap_remove_range(rb.cpointer, C.uint64_t(min), C.uint64_t(max)) + runtime.KeepAlive(rb) +} + +// RemoveRangeClosed removes all values in the range [min, max]. +func (rb *Bitmap) RemoveRangeClosed(min, max uint32) { + C.roaring_bitmap_remove_range_closed(rb.cpointer, C.uint32_t(min), C.uint32_t(max)) + runtime.KeepAlive(rb) +} + +// Clear removes all elements from the bitmap. +func (rb *Bitmap) Clear() { + C.roaring_bitmap_clear(rb.cpointer) + runtime.KeepAlive(rb) +} + +// BulkContext accelerates repeated accesses to a bitmap when the values are +// provided in ascending order. A context is tied to the bitmap it was last +// used with: it must not be reused across bitmaps, and it must be discarded +// whenever the bitmap is modified by anything other than AddBulk. +type BulkContext struct { + ctx C.roaring_bulk_context_t +} + +// NewBulkContext returns a fresh context for use with AddBulk and +// ContainsBulk. +func NewBulkContext() *BulkContext { + return &BulkContext{} +} + +// AddBulk adds the integer x to the bitmap, using ctx to remember the last +// container visited. Values should be provided in ascending order. +func (rb *Bitmap) AddBulk(ctx *BulkContext, x uint32) { + C.roaring_bitmap_add_bulk(rb.cpointer, &ctx.ctx, C.uint32_t(x)) + runtime.KeepAlive(rb) + runtime.KeepAlive(ctx) +} + +// ContainsBulk reports whether the integer x is in the bitmap, using ctx to +// remember the last container visited. Values should be provided in ascending +// order. +func (rb *Bitmap) ContainsBulk(ctx *BulkContext, x uint32) bool { + answer := bool(C.roaring_bitmap_contains_bulk(rb.cpointer, &ctx.ctx, C.uint32_t(x))) + runtime.KeepAlive(rb) + runtime.KeepAlive(ctx) + return answer +} + +//////////////////////////////////////////////////////////////////////////////// +// Queries +//////////////////////////////////////////////////////////////////////////////// + +// Contains returns true if the integer is contained in the bitmap. func (rb *Bitmap) Contains(x uint32) bool { answer := bool(C.roaring_bitmap_contains(rb.cpointer, C.uint32_t(x))) runtime.KeepAlive(rb) return answer } -// ContainsRange returns true if the integers in the range [x, y) are contained in the bitmap +// ContainsRange returns true if all the integers in the range [x, y) are +// contained in the bitmap. func (rb *Bitmap) ContainsRange(x, y uint64) bool { answer := bool(C.roaring_bitmap_contains_range(rb.cpointer, C.uint64_t(x), C.uint64_t(y))) runtime.KeepAlive(rb) return answer } -// Clear removes all elements from the bitmap -func (rb *Bitmap) Clear() { - C.roaring_bitmap_clear(rb.cpointer) - runtime.KeepAlive(rb) -} - -// Remove the integer x from the bitmap -func (rb *Bitmap) Remove(x uint32) { - C.roaring_bitmap_remove(rb.cpointer, C.uint32_t(x)) +// ContainsRangeClosed returns true if all the integers in the range [x, y] are +// contained in the bitmap. +func (rb *Bitmap) ContainsRangeClosed(x, y uint32) bool { + answer := bool(C.roaring_bitmap_contains_range_closed(rb.cpointer, C.uint32_t(x), C.uint32_t(y))) runtime.KeepAlive(rb) + return answer } -// Cardinality returns the number of integers contained in the bitmap +// Cardinality returns the number of integers contained in the bitmap. func (rb *Bitmap) Cardinality() uint64 { answer := uint64(C.roaring_bitmap_get_cardinality(rb.cpointer)) runtime.KeepAlive(rb) return answer } -// Cardinality returns the number of integers contained in the bitmap +// GetCardinality returns the number of integers contained in the bitmap. func (rb *Bitmap) GetCardinality() uint64 { - answer := uint64(C.roaring_bitmap_get_cardinality(rb.cpointer)) + return rb.Cardinality() +} + +// RangeCardinality returns the number of integers in the bitmap that fall in +// the range [min, max). +func (rb *Bitmap) RangeCardinality(min, max uint64) uint64 { + answer := uint64(C.roaring_bitmap_range_cardinality(rb.cpointer, C.uint64_t(min), C.uint64_t(max))) + runtime.KeepAlive(rb) + return answer +} + +// RangeCardinalityClosed returns the number of integers in the bitmap that +// fall in the range [min, max]. +func (rb *Bitmap) RangeCardinalityClosed(min, max uint32) uint64 { + answer := uint64(C.roaring_bitmap_range_cardinality_closed(rb.cpointer, C.uint32_t(min), C.uint32_t(max))) + runtime.KeepAlive(rb) + return answer +} + +// IsEmpty returns true if the Bitmap is empty (it is faster than doing +// Cardinality() == 0). +func (rb *Bitmap) IsEmpty() bool { + answer := bool(C.roaring_bitmap_is_empty(rb.cpointer)) runtime.KeepAlive(rb) return answer } -// Maximum returns the largest of the integers contained in the bitmap assuming that it is not empty +// Maximum returns the largest of the integers contained in the bitmap, +// or 0 if the bitmap is empty. func (rb *Bitmap) Maximum() uint32 { answer := uint32(C.roaring_bitmap_maximum(rb.cpointer)) runtime.KeepAlive(rb) return answer } -// Minimum returns the smallest of the integers contained in the bitmap assuming that it is not empty +// Minimum returns the smallest of the integers contained in the bitmap, +// or math.MaxUint32 if the bitmap is empty. func (rb *Bitmap) Minimum() uint32 { answer := uint32(C.roaring_bitmap_minimum(rb.cpointer)) runtime.KeepAlive(rb) return answer } -// Rank returns the number of values smaller or equal to x +// Rank returns the number of values smaller or equal to x. func (rb *Bitmap) Rank(x uint32) uint64 { answer := uint64(C.roaring_bitmap_rank(rb.cpointer, C.uint32_t(x))) runtime.KeepAlive(rb) return answer } -// Select returns the element having the designated rank, if it exists -func (rb *Bitmap) Select(rank uint32) (uint32, error) { - var element uint32 = 0 - exists := bool(C.roaring_bitmap_select(rb.cpointer, C.uint32_t(rank), (*C.uint32_t)(unsafe.Pointer(&element)))) - runtime.KeepAlive(rb) - if exists { - return element, nil - } else { - return element, errors.New("no such element") +// RankMany returns the rank of each value in vals. The values must be sorted +// in ascending order. It is faster than calling Rank repeatedly. +func (rb *Bitmap) RankMany(vals []uint32) []uint64 { + answer := make([]uint64, len(vals)) + if len(vals) == 0 { + return answer } + begin := (*C.uint32_t)(unsafe.Pointer(&vals[0])) + end := (*C.uint32_t)(unsafe.Pointer(&vals[len(vals)-1])) + // The C API takes a one-past-the-end pointer. + end = (*C.uint32_t)(unsafe.Add(unsafe.Pointer(end), unsafe.Sizeof(vals[0]))) + C.roaring_bitmap_rank_many(rb.cpointer, begin, end, (*C.uint64_t)(unsafe.Pointer(&answer[0]))) + runtime.KeepAlive(vals) + runtime.KeepAlive(answer) + runtime.KeepAlive(rb) + return answer } -// IsEmpty returns true if the Bitmap is empty (it is faster than doing (Cardinality() == 0)) -func (rb *Bitmap) IsEmpty() bool { - answer := bool(C.roaring_bitmap_is_empty(rb.cpointer)) +// GetIndex returns the index of x in the bitmap, or -1 if x is not present. +// Unlike Rank, it distinguishes a missing value from a value of rank zero. +func (rb *Bitmap) GetIndex(x uint32) int64 { + answer := int64(C.roaring_bitmap_get_index(rb.cpointer, C.uint32_t(x))) runtime.KeepAlive(rb) return answer } -// Equals returns true if the two bitmaps contain the same integers +// Select returns the element having the designated rank, if it exists. +func (rb *Bitmap) Select(rank uint32) (uint32, error) { + var element C.uint32_t + exists := bool(C.roaring_bitmap_select(rb.cpointer, C.uint32_t(rank), &element)) + runtime.KeepAlive(rb) + if !exists { + return 0, ErrNoSuchElement + } + return uint32(element), nil +} + +// Equals returns true if the two bitmaps contain the same integers. func (rb *Bitmap) Equals(o interface{}) bool { srb, ok := o.(*Bitmap) - if ok { - answer := bool(C.roaring_bitmap_equals(rb.cpointer, srb.cpointer)) - runtime.KeepAlive(rb) - runtime.KeepAlive(srb) - return answer + if !ok { + return false } - return false + answer := bool(C.roaring_bitmap_equals(rb.cpointer, srb.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(srb) + return answer } -// Clone creates a copy of the Bitmap -// This function may panic if the allocation failed. -func (rb *Bitmap) Clone() *Bitmap { - b := &Bitmap{C.roaring_bitmap_copy(rb.cpointer)} +// IsSubset returns true if every integer of rb is also in x2. +func (rb *Bitmap) IsSubset(x2 *Bitmap) bool { + answer := bool(C.roaring_bitmap_is_subset(rb.cpointer, x2.cpointer)) runtime.KeepAlive(rb) - if b.cpointer == nil { - panic("C code returned a null pointer.") + runtime.KeepAlive(x2) + return answer +} + +// IsStrictSubset returns true if every integer of rb is also in x2 and the two +// bitmaps differ. +func (rb *Bitmap) IsStrictSubset(x2 *Bitmap) bool { + answer := bool(C.roaring_bitmap_is_strict_subset(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// Intersect checks whether the two bitmaps intersect. +func (rb *Bitmap) Intersect(x2 *Bitmap) bool { + answer := bool(C.roaring_bitmap_intersect(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// IntersectWithRange checks whether the bitmap intersects the range [x, y). +func (rb *Bitmap) IntersectWithRange(x, y uint64) bool { + answer := bool(C.roaring_bitmap_intersect_with_range(rb.cpointer, C.uint64_t(x), C.uint64_t(y))) + runtime.KeepAlive(rb) + return answer +} + +// AndCardinality computes the size of the intersection between two bitmaps. +func (rb *Bitmap) AndCardinality(x2 *Bitmap) uint64 { + answer := uint64(C.roaring_bitmap_and_cardinality(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// OrCardinality computes the size of the union between two bitmaps. +func (rb *Bitmap) OrCardinality(x2 *Bitmap) uint64 { + answer := uint64(C.roaring_bitmap_or_cardinality(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// XorCardinality computes the size of the symmetric difference between two +// bitmaps. +func (rb *Bitmap) XorCardinality(x2 *Bitmap) uint64 { + answer := uint64(C.roaring_bitmap_xor_cardinality(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// AndNotCardinality computes the size of the difference between two bitmaps. +func (rb *Bitmap) AndNotCardinality(x2 *Bitmap) uint64 { + answer := uint64(C.roaring_bitmap_andnot_cardinality(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// JaccardIndex computes the Jaccard index between two bitmaps. +func (rb *Bitmap) JaccardIndex(x2 *Bitmap) float64 { + answer := float64(C.roaring_bitmap_jaccard_index(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// InternalValidate performs internal consistency checks. It is useful after +// deserializing bitmaps from untrusted sources. It returns nil if the bitmap +// is consistent, and an error describing the problem otherwise. +func (rb *Bitmap) InternalValidate() error { + var reason *C.char + ok := bool(C.roaring_bitmap_internal_validate(rb.cpointer, &reason)) + runtime.KeepAlive(rb) + if ok { + return nil } - runtime.SetFinalizer(b, free) - return b + return errors.New(C.GoString(reason)) } -// Assign let rb = x2 -func (rb *Bitmap) Assign(x2 *Bitmap) bool { - answer := bool(C.roaring_bitmap_overwrite(rb.cpointer, x2.cpointer)) +// GetCopyOnWrite reports whether the bitmap uses copy-on-write containers. +func (rb *Bitmap) GetCopyOnWrite() bool { + answer := bool(C.roaring_bitmap_get_copy_on_write(rb.cpointer)) runtime.KeepAlive(rb) return answer } -// And computes the intersection between two bitmaps and stores the result in the current bitmap +// SetCopyOnWrite turns copy-on-write on or off. Copy-on-write saves memory and +// avoids copies, but it requires more care in a threaded context. If you +// enable it, enable it on all of your bitmaps: mixing bitmaps with and without +// copy-on-write is unsafe. +func (rb *Bitmap) SetCopyOnWrite(cow bool) { + C.roaring_bitmap_set_copy_on_write(rb.cpointer, C.bool(cow)) + runtime.KeepAlive(rb) +} + +//////////////////////////////////////////////////////////////////////////////// +// In-place set operations +//////////////////////////////////////////////////////////////////////////////// + +// And computes the intersection between two bitmaps and stores the result in +// the current bitmap. func (rb *Bitmap) And(x2 *Bitmap) { C.roaring_bitmap_and_inplace(rb.cpointer, x2.cpointer) runtime.KeepAlive(rb) runtime.KeepAlive(x2) } -// Xor computes the symmetric difference between two bitmaps and stores the result in the current bitmap +// Xor computes the symmetric difference between two bitmaps and stores the +// result in the current bitmap. func (rb *Bitmap) Xor(x2 *Bitmap) { C.roaring_bitmap_xor_inplace(rb.cpointer, x2.cpointer) runtime.KeepAlive(rb) runtime.KeepAlive(x2) } -// Or computes the union between two bitmaps and stores the result in the current bitmap +// Or computes the union between two bitmaps and stores the result in the +// current bitmap. func (rb *Bitmap) Or(x2 *Bitmap) { C.roaring_bitmap_or_inplace(rb.cpointer, x2.cpointer) runtime.KeepAlive(rb) runtime.KeepAlive(x2) } -// AndNot computes the difference between two bitmaps and stores the result in the current bitmap +// AndNot computes the difference between two bitmaps and stores the result in +// the current bitmap. func (rb *Bitmap) AndNot(x2 *Bitmap) { C.roaring_bitmap_andnot_inplace(rb.cpointer, x2.cpointer) runtime.KeepAlive(rb) runtime.KeepAlive(x2) } -// Intersect checks whether the two bitmaps intersect -func (rb *Bitmap) Intersect(x2 *Bitmap) bool { - answer := bool(C.roaring_bitmap_intersect(rb.cpointer, x2.cpointer)) +// LazyOrInplace computes the union with x2 in place, leaving the bitmap in an +// invalid state until RepairAfterLazy is called. Set bitsetconversion to true +// to eagerly convert containers to bitsets when it might help. +func (rb *Bitmap) LazyOrInplace(x2 *Bitmap, bitsetconversion bool) { + C.roaring_bitmap_lazy_or_inplace(rb.cpointer, x2.cpointer, C.bool(bitsetconversion)) runtime.KeepAlive(rb) runtime.KeepAlive(x2) - return answer } -// JaccardIndex computes the Jaccard index between two bitmaps -func (rb *Bitmap) JaccardIndex(x2 *Bitmap) float64 { - answer := float64(C.roaring_bitmap_jaccard_index(rb.cpointer, x2.cpointer)) +// LazyXorInplace computes the symmetric difference with x2 in place, leaving +// the bitmap in an invalid state until RepairAfterLazy is called. The two +// bitmaps must be distinct. +func (rb *Bitmap) LazyXorInplace(x2 *Bitmap) { + C.roaring_bitmap_lazy_xor_inplace(rb.cpointer, x2.cpointer) runtime.KeepAlive(rb) runtime.KeepAlive(x2) - return answer } -// AndCardinality computes the size of the intersection between two bitmaps -func (rb *Bitmap) AndCardinality(x2 *Bitmap) uint64 { - answer := uint64(C.roaring_bitmap_and_cardinality(rb.cpointer, x2.cpointer)) +// RepairAfterLazy restores a bitmap produced by the lazy operations to a valid +// state. It must be called before any other operation. +func (rb *Bitmap) RepairAfterLazy() { + C.roaring_bitmap_repair_after_lazy(rb.cpointer) runtime.KeepAlive(rb) - runtime.KeepAlive(x2) - return answer } -// XorCardinality computes the size of the symmetric difference between two bitmaps -func (rb *Bitmap) XorCardinality(x2 *Bitmap) uint64 { - answer := uint64(C.roaring_bitmap_xor_cardinality(rb.cpointer, x2.cpointer)) +// Flip negates the bits in the given range (i.e., [rangeStart, rangeEnd)): any +// integer present in this range and in the bitmap is removed, and any integer +// in the range that was absent is added. +func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) { + C.roaring_bitmap_flip_inplace(rb.cpointer, C.uint64_t(rangeStart), C.uint64_t(rangeEnd)) + runtime.KeepAlive(rb) +} + +// FlipClosed negates the bits in the given range (i.e., [rangeStart, rangeEnd]). +func (rb *Bitmap) FlipClosed(rangeStart, rangeEnd uint32) { + C.roaring_bitmap_flip_inplace_closed(rb.cpointer, C.uint32_t(rangeStart), C.uint32_t(rangeEnd)) + runtime.KeepAlive(rb) +} + +// RunOptimize improves the compression of the bitmap (call this after +// populating a new bitmap); it returns true if the bitmap was modified. +func (rb *Bitmap) RunOptimize() bool { + answer := bool(C.roaring_bitmap_run_optimize(rb.cpointer)) runtime.KeepAlive(rb) - runtime.KeepAlive(x2) return answer } -// OrCardinality computes the size of the union between two bitmaps -func (rb *Bitmap) OrCardinality(x2 *Bitmap) uint64 { - answer := uint64(C.roaring_bitmap_or_cardinality(rb.cpointer, x2.cpointer)) +// RemoveRunCompression removes run-length encoding even when it is more space +// efficient; it returns whether a change was applied. +func (rb *Bitmap) RemoveRunCompression() bool { + answer := bool(C.roaring_bitmap_remove_run_compression(rb.cpointer)) runtime.KeepAlive(rb) - runtime.KeepAlive(x2) return answer } -// AndNotCardinality computes the size of the difference between two bitmaps -func (rb *Bitmap) AndNotCardinality(x2 *Bitmap) uint64 { - answer := uint64(C.roaring_bitmap_andnot_cardinality(rb.cpointer, x2.cpointer)) +// ShrinkToFit releases unused memory and returns how many bytes were freed. +func (rb *Bitmap) ShrinkToFit() int { + answer := int(C.roaring_bitmap_shrink_to_fit(rb.cpointer)) runtime.KeepAlive(rb) - runtime.KeepAlive(x2) return answer } -// Or computes the union between two bitmaps and returns the result +//////////////////////////////////////////////////////////////////////////////// +// Set operations returning a new bitmap +//////////////////////////////////////////////////////////////////////////////// + +// Or computes the union between two bitmaps and returns the result. // This function may panic if the allocation failed. func Or(x1, x2 *Bitmap) *Bitmap { - b := &Bitmap{C.roaring_bitmap_or(x1.cpointer, x2.cpointer)} + b := wrap(C.roaring_bitmap_or(x1.cpointer, x2.cpointer)) runtime.KeepAlive(x1) runtime.KeepAlive(x2) - if b.cpointer == nil { - panic("C code returned a null pointer.") - } - runtime.SetFinalizer(b, free) return b } -// And computes the intersection between two bitmaps and returns the result +// And computes the intersection between two bitmaps and returns the result. // This function may panic if the allocation failed. func And(x1, x2 *Bitmap) *Bitmap { - b := &Bitmap{C.roaring_bitmap_and(x1.cpointer, x2.cpointer)} + b := wrap(C.roaring_bitmap_and(x1.cpointer, x2.cpointer)) runtime.KeepAlive(x1) runtime.KeepAlive(x2) - if b.cpointer == nil { - panic("C code returned a null pointer.") - } - runtime.SetFinalizer(b, free) return b } -// Xor computes the symmetric difference between two bitmaps and returns the result +// Xor computes the symmetric difference between two bitmaps and returns the +// result. // This function may panic if the allocation failed. func Xor(x1, x2 *Bitmap) *Bitmap { - b := &Bitmap{C.roaring_bitmap_xor(x1.cpointer, x2.cpointer)} + b := wrap(C.roaring_bitmap_xor(x1.cpointer, x2.cpointer)) runtime.KeepAlive(x1) runtime.KeepAlive(x2) - if b.cpointer == nil { - panic("C code returned a null pointer.") - } - runtime.SetFinalizer(b, free) return b } -// AndNot computes the difference between two bitmaps and returns the result +// AndNot computes the difference between two bitmaps and returns the result. // This function may panic if the allocation failed. func AndNot(x1, x2 *Bitmap) *Bitmap { - b := &Bitmap{C.roaring_bitmap_andnot(x1.cpointer, x2.cpointer)} + b := wrap(C.roaring_bitmap_andnot(x1.cpointer, x2.cpointer)) runtime.KeepAlive(x1) runtime.KeepAlive(x2) - if b.cpointer == nil { - panic("C code returned a null pointer.") - } - runtime.SetFinalizer(b, free) return b } -// Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed. -func (rb *Bitmap) Flip(rangeStart, rangeEnd uint64) { - C.roaring_bitmap_flip_inplace(rb.cpointer, C.uint64_t(rangeStart), C.uint64_t(rangeEnd)) - runtime.KeepAlive(rb) +// LazyOr computes the union between two bitmaps, leaving the result in an +// invalid state until RepairAfterLazy is called on it. Set bitsetconversion to +// true to eagerly convert containers to bitsets when it might help. +// This function may panic if the allocation failed. +func LazyOr(x1, x2 *Bitmap, bitsetconversion bool) *Bitmap { + b := wrap(C.roaring_bitmap_lazy_or(x1.cpointer, x2.cpointer, C.bool(bitsetconversion))) + runtime.KeepAlive(x1) + runtime.KeepAlive(x2) + return b +} + +// LazyXor computes the symmetric difference between two bitmaps, leaving the +// result in an invalid state until RepairAfterLazy is called on it. +// This function may panic if the allocation failed. +func LazyXor(x1, x2 *Bitmap) *Bitmap { + b := wrap(C.roaring_bitmap_lazy_xor(x1.cpointer, x2.cpointer)) + runtime.KeepAlive(x1) + runtime.KeepAlive(x2) + return b } -// Flip negates the bits in the given range (i.e., [rangeStart,rangeEnd)), any integer present in this range and in the bitmap is removed. +// Flip negates the bits in the given range (i.e., [rangeStart, rangeEnd)) and +// returns the result. // This function may panic if the allocation failed. func Flip(bm *Bitmap, rangeStart, rangeEnd uint64) *Bitmap { - b := &Bitmap{C.roaring_bitmap_flip(bm.cpointer, C.uint64_t(rangeStart), C.uint64_t(rangeEnd))} - if b.cpointer == nil { - panic("C code returned a null pointer.") - } - runtime.SetFinalizer(b, free) + b := wrap(C.roaring_bitmap_flip(bm.cpointer, C.uint64_t(rangeStart), C.uint64_t(rangeEnd))) runtime.KeepAlive(bm) return b } -// SerializedSizeInBytes computes the serialized size in bytes the Bitmap. -func (rb *Bitmap) SerializedSizeInBytes() int { - answer := int(C.roaring_bitmap_portable_size_in_bytes(rb.cpointer)) +// FlipClosed negates the bits in the given range (i.e., [rangeStart, rangeEnd]) +// and returns the result. +// This function may panic if the allocation failed. +func FlipClosed(bm *Bitmap, rangeStart, rangeEnd uint32) *Bitmap { + b := wrap(C.roaring_bitmap_flip_closed(bm.cpointer, C.uint32_t(rangeStart), C.uint32_t(rangeEnd))) + runtime.KeepAlive(bm) + return b +} + +// AddOffset returns a copy of the bitmap with the given (possibly negative) +// offset added to every value. Values that fall outside the 32-bit range are +// dropped. +// This function may panic if the allocation failed. +func (rb *Bitmap) AddOffset(offset int64) *Bitmap { + b := wrap(C.roaring_bitmap_add_offset(rb.cpointer, C.int64_t(offset))) runtime.KeepAlive(rb) - return answer + return b } -// FrozenSizeInBytes computes the frozen serialized size in bytes -func (rb *Bitmap) FrozenSizeInBytes() int { - answer := int(C.roaring_bitmap_frozen_size_in_bytes(rb.cpointer)) +// cpointers collects the C pointers of a slice of bitmaps. +func cpointers(bitmaps []*Bitmap) []*C.roaring_bitmap_t { + po := make([]*C.roaring_bitmap_t, len(bitmaps)) + for i, v := range bitmaps { + po[i] = v.cpointer + } + return po +} + +// FastOr computes the union between many bitmaps quickly, as opposed to having +// to call Or repeatedly. +// This function may panic if the allocation failed. +func FastOr(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return New() + } + po := cpointers(bitmaps) + b := wrap(C.roaring_bitmap_or_many(C.size_t(len(po)), (**C.roaring_bitmap_t)(unsafe.Pointer(&po[0])))) + runtime.KeepAlive(bitmaps) + runtime.KeepAlive(po) + return b +} + +// FastOrHeap computes the union between many bitmaps using a heap. It can be +// faster than FastOr when the bitmaps are numerous and of uneven sizes. +// This function may panic if the allocation failed. +func FastOrHeap(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return New() + } + po := cpointers(bitmaps) + b := wrap(C.roaring_bitmap_or_many_heap(C.uint32_t(len(po)), (**C.roaring_bitmap_t)(unsafe.Pointer(&po[0])))) + runtime.KeepAlive(bitmaps) + runtime.KeepAlive(po) + return b +} + +// FastXor computes the symmetric difference between many bitmaps quickly, as +// opposed to having to call Xor repeatedly. +// This function may panic if the allocation failed. +func FastXor(bitmaps ...*Bitmap) *Bitmap { + if len(bitmaps) == 0 { + return New() + } + po := cpointers(bitmaps) + b := wrap(C.roaring_bitmap_xor_many(C.size_t(len(po)), (**C.roaring_bitmap_t)(unsafe.Pointer(&po[0])))) + runtime.KeepAlive(bitmaps) + runtime.KeepAlive(po) + return b +} + +//////////////////////////////////////////////////////////////////////////////// +// Serialization +//////////////////////////////////////////////////////////////////////////////// + +// SerializedSizeInBytes computes the serialized size in bytes of the Bitmap, +// using the portable format. +func (rb *Bitmap) SerializedSizeInBytes() int { + answer := int(C.roaring_bitmap_portable_size_in_bytes(rb.cpointer)) runtime.KeepAlive(rb) return answer - } -// IntIterable allows you to iterate over the values in a Bitmap -type IntIterable interface { - HasNext() bool - Next() uint32 +// Write writes a serialized version of this bitmap to b, using the portable +// format. The buffer must be at least SerializedSizeInBytes long. +func (rb *Bitmap) Write(b []byte) error { + if len(b) < rb.SerializedSizeInBytes() { + return ErrNotEnoughSpace + } + if len(b) == 0 { + return ErrNotEnoughSpace + } + C.roaring_bitmap_portable_serialize(rb.cpointer, (*C.char)(unsafe.Pointer(&b[0]))) + runtime.KeepAlive(b) + runtime.KeepAlive(rb) + return nil } -type intIterator struct { - pointertonext *C.roaring_uint32_iterator_t - current uint32 - has_next bool +// ToBytes returns a serialized version of this bitmap, using the portable +// format. +func (rb *Bitmap) ToBytes() []byte { + b := make([]byte, rb.SerializedSizeInBytes()) + if len(b) > 0 { + C.roaring_bitmap_portable_serialize(rb.cpointer, (*C.char)(unsafe.Pointer(&b[0]))) + runtime.KeepAlive(b) + } + runtime.KeepAlive(rb) + return b } -// Iterator creates a new IntIterable to iterate over the integers contained in the bitmap, in sorted order -func (rb *Bitmap) Iterator() IntIterable { - return newIntIterator(rb) +// Read reads a serialized version of the bitmap, in the portable format. The +// buffer is not retained. If the data comes from an untrusted source, prefer +// ReadValidated. +func Read(b []byte) (*Bitmap, error) { + if len(b) == 0 { + return nil, ErrEmptyBuffer + } + p := C.roaring_bitmap_portable_deserialize_safe((*C.char)(unsafe.Pointer(&b[0])), C.size_t(len(b))) + runtime.KeepAlive(b) + if p == nil { + return nil, ErrDeserialize + } + return wrap(p), nil } -// HasNext returns true if there are more integers to iterate over -func (ii *intIterator) HasNext() bool { - return ii.has_next +// ReadValidated reads a serialized version of the bitmap, in the portable +// format, and checks its internal consistency. Use it when the data comes from +// an untrusted source. The deserialization and the validation are performed in +// a single crossing of the Go/C boundary. +func ReadValidated(b []byte) (*Bitmap, error) { + if len(b) == 0 { + return nil, ErrEmptyBuffer + } + var reason *C.char + p := C.gocroaring_deserialize_validate((*C.char)(unsafe.Pointer(&b[0])), C.size_t(len(b)), &reason) + runtime.KeepAlive(b) + if p == nil { + return nil, fmt.Errorf("%w: %s", ErrDeserialize, C.GoString(reason)) + } + return wrap(p), nil } -// Next returns the next integer -func (ii *intIterator) Next() uint32 { - answer := ii.current - ii.has_next = bool(ii.pointertonext.has_value) - ii.current = uint32(ii.pointertonext.current_value) - C.roaring_uint32_iterator_advance(ii.pointertonext) - runtime.KeepAlive(ii) +// PortableDeserializeSize returns how many bytes would be read from b by Read, +// or zero if b does not start with a valid bitmap. +func PortableDeserializeSize(b []byte) int { + if len(b) == 0 { + return 0 + } + answer := int(C.roaring_bitmap_portable_deserialize_size((*C.char)(unsafe.Pointer(&b[0])), C.size_t(len(b)))) + runtime.KeepAlive(b) return answer } -func freeIntIterator(a *intIterator) { - C.roaring_uint32_iterator_free(a.pointertonext) - runtime.KeepAlive(a) +// NativeSerializedSizeInBytes computes the serialized size in bytes of the +// Bitmap, using the non-portable native format. The native format is not +// compatible with the Java and Go implementations; prefer the portable format +// unless you know what you are doing. +func (rb *Bitmap) NativeSerializedSizeInBytes() int { + answer := int(C.roaring_bitmap_size_in_bytes(rb.cpointer)) + runtime.KeepAlive(rb) + return answer } -// This function may panic if the allocation failed. -func newIntIterator(a *Bitmap) *intIterator { - p := new(intIterator) - p.pointertonext = C.roaring_iterator_create(a.cpointer) - p.has_next = bool(p.pointertonext.has_value) - p.current = uint32(p.pointertonext.current_value) - if p.has_next { - C.roaring_uint32_iterator_advance(p.pointertonext) - } - runtime.KeepAlive(a) - if p.pointertonext == nil { - panic("C code returned a null pointer.") +// WriteNative writes a serialized version of this bitmap to b, using the +// non-portable native format. +func (rb *Bitmap) WriteNative(b []byte) error { + if len(b) < rb.NativeSerializedSizeInBytes() { + return ErrNotEnoughSpace + } + if len(b) == 0 { + return ErrNotEnoughSpace } - runtime.SetFinalizer(p, freeIntIterator) - return p + C.roaring_bitmap_serialize(rb.cpointer, (*C.char)(unsafe.Pointer(&b[0]))) + runtime.KeepAlive(b) + runtime.KeepAlive(rb) + return nil } -// Write writes a serialized version of this bitmap to stream (you should have enough space) -func (rb *Bitmap) Write(b []byte) error { - if len(b) < rb.SerializedSizeInBytes() { - return errors.New("not enough space") +// ReadNative reads a bitmap written by WriteNative. The buffer is not +// retained. +func ReadNative(b []byte) (*Bitmap, error) { + if len(b) == 0 { + return nil, ErrEmptyBuffer } - bchar := (*C.char)(unsafe.Pointer(&b[0])) - C.roaring_bitmap_portable_serialize(rb.cpointer, bchar) + p := C.roaring_bitmap_deserialize_safe(unsafe.Pointer(&b[0]), C.size_t(len(b))) runtime.KeepAlive(b) + if p == nil { + return nil, ErrDeserialize + } + return wrap(p), nil +} + +// FrozenSizeInBytes computes the frozen serialized size in bytes. +func (rb *Bitmap) FrozenSizeInBytes() int { + answer := int(C.roaring_bitmap_frozen_size_in_bytes(rb.cpointer)) runtime.KeepAlive(rb) - return nil + return answer } -// WriteFrozen writes a serialized version of bitmap to the stream in the Frozen format +// WriteFrozen writes a serialized version of the bitmap to b in the frozen +// format. The buffer must be at least FrozenSizeInBytes long. The frozen +// format is endian-sensitive and version-specific; it is meant for fast +// reloading of data you produced yourself, not for interchange. func (rb *Bitmap) WriteFrozen(b []byte) error { if len(b) < rb.FrozenSizeInBytes() { - return errors.New("not enough space") + return ErrNotEnoughSpace } - bchar := (*C.char)(unsafe.Pointer(&b[0])) - C.roaring_bitmap_frozen_serialize(rb.cpointer, bchar) + if len(b) == 0 { + return ErrNotEnoughSpace + } + C.roaring_bitmap_frozen_serialize(rb.cpointer, (*C.char)(unsafe.Pointer(&b[0]))) runtime.KeepAlive(b) runtime.KeepAlive(rb) return nil } -// ToArray creates a new slice containing all of the integers stored in the Bitmap in sorted order +// AlignedBuffer returns a byte slice of the requested size whose first byte is +// aligned on a FrozenAlignment boundary, as required by ReadFrozenView. +func AlignedBuffer(size int) []byte { + if size == 0 { + return nil + } + b := make([]byte, size+FrozenAlignment) + offset := int(uintptr(unsafe.Pointer(&b[0])) % FrozenAlignment) + if offset != 0 { + offset = FrozenAlignment - offset + } + return b[offset : offset+size : offset+size] +} + +// isAligned reports whether the first byte of b sits on a FrozenAlignment +// boundary. +func isAligned(b []byte) bool { + return uintptr(unsafe.Pointer(&b[0]))%FrozenAlignment == 0 +} + +// ReadFrozenView reads a frozen serialized version of the bitmap, as written +// by WriteFrozen. The result is immutable: attempting to mutate it will fail +// catastrophically. The buffer must be aligned on a FrozenAlignment boundary +// (see AlignedBuffer) and its length must be exactly the length that was +// written. A reference to the buffer is retained for the lifetime of the view. +func ReadFrozenView(b []byte) (*Bitmap, error) { + if len(b) == 0 { + return nil, ErrEmptyBuffer + } + if !isAligned(b) { + return nil, ErrMisaligned + } + p := C.roaring_bitmap_frozen_view((*C.char)(unsafe.Pointer(&b[0])), C.size_t(len(b))) + if p == nil { + return nil, ErrDeserialize + } + rb := wrap(p) + rb.pinned = &b[0] + return rb, nil +} + +// ReadPortableFrozenView reads a bitmap written in the portable format (see +// Write) without copying the container data: the result is a read-only view +// over b. A reference to the buffer is retained for the lifetime of the view. +func ReadPortableFrozenView(b []byte) (*Bitmap, error) { + if len(b) == 0 { + return nil, ErrEmptyBuffer + } + p := C.roaring_bitmap_portable_deserialize_frozen((*C.char)(unsafe.Pointer(&b[0]))) + if p == nil { + return nil, ErrDeserialize + } + rb := wrap(p) + rb.pinned = &b[0] + return rb, nil +} + +//////////////////////////////////////////////////////////////////////////////// +// Conversion +//////////////////////////////////////////////////////////////////////////////// + +// ToArray creates a new slice containing all of the integers stored in the +// Bitmap in sorted order. func (rb *Bitmap) ToArray() []uint32 { card := rb.Cardinality() array := make([]uint32, card) if card > 0 { C.roaring_bitmap_to_uint32_array(rb.cpointer, (*C.uint32_t)(unsafe.Pointer(&array[0]))) + runtime.KeepAlive(array) } runtime.KeepAlive(rb) return array } -// String creates a string representation of the Bitmap -func (rb *Bitmap) String() string { - arr := rb.ToArray() // todo: replace with an iterator - var buffer bytes.Buffer - start := []byte("{") - buffer.Write(start) - l := len(arr) - for counter, i := range arr { - // to avoid exhausting the memory - if counter > 0x40000 { - buffer.WriteString("...") - break - } - buffer.WriteString(strconv.FormatInt(int64(i), 10)) - if counter+1 < l { // there is more - buffer.WriteString(",") - } +// RangeToArray returns at most limit integers from the bitmap, in sorted +// order, starting at the given offset (a rank, not a value). +func (rb *Bitmap) RangeToArray(offset, limit uint64) []uint32 { + card := rb.Cardinality() + if offset >= card || limit == 0 { + return []uint32{} } - buffer.WriteString("}") - return buffer.String() -} - -// Read reads a serialized version of the bitmap (you need to call Free on it once you are done) -func Read(b []byte) (*Bitmap, error) { - bchar := (*C.char)(unsafe.Pointer(&b[0])) - answer := &Bitmap{C.roaring_bitmap_portable_deserialize_safe(bchar, C.size_t(len(b)))} - runtime.KeepAlive(b) - if answer.cpointer == nil { - return nil, errors.New("failed to read roaring array") + if limit > card-offset { + limit = card - offset } - runtime.SetFinalizer(answer, free) - return answer, nil + array := make([]uint32, limit) + C.roaring_bitmap_range_uint32_array(rb.cpointer, C.size_t(offset), C.size_t(limit), + (*C.uint32_t)(unsafe.Pointer(&array[0]))) + runtime.KeepAlive(array) + runtime.KeepAlive(rb) + return array } -// ReadFrozenView reads a frozen serialized version of the bitmap -// this is immutable and attempting to mutate it will fail catastrophically -// It keeps a reference to the buffer internally to make sure it's alive for -// the complete lifetime of the view -func ReadFrozenView(b []byte) (*Bitmap, error) { - bchar := (*C.char)(unsafe.Pointer(&b[0])) - answer := &frozenBitmap{ - Bitmap{C.roaring_bitmap_frozen_view(bchar, C.size_t(len(b)))}, - &b[0], +// ToDenseBitset converts the bitmap to an uncompressed bitset, returned as a +// slice of 64-bit words in little-endian bit order: value x is present when +// bit x%64 of word x/64 is set. Beware that the result can be large: a bitmap +// containing a single large value produces a slice proportional to that value. +func (rb *Bitmap) ToDenseBitset() ([]uint64, error) { + bs := C.bitset_create() + if bs == nil { + return nil, errors.New("failed to allocate a bitset") + } + defer C.bitset_free(bs) + ok := bool(C.roaring_bitmap_to_bitset(rb.cpointer, bs)) + runtime.KeepAlive(rb) + if !ok { + return nil, errors.New("failed to convert the bitmap to a bitset") } - if answer.cpointer == nil { - return nil, errors.New("failed to read roaring array") + words := int(C.bitset_size_in_words(bs)) + answer := make([]uint64, words) + if words > 0 { + copy(answer, unsafe.Slice((*uint64)(unsafe.Pointer(bs.array)), words)) } - runtime.SetFinalizer(&answer.Bitmap, free) - return &answer.Bitmap, nil + return answer, nil } +//////////////////////////////////////////////////////////////////////////////// +// Statistics +//////////////////////////////////////////////////////////////////////////////// + // Stats returns some statistics about the roaring bitmap. func (rb *Bitmap) Stats() map[string]uint64 { var stat C.roaring_statistics_t @@ -566,9 +1250,13 @@ func (rb *Bitmap) Stats() map[string]uint64 { "n_values_array_containers": uint64(stat.n_values_array_containers), "n_values_run_containers": uint64(stat.n_values_run_containers), "n_values_bitset_containers": uint64(stat.n_values_bitset_containers), + + "min_value": uint64(stat.min_value), + "max_value": uint64(stat.max_value), } } +// Statistics describes the internal structure of a bitmap. type Statistics struct { Cardinality uint64 Containers uint64 @@ -584,13 +1272,19 @@ type Statistics struct { RunContainers uint64 RunContainerBytes uint64 RunContainerValues uint64 + + // MinValue and MaxValue are undefined when Cardinality is zero. + MinValue uint64 + MaxValue uint64 } -// StatsStruct - same as Stats but returns typed struct. See https://github.com/RoaringBitmap/roaring/pull/73 for rationale +// StatsStruct is the same as Stats but returns a typed struct. +// See https://github.com/RoaringBitmap/roaring/pull/73 for the rationale. func (rb *Bitmap) StatsStruct() Statistics { var stat C.roaring_statistics_t C.roaring_bitmap_statistics(rb.cpointer, &stat) - stats := Statistics{ + runtime.KeepAlive(rb) + return Statistics{ Cardinality: uint64(stat.cardinality), Containers: uint64(stat.n_containers), @@ -605,7 +1299,8 @@ func (rb *Bitmap) StatsStruct() Statistics { RunContainers: uint64(stat.n_run_containers), RunContainerBytes: uint64(stat.n_bytes_run_containers), RunContainerValues: uint64(stat.n_values_run_containers), - } - return stats + MinValue: uint64(stat.min_value), + MaxValue: uint64(stat.max_value), + } } diff --git a/gocroaring64.go b/gocroaring64.go new file mode 100644 index 0000000..afb93cb --- /dev/null +++ b/gocroaring64.go @@ -0,0 +1,1014 @@ +package gocroaring + +/* +#cgo CFLAGS: -O3 -std=c11 + +// None of the CRoaring entry points below calls back into Go, and none of them +// retains a pointer to the memory it is handed. Saying so lets cgo use the +// cheaper calling convention and keeps the Go buffers we pass from escaping to +// the heap. +// +// The frozen views are the exception: they keep the buffer they are given, so +// they are deliberately absent from the noescape list. +#cgo noescape gocroaring64_deserialize_validate +#cgo noescape roaring64_bitmap_add +#cgo noescape roaring64_bitmap_add_bulk +#cgo noescape roaring64_bitmap_add_checked +#cgo noescape roaring64_bitmap_add_many +#cgo noescape roaring64_bitmap_add_offset_signed +#cgo noescape roaring64_bitmap_add_range +#cgo noescape roaring64_bitmap_add_range_closed +#cgo noescape roaring64_bitmap_and +#cgo noescape roaring64_bitmap_and_cardinality +#cgo noescape roaring64_bitmap_and_inplace +#cgo noescape roaring64_bitmap_andnot +#cgo noescape roaring64_bitmap_andnot_cardinality +#cgo noescape roaring64_bitmap_andnot_inplace +#cgo noescape roaring64_bitmap_clear +#cgo noescape roaring64_bitmap_contains +#cgo noescape roaring64_bitmap_contains_bulk +#cgo noescape roaring64_bitmap_contains_range +#cgo noescape roaring64_bitmap_contains_range_closed +#cgo noescape roaring64_bitmap_copy +#cgo noescape roaring64_bitmap_create +#cgo noescape roaring64_bitmap_equals +#cgo noescape roaring64_bitmap_flip +#cgo noescape roaring64_bitmap_flip_closed +#cgo noescape roaring64_bitmap_flip_closed_inplace +#cgo noescape roaring64_bitmap_flip_inplace +#cgo noescape roaring64_bitmap_free +#cgo noescape roaring64_bitmap_from_range +#cgo noescape roaring64_bitmap_frozen_serialize +#cgo noescape roaring64_bitmap_frozen_size_in_bytes +#cgo noescape roaring64_bitmap_get_cardinality +#cgo noescape roaring64_bitmap_get_index +#cgo noescape roaring64_bitmap_internal_validate +#cgo noescape roaring64_bitmap_intersect +#cgo noescape roaring64_bitmap_intersect_with_range +#cgo noescape roaring64_bitmap_is_empty +#cgo noescape roaring64_bitmap_is_strict_subset +#cgo noescape roaring64_bitmap_is_subset +#cgo noescape roaring64_bitmap_jaccard_index +#cgo noescape roaring64_bitmap_maximum +#cgo noescape roaring64_bitmap_minimum +#cgo noescape roaring64_bitmap_move_from_roaring32 +#cgo noescape roaring64_bitmap_of_ptr +#cgo noescape roaring64_bitmap_or +#cgo noescape roaring64_bitmap_or_cardinality +#cgo noescape roaring64_bitmap_or_inplace +#cgo noescape roaring64_bitmap_overwrite +#cgo noescape roaring64_bitmap_portable_deserialize_safe +#cgo noescape roaring64_bitmap_portable_deserialize_size +#cgo noescape roaring64_bitmap_portable_serialize +#cgo noescape roaring64_bitmap_portable_size_in_bytes +#cgo noescape roaring64_bitmap_range_cardinality +#cgo noescape roaring64_bitmap_range_closed_cardinality +#cgo noescape roaring64_bitmap_rank +#cgo noescape roaring64_bitmap_remove +#cgo noescape roaring64_bitmap_remove_bulk +#cgo noescape roaring64_bitmap_remove_checked +#cgo noescape roaring64_bitmap_remove_many +#cgo noescape roaring64_bitmap_remove_range +#cgo noescape roaring64_bitmap_remove_range_closed +#cgo noescape roaring64_bitmap_remove_run_compression +#cgo noescape roaring64_bitmap_run_optimize +#cgo noescape roaring64_bitmap_select +#cgo noescape roaring64_bitmap_shrink_to_fit +#cgo noescape roaring64_bitmap_statistics +#cgo noescape roaring64_bitmap_to_uint64_array +#cgo noescape roaring64_bitmap_xor +#cgo noescape roaring64_bitmap_xor_cardinality +#cgo noescape roaring64_bitmap_xor_inplace + +#cgo nocallback gocroaring64_deserialize_validate +#cgo nocallback roaring64_bitmap_add +#cgo nocallback roaring64_bitmap_add_bulk +#cgo nocallback roaring64_bitmap_add_checked +#cgo nocallback roaring64_bitmap_add_many +#cgo nocallback roaring64_bitmap_add_offset_signed +#cgo nocallback roaring64_bitmap_add_range +#cgo nocallback roaring64_bitmap_add_range_closed +#cgo nocallback roaring64_bitmap_and +#cgo nocallback roaring64_bitmap_and_cardinality +#cgo nocallback roaring64_bitmap_and_inplace +#cgo nocallback roaring64_bitmap_andnot +#cgo nocallback roaring64_bitmap_andnot_cardinality +#cgo nocallback roaring64_bitmap_andnot_inplace +#cgo nocallback roaring64_bitmap_clear +#cgo nocallback roaring64_bitmap_contains +#cgo nocallback roaring64_bitmap_contains_bulk +#cgo nocallback roaring64_bitmap_contains_range +#cgo nocallback roaring64_bitmap_contains_range_closed +#cgo nocallback roaring64_bitmap_copy +#cgo nocallback roaring64_bitmap_create +#cgo nocallback roaring64_bitmap_equals +#cgo nocallback roaring64_bitmap_flip +#cgo nocallback roaring64_bitmap_flip_closed +#cgo nocallback roaring64_bitmap_flip_closed_inplace +#cgo nocallback roaring64_bitmap_flip_inplace +#cgo nocallback roaring64_bitmap_free +#cgo nocallback roaring64_bitmap_from_range +#cgo nocallback roaring64_bitmap_frozen_serialize +#cgo nocallback roaring64_bitmap_frozen_size_in_bytes +#cgo nocallback roaring64_bitmap_frozen_view +#cgo nocallback roaring64_bitmap_get_cardinality +#cgo nocallback roaring64_bitmap_get_index +#cgo nocallback roaring64_bitmap_internal_validate +#cgo nocallback roaring64_bitmap_intersect +#cgo nocallback roaring64_bitmap_intersect_with_range +#cgo nocallback roaring64_bitmap_is_empty +#cgo nocallback roaring64_bitmap_is_strict_subset +#cgo nocallback roaring64_bitmap_is_subset +#cgo nocallback roaring64_bitmap_jaccard_index +#cgo nocallback roaring64_bitmap_maximum +#cgo nocallback roaring64_bitmap_minimum +#cgo nocallback roaring64_bitmap_move_from_roaring32 +#cgo nocallback roaring64_bitmap_of_ptr +#cgo nocallback roaring64_bitmap_or +#cgo nocallback roaring64_bitmap_or_cardinality +#cgo nocallback roaring64_bitmap_or_inplace +#cgo nocallback roaring64_bitmap_overwrite +#cgo nocallback roaring64_bitmap_portable_deserialize_frozen +#cgo nocallback roaring64_bitmap_portable_deserialize_safe +#cgo nocallback roaring64_bitmap_portable_deserialize_size +#cgo nocallback roaring64_bitmap_portable_serialize +#cgo nocallback roaring64_bitmap_portable_size_in_bytes +#cgo nocallback roaring64_bitmap_range_cardinality +#cgo nocallback roaring64_bitmap_range_closed_cardinality +#cgo nocallback roaring64_bitmap_rank +#cgo nocallback roaring64_bitmap_remove +#cgo nocallback roaring64_bitmap_remove_bulk +#cgo nocallback roaring64_bitmap_remove_checked +#cgo nocallback roaring64_bitmap_remove_many +#cgo nocallback roaring64_bitmap_remove_range +#cgo nocallback roaring64_bitmap_remove_range_closed +#cgo nocallback roaring64_bitmap_remove_run_compression +#cgo nocallback roaring64_bitmap_run_optimize +#cgo nocallback roaring64_bitmap_select +#cgo nocallback roaring64_bitmap_shrink_to_fit +#cgo nocallback roaring64_bitmap_statistics +#cgo nocallback roaring64_bitmap_to_uint64_array +#cgo nocallback roaring64_bitmap_xor +#cgo nocallback roaring64_bitmap_xor_cardinality +#cgo nocallback roaring64_bitmap_xor_inplace +#include "roaring.h" + +// Deserialize and validate in a single cgo crossing. +static inline roaring64_bitmap_t *gocroaring64_deserialize_validate( + const char *buf, size_t maxbytes, const char **reason) { + roaring64_bitmap_t *r = roaring64_bitmap_portable_deserialize_safe(buf, maxbytes); + if (r == NULL) { + *reason = "deserialization failed"; + return NULL; + } + if (!roaring64_bitmap_internal_validate(r, reason)) { + roaring64_bitmap_free(r); + return NULL; + } + return r; +} +*/ +import "C" +import ( + "bytes" + "errors" + "fmt" + "runtime" + "strconv" + "unsafe" +) + +// Frozen64Alignment is the alignment that a buffer must have before it can +// back a 64-bit frozen view. See AlignedBuffer64 and ReadFrozenView64. +const Frozen64Alignment = 64 + +// ErrMisaligned64 is returned by ReadFrozenView64 when the buffer is not +// aligned on a Frozen64Alignment boundary. +var ErrMisaligned64 = fmt.Errorf("buffer is not aligned on a %d-byte boundary", Frozen64Alignment) + +// ErrNotShrunken is returned by Bitmap64.WriteFrozen when ShrinkToFit has not +// been called since the last modification of the bitmap. +var ErrNotShrunken = errors.New("the bitmap must be shrunk to fit before it can be frozen") + +// Bitmap64 is a compressed bitmap of 64-bit integers. +// +// A Bitmap64 is not safe for concurrent modification. +type Bitmap64 struct { + cpointer *C.roaring64_bitmap_t + cleanup runtime.Cleanup + // pinned keeps the buffer backing a frozen view alive for as long as the + // bitmap is alive. It is nil for ordinary bitmaps. + pinned *byte +} + +// wrap64 takes ownership of a C bitmap and arranges for it to be freed once +// the returned Bitmap64 becomes unreachable. It panics if p is nil. +func wrap64(p *C.roaring64_bitmap_t) *Bitmap64 { + if p == nil { + panic("C code returned a null pointer.") + } + rb := &Bitmap64{cpointer: p} + // See the comment in wrap: runtime.AddCleanup is cheaper than + // runtime.SetFinalizer, and the closure must not capture rb. + rb.cleanup = runtime.AddCleanup(rb, func(p *C.roaring64_bitmap_t) { + C.roaring64_bitmap_free(p) + }, p) + return rb +} + +// Free releases the memory held by the bitmap. Using the bitmap afterwards is +// a mistake. Calling Free more than once is harmless. +func (rb *Bitmap64) Free() { + if rb.cpointer == nil { + return + } + rb.cleanup.Stop() + C.roaring64_bitmap_free(rb.cpointer) + rb.cpointer = nil + rb.pinned = nil +} + +// New64 creates a new Bitmap64 with any number of initial values. +// This function may panic if the allocation failed. +func New64(x ...uint64) *Bitmap64 { + if len(x) == 0 { + return wrap64(C.roaring64_bitmap_create()) + } + rb := wrap64(C.roaring64_bitmap_of_ptr(C.size_t(len(x)), (*C.uint64_t)(unsafe.Pointer(&x[0])))) + runtime.KeepAlive(x) + return rb +} + +// FromRange64 creates a bitmap containing min, min+step, min+2*step... up to +// but not including max. The step must be strictly positive. +// This function may panic if the allocation failed. +func FromRange64(min, max, step uint64) *Bitmap64 { + if step == 0 { + panic("gocroaring: FromRange64 requires a strictly positive step") + } + return wrap64(C.roaring64_bitmap_from_range(C.uint64_t(min), C.uint64_t(max), C.uint64_t(step))) +} + +// MoveFrom32 builds a 64-bit bitmap by moving the containers out of a 32-bit +// bitmap. This avoids copying the container data, but it leaves the source +// bitmap empty. +// This function may panic if the allocation failed. +func MoveFrom32(from *Bitmap) *Bitmap64 { + b := wrap64(C.roaring64_bitmap_move_from_roaring32(from.cpointer)) + runtime.KeepAlive(from) + return b +} + +// Clone creates a copy of the Bitmap64. +// This function may panic if the allocation failed. +func (rb *Bitmap64) Clone() *Bitmap64 { + b := wrap64(C.roaring64_bitmap_copy(rb.cpointer)) + runtime.KeepAlive(rb) + return b +} + +// Assign copies x2 over rb. +func (rb *Bitmap64) Assign(x2 *Bitmap64) { + C.roaring64_bitmap_overwrite(rb.cpointer, x2.cpointer) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) +} + +// String creates a string representation of the Bitmap64. +func (rb *Bitmap64) String() string { + var buffer bytes.Buffer + buffer.WriteString("{") + i := rb.Iterator() + counter := 0 + for i.HasNext() { + // to avoid exhausting the memory + if counter > 0x40000 { + buffer.WriteString("...") + break + } + if counter > 0 { + buffer.WriteString(",") + } + buffer.WriteString(strconv.FormatUint(i.Next(), 10)) + counter++ + } + buffer.WriteString("}") + return buffer.String() +} + +//////////////////////////////////////////////////////////////////////////////// +// Adding and removing values +//////////////////////////////////////////////////////////////////////////////// + +// Add the integer(s) x to the bitmap. +func (rb *Bitmap64) Add(x ...uint64) { + switch len(x) { + case 0: + return + case 1: + C.roaring64_bitmap_add(rb.cpointer, C.uint64_t(x[0])) + default: + C.roaring64_bitmap_add_many(rb.cpointer, C.size_t(len(x)), (*C.uint64_t)(unsafe.Pointer(&x[0]))) + runtime.KeepAlive(x) + } + runtime.KeepAlive(rb) +} + +// AddChecked adds the integer x to the bitmap and reports whether a new value +// was actually added. +func (rb *Bitmap64) AddChecked(x uint64) bool { + answer := bool(C.roaring64_bitmap_add_checked(rb.cpointer, C.uint64_t(x))) + runtime.KeepAlive(rb) + return answer +} + +// AddRange adds all values in the range [min, max). +func (rb *Bitmap64) AddRange(min, max uint64) { + C.roaring64_bitmap_add_range(rb.cpointer, C.uint64_t(min), C.uint64_t(max)) + runtime.KeepAlive(rb) +} + +// AddRangeClosed adds all values in the range [min, max]. +func (rb *Bitmap64) AddRangeClosed(min, max uint64) { + C.roaring64_bitmap_add_range_closed(rb.cpointer, C.uint64_t(min), C.uint64_t(max)) + runtime.KeepAlive(rb) +} + +// Remove the integer x from the bitmap. +func (rb *Bitmap64) Remove(x uint64) { + C.roaring64_bitmap_remove(rb.cpointer, C.uint64_t(x)) + runtime.KeepAlive(rb) +} + +// RemoveChecked removes the integer x from the bitmap and reports whether a +// value was actually removed. +func (rb *Bitmap64) RemoveChecked(x uint64) bool { + answer := bool(C.roaring64_bitmap_remove_checked(rb.cpointer, C.uint64_t(x))) + runtime.KeepAlive(rb) + return answer +} + +// RemoveMany removes the integer(s) x from the bitmap. +func (rb *Bitmap64) RemoveMany(x ...uint64) { + if len(x) == 0 { + return + } + C.roaring64_bitmap_remove_many(rb.cpointer, C.size_t(len(x)), (*C.uint64_t)(unsafe.Pointer(&x[0]))) + runtime.KeepAlive(x) + runtime.KeepAlive(rb) +} + +// RemoveRange removes all values in the range [min, max). +func (rb *Bitmap64) RemoveRange(min, max uint64) { + C.roaring64_bitmap_remove_range(rb.cpointer, C.uint64_t(min), C.uint64_t(max)) + runtime.KeepAlive(rb) +} + +// RemoveRangeClosed removes all values in the range [min, max]. +func (rb *Bitmap64) RemoveRangeClosed(min, max uint64) { + C.roaring64_bitmap_remove_range_closed(rb.cpointer, C.uint64_t(min), C.uint64_t(max)) + runtime.KeepAlive(rb) +} + +// Clear removes all elements from the bitmap. +func (rb *Bitmap64) Clear() { + C.roaring64_bitmap_clear(rb.cpointer) + runtime.KeepAlive(rb) +} + +// BulkContext64 accelerates repeated accesses to a Bitmap64 when the values +// are provided in ascending order. A context is tied to the bitmap it was last +// used with: it must not be reused across bitmaps, and it must be discarded +// whenever the bitmap is modified by anything other than AddBulk or +// RemoveBulk. +type BulkContext64 struct { + ctx C.roaring64_bulk_context_t +} + +// NewBulkContext64 returns a fresh context for use with AddBulk, RemoveBulk +// and ContainsBulk. +func NewBulkContext64() *BulkContext64 { + return &BulkContext64{} +} + +// AddBulk adds the integer x to the bitmap, using ctx to remember the last +// container visited. Values should be provided in ascending order. +func (rb *Bitmap64) AddBulk(ctx *BulkContext64, x uint64) { + C.roaring64_bitmap_add_bulk(rb.cpointer, &ctx.ctx, C.uint64_t(x)) + runtime.KeepAlive(rb) + runtime.KeepAlive(ctx) +} + +// RemoveBulk removes the integer x from the bitmap, using ctx to remember the +// last container visited. Values should be provided in ascending order. +func (rb *Bitmap64) RemoveBulk(ctx *BulkContext64, x uint64) { + C.roaring64_bitmap_remove_bulk(rb.cpointer, &ctx.ctx, C.uint64_t(x)) + runtime.KeepAlive(rb) + runtime.KeepAlive(ctx) +} + +// ContainsBulk reports whether the integer x is in the bitmap, using ctx to +// remember the last container visited. Values should be provided in ascending +// order. +func (rb *Bitmap64) ContainsBulk(ctx *BulkContext64, x uint64) bool { + answer := bool(C.roaring64_bitmap_contains_bulk(rb.cpointer, &ctx.ctx, C.uint64_t(x))) + runtime.KeepAlive(rb) + runtime.KeepAlive(ctx) + return answer +} + +//////////////////////////////////////////////////////////////////////////////// +// Queries +//////////////////////////////////////////////////////////////////////////////// + +// Contains returns true if the integer is contained in the bitmap. +func (rb *Bitmap64) Contains(x uint64) bool { + answer := bool(C.roaring64_bitmap_contains(rb.cpointer, C.uint64_t(x))) + runtime.KeepAlive(rb) + return answer +} + +// ContainsRange returns true if all the integers in the range [x, y) are +// contained in the bitmap. +func (rb *Bitmap64) ContainsRange(x, y uint64) bool { + answer := bool(C.roaring64_bitmap_contains_range(rb.cpointer, C.uint64_t(x), C.uint64_t(y))) + runtime.KeepAlive(rb) + return answer +} + +// ContainsRangeClosed returns true if all the integers in the range [x, y] are +// contained in the bitmap. +func (rb *Bitmap64) ContainsRangeClosed(x, y uint64) bool { + answer := bool(C.roaring64_bitmap_contains_range_closed(rb.cpointer, C.uint64_t(x), C.uint64_t(y))) + runtime.KeepAlive(rb) + return answer +} + +// Cardinality returns the number of integers contained in the bitmap. +func (rb *Bitmap64) Cardinality() uint64 { + answer := uint64(C.roaring64_bitmap_get_cardinality(rb.cpointer)) + runtime.KeepAlive(rb) + return answer +} + +// GetCardinality returns the number of integers contained in the bitmap. +func (rb *Bitmap64) GetCardinality() uint64 { + return rb.Cardinality() +} + +// RangeCardinality returns the number of integers in the bitmap that fall in +// the range [min, max). +func (rb *Bitmap64) RangeCardinality(min, max uint64) uint64 { + answer := uint64(C.roaring64_bitmap_range_cardinality(rb.cpointer, C.uint64_t(min), C.uint64_t(max))) + runtime.KeepAlive(rb) + return answer +} + +// RangeCardinalityClosed returns the number of integers in the bitmap that +// fall in the range [min, max]. +func (rb *Bitmap64) RangeCardinalityClosed(min, max uint64) uint64 { + answer := uint64(C.roaring64_bitmap_range_closed_cardinality(rb.cpointer, C.uint64_t(min), C.uint64_t(max))) + runtime.KeepAlive(rb) + return answer +} + +// IsEmpty returns true if the Bitmap64 is empty (it is faster than doing +// Cardinality() == 0). +func (rb *Bitmap64) IsEmpty() bool { + answer := bool(C.roaring64_bitmap_is_empty(rb.cpointer)) + runtime.KeepAlive(rb) + return answer +} + +// Maximum returns the largest of the integers contained in the bitmap, +// or 0 if the bitmap is empty. +func (rb *Bitmap64) Maximum() uint64 { + answer := uint64(C.roaring64_bitmap_maximum(rb.cpointer)) + runtime.KeepAlive(rb) + return answer +} + +// Minimum returns the smallest of the integers contained in the bitmap, +// or math.MaxUint64 if the bitmap is empty. +func (rb *Bitmap64) Minimum() uint64 { + answer := uint64(C.roaring64_bitmap_minimum(rb.cpointer)) + runtime.KeepAlive(rb) + return answer +} + +// Rank returns the number of values smaller or equal to x. +func (rb *Bitmap64) Rank(x uint64) uint64 { + answer := uint64(C.roaring64_bitmap_rank(rb.cpointer, C.uint64_t(x))) + runtime.KeepAlive(rb) + return answer +} + +// GetIndex returns the index of x in the bitmap, or -1 if x is not present. +// Unlike Rank, it distinguishes a missing value from a value of rank zero. +func (rb *Bitmap64) GetIndex(x uint64) int64 { + var index C.uint64_t + found := bool(C.roaring64_bitmap_get_index(rb.cpointer, C.uint64_t(x), &index)) + runtime.KeepAlive(rb) + if !found { + return -1 + } + return int64(index) +} + +// Select returns the element having the designated rank, if it exists. +func (rb *Bitmap64) Select(rank uint64) (uint64, error) { + var element C.uint64_t + exists := bool(C.roaring64_bitmap_select(rb.cpointer, C.uint64_t(rank), &element)) + runtime.KeepAlive(rb) + if !exists { + return 0, ErrNoSuchElement + } + return uint64(element), nil +} + +// Equals returns true if the two bitmaps contain the same integers. +func (rb *Bitmap64) Equals(o interface{}) bool { + srb, ok := o.(*Bitmap64) + if !ok { + return false + } + answer := bool(C.roaring64_bitmap_equals(rb.cpointer, srb.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(srb) + return answer +} + +// IsSubset returns true if every integer of rb is also in x2. +func (rb *Bitmap64) IsSubset(x2 *Bitmap64) bool { + answer := bool(C.roaring64_bitmap_is_subset(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// IsStrictSubset returns true if every integer of rb is also in x2 and the two +// bitmaps differ. +func (rb *Bitmap64) IsStrictSubset(x2 *Bitmap64) bool { + answer := bool(C.roaring64_bitmap_is_strict_subset(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// Intersect checks whether the two bitmaps intersect. +func (rb *Bitmap64) Intersect(x2 *Bitmap64) bool { + answer := bool(C.roaring64_bitmap_intersect(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// IntersectWithRange checks whether the bitmap intersects the range [x, y). +func (rb *Bitmap64) IntersectWithRange(x, y uint64) bool { + answer := bool(C.roaring64_bitmap_intersect_with_range(rb.cpointer, C.uint64_t(x), C.uint64_t(y))) + runtime.KeepAlive(rb) + return answer +} + +// JaccardIndex computes the Jaccard index between two bitmaps. +func (rb *Bitmap64) JaccardIndex(x2 *Bitmap64) float64 { + answer := float64(C.roaring64_bitmap_jaccard_index(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// AndCardinality computes the size of the intersection between two bitmaps. +func (rb *Bitmap64) AndCardinality(x2 *Bitmap64) uint64 { + answer := uint64(C.roaring64_bitmap_and_cardinality(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// OrCardinality computes the size of the union between two bitmaps. +func (rb *Bitmap64) OrCardinality(x2 *Bitmap64) uint64 { + answer := uint64(C.roaring64_bitmap_or_cardinality(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// XorCardinality computes the size of the symmetric difference between two +// bitmaps. +func (rb *Bitmap64) XorCardinality(x2 *Bitmap64) uint64 { + answer := uint64(C.roaring64_bitmap_xor_cardinality(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// AndNotCardinality computes the size of the difference between two bitmaps. +func (rb *Bitmap64) AndNotCardinality(x2 *Bitmap64) uint64 { + answer := uint64(C.roaring64_bitmap_andnot_cardinality(rb.cpointer, x2.cpointer)) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) + return answer +} + +// InternalValidate performs internal consistency checks. It is useful after +// deserializing bitmaps from untrusted sources. It returns nil if the bitmap +// is consistent, and an error describing the problem otherwise. +func (rb *Bitmap64) InternalValidate() error { + var reason *C.char + ok := bool(C.roaring64_bitmap_internal_validate(rb.cpointer, &reason)) + runtime.KeepAlive(rb) + if ok { + return nil + } + return errors.New(C.GoString(reason)) +} + +//////////////////////////////////////////////////////////////////////////////// +// In-place set operations +//////////////////////////////////////////////////////////////////////////////// + +// And computes the intersection between two bitmaps and stores the result in +// the current bitmap. +func (rb *Bitmap64) And(x2 *Bitmap64) { + C.roaring64_bitmap_and_inplace(rb.cpointer, x2.cpointer) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) +} + +// Xor computes the symmetric difference between two bitmaps and stores the +// result in the current bitmap. +func (rb *Bitmap64) Xor(x2 *Bitmap64) { + C.roaring64_bitmap_xor_inplace(rb.cpointer, x2.cpointer) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) +} + +// Or computes the union between two bitmaps and stores the result in the +// current bitmap. +func (rb *Bitmap64) Or(x2 *Bitmap64) { + C.roaring64_bitmap_or_inplace(rb.cpointer, x2.cpointer) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) +} + +// AndNot computes the difference between two bitmaps and stores the result in +// the current bitmap. +func (rb *Bitmap64) AndNot(x2 *Bitmap64) { + C.roaring64_bitmap_andnot_inplace(rb.cpointer, x2.cpointer) + runtime.KeepAlive(rb) + runtime.KeepAlive(x2) +} + +// Flip negates the bits in the given range (i.e., [rangeStart, rangeEnd)). +func (rb *Bitmap64) Flip(rangeStart, rangeEnd uint64) { + C.roaring64_bitmap_flip_inplace(rb.cpointer, C.uint64_t(rangeStart), C.uint64_t(rangeEnd)) + runtime.KeepAlive(rb) +} + +// FlipClosed negates the bits in the given range (i.e., [rangeStart, rangeEnd]). +func (rb *Bitmap64) FlipClosed(rangeStart, rangeEnd uint64) { + C.roaring64_bitmap_flip_closed_inplace(rb.cpointer, C.uint64_t(rangeStart), C.uint64_t(rangeEnd)) + runtime.KeepAlive(rb) +} + +// RunOptimize improves the compression of the bitmap (call this after +// populating a new bitmap); it returns true if the bitmap was modified. +func (rb *Bitmap64) RunOptimize() bool { + answer := bool(C.roaring64_bitmap_run_optimize(rb.cpointer)) + runtime.KeepAlive(rb) + return answer +} + +// RemoveRunCompression removes run-length encoding even when it is more space +// efficient; it returns whether a change was applied. +func (rb *Bitmap64) RemoveRunCompression() bool { + answer := bool(C.roaring64_bitmap_remove_run_compression(rb.cpointer)) + runtime.KeepAlive(rb) + return answer +} + +// ShrinkToFit releases unused memory and returns how many bytes were freed. +func (rb *Bitmap64) ShrinkToFit() int { + answer := int(C.roaring64_bitmap_shrink_to_fit(rb.cpointer)) + runtime.KeepAlive(rb) + return answer +} + +//////////////////////////////////////////////////////////////////////////////// +// Set operations returning a new bitmap +//////////////////////////////////////////////////////////////////////////////// + +// Or64 computes the union between two bitmaps and returns the result. +// This function may panic if the allocation failed. +func Or64(x1, x2 *Bitmap64) *Bitmap64 { + b := wrap64(C.roaring64_bitmap_or(x1.cpointer, x2.cpointer)) + runtime.KeepAlive(x1) + runtime.KeepAlive(x2) + return b +} + +// And64 computes the intersection between two bitmaps and returns the result. +// This function may panic if the allocation failed. +func And64(x1, x2 *Bitmap64) *Bitmap64 { + b := wrap64(C.roaring64_bitmap_and(x1.cpointer, x2.cpointer)) + runtime.KeepAlive(x1) + runtime.KeepAlive(x2) + return b +} + +// Xor64 computes the symmetric difference between two bitmaps and returns the +// result. +// This function may panic if the allocation failed. +func Xor64(x1, x2 *Bitmap64) *Bitmap64 { + b := wrap64(C.roaring64_bitmap_xor(x1.cpointer, x2.cpointer)) + runtime.KeepAlive(x1) + runtime.KeepAlive(x2) + return b +} + +// AndNot64 computes the difference between two bitmaps and returns the result. +// This function may panic if the allocation failed. +func AndNot64(x1, x2 *Bitmap64) *Bitmap64 { + b := wrap64(C.roaring64_bitmap_andnot(x1.cpointer, x2.cpointer)) + runtime.KeepAlive(x1) + runtime.KeepAlive(x2) + return b +} + +// Flip64 negates the bits in the given range (i.e., [rangeStart, rangeEnd)) +// and returns the result. +// This function may panic if the allocation failed. +func Flip64(bm *Bitmap64, rangeStart, rangeEnd uint64) *Bitmap64 { + b := wrap64(C.roaring64_bitmap_flip(bm.cpointer, C.uint64_t(rangeStart), C.uint64_t(rangeEnd))) + runtime.KeepAlive(bm) + return b +} + +// FlipClosed64 negates the bits in the given range (i.e., [rangeStart, +// rangeEnd]) and returns the result. +// This function may panic if the allocation failed. +func FlipClosed64(bm *Bitmap64, rangeStart, rangeEnd uint64) *Bitmap64 { + b := wrap64(C.roaring64_bitmap_flip_closed(bm.cpointer, C.uint64_t(rangeStart), C.uint64_t(rangeEnd))) + runtime.KeepAlive(bm) + return b +} + +// AddOffset returns a copy of the bitmap with the given (possibly negative) +// offset added to every value. Values that fall outside the 64-bit range are +// dropped. +// This function may panic if the allocation failed. +func (rb *Bitmap64) AddOffset(offset int64) *Bitmap64 { + positive := offset >= 0 + var magnitude uint64 + if positive { + magnitude = uint64(offset) + } else { + // Negating math.MinInt64 overflows, so go through uint64. + magnitude = -uint64(offset) + } + b := wrap64(C.roaring64_bitmap_add_offset_signed(rb.cpointer, C.bool(positive), C.uint64_t(magnitude))) + runtime.KeepAlive(rb) + return b +} + +//////////////////////////////////////////////////////////////////////////////// +// Serialization +//////////////////////////////////////////////////////////////////////////////// + +// SerializedSizeInBytes computes the serialized size in bytes of the Bitmap64, +// using the portable format. +func (rb *Bitmap64) SerializedSizeInBytes() int { + answer := int(C.roaring64_bitmap_portable_size_in_bytes(rb.cpointer)) + runtime.KeepAlive(rb) + return answer +} + +// Write writes a serialized version of this bitmap to b, using the portable +// format. The buffer must be at least SerializedSizeInBytes long. +func (rb *Bitmap64) Write(b []byte) error { + if len(b) < rb.SerializedSizeInBytes() { + return ErrNotEnoughSpace + } + if len(b) == 0 { + return ErrNotEnoughSpace + } + C.roaring64_bitmap_portable_serialize(rb.cpointer, (*C.char)(unsafe.Pointer(&b[0]))) + runtime.KeepAlive(b) + runtime.KeepAlive(rb) + return nil +} + +// ToBytes returns a serialized version of this bitmap, using the portable +// format. +func (rb *Bitmap64) ToBytes() []byte { + b := make([]byte, rb.SerializedSizeInBytes()) + if len(b) > 0 { + C.roaring64_bitmap_portable_serialize(rb.cpointer, (*C.char)(unsafe.Pointer(&b[0]))) + runtime.KeepAlive(b) + } + runtime.KeepAlive(rb) + return b +} + +// Read64 reads a serialized version of the bitmap, in the portable format. The +// buffer is not retained. If the data comes from an untrusted source, prefer +// ReadValidated64. +func Read64(b []byte) (*Bitmap64, error) { + if len(b) == 0 { + return nil, ErrEmptyBuffer + } + p := C.roaring64_bitmap_portable_deserialize_safe((*C.char)(unsafe.Pointer(&b[0])), C.size_t(len(b))) + runtime.KeepAlive(b) + if p == nil { + return nil, ErrDeserialize + } + return wrap64(p), nil +} + +// ReadValidated64 reads a serialized version of the bitmap, in the portable +// format, and checks its internal consistency. Use it when the data comes from +// an untrusted source. The deserialization and the validation are performed in +// a single crossing of the Go/C boundary. +func ReadValidated64(b []byte) (*Bitmap64, error) { + if len(b) == 0 { + return nil, ErrEmptyBuffer + } + var reason *C.char + p := C.gocroaring64_deserialize_validate((*C.char)(unsafe.Pointer(&b[0])), C.size_t(len(b)), &reason) + runtime.KeepAlive(b) + if p == nil { + return nil, fmt.Errorf("%w: %s", ErrDeserialize, C.GoString(reason)) + } + return wrap64(p), nil +} + +// PortableDeserializeSize64 returns how many bytes would be read from b by +// Read64, or zero if b does not start with a valid bitmap. +func PortableDeserializeSize64(b []byte) int { + if len(b) == 0 { + return 0 + } + answer := int(C.roaring64_bitmap_portable_deserialize_size((*C.char)(unsafe.Pointer(&b[0])), C.size_t(len(b)))) + runtime.KeepAlive(b) + return answer +} + +// FrozenSizeInBytes computes the frozen serialized size in bytes. ShrinkToFit +// must have been called since the last modification of the bitmap; otherwise +// this returns zero. +func (rb *Bitmap64) FrozenSizeInBytes() int { + answer := int(C.roaring64_bitmap_frozen_size_in_bytes(rb.cpointer)) + runtime.KeepAlive(rb) + return answer +} + +// WriteFrozen writes a serialized version of the bitmap to b in the frozen +// format. ShrinkToFit must have been called since the last modification of the +// bitmap. The buffer must be at least FrozenSizeInBytes long. The frozen +// format is endian-sensitive and version-specific; it is meant for fast +// reloading of data you produced yourself, not for interchange. +func (rb *Bitmap64) WriteFrozen(b []byte) error { + size := rb.FrozenSizeInBytes() + if size == 0 { + return ErrNotShrunken + } + if len(b) < size { + return ErrNotEnoughSpace + } + C.roaring64_bitmap_frozen_serialize(rb.cpointer, (*C.char)(unsafe.Pointer(&b[0]))) + runtime.KeepAlive(b) + runtime.KeepAlive(rb) + return nil +} + +// isAligned64 reports whether the first byte of b sits on a +// Frozen64Alignment boundary. +func isAligned64(b []byte) bool { + return uintptr(unsafe.Pointer(&b[0]))%Frozen64Alignment == 0 +} + +// AlignedBuffer64 returns a byte slice of the requested size whose first byte +// is aligned on a Frozen64Alignment boundary, as required by ReadFrozenView64. +func AlignedBuffer64(size int) []byte { + if size == 0 { + return nil + } + b := make([]byte, size+Frozen64Alignment) + offset := int(uintptr(unsafe.Pointer(&b[0])) % Frozen64Alignment) + if offset != 0 { + offset = Frozen64Alignment - offset + } + return b[offset : offset+size : offset+size] +} + +// ReadFrozenView64 reads a frozen serialized version of the bitmap, as written +// by Bitmap64.WriteFrozen. The result is immutable: attempting to mutate it +// will fail catastrophically. The buffer must be aligned on a +// Frozen64Alignment boundary (see AlignedBuffer64). A reference to the buffer +// is retained for the lifetime of the view. +func ReadFrozenView64(b []byte) (*Bitmap64, error) { + if len(b) == 0 { + return nil, ErrEmptyBuffer + } + if !isAligned64(b) { + return nil, ErrMisaligned64 + } + p := C.roaring64_bitmap_frozen_view((*C.char)(unsafe.Pointer(&b[0])), C.size_t(len(b))) + if p == nil { + return nil, ErrDeserialize + } + rb := wrap64(p) + rb.pinned = &b[0] + return rb, nil +} + +// ReadPortableFrozenView64 reads a bitmap written in the portable format (see +// Bitmap64.Write) without copying the container data: the result is a +// read-only view over b. A reference to the buffer is retained for the +// lifetime of the view. It fails on big-endian systems, where the portable +// format cannot be viewed in place. +func ReadPortableFrozenView64(b []byte) (*Bitmap64, error) { + if len(b) == 0 { + return nil, ErrEmptyBuffer + } + p := C.roaring64_bitmap_portable_deserialize_frozen((*C.char)(unsafe.Pointer(&b[0])), C.size_t(len(b))) + if p == nil { + return nil, ErrDeserialize + } + rb := wrap64(p) + rb.pinned = &b[0] + return rb, nil +} + +//////////////////////////////////////////////////////////////////////////////// +// Conversion +//////////////////////////////////////////////////////////////////////////////// + +// ToArray creates a new slice containing all of the integers stored in the +// Bitmap64 in sorted order. +func (rb *Bitmap64) ToArray() []uint64 { + card := rb.Cardinality() + array := make([]uint64, card) + if card > 0 { + C.roaring64_bitmap_to_uint64_array(rb.cpointer, (*C.uint64_t)(unsafe.Pointer(&array[0]))) + runtime.KeepAlive(array) + } + runtime.KeepAlive(rb) + return array +} + +//////////////////////////////////////////////////////////////////////////////// +// Statistics +//////////////////////////////////////////////////////////////////////////////// + +// StatsStruct returns statistics describing the internal structure of the +// bitmap. +func (rb *Bitmap64) StatsStruct() Statistics { + var stat C.roaring64_statistics_t + C.roaring64_bitmap_statistics(rb.cpointer, &stat) + runtime.KeepAlive(rb) + return Statistics{ + Cardinality: uint64(stat.cardinality), + Containers: uint64(stat.n_containers), + + ArrayContainers: uint64(stat.n_array_containers), + ArrayContainerBytes: uint64(stat.n_bytes_array_containers), + ArrayContainerValues: uint64(stat.n_values_array_containers), + + BitmapContainers: uint64(stat.n_bitset_containers), + BitmapContainerBytes: uint64(stat.n_bytes_bitset_containers), + BitmapContainerValues: uint64(stat.n_values_bitset_containers), + + RunContainers: uint64(stat.n_run_containers), + RunContainerBytes: uint64(stat.n_bytes_run_containers), + RunContainerValues: uint64(stat.n_values_run_containers), + + MinValue: uint64(stat.min_value), + MaxValue: uint64(stat.max_value), + } +} + +// Stats returns some statistics about the roaring bitmap. +func (rb *Bitmap64) Stats() map[string]uint64 { + s := rb.StatsStruct() + return map[string]uint64{ + "cardinality": s.Cardinality, + "n_containers": s.Containers, + + "n_array_containers": s.ArrayContainers, + "n_run_containers": s.RunContainers, + "n_bitset_containers": s.BitmapContainers, + + "n_bytes_array_containers": s.ArrayContainerBytes, + "n_bytes_run_containers": s.RunContainerBytes, + "n_bytes_bitset_containers": s.BitmapContainerBytes, + + "n_values_array_containers": s.ArrayContainerValues, + "n_values_run_containers": s.RunContainerValues, + "n_values_bitset_containers": s.BitmapContainerValues, + + "min_value": s.MinValue, + "max_value": s.MaxValue, + } +} diff --git a/gocroaring64_test.go b/gocroaring64_test.go new file mode 100644 index 0000000..65466c5 --- /dev/null +++ b/gocroaring64_test.go @@ -0,0 +1,592 @@ +package gocroaring + +import ( + "math" + "reflect" + "testing" +) + +const big = uint64(1) << 40 + +func TestNew64(t *testing.T) { + rb := New64() + if !rb.IsEmpty() { + t.Error("expected an empty bitmap") + } + rb = New64(1, 2, big, big+1) + if rb.Cardinality() != 4 { + t.Errorf("expected 4, got %d", rb.Cardinality()) + } + for _, v := range []uint64{1, 2, big, big + 1} { + if !rb.Contains(v) { + t.Errorf("expected to contain %d", v) + } + } + if rb.Contains(3) { + t.Error("did not expect to contain 3") + } + if got, want := rb.Minimum(), uint64(1); got != want { + t.Errorf("expected %d, got %d", want, got) + } + if got, want := rb.Maximum(), big+1; got != want { + t.Errorf("expected %d, got %d", want, got) + } +} + +func TestNew64Empty(t *testing.T) { + rb := New64() + if rb.Minimum() != math.MaxUint64 { + t.Errorf("expected MaxUint64, got %d", rb.Minimum()) + } + if rb.Maximum() != 0 { + t.Errorf("expected 0, got %d", rb.Maximum()) + } +} + +func TestFromRange64(t *testing.T) { + rb := FromRange64(big, big+10, 3) + want := []uint64{big, big + 3, big + 6, big + 9} + if got := rb.ToArray(); !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + defer func() { + if recover() == nil { + t.Error("expected a panic on a zero step") + } + }() + FromRange64(0, 10, 0) +} + +func TestMoveFrom32(t *testing.T) { + src := New(1, 2, 3) + rb := MoveFrom32(src) + if got, want := rb.ToArray(), []uint64{1, 2, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + // The source is emptied but stays usable. + if !src.IsEmpty() { + t.Error("expected the source bitmap to be emptied") + } + src.Add(9) + if !src.Contains(9) { + t.Error("expected the source bitmap to remain usable") + } +} + +func TestAddRemoveChecked64(t *testing.T) { + rb := New64() + if !rb.AddChecked(big) { + t.Error("expected the value to be new") + } + if rb.AddChecked(big) { + t.Error("expected the value to already be present") + } + if !rb.RemoveChecked(big) { + t.Error("expected the value to be removed") + } + if rb.RemoveChecked(big) { + t.Error("expected the value to already be gone") + } +} + +func TestRanges64(t *testing.T) { + rb := New64() + rb.AddRange(big, big+5) + if rb.Cardinality() != 5 { + t.Errorf("expected 5, got %d", rb.Cardinality()) + } + if !rb.ContainsRange(big, big+5) { + t.Error("expected to contain the range") + } + if rb.ContainsRange(big, big+6) { + t.Error("did not expect to contain the wider range") + } + rb.RemoveRange(big+1, big+3) + if got, want := rb.ToArray(), []uint64{big, big + 3, big + 4}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + + closed := New64() + closed.AddRangeClosed(big, big+4) + if !closed.ContainsRangeClosed(big, big+4) { + t.Error("expected to contain the closed range") + } + if got := closed.RangeCardinalityClosed(big, big+2); got != 3 { + t.Errorf("expected 3, got %d", got) + } + if got := closed.RangeCardinality(big, big+2); got != 2 { + t.Errorf("expected 2, got %d", got) + } + closed.RemoveRangeClosed(big+1, big+3) + if got, want := closed.ToArray(), []uint64{big, big + 4}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if !closed.IntersectWithRange(big, big+1) { + t.Error("expected an intersection") + } + if closed.IntersectWithRange(big+1, big+4) { + t.Error("did not expect an intersection") + } +} + +func TestRemoveMany64(t *testing.T) { + rb := New64(1, 2, 3, big, big+1) + rb.RemoveMany(2, big) + if got, want := rb.ToArray(), []uint64{1, 3, big + 1}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + rb.RemoveMany() + if rb.Cardinality() != 3 { + t.Errorf("expected 3, got %d", rb.Cardinality()) + } +} + +func TestBulkContext64(t *testing.T) { + rb := New64() + ctx := NewBulkContext64() + for i := uint64(0); i < 100000; i += 3 { + rb.AddBulk(ctx, big+i) + } + if rb.Cardinality() != 33334 { + t.Errorf("expected 33334, got %d", rb.Cardinality()) + } + ctx = NewBulkContext64() + for i := uint64(0); i < 100000; i += 3 { + if !rb.ContainsBulk(ctx, big+i) { + t.Fatalf("expected to contain %d", big+i) + } + } + ctx = NewBulkContext64() + for i := uint64(0); i < 100000; i += 3 { + rb.RemoveBulk(ctx, big+i) + } + if !rb.IsEmpty() { + t.Errorf("expected an empty bitmap, got %d values", rb.Cardinality()) + } +} + +func TestRankSelectIndex64(t *testing.T) { + rb := New64(2, 4, big, big+8) + if got := rb.Rank(big); got != 3 { + t.Errorf("expected 3, got %d", got) + } + if got := rb.GetIndex(big); got != 2 { + t.Errorf("expected 2, got %d", got) + } + if got := rb.GetIndex(big + 1); got != -1 { + t.Errorf("expected -1, got %d", got) + } + if got, err := rb.Select(2); err != nil || got != big { + t.Errorf("expected %d, got %d (%v)", big, got, err) + } + if _, err := rb.Select(4); err != ErrNoSuchElement { + t.Errorf("expected ErrNoSuchElement, got %v", err) + } +} + +func TestSetOperations64(t *testing.T) { + rb1 := New64(1, 2, big) + rb2 := New64(2, big, big+1) + + if got, want := And64(rb1, rb2).ToArray(), []uint64{2, big}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if got, want := Or64(rb1, rb2).ToArray(), []uint64{1, 2, big, big + 1}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if got, want := Xor64(rb1, rb2).ToArray(), []uint64{1, big + 1}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if got, want := AndNot64(rb1, rb2).ToArray(), []uint64{1}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + + if got, want := rb1.AndCardinality(rb2), uint64(2); got != want { + t.Errorf("expected %d, got %d", want, got) + } + if got, want := rb1.OrCardinality(rb2), uint64(4); got != want { + t.Errorf("expected %d, got %d", want, got) + } + if got, want := rb1.XorCardinality(rb2), uint64(2); got != want { + t.Errorf("expected %d, got %d", want, got) + } + if got, want := rb1.AndNotCardinality(rb2), uint64(1); got != want { + t.Errorf("expected %d, got %d", want, got) + } + if got, want := rb1.JaccardIndex(rb2), 0.5; got != want { + t.Errorf("expected %v, got %v", want, got) + } + if !rb1.Intersect(rb2) { + t.Error("expected an intersection") + } + + inplace := rb1.Clone() + inplace.Or(rb2) + if got, want := inplace.ToArray(), []uint64{1, 2, big, big + 1}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + inplace.And(rb1) + if !inplace.Equals(rb1) { + t.Error("expected the intersection to be rb1") + } + inplace.Xor(rb1) + if !inplace.IsEmpty() { + t.Error("expected an empty bitmap") + } + inplace.Or(rb1) + inplace.AndNot(rb1) + if !inplace.IsEmpty() { + t.Error("expected an empty bitmap") + } +} + +func TestSubsets64(t *testing.T) { + rb := New64(1, 2, big) + sub := New64(1, big) + if !sub.IsSubset(rb) || !sub.IsStrictSubset(rb) { + t.Error("expected sub to be a strict subset of rb") + } + if !rb.IsSubset(rb) || rb.IsStrictSubset(rb) { + t.Error("a bitmap is a subset of itself, but not a strict one") + } + if rb.Equals("not a bitmap") { + t.Error("expected a bitmap not to equal a string") + } +} + +func TestFlip64(t *testing.T) { + rb := New64(big, big+1, big+2) + if got, want := Flip64(rb, big+1, big+4).ToArray(), []uint64{big, big + 3}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if got, want := FlipClosed64(rb, big+1, big+3).ToArray(), []uint64{big, big + 3}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + inplace := rb.Clone() + inplace.Flip(big+1, big+4) + if got, want := inplace.ToArray(), []uint64{big, big + 3}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + inplace = rb.Clone() + inplace.FlipClosed(big+1, big+3) + if got, want := inplace.ToArray(), []uint64{big, big + 3}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } +} + +func TestAddOffset64(t *testing.T) { + rb := New64(big, big+1) + if got, want := rb.AddOffset(10).ToArray(), []uint64{big + 10, big + 11}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + if got, want := rb.AddOffset(-10).ToArray(), []uint64{big - 10, big - 9}; !reflect.DeepEqual(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + // A negative offset larger than any value drops everything. + if got := New64(1, 2).AddOffset(math.MinInt64); !got.IsEmpty() { + t.Errorf("expected an empty bitmap, got %v", got.ToArray()) + } +} + +func TestOptimize64(t *testing.T) { + rb := New64() + rb.AddRange(big, big+100000) + if !rb.RunOptimize() { + t.Error("expected run optimization to change the bitmap") + } + stats := rb.StatsStruct() + if stats.RunContainers == 0 { + t.Error("expected at least one run container") + } + if stats.Cardinality != rb.Cardinality() { + t.Errorf("expected %d, got %d", rb.Cardinality(), stats.Cardinality) + } + if stats.MinValue != big || stats.MaxValue != big+99999 { + t.Errorf("unexpected min/max: %d, %d", stats.MinValue, stats.MaxValue) + } + if m := rb.Stats(); m["cardinality"] != rb.Cardinality() { + t.Errorf("expected %d, got %d", rb.Cardinality(), m["cardinality"]) + } + if !rb.RemoveRunCompression() { + t.Error("expected run compression to be removed") + } + rb.ShrinkToFit() + if err := rb.InternalValidate(); err != nil { + t.Errorf("expected a valid bitmap, got %v", err) + } +} + +func TestAssignClearFree64(t *testing.T) { + rb := New64(1, 2, big) + other := New64(9) + other.Assign(rb) + if !other.Equals(rb) { + t.Error("expected the two bitmaps to be equal") + } + other.Clear() + if !other.IsEmpty() { + t.Error("expected an empty bitmap") + } + other.Free() + other.Free() +} + +func TestString64(t *testing.T) { + if got, want := New64(1, 2, 3).String(), "{1,2,3}"; got != want { + t.Errorf("expected %q, got %q", want, got) + } + if got, want := New64().String(), "{}"; got != want { + t.Errorf("expected %q, got %q", want, got) + } +} + +func TestSerialization64(t *testing.T) { + rb := New64() + rb.AddRange(big, big+100000) + rb.Add(1, 2, 3) + rb.RunOptimize() + + buf := rb.ToBytes() + if got := PortableDeserializeSize64(buf); got != len(buf) { + t.Errorf("expected %d, got %d", len(buf), got) + } + back, err := Read64(buf) + if err != nil { + t.Fatal(err) + } + if !rb.Equals(back) { + t.Error("round trip through the portable format failed") + } + + validated, err := ReadValidated64(buf) + if err != nil { + t.Fatal(err) + } + if !rb.Equals(validated) { + t.Error("round trip through ReadValidated64 failed") + } + if _, err := ReadValidated64([]byte{1, 2, 3, 4, 5, 6, 7, 8}); err == nil { + t.Error("expected garbage to be rejected") + } + + explicit := make([]byte, rb.SerializedSizeInBytes()) + if err := rb.Write(explicit); err != nil { + t.Fatal(err) + } + if err := rb.Write(explicit[:1]); err != ErrNotEnoughSpace { + t.Errorf("expected ErrNotEnoughSpace, got %v", err) + } + if _, err := Read64(nil); err != ErrEmptyBuffer { + t.Errorf("expected ErrEmptyBuffer, got %v", err) + } + if PortableDeserializeSize64(nil) != 0 { + t.Error("expected 0 for an empty buffer") + } +} + +func TestFrozenViews64(t *testing.T) { + rb := New64() + rb.AddRange(big, big+100000) + rb.RunOptimize() + + // The 64-bit frozen format requires a shrunken bitmap. + if err := rb.WriteFrozen(make([]byte, 1024)); err != ErrNotShrunken { + t.Errorf("expected ErrNotShrunken, got %v", err) + } + rb.ShrinkToFit() + + frozen := AlignedBuffer64(rb.FrozenSizeInBytes()) + if err := rb.WriteFrozen(frozen); err != nil { + t.Fatal(err) + } + view, err := ReadFrozenView64(frozen) + if err != nil { + t.Fatal(err) + } + if !rb.Equals(view) { + t.Error("the frozen view differs from the original") + } + if err := rb.WriteFrozen(frozen[:1]); err != ErrNotEnoughSpace { + t.Errorf("expected ErrNotEnoughSpace, got %v", err) + } + + misaligned := make([]byte, len(frozen)+1) + copy(misaligned[1:], frozen) + if _, err := ReadFrozenView64(misaligned[1:]); err == nil { + t.Error("expected a misaligned buffer to be rejected") + } + if _, err := ReadFrozenView64(nil); err != ErrEmptyBuffer { + t.Errorf("expected ErrEmptyBuffer, got %v", err) + } + + pview, err := ReadPortableFrozenView64(rb.ToBytes()) + if err != nil { + t.Fatal(err) + } + if !rb.Equals(pview) { + t.Error("the portable frozen view differs from the original") + } + if _, err := ReadPortableFrozenView64(nil); err != ErrEmptyBuffer { + t.Errorf("expected ErrEmptyBuffer, got %v", err) + } +} + +func TestAlignedBuffer64(t *testing.T) { + if AlignedBuffer64(0) != nil { + t.Error("expected nil for a zero-sized buffer") + } + for size := 1; size < 200; size++ { + b := AlignedBuffer64(size) + if len(b) != size { + t.Fatalf("expected a buffer of size %d, got %d", size, len(b)) + } + if !isAligned64(b) { + t.Fatalf("buffer of size %d is not aligned", size) + } + } +} + +func TestIterate64(t *testing.T) { + rb := New64() + rb.AddRange(big, big+5000) + var collected []uint64 + rb.Iterate(func(x uint64) bool { + collected = append(collected, x) + return true + }) + if !reflect.DeepEqual(collected, rb.ToArray()) { + t.Error("Iterate disagrees with ToArray") + } + + count := 0 + rb.Iterate(func(x uint64) bool { + count++ + return count < 10 + }) + if count != 10 { + t.Errorf("expected the iteration to stop after 10 values, got %d", count) + } +} + +func TestBufferedIterator64CrossesBlocks(t *testing.T) { + rb := New64() + rb.AddRange(big, big+uint64(iterBufferSize)*3+7) + expected := rb.ToArray() + var got []uint64 + it := rb.Iterator() + for it.HasNext() { + got = append(got, it.Next()) + } + if !reflect.DeepEqual(got, expected) { + t.Errorf("expected %d values, got %d", len(expected), len(got)) + } + if it.HasNext() { + t.Error("expected the iterator to be exhausted") + } +} + +func TestIterator64Navigation(t *testing.T) { + rb := New64(1, 2, 3, big, big+1000) + + it := rb.NewIterator() + if !it.HasValue() || it.Value() != 1 { + t.Fatalf("expected to start at 1, got %d", it.Value()) + } + if !it.Next() || it.Value() != 2 { + t.Fatalf("expected 2, got %d", it.Value()) + } + if !it.AdvanceIfNeeded(big) || it.Value() != big { + t.Fatalf("expected %d, got %d", big, it.Value()) + } + if !it.Previous() || it.Value() != 3 { + t.Fatalf("expected 3, got %d", it.Value()) + } + + clone := it.Clone() + if clone.Value() != it.Value() { + t.Error("the clone should point at the same value") + } + clone.Free() + clone.Free() + + it.Reset() + if it.Value() != 1 { + t.Errorf("expected 1 after Reset, got %d", it.Value()) + } + it.ResetToLast() + if it.Value() != big+1000 { + t.Errorf("expected %d after ResetToLast, got %d", big+1000, it.Value()) + } + + rev := rb.ReverseIterator() + var backwards []uint64 + for rev.HasValue() { + backwards = append(backwards, rev.Value()) + rev.Previous() + } + if !reflect.DeepEqual(backwards, []uint64{big + 1000, big, 3, 2, 1}) { + t.Errorf("unexpected reverse iteration: %v", backwards) + } +} + +func TestIterator64Read(t *testing.T) { + rb := New64() + rb.AddRange(big, big+1000) + + it := rb.NewIterator() + buf := make([]uint64, 300) + total := 0 + for { + n := it.Read(buf) + if n == 0 { + break + } + total += n + } + if total != 1000 { + t.Errorf("expected 1000 values, got %d", total) + } + if it.Read(nil) != 0 { + t.Error("expected zero values for an empty buffer") + } + + rev := rb.ReverseIterator() + if n := rev.ReadBackward(buf); n != 300 || buf[0] != big+999 { + t.Errorf("expected to read 300 values starting at %d, got %d starting at %d", big+999, n, buf[0]) + } + if rev.ReadBackward(nil) != 0 { + t.Error("expected zero values for an empty buffer") + } +} + +func TestIterator64ReadRanges(t *testing.T) { + rb := New64() + rb.AddRange(big, big+10) + rb.AddRange(big+100, big+110) + rb.RunOptimize() + + it := rb.NewIterator() + ranges := make([]Range64, 4) + if n := it.ReadRanges(ranges); n != 2 { + t.Fatalf("expected 2 ranges, got %d", n) + } + want := []Range64{{big, big + 9}, {big + 100, big + 109}} + if !reflect.DeepEqual(ranges[:2], want) { + t.Errorf("expected %v, got %v", want, ranges[:2]) + } + if it.ReadRanges(nil) != 0 { + t.Error("expected zero ranges for an empty buffer") + } + + rev := rb.ReverseIterator() + if n := rev.ReadPreviousRanges(ranges); n != 2 { + t.Fatalf("expected 2 ranges, got %d", n) + } + want = []Range64{{big + 100, big + 109}, {big, big + 9}} + if !reflect.DeepEqual(ranges[:2], want) { + t.Errorf("expected %v, got %v", want, ranges[:2]) + } + if rev.ReadPreviousRanges(nil) != 0 { + t.Error("expected zero ranges for an empty buffer") + } +} diff --git a/gocroaring_test.go b/gocroaring_test.go index 320b624..b4478a9 100644 --- a/gocroaring_test.go +++ b/gocroaring_test.go @@ -2,6 +2,7 @@ package gocroaring import ( "fmt" + "math" "math/rand" "os/exec" "reflect" @@ -264,8 +265,9 @@ func TestWriteFrozen(t *testing.T) { rb.Add(j) } - // frozen serialization - buf := make([]byte, rb.FrozenSizeInBytes()) + // frozen serialization: the buffer backing a frozen view must be + // aligned, see AlignedBuffer. + buf := AlignedBuffer(rb.FrozenSizeInBytes()) rb.WriteFrozen(buf) // we omit error handling newrb, err := ReadFrozenView(buf) @@ -283,7 +285,7 @@ func TestWriteFrozen(t *testing.T) { func TestStatsStruct(t *testing.T) { t.Run("Test Stats with empty bitmap", func(t *testing.T) { - expectedStats := Statistics{} + expectedStats := Statistics{MinValue: math.MaxUint32} rr := New() if !reflect.DeepEqual(expectedStats, rr.StatsStruct()) { t.Errorf("expected %#v, got %#v", expectedStats, rr.StatsStruct()) @@ -299,6 +301,9 @@ func TestStatsStruct(t *testing.T) { BitmapContainers: 1, BitmapContainerValues: 60000, BitmapContainerBytes: 8192, + + MinValue: 0, + MaxValue: 59999, } rr := New() for i := uint32(0); i < 60000; i++ { @@ -318,6 +323,9 @@ func TestStatsStruct(t *testing.T) { ArrayContainers: 1, ArrayContainerValues: 2, ArrayContainerBytes: 4, + + MinValue: 2, + MaxValue: 4, } rr := New() rr.Add(2) diff --git a/iterator.go b/iterator.go new file mode 100644 index 0000000..e5b79ea --- /dev/null +++ b/iterator.go @@ -0,0 +1,316 @@ +package gocroaring + +/* +#cgo CFLAGS: -O3 -std=c11 + +// None of the CRoaring entry points below calls back into Go, and none of them +// retains a pointer to the memory it is handed. Saying so lets cgo use the +// cheaper calling convention and keeps the Go buffers we pass from escaping to +// the heap. +// +// The frozen views are the exception: they keep the buffer they are given, so +// they are deliberately absent from the noescape list. +#cgo noescape roaring_iterator_create +#cgo noescape roaring_iterator_init +#cgo noescape roaring_iterator_init_last +#cgo noescape roaring_uint32_iterator_advance +#cgo noescape roaring_uint32_iterator_copy +#cgo noescape roaring_uint32_iterator_free +#cgo noescape roaring_uint32_iterator_move_equalorlarger +#cgo noescape roaring_uint32_iterator_previous +#cgo noescape roaring_uint32_iterator_read +#cgo noescape roaring_uint32_iterator_read_backward +#cgo noescape roaring_uint32_iterator_read_prev_ranges +#cgo noescape roaring_uint32_iterator_read_ranges +#cgo noescape roaring_uint32_iterator_skip +#cgo noescape roaring_uint32_iterator_skip_backward + +#cgo nocallback roaring_iterator_create +#cgo nocallback roaring_iterator_init +#cgo nocallback roaring_iterator_init_last +#cgo nocallback roaring_uint32_iterator_advance +#cgo nocallback roaring_uint32_iterator_copy +#cgo nocallback roaring_uint32_iterator_free +#cgo nocallback roaring_uint32_iterator_move_equalorlarger +#cgo nocallback roaring_uint32_iterator_previous +#cgo nocallback roaring_uint32_iterator_read +#cgo nocallback roaring_uint32_iterator_read_backward +#cgo nocallback roaring_uint32_iterator_read_prev_ranges +#cgo nocallback roaring_uint32_iterator_read_ranges +#cgo nocallback roaring_uint32_iterator_skip +#cgo nocallback roaring_uint32_iterator_skip_backward +#include "roaring.h" +*/ +import "C" +import ( + "runtime" + "unsafe" +) + +// iterBufferSize is how many values a buffered iterator pulls across the +// Go/C boundary at a time. Crossing that boundary is what makes iteration +// expensive, so we amortize it over a whole block of values. +const iterBufferSize = 512 + +// IntIterable allows you to iterate over the values in a Bitmap. +type IntIterable interface { + HasNext() bool + Next() uint32 +} + +// intIterator is a forward-only iterator that reads values in blocks. +type intIterator struct { + it *C.roaring_uint32_iterator_t + cleanup runtime.Cleanup + parent *Bitmap + buf [iterBufferSize]uint32 + pos int + n int +} + +// Iterator creates a new IntIterable to iterate over the integers contained in +// the bitmap, in sorted order. +// This function may panic if the allocation failed. +func (rb *Bitmap) Iterator() IntIterable { + return newIntIterator(rb) +} + +func newIntIterator(rb *Bitmap) *intIterator { + p := C.roaring_iterator_create(rb.cpointer) + runtime.KeepAlive(rb) + if p == nil { + panic("C code returned a null pointer.") + } + ii := &intIterator{it: p, parent: rb} + ii.cleanup = runtime.AddCleanup(ii, func(p *C.roaring_uint32_iterator_t) { + C.roaring_uint32_iterator_free(p) + }, p) + ii.fill() + return ii +} + +func (ii *intIterator) fill() { + ii.n = int(C.roaring_uint32_iterator_read(ii.it, + (*C.uint32_t)(unsafe.Pointer(&ii.buf[0])), C.uint32_t(len(ii.buf)))) + ii.pos = 0 + runtime.KeepAlive(ii) +} + +// HasNext returns true if there are more integers to iterate over. +func (ii *intIterator) HasNext() bool { + return ii.pos < ii.n +} + +// Next returns the next integer. It must not be called when HasNext is false. +func (ii *intIterator) Next() uint32 { + answer := ii.buf[ii.pos] + ii.pos++ + if ii.pos == ii.n { + ii.fill() + } + return answer +} + +// Iterate calls cb with every integer in the bitmap, in sorted order, stopping +// early if cb returns false. +func (rb *Bitmap) Iterate(cb func(x uint32) bool) { + it := newIntIterator(rb) + for it.HasNext() { + if !cb(it.Next()) { + return + } + } +} + +// Iterator is a full-featured iterator over the values of a Bitmap. Unlike the +// iterator returned by Bitmap.Iterator, it can move backwards and can seek. +// +// An Iterator points at a value or is exhausted; use HasValue to tell the two +// apart. It is invalidated by any modification of the underlying bitmap. +type Iterator struct { + it *C.roaring_uint32_iterator_t + cleanup runtime.Cleanup + parent *Bitmap +} + +func newIterator(rb *Bitmap, p *C.roaring_uint32_iterator_t) *Iterator { + if p == nil { + panic("C code returned a null pointer.") + } + i := &Iterator{it: p, parent: rb} + i.cleanup = runtime.AddCleanup(i, func(p *C.roaring_uint32_iterator_t) { + C.roaring_uint32_iterator_free(p) + }, p) + return i +} + +// NewIterator returns an iterator positioned on the smallest value of the +// bitmap. +// This function may panic if the allocation failed. +func (rb *Bitmap) NewIterator() *Iterator { + p := C.roaring_iterator_create(rb.cpointer) + runtime.KeepAlive(rb) + return newIterator(rb, p) +} + +// ReverseIterator returns an iterator positioned on the largest value of the +// bitmap, meant to be walked with Previous. +// This function may panic if the allocation failed. +func (rb *Bitmap) ReverseIterator() *Iterator { + p := C.roaring_iterator_create(rb.cpointer) + runtime.KeepAlive(rb) + i := newIterator(rb, p) + C.roaring_iterator_init_last(rb.cpointer, p) + runtime.KeepAlive(rb) + runtime.KeepAlive(i) + return i +} + +// Free releases the memory held by the iterator. Using the iterator afterwards +// is a mistake. Calling Free more than once is harmless. +func (i *Iterator) Free() { + if i.it == nil { + return + } + i.cleanup.Stop() + C.roaring_uint32_iterator_free(i.it) + i.it = nil + i.parent = nil +} + +// HasValue reports whether the iterator points at a value. +func (i *Iterator) HasValue() bool { + answer := bool(i.it.has_value) + runtime.KeepAlive(i) + return answer +} + +// Value returns the value the iterator points at. It is meaningless when +// HasValue is false. +func (i *Iterator) Value() uint32 { + answer := uint32(i.it.current_value) + runtime.KeepAlive(i) + return answer +} + +// Next moves the iterator to the next value and reports whether it points at +// one. +func (i *Iterator) Next() bool { + answer := bool(C.roaring_uint32_iterator_advance(i.it)) + runtime.KeepAlive(i) + return answer +} + +// Previous moves the iterator to the previous value and reports whether it +// points at one. +func (i *Iterator) Previous() bool { + answer := bool(C.roaring_uint32_iterator_previous(i.it)) + runtime.KeepAlive(i) + return answer +} + +// AdvanceIfNeeded moves the iterator to the smallest value that is greater +// than or equal to x, and reports whether it points at a value. The iterator +// does not move if it already points at such a value. +func (i *Iterator) AdvanceIfNeeded(x uint32) bool { + answer := bool(C.roaring_uint32_iterator_move_equalorlarger(i.it, C.uint32_t(x))) + runtime.KeepAlive(i) + return answer +} + +// Skip advances the iterator by count values and returns how many values were +// actually skipped. +func (i *Iterator) Skip(count uint32) uint32 { + answer := uint32(C.roaring_uint32_iterator_skip(i.it, C.uint32_t(count))) + runtime.KeepAlive(i) + return answer +} + +// SkipBackward moves the iterator back by count values and returns how many +// values were actually skipped. +func (i *Iterator) SkipBackward(count uint32) uint32 { + answer := uint32(C.roaring_uint32_iterator_skip_backward(i.it, C.uint32_t(count))) + runtime.KeepAlive(i) + return answer +} + +// Read fills buf with up to len(buf) values, in ascending order, and returns +// how many were written. A return value smaller than len(buf) means the +// iterator is exhausted. +func (i *Iterator) Read(buf []uint32) int { + if len(buf) == 0 { + return 0 + } + n := int(C.roaring_uint32_iterator_read(i.it, + (*C.uint32_t)(unsafe.Pointer(&buf[0])), C.uint32_t(len(buf)))) + runtime.KeepAlive(buf) + runtime.KeepAlive(i) + return n +} + +// ReadBackward fills buf with up to len(buf) values, in descending order, and +// returns how many were written. +func (i *Iterator) ReadBackward(buf []uint32) int { + if len(buf) == 0 { + return 0 + } + n := int(C.roaring_uint32_iterator_read_backward(i.it, + (*C.uint32_t)(unsafe.Pointer(&buf[0])), C.uint32_t(len(buf)))) + runtime.KeepAlive(buf) + runtime.KeepAlive(i) + return n +} + +// Range is a closed interval [Min, Max] of 32-bit values. +type Range struct { + Min uint32 + Max uint32 +} + +// ReadRanges fills buf with up to len(buf) runs of consecutive values, in +// ascending order, and returns how many were written. +func (i *Iterator) ReadRanges(buf []Range) int { + if len(buf) == 0 { + return 0 + } + n := int(C.roaring_uint32_iterator_read_ranges(i.it, + (*C.roaring_uint32_range_closed_t)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)))) + runtime.KeepAlive(buf) + runtime.KeepAlive(i) + return n +} + +// ReadPreviousRanges fills buf with up to len(buf) runs of consecutive values, +// in descending order, and returns how many were written. +func (i *Iterator) ReadPreviousRanges(buf []Range) int { + if len(buf) == 0 { + return 0 + } + n := int(C.roaring_uint32_iterator_read_prev_ranges(i.it, + (*C.roaring_uint32_range_closed_t)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)))) + runtime.KeepAlive(buf) + runtime.KeepAlive(i) + return n +} + +// Clone returns a copy of the iterator, positioned on the same value. +// This function may panic if the allocation failed. +func (i *Iterator) Clone() *Iterator { + p := C.roaring_uint32_iterator_copy(i.it) + runtime.KeepAlive(i) + return newIterator(i.parent, p) +} + +// Reset repositions the iterator on the smallest value of the bitmap it was +// created from. +func (i *Iterator) Reset() { + C.roaring_iterator_init(i.parent.cpointer, i.it) + runtime.KeepAlive(i) +} + +// ResetToLast repositions the iterator on the largest value of the bitmap it +// was created from. +func (i *Iterator) ResetToLast() { + C.roaring_iterator_init_last(i.parent.cpointer, i.it) + runtime.KeepAlive(i) +} diff --git a/iterator64.go b/iterator64.go new file mode 100644 index 0000000..9b4dc1b --- /dev/null +++ b/iterator64.go @@ -0,0 +1,295 @@ +package gocroaring + +/* +#cgo CFLAGS: -O3 -std=c11 + +// None of the CRoaring entry points below calls back into Go, and none of them +// retains a pointer to the memory it is handed. Saying so lets cgo use the +// cheaper calling convention and keeps the Go buffers we pass from escaping to +// the heap. +// +// The frozen views are the exception: they keep the buffer they are given, so +// they are deliberately absent from the noescape list. +#cgo noescape roaring64_iterator_advance +#cgo noescape roaring64_iterator_copy +#cgo noescape roaring64_iterator_create +#cgo noescape roaring64_iterator_create_last +#cgo noescape roaring64_iterator_free +#cgo noescape roaring64_iterator_has_value +#cgo noescape roaring64_iterator_move_equalorlarger +#cgo noescape roaring64_iterator_previous +#cgo noescape roaring64_iterator_read +#cgo noescape roaring64_iterator_read_backward +#cgo noescape roaring64_iterator_read_prev_ranges +#cgo noescape roaring64_iterator_read_ranges +#cgo noescape roaring64_iterator_reinit +#cgo noescape roaring64_iterator_reinit_last +#cgo noescape roaring64_iterator_value + +#cgo nocallback roaring64_iterator_advance +#cgo nocallback roaring64_iterator_copy +#cgo nocallback roaring64_iterator_create +#cgo nocallback roaring64_iterator_create_last +#cgo nocallback roaring64_iterator_free +#cgo nocallback roaring64_iterator_has_value +#cgo nocallback roaring64_iterator_move_equalorlarger +#cgo nocallback roaring64_iterator_previous +#cgo nocallback roaring64_iterator_read +#cgo nocallback roaring64_iterator_read_backward +#cgo nocallback roaring64_iterator_read_prev_ranges +#cgo nocallback roaring64_iterator_read_ranges +#cgo nocallback roaring64_iterator_reinit +#cgo nocallback roaring64_iterator_reinit_last +#cgo nocallback roaring64_iterator_value +#include "roaring.h" +*/ +import "C" +import ( + "runtime" + "unsafe" +) + +// IntIterable64 allows you to iterate over the values in a Bitmap64. +type IntIterable64 interface { + HasNext() bool + Next() uint64 +} + +// intIterator64 is a forward-only iterator that reads values in blocks, so +// that the cost of crossing the Go/C boundary is amortized over a whole block. +type intIterator64 struct { + it *C.roaring64_iterator_t + cleanup runtime.Cleanup + parent *Bitmap64 + buf [iterBufferSize]uint64 + pos int + n int +} + +// Iterator creates a new IntIterable64 to iterate over the integers contained +// in the bitmap, in sorted order. +// This function may panic if the allocation failed. +func (rb *Bitmap64) Iterator() IntIterable64 { + return newIntIterator64(rb) +} + +func newIntIterator64(rb *Bitmap64) *intIterator64 { + p := C.roaring64_iterator_create(rb.cpointer) + runtime.KeepAlive(rb) + if p == nil { + panic("C code returned a null pointer.") + } + ii := &intIterator64{it: p, parent: rb} + ii.cleanup = runtime.AddCleanup(ii, func(p *C.roaring64_iterator_t) { + C.roaring64_iterator_free(p) + }, p) + ii.fill() + return ii +} + +func (ii *intIterator64) fill() { + ii.n = int(C.roaring64_iterator_read(ii.it, + (*C.uint64_t)(unsafe.Pointer(&ii.buf[0])), C.uint64_t(len(ii.buf)))) + ii.pos = 0 + runtime.KeepAlive(ii) +} + +// HasNext returns true if there are more integers to iterate over. +func (ii *intIterator64) HasNext() bool { + return ii.pos < ii.n +} + +// Next returns the next integer. It must not be called when HasNext is false. +func (ii *intIterator64) Next() uint64 { + answer := ii.buf[ii.pos] + ii.pos++ + if ii.pos == ii.n { + ii.fill() + } + return answer +} + +// Iterate calls cb with every integer in the bitmap, in sorted order, stopping +// early if cb returns false. +func (rb *Bitmap64) Iterate(cb func(x uint64) bool) { + it := newIntIterator64(rb) + for it.HasNext() { + if !cb(it.Next()) { + return + } + } +} + +// Iterator64 is a full-featured iterator over the values of a Bitmap64. Unlike +// the iterator returned by Bitmap64.Iterator, it can move backwards and can +// seek. +// +// An Iterator64 points at a value or is exhausted; use HasValue to tell the +// two apart. It is invalidated by any modification of the underlying bitmap. +type Iterator64 struct { + it *C.roaring64_iterator_t + cleanup runtime.Cleanup + parent *Bitmap64 +} + +func newIterator64(rb *Bitmap64, p *C.roaring64_iterator_t) *Iterator64 { + if p == nil { + panic("C code returned a null pointer.") + } + i := &Iterator64{it: p, parent: rb} + i.cleanup = runtime.AddCleanup(i, func(p *C.roaring64_iterator_t) { + C.roaring64_iterator_free(p) + }, p) + return i +} + +// NewIterator returns an iterator positioned on the smallest value of the +// bitmap. +// This function may panic if the allocation failed. +func (rb *Bitmap64) NewIterator() *Iterator64 { + p := C.roaring64_iterator_create(rb.cpointer) + runtime.KeepAlive(rb) + return newIterator64(rb, p) +} + +// ReverseIterator returns an iterator positioned on the largest value of the +// bitmap, meant to be walked with Previous. +// This function may panic if the allocation failed. +func (rb *Bitmap64) ReverseIterator() *Iterator64 { + p := C.roaring64_iterator_create_last(rb.cpointer) + runtime.KeepAlive(rb) + return newIterator64(rb, p) +} + +// Free releases the memory held by the iterator. Using the iterator afterwards +// is a mistake. Calling Free more than once is harmless. +func (i *Iterator64) Free() { + if i.it == nil { + return + } + i.cleanup.Stop() + C.roaring64_iterator_free(i.it) + i.it = nil + i.parent = nil +} + +// HasValue reports whether the iterator points at a value. +func (i *Iterator64) HasValue() bool { + answer := bool(C.roaring64_iterator_has_value(i.it)) + runtime.KeepAlive(i) + return answer +} + +// Value returns the value the iterator points at. It is meaningless when +// HasValue is false. +func (i *Iterator64) Value() uint64 { + answer := uint64(C.roaring64_iterator_value(i.it)) + runtime.KeepAlive(i) + return answer +} + +// Next moves the iterator to the next value and reports whether it points at +// one. +func (i *Iterator64) Next() bool { + answer := bool(C.roaring64_iterator_advance(i.it)) + runtime.KeepAlive(i) + return answer +} + +// Previous moves the iterator to the previous value and reports whether it +// points at one. +func (i *Iterator64) Previous() bool { + answer := bool(C.roaring64_iterator_previous(i.it)) + runtime.KeepAlive(i) + return answer +} + +// AdvanceIfNeeded moves the iterator to the smallest value that is greater +// than or equal to x, and reports whether it points at a value. The iterator +// does not move if it already points at such a value. +func (i *Iterator64) AdvanceIfNeeded(x uint64) bool { + answer := bool(C.roaring64_iterator_move_equalorlarger(i.it, C.uint64_t(x))) + runtime.KeepAlive(i) + return answer +} + +// Read fills buf with up to len(buf) values, in ascending order, and returns +// how many were written. A return value smaller than len(buf) means the +// iterator is exhausted. +func (i *Iterator64) Read(buf []uint64) int { + if len(buf) == 0 { + return 0 + } + n := int(C.roaring64_iterator_read(i.it, + (*C.uint64_t)(unsafe.Pointer(&buf[0])), C.uint64_t(len(buf)))) + runtime.KeepAlive(buf) + runtime.KeepAlive(i) + return n +} + +// ReadBackward fills buf with up to len(buf) values, in descending order, and +// returns how many were written. +func (i *Iterator64) ReadBackward(buf []uint64) int { + if len(buf) == 0 { + return 0 + } + n := int(C.roaring64_iterator_read_backward(i.it, + (*C.uint64_t)(unsafe.Pointer(&buf[0])), C.uint64_t(len(buf)))) + runtime.KeepAlive(buf) + runtime.KeepAlive(i) + return n +} + +// Range64 is a closed interval [Min, Max] of 64-bit values. +type Range64 struct { + Min uint64 + Max uint64 +} + +// ReadRanges fills buf with up to len(buf) runs of consecutive values, in +// ascending order, and returns how many were written. +func (i *Iterator64) ReadRanges(buf []Range64) int { + if len(buf) == 0 { + return 0 + } + n := int(C.roaring64_iterator_read_ranges(i.it, + (*C.roaring64_range_closed_t)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)))) + runtime.KeepAlive(buf) + runtime.KeepAlive(i) + return n +} + +// ReadPreviousRanges fills buf with up to len(buf) runs of consecutive values, +// in descending order, and returns how many were written. +func (i *Iterator64) ReadPreviousRanges(buf []Range64) int { + if len(buf) == 0 { + return 0 + } + n := int(C.roaring64_iterator_read_prev_ranges(i.it, + (*C.roaring64_range_closed_t)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)))) + runtime.KeepAlive(buf) + runtime.KeepAlive(i) + return n +} + +// Clone returns a copy of the iterator, positioned on the same value. +// This function may panic if the allocation failed. +func (i *Iterator64) Clone() *Iterator64 { + p := C.roaring64_iterator_copy(i.it) + runtime.KeepAlive(i) + return newIterator64(i.parent, p) +} + +// Reset repositions the iterator on the smallest value of the bitmap it was +// created from. +func (i *Iterator64) Reset() { + C.roaring64_iterator_reinit(i.parent.cpointer, i.it) + runtime.KeepAlive(i) +} + +// ResetToLast repositions the iterator on the largest value of the bitmap it +// was created from. +func (i *Iterator64) ResetToLast() { + C.roaring64_iterator_reinit_last(i.parent.cpointer, i.it) + runtime.KeepAlive(i) +} diff --git a/roaring.c b/roaring.c index 56e04f2..f753a21 100644 --- a/roaring.c +++ b/roaring.c @@ -1,5 +1,5 @@ // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! -// Created by amalgamation.sh on 2025-12-05T04:42:38Z +// Created by amalgamation.sh on 2026-08-22T02:17:56Z /* * The CRoaring project is under a dual license (Apache/MIT). @@ -63,6 +63,17 @@ #include "roaring.h" /* include public API definitions */ /* begin file include/roaring/containers/perfparameters.h */ +/* + * perfparameters.h + * + * This header centralizes a small set of performance-tuning constants and + * heuristic defaults used by CRoaring container code. These parameters control + * decisions such as initial container sizing and when lazy or eager operations + * may convert between container representations. + * + * In practice, these values encode trade-offs between memory use, allocation + * overhead, and execution speed for common workloads. + */ #ifndef PERFPARAMETERS_H_ #define PERFPARAMETERS_H_ @@ -117,6 +128,15 @@ enum { ARRAY_DEFAULT_INIT_SIZE = 0 }; /* * utilasm.h * + * This file provides optional inline-assembly helpers for low-level bit + * manipulation on supported x86/x64 targets. These macros are used to map a + * few performance-sensitive operations, such as shifting, testing, setting, + * and clearing bits, to specific machine instructions when inline assembly is + * enabled. + * + * The intent is to centralize these architecture-specific primitives behind a + * small interface so the rest of the codebase can use them conditionally while + * keeping the generic implementation paths separate. */ #ifndef INCLUDE_UTILASM_H_ @@ -819,9 +839,8 @@ static const uint8_t shuffle_mask16[] = { * Optimized by D. Lemire on May 3rd 2013 */ CROARING_TARGET_AVX2 -int32_t intersect_vector16(const uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b, - uint16_t *C) { +int32_t intersect_vector16(const uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b, uint16_t *C) { size_t count = 0; size_t i_a = 0, i_b = 0; const int vectorlength = sizeof(__m128i) / sizeof(uint16_t); @@ -920,8 +939,8 @@ int array_container_to_uint32_array_vector16(void *vout, const uint16_t *array, return outpos; } -int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b) { +int32_t intersect_vector16_inplace(uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b) { size_t count = 0; size_t i_a = 0, i_b = 0; const int vectorlength = sizeof(__m128i) / sizeof(uint16_t); @@ -1015,10 +1034,8 @@ int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, CROARING_UNTARGET_AVX2 CROARING_TARGET_AVX2 -int32_t intersect_vector16_cardinality(const uint16_t *__restrict__ A, - size_t s_a, - const uint16_t *__restrict__ B, - size_t s_b) { +int32_t intersect_vector16_cardinality(const uint16_t *A, size_t s_a, + const uint16_t *B, size_t s_b) { size_t count = 0; size_t i_a = 0, i_b = 0; const int vectorlength = sizeof(__m128i) / sizeof(uint16_t); @@ -1091,9 +1108,8 @@ CROARING_TARGET_AVX2 // Warning: // This function may not be safe if A == C or B == C. ///////// -int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b, - uint16_t *C) { +int32_t difference_vector16(const uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b, uint16_t *C) { // we handle the degenerate case if (s_a == 0) return 0; if (s_b == 0) { @@ -2082,17 +2098,27 @@ static inline uint32_t unique(uint16_t *out, uint32_t len) { return pos; } -// use with qsort, could be avoided -static int uint16_compare(const void *a, const void *b) { - return (*(uint16_t *)a - *(uint16_t *)b); +// Sort a very short run of uint16 values in place. The callers below feed this +// at most 16 values; calling qsort() for that costs more in glibc merge-sort +// setup, function-pointer compares and memmove traffic than the sort itself. +static inline void sort_uint16_short(uint16_t *z, uint32_t n) { + for (uint32_t i = 1; i < n; i++) { + uint16_t v = z[i]; + uint32_t j = i; + while (j > 0 && z[j - 1] > v) { + z[j] = z[j - 1]; + j--; + } + z[j] = v; + } } CROARING_TARGET_AVX2 // a one-pass SSE union algorithm // This function may not be safe if array1 == output or array2 == output. -uint32_t union_vector16(const uint16_t *__restrict__ array1, uint32_t length1, - const uint16_t *__restrict__ array2, uint32_t length2, - uint16_t *__restrict__ output) { +uint32_t union_vector16(const uint16_t *array1, uint32_t length1, + const uint16_t *array2, uint32_t length2, + uint16_t *output) { if ((length1 < 8) || (length2 < 8)) { return (uint32_t)union_uint16(array1, length1, array2, length2, output); } @@ -2153,7 +2179,7 @@ uint32_t union_vector16(const uint16_t *__restrict__ array1, uint32_t length1, memcpy(buffer + leftoversize, array1 + 8 * pos1, (length1 - 8 * len1) * sizeof(uint16_t)); leftoversize += length1 - 8 * len1; - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique(buffer, leftoversize); len += (uint32_t)union_uint16(buffer, leftoversize, array2 + 8 * pos2, @@ -2162,7 +2188,7 @@ uint32_t union_vector16(const uint16_t *__restrict__ array1, uint32_t length1, memcpy(buffer + leftoversize, array2 + 8 * pos2, (length2 - 8 * len2) * sizeof(uint16_t)); leftoversize += length2 - 8 * len2; - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique(buffer, leftoversize); len += (uint32_t)union_uint16(buffer, leftoversize, array1 + 8 * pos1, length1 - 8 * pos1, output); @@ -2176,6 +2202,193 @@ CROARING_UNTARGET_AVX2 * */ +/** + * Start of the AVX-512 16-bit union code. + * + * union_vector16 above merges 8 lanes at a time with an odd-even transposition + * network: 8 dependent min/max stages per 8 output values, which measures at + * ~2.3 cycles per input element -- no better than the scalar union_uint16. + * + * With AVX-512 we can merge 32+32 lanes with a Batcher bitonic network: one + * reverse plus 5 compare-exchange stages per sorted half, i.e. ~4x fewer + * operations per output value. This needs cross-lane 16-bit permutes (vpermw, + * vpermt2w) and a 16-bit compress store (vpcompressw); AVX2 has none of these, + * so the algorithm is genuinely AVX-512-only rather than a wider rerun of the + * SSE code. + */ +#if CROARING_COMPILER_SUPPORTS_AVX512 + +CROARING_TARGET_AVX512 + +// A compare-exchange at distance d pairs lane i with lane i^d and keeps the +// smaller value in whichever lane has its d-bit clear. `hi` selects the lanes +// whose d-bit is set. +static inline __m512i avx512_cx16(__m512i v, __m512i t, __mmask32 hi) { + return _mm512_mask_mov_epi16(_mm512_min_epu16(v, t), hi, + _mm512_max_epu16(v, t)); +} + +// Sort a bitonic 32-lane sequence into ascending order. Each distance has a +// dedicated cheap shuffle, so no stage needs a full vpermw. +static inline __m512i avx512_bitonic_sort32(__m512i v) { + v = avx512_cx16(v, _mm512_shuffle_i64x2(v, v, 0x4E), 0xFFFF0000u); // d=16 + v = avx512_cx16(v, _mm512_shuffle_i64x2(v, v, 0xB1), 0xFF00FF00u); // d=8 + v = avx512_cx16(v, _mm512_shuffle_epi32(v, 0x4E), 0xF0F0F0F0u); // d=4 + v = avx512_cx16(v, _mm512_shuffle_epi32(v, 0xB1), 0xCCCCCCCCu); // d=2 + v = avx512_cx16(v, _mm512_rol_epi32(v, 16), 0xAAAAAAAAu); // d=1 + return v; +} + +// Merge two ascending 32-lane vectors: *lo receives the 32 smallest values in +// ascending order, *hi the 32 largest, also ascending. +static inline void avx512_bitonic_merge32(__m512i a, __m512i b, __m512i *lo, + __m512i *hi) { + static const uint16_t revtab[32] = { + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + __m512i br = _mm512_permutexvar_epi16( + _mm512_loadu_si512((const __m512i *)revtab), b); + *lo = avx512_bitonic_sort32(_mm512_min_epu16(a, br)); + *hi = avx512_bitonic_sort32(_mm512_max_epu16(a, br)); +} + +// Write the values of the ascending vector `v` that differ from their +// predecessor, where the predecessor of lane 0 is *last. Updates *last to the +// largest value emitted and returns how many values were written. +static inline int avx512_emit_unique16(__m512i v, uint16_t *out, + uint16_t *last) { + static const uint16_t shift1[32] = { + 32, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; + __m512i prev = _mm512_permutex2var_epi16( + v, _mm512_loadu_si512((const __m512i *)shift1), + _mm512_set1_epi16((short)*last)); + __mmask32 keep = _mm512_cmpneq_epi16_mask(v, prev); + _mm512_mask_compressstoreu_epi16(out, keep, v); + *last = (uint16_t)_mm_extract_epi16(_mm512_extracti32x4_epi32(v, 3), 7); + return (int)roaring_hamming(keep); +} + +/** + * A one-pass AVX-512 union of two sorted uint16 arrays. + * + * As with union_vector16, the caller must guarantee that `output` has room for + * size_1 + size_2 values. Partial overlap of `output` with the inputs is + * tolerated in exactly the same way union_vector16 tolerates it (see the long + * comment in array_run_container_inplace_union): after consuming p1 + p2 whole + * 32-lane blocks this routine has written at most 32*(p1+p2) - 32 values, + * because one merged block is always held back in `vmax`. The output pointer + * therefore stays strictly behind the read pointers. + * + * That bound is exactly tight rather than merely satisfied: in the worst case + * the final merge below writes its last value to precisely the slot the last + * input value was read from. Anything that reduces how much the tail holds back + * -- shrinking `pending`, emitting `vmax` sooner -- silently breaks aliased + * callers, so re-derive the bound before changing the buffering here. + */ +uint32_t avx512_union_uint16(const uint16_t *array1, uint32_t length1, + const uint16_t *array2, uint32_t length2, + uint16_t *output) { + const uint32_t W = 32; // lanes per 512-bit register + if (length1 < W || length2 < W) { + // Not enough on one side to fill a block. union_vector16 keeps + // vectorizing at 8-lane granularity, so it still beats the scalar + // merge once there is a worthwhile amount of data on the other side; + // below that its own fixed overhead dominates. + if (length1 + length2 >= 64) { + return union_vector16(array1, length1, array2, length2, output); + } + return (uint32_t)union_uint16(array1, length1, array2, length2, output); + } + const uint32_t blocks1 = length1 / W, blocks2 = length2 / W; + uint32_t p1 = 0, p2 = 0; + uint16_t *out = output; + __m512i vmin, vmax; + + avx512_bitonic_merge32(_mm512_loadu_si512((const __m512i *)array1), + _mm512_loadu_si512((const __m512i *)array2), &vmin, + &vmax); + p1 = 1; + p2 = 1; + // Lane 0 of the very first block has no predecessor. Seeding `last` with a + // value that differs from it in one bit keeps the first value. + uint16_t last = + (uint16_t)(_mm_extract_epi16(_mm512_castsi512_si128(vmin), 0) ^ 1); + out += avx512_emit_unique16(vmin, out, &last); + + while (p1 < blocks1 && p2 < blocks2) { + // Which side advances is essentially unpredictable, so select the + // source pointer without branching. + const uint16_t *pa = array1 + W * p1; + const uint16_t *pb = array2 + W * p2; + const uint32_t take1 = (*pa <= *pb) ? 1 : 0; + const __m512i v = + _mm512_loadu_si512((const __m512i *)(take1 ? pa : pb)); + p1 += take1; + p2 += 1 - take1; + avx512_bitonic_merge32(v, vmax, &vmin, &vmax); + out += avx512_emit_unique16(vmin, out, &last); + } + + // The ragged tail: the values still held in vmax, plus at most W-1 leftover + // values from the exhausted side, plus the rest of the other side. Two + // two-way merges branch-predict far better than one three-way merge. + // + // vmax can hold the same value twice (once from each input), so it goes + // through avx512_emit_unique16 rather than a raw store: that dedups within + // the block and against `last` at the same time. Every remaining tail value + // is >= last and the tail is sorted, so for the two scalar inputs only the + // very first value can still duplicate `last`. + uint16_t pending[32]; + uint16_t pending_last = last; + size_t npending = + (size_t)avx512_emit_unique16(vmax, pending, &pending_last); + + const uint16_t *rest1 = array1 + W * p1, *rest2 = array2 + W * p2; + size_t nrest1 = length1 - W * p1, nrest2 = length2 - W * p2; + const uint16_t *shortrest, *longrest; + size_t nshort, nlong; + if (p1 == blocks1) { + shortrest = rest1; + nshort = nrest1; + longrest = rest2; + nlong = nrest2; + } else { + shortrest = rest2; + nshort = nrest2; + longrest = rest1; + nlong = nrest1; + } + if (nshort > 0 && shortrest[0] == last) { + shortrest++; + nshort--; + } + if (nlong > 0 && longrest[0] == last) { + longrest++; + nlong--; + } + + uint16_t merged[2 * 32]; // <= W pending + (W-1) leftover + size_t nmerged = union_uint16(pending, npending, shortrest, nshort, merged); + // When the small side held fewer than two blocks the loop above ran at most + // once, so `longrest` can still be most of the larger input. Finishing that + // scalar would give away more than the block loop won, hence the same + // 8-lane handoff as in the short-input case. + if (nlong >= 64) { + out += union_vector16(merged, (uint32_t)nmerged, longrest, + (uint32_t)nlong, out); + } else { + out += union_uint16(merged, nmerged, longrest, nlong, out); + } + return (uint32_t)(out - output); +} +CROARING_UNTARGET_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + +/** + * End of the AVX-512 16-bit union code. + */ + /** * Start of SIMD 16-bit XOR code */ @@ -2214,9 +2427,9 @@ static inline uint32_t unique_xor(uint16_t *out, uint32_t len) { } CROARING_TARGET_AVX2 // a one-pass SSE xor algorithm -uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, - const uint16_t *__restrict__ array2, uint32_t length2, - uint16_t *__restrict__ output) { +uint32_t xor_vector16(const uint16_t *array1, uint32_t length1, + const uint16_t *array2, uint32_t length2, + uint16_t *output) { if ((length1 < 8) || (length2 < 8)) { return xor_uint16(array1, length1, array2, length2, output); } @@ -2294,7 +2507,7 @@ uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, (length2 - 8 * pos2) * sizeof(uint16_t)); len += (length2 - 8 * pos2); } else { - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique_xor(buffer, leftoversize); len += xor_uint16(buffer, leftoversize, array2 + 8 * pos2, length2 - 8 * pos2, output); @@ -2308,7 +2521,7 @@ uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, (length1 - 8 * pos1) * sizeof(uint16_t)); len += (length1 - 8 * pos1); } else { - qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + sort_uint16_short(buffer, leftoversize); leftoversize = unique_xor(buffer, leftoversize); len += xor_uint16(buffer, leftoversize, array1 + 8 * pos1, length1 - 8 * pos1, output); @@ -2420,7 +2633,20 @@ size_t fast_union_uint16(const uint16_t *set_1, size_t size_1, const uint16_t *set_2, size_t size_2, uint16_t *buffer) { #if CROARING_IS_X64 - if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + const unsigned support = (unsigned)croaring_hardware_support(); +#if CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX512) { + // compute union with smallest array first + if (size_1 < size_2) { + return avx512_union_uint16(set_1, (uint32_t)size_1, set_2, + (uint32_t)size_2, buffer); + } else { + return avx512_union_uint16(set_2, (uint32_t)size_2, set_1, + (uint32_t)size_1, buffer); + } + } +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX2) { // compute union with smallest array first if (size_1 < size_2) { return union_vector16(set_1, (uint32_t)size_1, set_2, @@ -2609,7 +2835,6 @@ CROARING_UNTARGET_AVX512 #endif/* end file src/array_util.c */ /* begin file src/art/art.c */ #include -#include #include #include @@ -4150,7 +4375,7 @@ static void art_node_print_type(art_ref_t ref) { } } -void art_node_printf(const art_t *art, art_ref_t ref, uint8_t depth) { +static void art_node_printf(const art_t *art, art_ref_t ref, uint8_t depth) { if (art_is_leaf(ref)) { printf("{ type: Leaf, key: "); art_leaf_t *leaf = (art_leaf_t *)art_deref(art, ref); @@ -5584,6 +5809,7 @@ const uint8_t vbmi2_table[64] = { size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, uint32_t *vout, size_t outcapacity, uint32_t base) { + if (outcapacity == 0) return 0; uint32_t *out = (uint32_t *)vout; uint32_t *initout = out; uint32_t *safeout = out + outcapacity; @@ -5640,6 +5866,7 @@ size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, size_t bitset_extract_setbits_avx512_uint16(const uint64_t *array, size_t length, uint16_t *vout, size_t capacity, uint16_t base) { + if (capacity == 0) return 0; uint16_t *out = (uint16_t *)vout; uint16_t *initout = out; uint16_t *safeout = vout + capacity; @@ -5691,6 +5918,7 @@ CROARING_TARGET_AVX2 size_t bitset_extract_setbits_avx2(const uint64_t *words, size_t length, uint32_t *out, size_t outcapacity, uint32_t base) { + if (outcapacity == 0) return 0; uint32_t *initout = out; __m256i baseVec = _mm256_set1_epi32(base - 1); __m256i incVec = _mm256_set1_epi32(64); @@ -5794,6 +6022,7 @@ CROARING_TARGET_AVX2 size_t bitset_extract_setbits_sse_uint16(const uint64_t *words, size_t length, uint16_t *out, size_t outcapacity, uint16_t base) { + if (outcapacity == 0) return 0; uint16_t *initout = out; __m128i baseVec = _mm_set1_epi16(base - 1); __m128i incVec = _mm_set1_epi16(64); @@ -6755,17 +6984,19 @@ void array_container_offset(const array_container_t *c, container_t **loc, if (loc && lo_cap) { lo = array_container_create_given_capacity(lo_cap); for (int i = 0; i < lo_cap; ++i) { - array_container_add(lo, c->array[i] + offset); + lo->array[i] = c->array[i] + offset; } + lo->cardinality = lo_cap; *loc = (container_t *)lo; } hi_cap = c->cardinality - lo_cap; if (hic && hi_cap) { hi = array_container_create_given_capacity(hi_cap); - for (int i = lo_cap; i < c->cardinality; ++i) { - array_container_add(hi, c->array[i] + offset); + for (int i = 0; i < hi_cap; ++i) { + hi->array[i] = c->array[lo_cap + i] + offset; } + hi->cardinality = hi_cap; *hic = (container_t *)hi; } } @@ -7152,7 +7383,14 @@ int32_t array_container_number_of_runs(const array_container_t *ac) { * */ int32_t array_container_write(const array_container_t *container, char *buf) { +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < container->cardinality; ++i) { + uint16_t v_le = croaring_htole16(container->array[i]); + memcpy(buf + i * sizeof(uint16_t), &v_le, sizeof(uint16_t)); + } +#else memcpy(buf, container->array, container->cardinality * sizeof(uint16_t)); +#endif return array_container_size_in_bytes(container); } @@ -7185,7 +7423,15 @@ int32_t array_container_read(int32_t cardinality, array_container_t *container, array_container_grow(container, cardinality, false); } container->cardinality = cardinality; +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < cardinality; ++i) { + uint16_t v_le; + memcpy(&v_le, buf + i * sizeof(uint16_t), sizeof(uint16_t)); + container->array[i] = croaring_letoh16(v_le); + } +#else memcpy(container->array, buf, container->cardinality * sizeof(uint16_t)); +#endif return array_container_size_in_bytes(container); } @@ -7268,8 +7514,7 @@ void bitset_container_set_all(bitset_container_t *bitset) { bitset->cardinality = (1 << 16); } -/* Create a new bitset. Return NULL in case of failure. */ -bitset_container_t *bitset_container_create(void) { +static bitset_container_t *bitset_container_allocate(void) { bitset_container_t *bitset = (bitset_container_t *)roaring_malloc(sizeof(bitset_container_t)); @@ -7294,7 +7539,23 @@ bitset_container_t *bitset_container_create(void) { roaring_free(bitset); return NULL; } - bitset_container_clear(bitset); + return bitset; +} + +/* Create a new bitset. Return NULL in case of failure. */ +bitset_container_t *bitset_container_create(void) { + bitset_container_t *bitset = bitset_container_allocate(); + if (bitset) { + bitset_container_clear(bitset); + } + return bitset; +} + +bitset_container_t *bitset_container_create_uninitialized(void) { + bitset_container_t *bitset = bitset_container_allocate(); + if (bitset) { + bitset->cardinality = 0; + } return bitset; } @@ -8257,7 +8518,14 @@ int bitset_container_number_of_runs(bitset_container_t *bc) { int32_t bitset_container_write(const bitset_container_t *container, char *buf) { +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i) { + uint64_t w_le = croaring_htole64(container->words[i]); + memcpy(buf + i * sizeof(uint64_t), &w_le, sizeof(uint64_t)); + } +#else memcpy(buf, container->words, BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); +#endif return bitset_container_size_in_bytes(container); } @@ -8265,7 +8533,15 @@ int32_t bitset_container_write(const bitset_container_t *container, int32_t bitset_container_read(int32_t cardinality, bitset_container_t *container, const char *buf) { container->cardinality = cardinality; +#if CROARING_IS_BIG_ENDIAN + for (int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i) { + uint64_t w_le; + memcpy(&w_le, buf + i * sizeof(uint64_t), sizeof(uint64_t)); + container->words[i] = croaring_letoh64(w_le); + } +#else memcpy(container->words, buf, BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); +#endif return bitset_container_size_in_bytes(container); } @@ -8561,6 +8837,10 @@ extern bool container_iterator_next(const container_t *c, uint8_t typecode, extern bool container_iterator_prev(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint16_t *value); +extern bool container_contains( + const container_t *c, uint16_t val, + uint8_t typecode // !!! should be second argument? +); void container_free(container_t *c, uint8_t type) { switch (type) { @@ -8806,6 +9086,7 @@ extern inline container_t *container_andnot(const container_t *c1, uint8_t type2, uint8_t *result_type); +CROARING_ALLOW_UNALIGNED roaring_container_iterator_t container_init_iterator(const container_t *c, uint8_t typecode, uint16_t *value) { @@ -8845,6 +9126,7 @@ roaring_container_iterator_t container_init_iterator(const container_t *c, } } +CROARING_ALLOW_UNALIGNED roaring_container_iterator_t container_init_iterator_last(const container_t *c, uint8_t typecode, uint16_t *value) { @@ -8924,6 +9206,7 @@ bool container_iterator_lower_bound(const container_t *c, uint8_t typecode, } } +CROARING_ALLOW_UNALIGNED bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint32_t high16, uint32_t *buf, @@ -8967,8 +9250,10 @@ bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, const array_container_t *ac = const_CAST_array(c); uint32_t num_values = minimum_uint32(ac->cardinality - it->index, count); + // Hoist so GCC can vectorize the uint16->uint32 widen-or. + const uint16_t *src = ac->array + it->index; for (uint32_t i = 0; i < num_values; i++) { - buf[i] = high16 | ac->array[it->index + i]; + buf[i] = high16 | src[i]; } *consumed += num_values; it->index += num_values; @@ -9013,6 +9298,7 @@ bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, } } +CROARING_ALLOW_UNALIGNED bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint64_t high48, uint64_t *buf, @@ -9056,8 +9342,10 @@ bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, const array_container_t *ac = const_CAST_array(c); uint32_t num_values = minimum_uint32(ac->cardinality - it->index, count); + // Hoist so GCC can vectorize the uint16->uint64 widen-or. + const uint16_t *src = ac->array + it->index; for (uint32_t i = 0; i < num_values; i++) { - buf[i] = high48 | ac->array[it->index + i]; + buf[i] = high48 | src[i]; } *consumed += num_values; it->index += num_values; @@ -9102,6 +9390,186 @@ bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, } } +CROARING_ALLOW_UNALIGNED +bool container_iterator_read_backward_into_uint32( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out) { + *consumed = 0; + if (count == 0) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX >> (63 - (it->index % 64))); + do { + // Read set bits. + while (word != 0 && *consumed < count) { + uint32_t bit = 63 - roaring_leading_zeroes(word); + *buf = high16 | (wordindex * 64 + bit); + word &= ~(UINT64_C(1) << bit); + buf++; + (*consumed)++; + } + // Skip unset bits. + while (word == 0 && wordindex > 0) { + wordindex--; + word = bc->words[wordindex]; + } + } while (word != 0 && *consumed < count); + + if (word != 0) { + it->index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); + *value_out = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint32_t num_values = + minimum_uint32((uint32_t)(it->index + 1), count); + // Walk backwards so GCC can vectorize the uint16->uint32 widen-or. + const uint16_t *src = ac->array + it->index + 1; + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high16 | *--src; + } + *consumed += num_values; + it->index -= num_values; + if (it->index >= 0) { + *value_out = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + do { + uint32_t run_start = rc->runs[it->index].value; + uint32_t num_values = minimum_uint32(*value_out - run_start + 1, + count - *consumed); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high16 | (*value_out - i); + } + *value_out -= num_values; + buf += num_values; + *consumed += num_values; + + // We check for `value == UINT16_MAX` because + // `*value_out -= num_values` can underflow when + // `value == 0` (run_start == 0). In this case `value` + // will underflow to UINT16_MAX. + if (*value_out < run_start || *value_out == UINT16_MAX) { + it->index--; + if (it->index >= 0) { + *value_out = rc->runs[it->index].value + + rc->runs[it->index].length; + } else { + return false; + } + } + } while (*consumed < count); + return true; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + +CROARING_ALLOW_UNALIGNED +bool container_iterator_read_backward_into_uint64( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out) { + *consumed = 0; + if (count == 0) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX >> (63 - (it->index % 64))); + do { + // Read set bits. + while (word != 0 && *consumed < count) { + uint32_t bit = 63 - roaring_leading_zeroes(word); + *buf = high48 | (wordindex * 64 + bit); + word &= ~(UINT64_C(1) << bit); + buf++; + (*consumed)++; + } + // Skip unset bits. + while (word == 0 && wordindex > 0) { + wordindex--; + word = bc->words[wordindex]; + } + } while (word != 0 && *consumed < count); + + if (word != 0) { + it->index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); + *value_out = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint32_t num_values = + minimum_uint32((uint32_t)(it->index + 1), count); + // Walk backwards so GCC can vectorize the uint16->uint64 widen-or. + const uint16_t *src = ac->array + it->index + 1; + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high48 | *--src; + } + *consumed += num_values; + it->index -= num_values; + if (it->index >= 0) { + *value_out = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + do { + uint32_t run_start = rc->runs[it->index].value; + uint32_t num_values = minimum_uint32(*value_out - run_start + 1, + count - *consumed); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high48 | (*value_out - i); + } + *value_out -= num_values; + buf += num_values; + *consumed += num_values; + + if (*value_out < run_start || *value_out == UINT16_MAX) { + it->index--; + if (it->index >= 0) { + *value_out = rc->runs[it->index].value + + rc->runs[it->index].length; + } else { + return false; + } + } + } while (*consumed < count); + return true; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + bool container_iterator_skip(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint32_t skip_count, uint32_t *consumed_count, @@ -9301,34 +9769,199 @@ bool container_iterator_skip_backward(const container_t *c, uint8_t typecode, return has_value; } -#ifdef __cplusplus -} +uint16_t container_iterator_find_run_end(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more) { + switch (typecode) { + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + uint16_t run_end = + rc->runs[it->index].value + rc->runs[it->index].length; + it->index++; + if (it->index < rc->n_runs) { + *has_more = true; + *value = rc->runs[it->index].value; + } else { + *has_more = false; + } + return run_end; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint16_t v = *value; + while (it->index + 1 < ac->cardinality && + ac->array[it->index + 1] == (uint16_t)(v + 1)) { + it->index++; + v++; + } + it->index++; + if (it->index < ac->cardinality) { + *has_more = true; + *value = ac->array[it->index]; + } else { + *has_more = false; + } + return v; + } + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t pos = (uint32_t)*value + 1; + uint16_t run_end; + if (pos >= (1 << 16)) { + *has_more = false; + return UINT16_MAX; + } + uint32_t wordindex = pos / 64; + uint64_t word = ~bc->words[wordindex] & (UINT64_MAX << (pos % 64)); + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = ~bc->words[wordindex]; + } + if (word != 0) { + run_end = (uint16_t)(wordindex * 64 + + roaring_trailing_zeroes(word) - 1); + } else { + run_end = UINT16_MAX; + } + uint32_t next_pos = (uint32_t)run_end + 1; + if (next_pos >= (1 << 16)) { + *has_more = false; + } else { + wordindex = next_pos / 64; + word = bc->words[wordindex] & (UINT64_MAX << (next_pos % 64)); + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = bc->words[wordindex]; + } + if (word != 0) { + *has_more = true; + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value = (uint16_t)it->index; + } else { + *has_more = false; + } + } + return run_end; + } + default: + assert(false); + roaring_unreachable; + return 0; + } } -} // extern "C" { namespace roaring { namespace internal { -#endif - -#undef ROARING_INIT_ROARING_CONTAINER_ITERATOR_T -/* end file src/containers/containers.c */ -/* begin file src/containers/convert.c */ -#include - -#if CROARING_IS_X64 -#ifndef CROARING_COMPILER_SUPPORTS_AVX512 -#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." -#endif // CROARING_COMPILER_SUPPORTS_AVX512 -#endif - -#ifdef __cplusplus -extern "C" { -namespace roaring { -namespace internal { -#endif - -// file contains grubby stuff that must know impl. details of all container -// types. -bitset_container_t *bitset_container_from_array(const array_container_t *ac) { - bitset_container_t *ans = bitset_container_create(); +uint16_t container_iterator_find_run_start(const container_t *c, + uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more) { + switch (typecode) { + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + uint16_t run_start = rc->runs[it->index].value; + it->index--; + if (it->index >= 0) { + *has_more = true; + *value = rc->runs[it->index].value + rc->runs[it->index].length; + } else { + *has_more = false; + } + return run_start; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint16_t v = *value; + while (it->index > 0 && + ac->array[it->index - 1] == (uint16_t)(v - 1)) { + it->index--; + v--; + } + it->index--; + if (it->index >= 0) { + *has_more = true; + *value = ac->array[it->index]; + } else { + *has_more = false; + } + return v; + } + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + if (*value == 0) { + *has_more = false; + return 0; + } + uint32_t pos = (uint32_t)*value - 1; + int32_t wordindex = (int32_t)(pos / 64); + uint64_t word = + ~bc->words[wordindex] & (UINT64_MAX >> (63 - (pos % 64))); + while (word == 0 && --wordindex >= 0) { + word = ~bc->words[wordindex]; + } + uint16_t run_start; + if (word != 0) { + run_start = (uint16_t)(wordindex * 64 + + (63 - roaring_leading_zeroes(word)) + 1); + } else { + run_start = 0; + } + if (run_start == 0) { + *has_more = false; + } else { + int32_t prev_pos = (int32_t)run_start - 1; + wordindex = prev_pos / 64; + word = bc->words[wordindex] & + (UINT64_MAX >> (63 - (prev_pos % 64))); + while (word == 0 && --wordindex >= 0) { + word = bc->words[wordindex]; + } + if (word != 0) { + *has_more = true; + it->index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); + *value = (uint16_t)it->index; + } else { + *has_more = false; + } + } + return run_start; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#undef ROARING_INIT_ROARING_CONTAINER_ITERATOR_T +/* end file src/containers/containers.c */ +/* begin file src/containers/convert.c */ +#include + + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +// file contains grubby stuff that must know impl. details of all container +// types. +bitset_container_t *bitset_container_from_array(const array_container_t *ac) { + bitset_container_t *ans = bitset_container_create(); int limit = array_container_cardinality(ac); for (int i = 0; i < limit; ++i) bitset_container_set(ans, ac->array[i]); return ans; @@ -12581,9 +13214,21 @@ bool run_container_validate(const run_container_t *run, const char **reason) { int32_t run_container_write(const run_container_t *container, char *buf) { uint16_t cast_16 = container->n_runs; - memcpy(buf, &cast_16, sizeof(uint16_t)); + uint16_t n_runs_le = croaring_htole16(cast_16); + memcpy(buf, &n_runs_le, sizeof(uint16_t)); +#if CROARING_IS_BIG_ENDIAN + char *out = buf + sizeof(uint16_t); + for (int32_t i = 0; i < container->n_runs; ++i) { + uint16_t v_le = croaring_htole16(container->runs[i].value); + uint16_t l_le = croaring_htole16(container->runs[i].length); + memcpy(out, &v_le, sizeof(uint16_t)); + memcpy(out + sizeof(uint16_t), &l_le, sizeof(uint16_t)); + out += sizeof(rle16_t); + } +#else memcpy(buf + sizeof(uint16_t), container->runs, container->n_runs * sizeof(rle16_t)); +#endif return run_container_size_in_bytes(container); } @@ -12592,12 +13237,24 @@ int32_t run_container_read(int32_t cardinality, run_container_t *container, (void)cardinality; uint16_t cast_16; memcpy(&cast_16, buf, sizeof(uint16_t)); - container->n_runs = cast_16; + container->n_runs = croaring_letoh16(cast_16); if (container->n_runs > container->capacity) run_container_grow(container, container->n_runs, false); if (container->n_runs > 0) { +#if CROARING_IS_BIG_ENDIAN + const char *in = buf + sizeof(uint16_t); + for (int32_t i = 0; i < container->n_runs; ++i) { + uint16_t v_le, l_le; + memcpy(&v_le, in, sizeof(uint16_t)); + memcpy(&l_le, in + sizeof(uint16_t), sizeof(uint16_t)); + container->runs[i].value = croaring_letoh16(v_le); + container->runs[i].length = croaring_letoh16(l_le); + in += sizeof(rle16_t); + } +#else memcpy(container->runs, buf + sizeof(uint16_t), container->n_runs * sizeof(rle16_t)); +#endif } return run_container_size_in_bytes(container); } @@ -12792,9 +13449,11 @@ int run_container_get_index(const run_container_t *container, uint16_t x) { return -1; } } +#define CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY 0 #if defined(CROARING_IS_X64) && CROARING_COMPILER_SUPPORTS_AVX512 +#if CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY CROARING_TARGET_AVX512 CROARING_ALLOW_UNALIGNED /* Get the cardinality of `run'. Requires an actual computation. */ @@ -12821,6 +13480,7 @@ static inline int _avx512_run_container_cardinality( } CROARING_UNTARGET_AVX512 +#endif // CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY CROARING_TARGET_AVX2 CROARING_ALLOW_UNALIGNED @@ -12913,7 +13573,6 @@ static inline int _scalar_run_container_cardinality( int run_container_cardinality(const run_container_t *run) { // Empirically AVX-512 is not always faster than AVX2 -#define CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY 0 #if CROARING_COMPILER_SUPPORTS_AVX512 && \ CROARING_ENABLE_AVX512_RUN_CONTAINER_CARDINALITY if (croaring_hardware_support() & ROARING_SUPPORTS_AVX512) { @@ -13061,11 +13720,16 @@ POSSIBILITY OF SUCH DAMAGE. #endif // _MSC_VER == 1938 #endif // __clang__ +#ifdef __FILC__ +#include +#endif + // We need portability.h to be included first, see // https://github.com/RoaringBitmap/CRoaring/issues/394 #if CROARING_REGULAR_VISUAL_STUDIO #include -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || \ + defined(__FILC__) #include #endif // CROARING_REGULAR_VISUAL_STUDIO @@ -13104,7 +13768,7 @@ unsigned int CROARING_AVX512_REQUIRED = CROARING_AVX512VBMI2 | CROARING_AVX512BITALG | CROARING_AVX512VPOPCNTDQ); #endif -#if defined(__x86_64__) || defined(_M_AMD64) // x64 +#if CROARING_IS_X64 // x64 static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx) { @@ -13115,7 +13779,8 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, *ebx = cpu_info[1]; *ecx = cpu_info[2]; *edx = cpu_info[3]; -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || \ + defined(__FILC__) uint32_t level = *eax; __get_cpuid(level, eax, ebx, ecx, edx); #else @@ -13131,6 +13796,8 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, static inline uint64_t xgetbv(void) { #if defined(_MSC_VER) return _xgetbv(0); +#elif defined(__FILC__) + return zxgetbv(); #else uint32_t xcr0_lo, xcr0_hi; __asm__("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); @@ -13249,7 +13916,7 @@ static inline uint32_t dynamic_croaring_detect_supported_architectures(void) { #endif // end SIMD extension detection code -#if defined(__x86_64__) || defined(_M_AMD64) // x64 +#if CROARING_IS_X64 // x64 #if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_CPP static inline uint32_t croaring_detect_supported_architectures(void) { @@ -13331,7 +13998,7 @@ int croaring_hardware_support(void) { } #endif -#endif // defined(__x86_64__) || defined(_M_AMD64) // x64 +#endif // CROARING_IS_X64 // x64 #ifdef __cplusplus } } @@ -13866,15 +14533,17 @@ size_t ra_portable_size_in_bytes(const roaring_array_t *ra) { return count; } -// This function is endian-sensitive. +// The portable serialization format is little-endian. On big-endian hosts we +// byte-swap multi-byte fields before writing them to the buffer. size_t ra_portable_serialize(const roaring_array_t *ra, char *buf) { char *initbuf = buf; uint32_t startOffset = 0; bool hasrun = ra_has_run_container(ra); if (hasrun) { uint32_t cookie = SERIAL_COOKIE | ((uint32_t)(ra->size - 1) << 16); - memcpy(buf, &cookie, sizeof(cookie)); - buf += sizeof(cookie); + uint32_t cookie_le = croaring_htole32(cookie); + memcpy(buf, &cookie_le, sizeof(cookie_le)); + buf += sizeof(cookie_le); uint32_t s = (ra->size + 7) / 8; memset(buf, 0, s); for (int32_t i = 0; i < ra->size; ++i) { @@ -13891,30 +14560,34 @@ size_t ra_portable_serialize(const roaring_array_t *ra, char *buf) { } } else { // backwards compatibility uint32_t cookie = SERIAL_COOKIE_NO_RUNCONTAINER; - - memcpy(buf, &cookie, sizeof(cookie)); - buf += sizeof(cookie); - memcpy(buf, &ra->size, sizeof(ra->size)); - buf += sizeof(ra->size); + uint32_t cookie_le = croaring_htole32(cookie); + memcpy(buf, &cookie_le, sizeof(cookie_le)); + buf += sizeof(cookie_le); + uint32_t size_le = croaring_htole32((uint32_t)ra->size); + memcpy(buf, &size_le, sizeof(size_le)); + buf += sizeof(size_le); startOffset = 4 + 4 + 4 * ra->size + 4 * ra->size; } for (int32_t k = 0; k < ra->size; ++k) { - memcpy(buf, &ra->keys[k], sizeof(ra->keys[k])); - buf += sizeof(ra->keys[k]); + uint16_t key_le = croaring_htole16(ra->keys[k]); + memcpy(buf, &key_le, sizeof(key_le)); + buf += sizeof(key_le); // get_cardinality returns a value in [1,1<<16], subtracting one // we get [0,1<<16 - 1] which fits in 16 bits uint16_t card = (uint16_t)(container_get_cardinality(ra->containers[k], ra->typecodes[k]) - 1); - memcpy(buf, &card, sizeof(card)); - buf += sizeof(card); + uint16_t card_le = croaring_htole16(card); + memcpy(buf, &card_le, sizeof(card_le)); + buf += sizeof(card_le); } if ((!hasrun) || (ra->size >= NO_OFFSET_THRESHOLD)) { // writing the containers offsets for (int32_t k = 0; k < ra->size; k++) { - memcpy(buf, &startOffset, sizeof(startOffset)); - buf += sizeof(startOffset); + uint32_t off_le = croaring_htole32(startOffset); + memcpy(buf, &off_le, sizeof(off_le)); + buf += sizeof(off_le); startOffset = startOffset + container_size_in_bytes(ra->containers[k], ra->typecodes[k]); @@ -13938,6 +14611,7 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { if (bytestotal > maxbytes) return 0; uint32_t cookie; memcpy(&cookie, buf, sizeof(int32_t)); + cookie = croaring_letoh32(cookie); buf += sizeof(uint32_t); if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { @@ -13950,7 +14624,9 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { else { bytestotal += sizeof(int32_t); if (bytestotal > maxbytes) return 0; - memcpy(&size, buf, sizeof(int32_t)); + uint32_t size_le; + memcpy(&size_le, buf, sizeof(int32_t)); + size = (int32_t)croaring_letoh32(size_le); buf += sizeof(uint32_t); } if (size > (1 << 16) || size < 0) { @@ -13979,6 +14655,7 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { for (int32_t k = 0; k < size; ++k) { uint16_t tmp; memcpy(&tmp, keyscards + 4 * k + 2, sizeof(tmp)); + tmp = croaring_letoh16(tmp); uint32_t thiscard = tmp + 1; bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); bool isrun = false; @@ -13999,6 +14676,7 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { if (bytestotal > maxbytes) return 0; uint16_t n_runs; memcpy(&n_runs, buf, sizeof(uint16_t)); + n_runs = croaring_letoh16(n_runs); buf += sizeof(uint16_t); size_t containersize = n_runs * sizeof(rle16_t); bytestotal += containersize; @@ -14019,7 +14697,8 @@ size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { // cannot be found. If it returns true, readbytes is populated by how many bytes // were read, we have that *readbytes <= maxbytes. // -// This function is endian-sensitive. +// The portable serialization format is little-endian. On big-endian hosts we +// byte-swap multi-byte fields after reading them from the buffer. bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, const size_t maxbytes, size_t *readbytes) { *readbytes = sizeof(int32_t); // for cookie @@ -14029,6 +14708,7 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, } uint32_t cookie; memcpy(&cookie, buf, sizeof(int32_t)); + cookie = croaring_letoh32(cookie); buf += sizeof(uint32_t); if ((cookie & 0xFFFF) != SERIAL_COOKIE && cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { @@ -14045,7 +14725,9 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, // Ran out of bytes while reading second part of the cookie. return false; } - memcpy(&size, buf, sizeof(int32_t)); + uint32_t size_le; + memcpy(&size_le, buf, sizeof(int32_t)); + size = (int32_t)croaring_letoh32(size_le); buf += sizeof(uint32_t); } if (size < 0) { @@ -14087,7 +14769,7 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, for (int32_t k = 0; k < size; ++k) { uint16_t tmp; memcpy(&tmp, keyscards + 4 * k, sizeof(tmp)); - answer->keys[k] = tmp; + answer->keys[k] = croaring_letoh16(tmp); } if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { *readbytes += size * 4; @@ -14105,6 +14787,7 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, for (int32_t k = 0; k < size; ++k) { uint16_t tmp; memcpy(&tmp, keyscards + 4 * k + 2, sizeof(tmp)); + tmp = croaring_letoh16(tmp); uint32_t thiscard = tmp + 1; bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); bool isrun = false; @@ -14126,7 +14809,7 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, return false; } // it is now safe to read - bitset_container_t *c = bitset_container_create(); + bitset_container_t *c = bitset_container_create_uninitialized(); if (c == NULL) { // memory allocation failure // Failed to allocate memory for a bitset container. ra_clear(answer); // we need to clear the containers already @@ -14148,6 +14831,7 @@ bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, } uint16_t n_runs; memcpy(&n_runs, buf, sizeof(uint16_t)); + n_runs = croaring_letoh16(n_runs); size_t containersize = n_runs * sizeof(rle16_t); *readbytes += containersize; if (*readbytes > maxbytes) { // data is corrupted? @@ -15406,6 +16090,113 @@ roaring_bitmap_t *roaring_bitmap_or(const roaring_bitmap_t *x1, return answer; } +static void roaring_inplace_merge_bulk(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2, int dst, + int left, int right, bool is_xor) { + roaring_array_t *ra1 = &x1->high_low_container; + const roaring_array_t *ra2 = &x2->high_low_container; + const bool cow2 = is_cow(x2); + const int length1 = ra1->size; + const int length2 = ra2->size; + + int distinct = 0; + { + int l = left, r = right; + while (l < length1 && r < length2) { + uint16_t k1 = ra1->keys[l]; + uint16_t k2 = ra2->keys[r]; + if (k1 < k2) { + l++; + } else if (k1 > k2) { + r++; + } else { + l++; + r++; + } + distinct++; + } + distinct += (length1 - l) + (length2 - r); + } + const int total = dst + distinct; + + roaring_array_t merged; + ra_init_with_capacity(&merged, total > 0 ? (uint32_t)total : 1); + + for (int i = 0; i < dst; i++) { + ra_append(&merged, ra1->keys[i], ra1->containers[i], ra1->typecodes[i]); + } + + uint8_t result_type = 0; + while (left < length1 && right < length2) { + uint16_t k1 = ra1->keys[left]; + uint16_t k2 = ra2->keys[right]; + if (k1 < k2) { + ra_append(&merged, k1, ra1->containers[left], ra1->typecodes[left]); + left++; + } else if (k1 > k2) { + uint8_t type2 = ra2->typecodes[right]; + container_t *c2 = + get_copy_of_container(ra2->containers[right], &type2, cow2); + if (cow2) { + ra_set_container_at_index(ra2, right, c2, type2); + } + ra_append(&merged, k2, c2, type2); + right++; + } else { + uint8_t type1 = ra1->typecodes[left]; + container_t *c1 = ra1->containers[left]; + uint8_t type2 = ra2->typecodes[right]; + container_t *c2 = ra2->containers[right]; + if (is_xor) { + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_xor(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); + } else { + c = container_ixor(c1, type1, c2, type2, &result_type); + } + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&merged, k1, c, result_type); + } else { + container_free(c, result_type); + } + } else { + if (container_is_full(c1, type1)) { + ra_append(&merged, k1, c1, type1); + } else { + container_t *c = + (type1 == SHARED_CONTAINER_TYPE) + ? container_or(c1, type1, c2, type2, &result_type) + : container_ior(c1, type1, c2, type2, &result_type); + if (c != c1) { + container_free(c1, type1); + } + ra_append(&merged, k1, c, result_type); + } + } + left++; + right++; + } + } + for (; left < length1; left++) { + ra_append(&merged, ra1->keys[left], ra1->containers[left], + ra1->typecodes[left]); + } + for (; right < length2; right++) { + uint8_t type2 = ra2->typecodes[right]; + container_t *c2 = + get_copy_of_container(ra2->containers[right], &type2, cow2); + if (cow2) { + ra_set_container_at_index(ra2, right, c2, type2); + } + ra_append(&merged, ra2->keys[right], c2, type2); + } + + merged.flags = ra1->flags; + ra_clear_without_containers(ra1); + *ra1 = merged; +} + // inplace or (modifies its first argument). void roaring_bitmap_or_inplace(roaring_bitmap_t *x1, const roaring_bitmap_t *x2) { @@ -15455,22 +16246,8 @@ void roaring_bitmap_or_inplace(roaring_bitmap_t *x1, s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - - // container_t *c2_clone = container_clone(c2, type2); - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + roaring_inplace_merge_bulk(x1, x2, pos1, pos1, pos2, false); + return; } } if (pos1 == length1) { @@ -15606,8 +16383,9 @@ void roaring_bitmap_xor_inplace(roaring_bitmap_t *x1, ++pos1; } else { container_free(c, result_type); - ra_remove_at_index(&x1->high_low_container, pos1); - --length1; + roaring_inplace_merge_bulk(x1, x2, pos1, pos1 + 1, pos2 + 1, + true); + return; } ++pos2; @@ -15622,21 +16400,8 @@ void roaring_bitmap_xor_inplace(roaring_bitmap_t *x1, s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); } else { // s1 > s2 - container_t *c2 = ra_get_container_at_index(&x2->high_low_container, - (uint16_t)pos2, &type2); - c2 = get_copy_of_container(c2, &type2, is_cow(x2)); - if (is_cow(x2)) { - ra_set_container_at_index(&x2->high_low_container, pos2, c2, - type2); - } - - ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, - type2); - pos1++; - length1++; - pos2++; - if (pos2 == length2) break; - s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + roaring_inplace_merge_bulk(x1, x2, pos1, pos1, pos2, true); + return; } } if (pos1 == length1) { @@ -15973,9 +16738,18 @@ size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf) { return roaring_bitmap_portable_serialize(r, buf + 1) + 1; } else { buf[0] = CROARING_SERIALIZATION_ARRAY_UINT32; - memcpy(buf + 1, &cardinality, sizeof(uint32_t)); - roaring_bitmap_to_uint32_array( - r, (uint32_t *)(buf + 1 + sizeof(uint32_t))); + uint32_t card_le = croaring_htole32((uint32_t)cardinality); + memcpy(buf + 1, &card_le, sizeof(uint32_t)); + uint32_t *out = (uint32_t *)(buf + 1 + sizeof(uint32_t)); + roaring_bitmap_to_uint32_array(r, out); +#if CROARING_IS_BIG_ENDIAN + for (uint64_t i = 0; i < cardinality; ++i) { + uint32_t v; + memcpy(&v, out + i, sizeof(uint32_t)); + v = croaring_htole32(v); + memcpy(out + i, &v, sizeof(uint32_t)); + } +#endif return 1 + (size_t)sizeasarray; } } @@ -16034,6 +16808,7 @@ roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf) { uint32_t card; memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + card = croaring_letoh32(card); const uint32_t *elems = (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); @@ -16047,6 +16822,7 @@ roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf) { // elems may not be aligned, read with memcpy uint32_t elem; memcpy(&elem, elems + i, sizeof(elem)); + elem = croaring_letoh32(elem); roaring_bitmap_add_bulk(bitmap, &context, elem); } return bitmap; @@ -16072,6 +16848,7 @@ roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, /* This looks like a compressed set of uint32_t elements */ uint32_t card; memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + card = croaring_letoh32(card); // Check the buffer is big enough to contain card uint32_t elements if (maxbytes < 1 + sizeof(uint32_t) + card * sizeof(uint32_t)) { @@ -16090,6 +16867,7 @@ roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, // elems may not be aligned, read with memcpy uint32_t elem; memcpy((char *)&elem, (char *)(elems + i), sizeof(elem)); + elem = croaring_letoh32(elem); roaring_bitmap_add_bulk(bitmap, &context, elem); } return bitmap; @@ -16326,7 +17104,7 @@ uint32_t roaring_uint32_iterator_read(roaring_uint32_iterator_t *it, it->has_value = true; it->current_value = it->highbits | low16; // If the container still has values, we must have stopped because - // we skipped enough values. + // we read enough values. assert(ret == count); return ret; } @@ -16336,6 +17114,31 @@ uint32_t roaring_uint32_iterator_read(roaring_uint32_iterator_t *it, return ret; } +uint32_t roaring_uint32_iterator_read_backward(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count) { + uint32_t ret = 0; + while (it->has_value && ret < count) { + uint32_t consumed; + uint16_t low16 = (uint16_t)it->current_value; + bool has_value = container_iterator_read_backward_into_uint32( + it->container, it->typecode, &it->container_it, it->highbits, buf, + count - ret, &consumed, &low16); + ret += consumed; + buf += consumed; + if (has_value) { + it->has_value = true; + it->current_value = it->highbits | low16; + // If the container still has values, we must have stopped because + // we read enough values. + assert(ret == count); + return ret; + } + it->container_index--; + it->has_value = loadlastvalue(it); + } + return ret; +} + uint32_t roaring_uint32_iterator_skip(roaring_uint32_iterator_t *it, uint32_t count) { uint32_t ret = 0; @@ -16386,6 +17189,72 @@ uint32_t roaring_uint32_iterator_skip_backward(roaring_uint32_iterator_t *it, return ret; } +size_t roaring_uint32_iterator_read_ranges(roaring_uint32_iterator_t *it, + roaring_uint32_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->has_value && ret < count) { + buf[ret].min = it->current_value; + for (;;) { + uint16_t low16 = (uint16_t)it->current_value; + bool container_has_more; + uint16_t run_end_low16 = container_iterator_find_run_end( + it->container, it->typecode, &it->container_it, &low16, + &container_has_more); + buf[ret].max = it->highbits | run_end_low16; + + if (container_has_more) { + it->current_value = it->highbits | low16; + break; + } + // Move to next container + it->container_index++; + it->has_value = loadfirstvalue(it); + // Continue merging only if the run reached the container + // boundary and the next container starts exactly at max+1. + if (run_end_low16 != UINT16_MAX || !it->has_value || + it->current_value != buf[ret].max + 1) { + break; + } + } + ret++; + } + return ret; +} + +size_t roaring_uint32_iterator_read_prev_ranges( + roaring_uint32_iterator_t *it, roaring_uint32_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->has_value && ret < count) { + buf[ret].max = it->current_value; + for (;;) { + uint16_t low16 = (uint16_t)it->current_value; + bool container_has_more; + uint16_t run_start_low16 = container_iterator_find_run_start( + it->container, it->typecode, &it->container_it, &low16, + &container_has_more); + buf[ret].min = it->highbits | run_start_low16; + + if (container_has_more) { + it->current_value = it->highbits | low16; + break; + } + // Move to previous container + it->container_index--; + it->has_value = loadlastvalue(it); + // Continue merging only if the run reached the container + // boundary and the previous container ends exactly at min-1. + if (run_start_low16 != 0 || !it->has_value || + it->current_value != buf[ret].min - 1) { + break; + } + } + ret++; + } + return ret; +} + void roaring_uint32_iterator_free(roaring_uint32_iterator_t *it) { roaring_free(it); } @@ -17173,6 +18042,10 @@ void roaring_bitmap_rank_many(const roaring_bitmap_t *bm, const uint32_t *begin, iter++; } } + while (iter != end) { // must have N outputs for N inputs... + *(ans++) = size; // ...everything left is beyond all containers + iter++; + } } /** @@ -17722,6 +18595,15 @@ const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, CROARING_ALLOW_UNALIGNED roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf) { +#if CROARING_IS_BIG_ENDIAN + // The portable format is little-endian on every host, and this function + // uses the container payloads where they sit rather than converting them. + // There is therefore no correct in-place view of them here: refuse rather + // than hand back a bitmap that silently reads byte-swapped values. Use + // roaring_bitmap_portable_deserialize_safe(), which converts as it copies. + (void)buf; + return NULL; +#else char *start_of_buf = (char *)buf; uint32_t cookie; int32_t num_containers; @@ -17878,6 +18760,7 @@ roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf) { } return rb; +#endif } bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t *bitset) { @@ -17930,8 +18813,8 @@ bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t *bitset) { /* end file src/roaring.c */ /* begin file src/roaring64.c */ #include -#include #include +#include #include #include @@ -17962,6 +18845,9 @@ typedef struct roaring64_bitmap_s { uint64_t first_free; uint64_t capacity; container_t **containers; + // Parallel to containers[]. Live slots (non-NULL pointers) have the + // matching typecode; NULL slots are skipped and their typecodes ignored. + uint8_t *typecodes; } roaring64_bitmap_t; // Leaf type of the ART used to keep the high 48 bits of each entry. @@ -17971,25 +18857,76 @@ typedef roaring64_leaf_t leaf_t; // Iterator struct to hold iteration state. typedef struct roaring64_iterator_s { - const roaring64_bitmap_t *r; - art_iterator_t art_it; - roaring_container_iterator_t container_it; + // The order here is deliberate: everything `roaring64_iterator_advance` + // touches per value is packed into the first 64 bytes, and `art_it` -- + // 136 bytes, of which only `art_it.value` is read per value -- is last. + // Putting `art_it` earlier pushes the rest past the first cache line and + // costs ~18% on a scalar iteration loop. + + // Must stay first, and must stay a `roaring64_iterator_public_t`: the + // inline `roaring64_iterator_value` / `roaring64_iterator_has_value` in + // the public header reach these two members by converting a + // `roaring64_iterator_t *` to a pointer to its initial member. + roaring64_iterator_public_t pub; + uint64_t high48; // Key that art_it points to. + roaring_container_iterator_t container_it; - uint64_t value; - bool has_value; + // Forward-iteration cache for bitset containers. `container_iterator_next` + // recomputes the word index from `container_it.index`, reloads the word + // and re-masks it for every value; keeping the remaining bits of the + // current word here turns that into a `tzcnt` and a `blsr`. Array and run + // containers already advance by a single increment and gain nothing from + // a cache, so they stay on the ordinary path. + // + // Every field is a pure function of the ART position and + // `container_it.index`, so the cache describes where the iterator is + // exactly when both still match what it was built from. `fast_type` is + // BITSET_CONTAINER_TYPE, or 0 when there is no usable cache. + uint32_t fast_wordindex; + const art_val_t *fast_art_value; + const uint64_t *fast_words; + uint64_t fast_word; + int32_t fast_index; + uint8_t fast_type; // If has_value is false, then the iterator is saturated. This field // indicates the direction of saturation. If true, there are no more values // in the forward direction. If false, there are no more values in the // backward direction. bool saturated_forward; + + const roaring64_bitmap_t *r; + art_iterator_t art_it; } roaring64_iterator_t; static inline bool is_frozen64(const roaring64_bitmap_t *r) { return r->flags & ROARING_FLAG_FROZEN; } +static inline bool is_frozen_art64(const roaring64_bitmap_t *r) { + return r->flags & ROARING_FLAG_FROZEN_ART; +} + +typedef union { + bitset_container_t bitset; + array_container_t array; + run_container_t run; +} frozen_container_header_t; + +static void *roaring64_arena_alloc(char **arena, size_t num_bytes) { + char *res = *arena; + *arena += num_bytes; + return res; +} + +static char *roaring64_arena_pad(char *cursor, const char *base, + size_t alignment) { + uint64_t off = (uint64_t)(cursor - base); + uint64_t aligned = (off + alignment - 1) & ~(uint64_t)(alignment - 1); + return (char *)base + aligned; +} + // Splits the given uint64 key into high 48 bit and low 16 bit components. // Expects high48_out to be of length ART_KEY_BYTES. static inline uint16_t split_key(uint64_t key, uint8_t high48_out[]) { @@ -18023,19 +18960,26 @@ static inline container_t *get_container(const roaring64_bitmap_t *r, return r->containers[get_index(leaf)]; } +// Writes the pointer and its typecode together so they cannot drift. +static inline void set_container_at(roaring64_bitmap_t *r, uint64_t index, + container_t *container, uint8_t typecode) { + r->containers[index] = container; + r->typecodes[index] = typecode; +} + // Replaces the container of `leaf` with the given container. Returns the // modified leaf for convenience. static inline leaf_t replace_container(roaring64_bitmap_t *r, leaf_t *leaf, container_t *container, uint8_t typecode) { uint64_t index = get_index(*leaf); - r->containers[index] = container; + set_container_at(r, index, container, typecode); *leaf = create_leaf(index, typecode); return *leaf; } /** - * Extends the array of container pointers. + * Extends the array of container pointers (and the parallel typecode array). */ static void extend_containers(roaring64_bitmap_t *r) { uint64_t size = r->first_free; @@ -18054,6 +18998,8 @@ static void extend_containers(roaring64_bitmap_t *r) { r->containers = (container_t **)roaring_realloc( r->containers, new_capacity * sizeof(container_t *)); memset(r->containers + r->capacity, 0, increase * sizeof(container_t *)); + r->typecodes = (uint8_t *)roaring_realloc(r->typecodes, + new_capacity * sizeof(uint8_t)); r->capacity = new_capacity; } @@ -18078,10 +19024,41 @@ static uint64_t allocate_index(roaring64_bitmap_t *r) { static leaf_t add_container(roaring64_bitmap_t *r, container_t *container, uint8_t typecode) { uint64_t index = allocate_index(r); - r->containers[index] = container; + set_container_at(r, index, container, typecode); return create_leaf(index, typecode); } +static void ensure_container_capacity(roaring64_bitmap_t *r, uint64_t extra) { + uint64_t needed = r->first_free + extra; + if (needed <= r->capacity) { + return; + } + uint64_t new_capacity = r->capacity; + if (new_capacity == 0) { + new_capacity = 2; + } + while (new_capacity < needed) { + uint64_t grown; + if (new_capacity < 1024) { + grown = 2 * new_capacity; + } else { + grown = new_capacity + new_capacity / 4; + } + if (grown <= new_capacity) { + new_capacity = needed; + break; + } + new_capacity = grown; + } + uint64_t increase = new_capacity - r->capacity; + r->containers = (container_t **)roaring_realloc( + r->containers, new_capacity * sizeof(container_t *)); + memset(r->containers + r->capacity, 0, increase * sizeof(container_t *)); + r->typecodes = (uint8_t *)roaring_realloc(r->typecodes, + new_capacity * sizeof(uint8_t)); + r->capacity = new_capacity; +} + static void remove_container(roaring64_bitmap_t *r, leaf_t leaf) { uint64_t index = get_index(leaf); r->containers[index] = NULL; @@ -18105,6 +19082,36 @@ static inline int compare_high48(art_key_chunk_t key1[], return art_compare_keys(key1, key2); } +// Cache the current container so advance() need not re-read the ART leaf, +// reload the container pointer, or (for bitsets) re-mask the current word. +// Called after any positioning that sets container_it.index. +// fast_type is 0 when there is no usable cache (run containers, exhausted). +static inline void roaring64_iterator_prime(roaring64_iterator_t *it) { + it->fast_type = 0; + if (it->art_it.value == NULL) { + return; + } + leaf_t leaf = (leaf_t)*it->art_it.value; + if (get_typecode(leaf) != BITSET_CONTAINER_TYPE) { + return; + } + const bitset_container_t *bc = + const_CAST_bitset(get_container(it->r, leaf)); + int32_t index = it->container_it.index; + uint32_t wordindex = (uint32_t)index >> 6; + uint32_t bit = (uint32_t)index & 63u; + uint64_t word = bc->words[wordindex]; + // Bits strictly after the current value in this word. A shift of 64 is + // undefined, so the last bit of a word is a special case. + it->fast_word = + (bit == 63u) ? UINT64_C(0) : (word & (UINT64_MAX << (bit + 1))); + it->fast_words = bc->words; + it->fast_wordindex = wordindex; + it->fast_index = index; + it->fast_art_value = it->art_it.value; + it->fast_type = BITSET_CONTAINER_TYPE; +} + static inline bool roaring64_iterator_init_at_leaf_first( roaring64_iterator_t *it) { it->high48 = combine_key(it->art_it.key, 0); @@ -18112,8 +19119,10 @@ static inline bool roaring64_iterator_init_at_leaf_first( uint16_t low16 = 0; it->container_it = container_init_iterator(get_container(it->r, leaf), get_typecode(leaf), &low16); - it->value = it->high48 | low16; - return (it->has_value = true); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + roaring64_iterator_prime(it); + return true; } static inline bool roaring64_iterator_init_at_leaf_last( @@ -18123,16 +19132,18 @@ static inline bool roaring64_iterator_init_at_leaf_last( uint16_t low16 = 0; it->container_it = container_init_iterator_last(get_container(it->r, leaf), get_typecode(leaf), &low16); - it->value = it->high48 | low16; - return (it->has_value = true); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + roaring64_iterator_prime(it); + return true; } static inline roaring64_iterator_t *roaring64_iterator_init_at( const roaring64_bitmap_t *r, roaring64_iterator_t *it, bool first) { it->r = r; it->art_it = art_init_iterator((art_t *)&r->art, first); - it->has_value = it->art_it.value != NULL; - if (it->has_value) { + it->pub.has_value = it->art_it.value != NULL; + if (it->pub.has_value) { if (first) { roaring64_iterator_init_at_leaf_first(it); } else { @@ -18140,6 +19151,7 @@ static inline roaring64_iterator_t *roaring64_iterator_init_at( } } else { it->saturated_forward = first; + it->fast_type = 0; } return it; } @@ -18152,6 +19164,7 @@ roaring64_bitmap_t *roaring64_bitmap_create(void) { r->capacity = 0; r->first_free = 0; r->containers = NULL; + r->typecodes = NULL; return r; } @@ -18159,22 +19172,24 @@ void roaring64_bitmap_free(roaring64_bitmap_t *r) { if (!r) { return; } + if (is_frozen64(r)) { + // Headers, containers[], and typecodes[] live in the same allocation + // as `r`. Payloads alias a caller buffer. + if (!is_frozen_art64(r)) { + art_free(&r->art); + } + roaring_free(r); + return; + } art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); while (it.value != NULL) { leaf_t leaf = (leaf_t)*it.value; - if (is_frozen64(r)) { - // Only free the container itself, not the buffer-backed contents - // within. - roaring_free(get_container(r, leaf)); - } else { - container_free(get_container(r, leaf), get_typecode(leaf)); - } + container_free(get_container(r, leaf), get_typecode(leaf)); art_iterator_next(&it); } - if (!is_frozen64(r)) { - art_free(&r->art); - } + art_free(&r->art); roaring_free(r->containers); + roaring_free(r->typecodes); roaring_free(r); } @@ -18195,18 +19210,56 @@ roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r) { return result; } -/** - * Steal the containers from a 32-bit bitmap and insert them into a 64-bit - * bitmap (with an offset) - * - * After calling this function, the original bitmap will be empty, and the - * returned bitmap will contain all the values from the original bitmap. - */ -static void move_from_roaring32_offset(roaring64_bitmap_t *dst, +void roaring64_bitmap_overwrite(roaring64_bitmap_t *dest, + const roaring64_bitmap_t *src) { + if (dest == src) { + return; + } + + // Free dest's containers. + art_iterator_t it = art_init_iterator(&dest->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + container_free(get_container(dest, leaf), get_typecode(leaf)); + art_iterator_next(&it); + } + art_free(&dest->art); + + // Reinitialize dest. + art_init_cleared(&dest->art); + dest->flags = 0; + dest->first_free = 0; + if (dest->capacity > 0) { + memset(dest->containers, 0, + sizeof(dest->containers[0]) * dest->capacity); + } + + // Copy src's containers into dest. + it = art_init_iterator((art_t *)&src->art, /*first=*/true); + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + uint8_t typecode = get_typecode(leaf); + container_t *container = get_copy_of_container( + get_container(src, leaf), &typecode, /*copy_on_write=*/false); + leaf_t dest_leaf = add_container(dest, container, typecode); + art_insert(&dest->art, it.key, (art_val_t)dest_leaf); + art_iterator_next(&it); + } +} + +/** + * Steal the containers from a 32-bit bitmap and insert them into a 64-bit + * bitmap (with an offset) + * + * After calling this function, the original bitmap will be empty, and the + * returned bitmap will contain all the values from the original bitmap. + */ +static void move_from_roaring32_offset(roaring64_bitmap_t *dst, roaring_bitmap_t *src, uint32_t high_bits) { uint64_t key_base = ((uint64_t)high_bits) << 32; uint32_t r32_size = ra_get_size(&src->high_low_container); + ensure_container_capacity(dst, r32_size); for (uint32_t i = 0; i < r32_size; ++i) { uint16_t key = ra_get_key_at_index(&src->high_low_container, i); uint8_t typecode; @@ -18447,12 +19500,20 @@ bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, if (min >= max) { return true; } + return roaring64_bitmap_contains_range_closed(r, min, max - 1); +} + +bool roaring64_bitmap_contains_range_closed(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min > max) { + return true; + } uint8_t min_high48[ART_KEY_BYTES]; uint16_t min_low16 = split_key(min, min_high48); uint8_t max_high48[ART_KEY_BYTES]; uint16_t max_low16 = split_key(max, max_high48); - uint64_t max_high48_bits = (max - 1) & 0xFFFFFFFFFFFF0000; // Inclusive + uint64_t max_high48_bits = max & 0xFFFFFFFFFFFF0000; art_iterator_t it = art_lower_bound((art_t *)&r->art, min_high48); if (it.value == NULL || combine_key(it.key, 0) > min) { @@ -18478,7 +19539,7 @@ bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, } uint32_t container_max = 0xFFFF + 1; // Exclusive if (compare_high48(it.key, max_high48) == 0) { - container_max = max_low16; + container_max = (uint32_t)max_low16 + 1; } // For the first and last containers we use container_contains_range, @@ -18674,7 +19735,7 @@ void roaring64_bitmap_remove_bulk(roaring64_bitmap_t *r, } if (!container_nonzero_cardinality(container2, typecode2)) { container_free(container2, typecode2); - leaf_t leaf; + leaf_t leaf = 0; bool erased = art_erase(art, high48, (art_val_t *)&leaf); assert(erased); (void)erased; @@ -18760,7 +19821,7 @@ void roaring64_bitmap_remove_range_closed(roaring64_bitmap_t *r, uint64_t min, art_iterator_t it = art_upper_bound(art, min_high48); while (it.value != NULL && art_compare_keys(it.key, max_high48) < 0) { - leaf_t leaf; + leaf_t leaf = 0; bool erased = art_iterator_erase(&it, (art_val_t *)&leaf); assert(erased); (void)erased; @@ -18775,13 +19836,15 @@ void roaring64_bitmap_clear(roaring64_bitmap_t *r) { } uint64_t roaring64_bitmap_get_cardinality(const roaring64_bitmap_t *r) { - art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + // Scan the pointer array rather than the ART: the arrays are sequential + // and the ART is not. first_free is a free-list head, not a size, so + // after deletes live containers can sit above it; skip NULL slots. uint64_t cardinality = 0; - while (it.value != NULL) { - leaf_t leaf = (leaf_t)*it.value; - cardinality += container_get_cardinality(get_container(r, leaf), - get_typecode(leaf)); - art_iterator_next(&it); + for (uint64_t i = 0; i < r->capacity; ++i) { + if (r->containers[i] != NULL) { + cardinality += + container_get_cardinality(r->containers[i], r->typecodes[i]); + } } return cardinality; } @@ -18863,6 +19926,26 @@ uint64_t roaring64_bitmap_maximum(const roaring64_bitmap_t *r) { it.key, container_maximum(get_container(r, leaf), get_typecode(leaf))); } +bool roaring64_bitmap_remove_run_compression(roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + bool removed = false; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + if (get_typecode(*leaf) == RUN_CONTAINER_TYPE) { + run_container_t *run = CAST_run(get_container(r, *leaf)); + int32_t card = run_container_cardinality(run); + uint8_t new_typecode; + container_t *new_container = + convert_to_bitset_or_array_container(run, card, &new_typecode); + run_container_free(run); + replace_container(r, leaf, new_container, new_typecode); + removed = true; + } + art_iterator_next(&it); + } + return removed; +} + bool roaring64_bitmap_run_optimize(roaring64_bitmap_t *r) { art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); bool has_run_container = false; @@ -18885,9 +19968,10 @@ static void move_to_shrink(roaring64_bitmap_t *r, leaf_t *leaf) { if (idx < r->first_free) { return; } - r->containers[r->first_free] = get_container(r, *leaf); + uint8_t typecode = get_typecode(*leaf); + set_container_at(r, r->first_free, get_container(r, *leaf), typecode); r->containers[idx] = NULL; - *leaf = create_leaf(r->first_free, get_typecode(*leaf)); + *leaf = create_leaf(r->first_free, typecode); r->first_free = next_free_container_idx(r); } @@ -18912,7 +19996,10 @@ size_t roaring64_bitmap_shrink_to_fit(roaring64_bitmap_t *r) { if (new_capacity < r->capacity) { r->containers = (container_t **)roaring_realloc( r->containers, new_capacity * sizeof(container_t *)); - freed += (r->capacity - new_capacity) * sizeof(container_t *); + r->typecodes = (uint8_t *)roaring_realloc( + r->typecodes, new_capacity * sizeof(uint8_t)); + freed += (r->capacity - new_capacity) * + (sizeof(container_t *) + sizeof(uint8_t)); r->capacity = new_capacity; } return freed; @@ -18968,8 +20055,17 @@ static bool roaring64_leaf_internal_validate(const art_val_t val, void *context) { leaf_t leaf = (leaf_t)val; roaring64_bitmap_t *r = (roaring64_bitmap_t *)context; - return container_internal_validate(get_container(r, leaf), - get_typecode(leaf), reason); + uint64_t index = get_index(leaf); + uint8_t typecode = get_typecode(leaf); + if (index >= r->capacity || r->containers[index] == NULL) { + *reason = "ART leaf points at an empty container slot"; + return false; + } + if (r->typecodes[index] != typecode) { + *reason = "typecode array does not match ART leaf"; + return false; + } + return container_internal_validate(r->containers[index], typecode, reason); } bool roaring64_bitmap_internal_validate(const roaring64_bitmap_t *r, @@ -19180,7 +20276,7 @@ void roaring64_bitmap_and_inplace(roaring64_bitmap_t *r1, if (!it2_present || compare_result < 0) { // Cases 1 and 3a: it1 is the only iterator or is before it2. - leaf_t leaf; + leaf_t leaf = 0; bool erased = art_iterator_erase(&it1, (art_val_t *)&leaf); assert(erased); (void)erased; @@ -19821,6 +20917,131 @@ void roaring64_bitmap_flip_closed_inplace(roaring64_bitmap_t *r, uint64_t min, } } +roaring64_bitmap_t *roaring64_bitmap_add_offset_signed( + const roaring64_bitmap_t *r, bool positive, uint64_t offset) { + if (offset == 0) { + return roaring64_bitmap_copy(r); + } + + roaring64_bitmap_t *answer = roaring64_bitmap_create(); + + // Decompose the offset into a signed container-level shift and an + // intra-container shift. For negative offsets the low 16 bits wrap: e.g. + // -1 = container_offset(-1) + in_offset(0xffff), because shifting by -1 + // container is a shift of -0x1_0000, so we need to shift up within + // containers to get back to -1 + uint16_t low16 = (uint16_t)offset; + int64_t container_offset; + uint16_t in_offset; + if (positive) { + container_offset = (int64_t)(offset >> 16); + in_offset = low16; + } else if (low16 == 0) { + container_offset = -(int64_t)(offset >> 16); + in_offset = 0; + } else { + container_offset = -(int64_t)(offset >> 16) - 1; + in_offset = (uint16_t)-low16; + } + + art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); + + if (in_offset == 0) { + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + int64_t k = + (int64_t)(combine_key(it.key, 0) >> 16) + container_offset; + if ((uint64_t)k < (uint64_t)1 << 48) { + uint8_t new_high48[ART_KEY_BYTES]; + split_key((uint64_t)k << 16, new_high48); + uint8_t typecode = get_typecode(leaf); + container_t *container = + get_copy_of_container(get_container(r, leaf), &typecode, + /*copy_on_write=*/false); + leaf_t new_leaf = add_container(answer, container, typecode); + art_insert(&answer->art, new_high48, (art_val_t)new_leaf); + } + art_iterator_next(&it); + } + return answer; + } + + // Track the most recently inserted hi container so that the next + // iteration's lo can merge with it without re-searching the ART. + leaf_t *prev_hi_leaf = NULL; + int64_t prev_hi_k = -1; + + while (it.value != NULL) { + leaf_t leaf = (leaf_t)*it.value; + int64_t k = (int64_t)(combine_key(it.key, 0) >> 16) + container_offset; + + container_t *lo = NULL, *hi = NULL; + container_t **lo_ptr = NULL, **hi_ptr = NULL; + + if ((uint64_t)k < (uint64_t)1 << 48) { + lo_ptr = &lo; + } + if ((uint64_t)(k + 1) < (uint64_t)1 << 48) { + hi_ptr = &hi; + } + if (lo_ptr == NULL && hi_ptr == NULL) { + art_iterator_next(&it); + continue; + } + + uint8_t typecode = get_typecode(leaf); + const container_t *c = + container_unwrap_shared(get_container(r, leaf), &typecode); + container_add_offset(c, typecode, lo_ptr, hi_ptr, in_offset); + + if (lo != NULL) { + if (prev_hi_leaf != NULL && prev_hi_k == k) { + uint8_t existing_type = get_typecode(*prev_hi_leaf); + container_t *existing_c = get_container(answer, *prev_hi_leaf); + uint8_t merged_type; + container_t *merged_c = container_ior( + existing_c, existing_type, lo, typecode, &merged_type); + if (merged_c != existing_c) { + container_free(existing_c, existing_type); + } + replace_container(answer, prev_hi_leaf, merged_c, merged_type); + container_free(lo, typecode); + } else { + uint8_t lo_high48[ART_KEY_BYTES]; + split_key((uint64_t)k << 16, lo_high48); + leaf_t new_leaf = add_container(answer, lo, typecode); + art_insert(&answer->art, lo_high48, (art_val_t)new_leaf); + } + } + + prev_hi_leaf = NULL; + if (hi != NULL) { + uint8_t hi_high48[ART_KEY_BYTES]; + split_key((uint64_t)(k + 1) << 16, hi_high48); + leaf_t new_leaf = add_container(answer, hi, typecode); + prev_hi_leaf = (leaf_t *)art_insert(&answer->art, hi_high48, + (art_val_t)new_leaf); + prev_hi_k = k + 1; + } + + art_iterator_next(&it); + } + + // Repair containers (e.g., convert low-cardinality bitset containers to + // array containers after lazy union operations). + art_iterator_t repair_it = art_init_iterator(&answer->art, /*first=*/true); + while (repair_it.value != NULL) { + leaf_t *leaf_ptr = (leaf_t *)repair_it.value; + uint8_t typecode = get_typecode(*leaf_ptr); + container_t *repaired = container_repair_after_lazy( + get_container(answer, *leaf_ptr), &typecode); + replace_container(answer, leaf_ptr, repaired, typecode); + art_iterator_next(&repair_it); + } + + return answer; +} + // Returns the number of distinct high 32-bit entries in the bitmap. static inline uint64_t count_high32(const roaring64_bitmap_t *r) { art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); @@ -19915,7 +21136,8 @@ size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, // Write as uint64 the distinct number of "buckets", where a bucket is // defined as the most significant 32 bits of an element. uint64_t high32_count = count_high32(r); - memcpy(buf, &high32_count, sizeof(high32_count)); + uint64_t high32_count_le = croaring_htole64(high32_count); + memcpy(buf, &high32_count_le, sizeof(high32_count_le)); buf += sizeof(high32_count); art_iterator_t it = art_init_iterator((art_t *)&r->art, /*first=*/true); @@ -19930,7 +21152,8 @@ size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, if (bitmap32 != NULL) { // Write as uint32 the most significant 32 bits of the // bucket. - memcpy(buf, &prev_high32, sizeof(prev_high32)); + uint32_t prev_high32_le = croaring_htole32(prev_high32); + memcpy(buf, &prev_high32_le, sizeof(prev_high32_le)); buf += sizeof(prev_high32); // Write the 32-bit Roaring bitmaps representing the least @@ -19961,7 +21184,8 @@ size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, if (bitmap32 != NULL) { // Write as uint32 the most significant 32 bits of the bucket. - memcpy(buf, &prev_high32, sizeof(prev_high32)); + uint32_t prev_high32_le = croaring_htole32(prev_high32); + memcpy(buf, &prev_high32_le, sizeof(prev_high32_le)); buf += sizeof(prev_high32); // Write the 32-bit Roaring bitmaps representing the least @@ -19988,6 +21212,7 @@ size_t roaring64_bitmap_portable_deserialize_size(const char *buf, return 0; } memcpy(&buckets, buf, sizeof(buckets)); + buckets = croaring_letoh64(buckets); buf += sizeof(buckets); read_bytes += sizeof(buckets); @@ -20034,6 +21259,7 @@ roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( return NULL; } memcpy(&buckets, buf, sizeof(buckets)); + buckets = croaring_letoh64(buckets); buf += sizeof(buckets); read_bytes += sizeof(buckets); @@ -20053,6 +21279,7 @@ roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( return NULL; } memcpy(&high32, buf, sizeof(high32)); + high32 = croaring_letoh32(high32); buf += sizeof(high32); read_bytes += sizeof(high32); // High 32 bits must be strictly increasing. @@ -20063,22 +21290,26 @@ roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( previous_high32 = high32; // Read the 32-bit Roaring bitmaps representing the least - // significant bits of a set of elements. - size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( - buf, maxbytes - read_bytes); - if (bitmap32_size == 0) { + // significant bits of a set of elements. ra_portable_deserialize + // already reports bytes consumed, so we do not walk the 32-bit + // format a second time with deserialize_size. + roaring_bitmap_t *bitmap32 = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (bitmap32 == NULL) { roaring64_bitmap_free(r); return NULL; } - - roaring_bitmap_t *bitmap32 = roaring_bitmap_portable_deserialize_safe( - buf, maxbytes - read_bytes); - if (bitmap32 == NULL) { + size_t bytesread = 0; + bool is_ok = ra_portable_deserialize(&bitmap32->high_low_container, buf, + maxbytes - read_bytes, &bytesread); + if (!is_ok) { + roaring_free(bitmap32); roaring64_bitmap_free(r); return NULL; } - buf += bitmap32_size; - read_bytes += bitmap32_size; + roaring_bitmap_set_copy_on_write(bitmap32, false); + buf += bytesread; + read_bytes += bytesread; // While we don't attempt to validate much, we must ensure that there // is no duplication in the high 48 bits - inserting into the ART @@ -20307,22 +21538,68 @@ size_t roaring64_bitmap_frozen_serialize(const roaring64_bitmap_t *r, return buf - initial_buf; } -static container_t *container_frozen_view(uint8_t typecode, uint32_t elem_count, - const uint64_t **bitsets, - const uint16_t **arrays, - const rle16_t **runs) { +static roaring64_bitmap_t *alloc_frozen_bitmap( + uint64_t capacity, frozen_container_header_t **headers_out) { + if (capacity > SIZE_MAX / sizeof(frozen_container_header_t)) { + return NULL; + } + size_t ptrs = (size_t)capacity * sizeof(container_t *); + size_t codes = (size_t)capacity * sizeof(uint8_t); + size_t headers = (size_t)capacity * sizeof(frozen_container_header_t); + size_t pad = alignof(container_t *) + alignof(frozen_container_header_t); + if (sizeof(roaring64_bitmap_t) > SIZE_MAX - ptrs || + sizeof(roaring64_bitmap_t) + ptrs > SIZE_MAX - codes || + sizeof(roaring64_bitmap_t) + ptrs + codes > SIZE_MAX - headers || + sizeof(roaring64_bitmap_t) + ptrs + codes + headers > SIZE_MAX - pad) { + return NULL; + } + size_t sz = sizeof(roaring64_bitmap_t) + ptrs + codes + headers + pad; + char *base = (char *)roaring_malloc(sz); + if (base == NULL) { + return NULL; + } + char *cursor = base; + roaring64_bitmap_t *r = (roaring64_bitmap_t *)roaring64_arena_alloc( + &cursor, sizeof(roaring64_bitmap_t)); + art_init_cleared(&r->art); + r->flags = ROARING_FLAG_FROZEN; + r->capacity = capacity; + r->first_free = 0; + cursor = roaring64_arena_pad(cursor, base, alignof(container_t *)); + if (capacity == 0) { + r->containers = NULL; + r->typecodes = NULL; + if (headers_out != NULL) { + *headers_out = NULL; + } + return r; + } + r->containers = (container_t **)roaring64_arena_alloc(&cursor, ptrs); + memset(r->containers, 0, ptrs); + r->typecodes = (uint8_t *)roaring64_arena_alloc(&cursor, codes); + cursor = + roaring64_arena_pad(cursor, base, alignof(frozen_container_header_t)); + frozen_container_header_t *hdrs = + (frozen_container_header_t *)roaring64_arena_alloc(&cursor, headers); + if (headers_out != NULL) { + *headers_out = hdrs; + } + return r; +} + +static container_t *container_frozen_view_at( + frozen_container_header_t *header, uint8_t typecode, uint32_t elem_count, + const uint64_t **bitsets, const uint16_t **arrays, const rle16_t **runs) { switch (typecode) { case BITSET_CONTAINER_TYPE: { - bitset_container_t *c = (bitset_container_t *)roaring_malloc( - sizeof(bitset_container_t)); + bitset_container_t *c = &header->bitset; c->cardinality = elem_count; c->words = (uint64_t *)*bitsets; *bitsets += BITSET_CONTAINER_SIZE_IN_WORDS; return (container_t *)c; } case ARRAY_CONTAINER_TYPE: { - array_container_t *c = - (array_container_t *)roaring_malloc(sizeof(array_container_t)); + array_container_t *c = &header->array; c->cardinality = elem_count; c->capacity = elem_count; c->array = (uint16_t *)*arrays; @@ -20330,19 +21607,15 @@ static container_t *container_frozen_view(uint8_t typecode, uint32_t elem_count, return (container_t *)c; } case RUN_CONTAINER_TYPE: { - run_container_t *c = - (run_container_t *)roaring_malloc(sizeof(run_container_t)); + run_container_t *c = &header->run; c->n_runs = elem_count; c->capacity = elem_count; c->runs = (rle16_t *)*runs; *runs += elem_count; return (container_t *)c; } - default: { - assert(false); - roaring_unreachable; + default: return NULL; - } } } @@ -20355,38 +21628,43 @@ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, return NULL; } - roaring64_bitmap_t *r = roaring64_bitmap_create(); - - // Flags. - if (maxbytes < sizeof(r->flags)) { - roaring64_bitmap_free(r); + uint8_t flags; + uint64_t capacity; + if (maxbytes < sizeof(flags) + sizeof(capacity)) { return NULL; } - memcpy(&r->flags, buf, sizeof(r->flags)); - buf += sizeof(r->flags); - maxbytes -= sizeof(r->flags); - r->flags |= ROARING_FLAG_FROZEN; - - // Container count. - if (maxbytes < sizeof(r->capacity)) { - roaring64_bitmap_free(r); + memcpy(&flags, buf, sizeof(flags)); + buf += sizeof(flags); + maxbytes -= sizeof(flags); + memcpy(&capacity, buf, sizeof(capacity)); + buf += sizeof(capacity); + maxbytes -= sizeof(capacity); + + // The element counts alone need two bytes per container, so a capacity + // larger than that cannot be satisfied by this buffer. Checked before + // allocating, so a short buffer claiming a huge count cannot make us + // reserve (and clear) an arena sized from attacker-controlled bytes. + if (capacity > maxbytes / sizeof(uint16_t)) { return NULL; } - memcpy(&r->capacity, buf, sizeof(r->capacity)); - buf += sizeof(r->capacity); - maxbytes -= sizeof(r->capacity); - r->containers = - (container_t **)roaring_malloc(r->capacity * sizeof(container_t *)); + frozen_container_header_t *headers = NULL; + roaring64_bitmap_t *r = alloc_frozen_bitmap(capacity, &headers); + if (r == NULL) { + return NULL; + } + // Only flags the format actually defines; the byte comes from the buffer. + r->flags = (uint8_t)(flags & ROARING_FLAG_COW) | ROARING_FLAG_FROZEN | + ROARING_FLAG_FROZEN_ART; // Container element counts. - if (maxbytes < r->capacity * sizeof(uint16_t)) { + if (maxbytes < capacity * sizeof(uint16_t)) { roaring64_bitmap_free(r); return NULL; } const char *elem_counts = buf; - buf += r->capacity * sizeof(uint16_t); - maxbytes -= r->capacity * sizeof(uint16_t); + buf += capacity * sizeof(uint16_t); + maxbytes -= capacity * sizeof(uint16_t); // Total container sizes. uint64_t total_sizes[4]; @@ -20434,6 +21712,10 @@ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, // Deserialize in ART iteration order. art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); for (size_t i = 0; it.value != NULL; ++i) { + if (i >= capacity) { + roaring64_bitmap_free(r); + return NULL; + } leaf_t leaf = (leaf_t)*it.value; uint8_t typecode = get_typecode(leaf); @@ -20444,8 +21726,17 @@ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, // The container index is unrelated to the iteration order. uint64_t index = get_index(leaf); - r->containers[index] = container_frozen_view(typecode, elem_count, - &bitsets, &arrays, &runs); + if (index >= capacity) { + roaring64_bitmap_free(r); + return NULL; + } + container_t *c = container_frozen_view_at( + headers + index, typecode, elem_count, &bitsets, &arrays, &runs); + if (c == NULL) { + roaring64_bitmap_free(r); + return NULL; + } + set_container_at(r, index, c, typecode); art_iterator_next(&it); } @@ -20453,7 +21744,288 @@ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, // Padding to make overall size a multiple of required alignment. buf = CROARING_ALIGN_BUF(buf, CROARING_BITSET_ALIGNMENT); + r->first_free = r->capacity; + return r; +} + +static bool view_one_portable32(roaring64_bitmap_t *r, + frozen_container_header_t *headers, + uint32_t high32, const char *buf, + size_t maxbytes, size_t *consumed) { + *consumed = roaring_bitmap_portable_deserialize_size(buf, maxbytes); + if (*consumed == 0 || *consumed > maxbytes) { + return false; + } + const char *start = buf; + size_t remaining = *consumed; + + uint32_t cookie; + memcpy(&cookie, buf, sizeof(cookie)); + cookie = croaring_letoh32(cookie); + buf += sizeof(cookie); + remaining -= sizeof(cookie); + + int32_t num_containers; + const char *run_flag_bitset = NULL; + bool hasrun = false; + bool has_offsets; + + if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { + if (remaining < sizeof(uint32_t)) { + return false; + } + uint32_t n_le; + memcpy(&n_le, buf, sizeof(n_le)); + num_containers = (int32_t)croaring_letoh32(n_le); + buf += sizeof(uint32_t); + remaining -= sizeof(uint32_t); + has_offsets = true; + } else if ((cookie & 0xFFFF) == SERIAL_COOKIE) { + num_containers = (int32_t)(cookie >> 16) + 1; + hasrun = true; + int32_t run_flag_bitset_size = (num_containers + 7) / 8; + if (num_containers < 0 || remaining < (size_t)run_flag_bitset_size) { + return false; + } + run_flag_bitset = buf; + buf += run_flag_bitset_size; + remaining -= (size_t)run_flag_bitset_size; + has_offsets = num_containers >= NO_OFFSET_THRESHOLD; + } else { + return false; + } + if (num_containers < 0 || num_containers > (1 << 16)) { + return false; + } + + size_t desc_bytes = (size_t)num_containers * 2 * sizeof(uint16_t); + if (remaining < desc_bytes) { + return false; + } + const char *keyscards = buf; + buf += desc_bytes; + remaining -= desc_bytes; + + const char *offset_bytes = NULL; + if (has_offsets) { + size_t off_bytes = (size_t)num_containers * sizeof(uint32_t); + if (remaining < off_bytes) { + return false; + } + offset_bytes = buf; + buf += off_bytes; + remaining -= off_bytes; + } + + int32_t last_key = -1; + uint64_t key_base = ((uint64_t)high32) << 32; + + for (int32_t i = 0; i < num_containers; ++i) { + uint16_t key, card_m1; + memcpy(&key, keyscards + 4 * (size_t)i, sizeof(key)); + key = croaring_letoh16(key); + memcpy(&card_m1, keyscards + 4 * (size_t)i + 2, sizeof(card_m1)); + card_m1 = croaring_letoh16(card_m1); + if ((int32_t)key <= last_key) { + return false; + } + last_key = (int32_t)key; + + uint32_t cardinality = (uint32_t)card_m1 + 1; + bool isbitmap = cardinality > DEFAULT_MAX_SIZE; + bool isrun = false; + if (hasrun && (run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { + isbitmap = false; + isrun = true; + } + + const char *payload; + if (offset_bytes != NULL) { + uint32_t off; + memcpy(&off, offset_bytes + (size_t)i * sizeof(uint32_t), + sizeof(off)); + off = croaring_letoh32(off); + if ((size_t)off >= *consumed) { + return false; + } + payload = start + off; + } else { + payload = buf; + } + + uint8_t typecode; + size_t payload_size; + if (isbitmap) { + typecode = BITSET_CONTAINER_TYPE; + payload_size = BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + } else if (isrun) { + typecode = RUN_CONTAINER_TYPE; + if ((size_t)(payload - start) + sizeof(uint16_t) > *consumed) { + return false; + } + uint16_t n_runs; + memcpy(&n_runs, payload, sizeof(n_runs)); + n_runs = croaring_letoh16(n_runs); + payload_size = sizeof(uint16_t) + (size_t)n_runs * sizeof(rle16_t); + } else { + typecode = ARRAY_CONTAINER_TYPE; + payload_size = (size_t)cardinality * sizeof(uint16_t); + } + if ((size_t)(payload - start) + payload_size > *consumed) { + return false; + } + + if (r->first_free >= r->capacity) { + return false; + } + uint64_t index = allocate_index(r); + frozen_container_header_t *header = headers + index; + container_t *c; + if (isbitmap) { + header->bitset.cardinality = (int32_t)cardinality; + header->bitset.words = (uint64_t *)payload; + c = (container_t *)&header->bitset; + } else if (isrun) { + uint16_t n_runs; + memcpy(&n_runs, payload, sizeof(n_runs)); + n_runs = croaring_letoh16(n_runs); + header->run.n_runs = n_runs; + header->run.capacity = n_runs; + header->run.runs = (rle16_t *)(payload + sizeof(uint16_t)); + c = (container_t *)&header->run; + } else { + header->array.cardinality = (int32_t)cardinality; + header->array.capacity = (int32_t)cardinality; + header->array.array = (uint16_t *)payload; + c = (container_t *)&header->array; + } + set_container_at(r, index, c, typecode); + + uint8_t high48[ART_KEY_BYTES]; + uint64_t high48_bits = key_base | ((uint64_t)key << 16); + split_key(high48_bits, high48); + art_insert(&r->art, high48, (art_val_t)create_leaf(index, typecode)); + + if (offset_bytes == NULL) { + buf += payload_size; + remaining -= payload_size; + } + } + return true; +} + +CROARING_ALLOW_UNALIGNED +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_frozen( + const char *buf, size_t maxbytes) { + if (buf == NULL) { + return NULL; + } +#if CROARING_IS_BIG_ENDIAN + // The portable format is little-endian and this function uses the payload + // bytes where they sit, so there is no correct view of them here. Refuse + // rather than hand back a bitmap that silently reads byte-swapped values. + (void)maxbytes; + return NULL; +#else + size_t remaining = maxbytes; + + if (remaining < sizeof(uint64_t)) { + return NULL; + } + uint64_t buckets; + memcpy(&buckets, buf, sizeof(buckets)); + buckets = croaring_letoh64(buckets); + buf += sizeof(buckets); + remaining -= sizeof(buckets); + if (buckets > UINT32_MAX) { + return NULL; + } + + uint64_t ncontainers = 0; + const char *count_buf = buf; + size_t count_remaining = remaining; + int64_t previous_high32 = -1; + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + if (count_remaining < sizeof(uint32_t)) { + return NULL; + } + uint32_t high32; + memcpy(&high32, count_buf, sizeof(high32)); + high32 = croaring_letoh32(high32); + count_buf += sizeof(high32); + count_remaining -= sizeof(high32); + if (high32 <= previous_high32) { + return NULL; + } + previous_high32 = high32; + size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( + count_buf, count_remaining); + if (bitmap32_size == 0) { + return NULL; + } + uint32_t cookie; + if (count_remaining < sizeof(cookie)) { + return NULL; + } + memcpy(&cookie, count_buf, sizeof(cookie)); + cookie = croaring_letoh32(cookie); + int32_t size; + if ((cookie & 0xFFFF) == SERIAL_COOKIE) { + size = (int32_t)(cookie >> 16) + 1; + } else if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { + if (count_remaining < 2 * sizeof(uint32_t)) { + return NULL; + } + uint32_t size_le; + memcpy(&size_le, count_buf + sizeof(uint32_t), sizeof(size_le)); + size = (int32_t)croaring_letoh32(size_le); + } else { + return NULL; + } + if (size < 0) { + return NULL; + } + ncontainers += (uint64_t)size; + count_buf += bitmap32_size; + count_remaining -= bitmap32_size; + } + + frozen_container_header_t *headers = NULL; + roaring64_bitmap_t *r = alloc_frozen_bitmap(ncontainers, &headers); + if (r == NULL) { + return NULL; + } + // ART is owned; payloads alias buf. + r->flags = ROARING_FLAG_FROZEN; + + previous_high32 = -1; + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + if (remaining < sizeof(uint32_t)) { + roaring64_bitmap_free(r); + return NULL; + } + uint32_t high32; + memcpy(&high32, buf, sizeof(high32)); + high32 = croaring_letoh32(high32); + buf += sizeof(high32); + remaining -= sizeof(high32); + if (high32 <= previous_high32) { + roaring64_bitmap_free(r); + return NULL; + } + previous_high32 = high32; + + size_t consumed = 0; + if (!view_one_portable32(r, headers, high32, buf, remaining, + &consumed)) { + roaring64_bitmap_free(r); + return NULL; + } + buf += consumed; + remaining -= consumed; + } return r; +#endif } bool roaring64_bitmap_iterate(const roaring64_bitmap_t *r, @@ -20512,71 +22084,116 @@ roaring64_iterator_t *roaring64_iterator_copy(const roaring64_iterator_t *it) { void roaring64_iterator_free(roaring64_iterator_t *it) { roaring_free(it); } -bool roaring64_iterator_has_value(const roaring64_iterator_t *it) { - return it->has_value; -} +CROARING_STATIC_ASSERT(offsetof(roaring64_iterator_t, pub) == 0, + "the public members must be first in the iterator"); -uint64_t roaring64_iterator_value(const roaring64_iterator_t *it) { - return it->value; +extern inline bool roaring64_iterator_has_value(const roaring64_iterator_t *it); +extern inline uint64_t roaring64_iterator_value(const roaring64_iterator_t *it); + +// The current container is exhausted: step the ART to the next leaf. +static inline bool roaring64_iterator_next_leaf(roaring64_iterator_t *it) { + if (art_iterator_next(&it->art_it)) { + return roaring64_iterator_init_at_leaf_first(it); + } + it->saturated_forward = true; + it->fast_type = 0; + return (it->pub.has_value = false); } -bool roaring64_iterator_advance(roaring64_iterator_t *it) { +// Everything `advance` needs when the bitset cache does not describe where the +// iterator is: a restart, or a step through the container iterator. +static inline bool roaring64_iterator_advance_slow(roaring64_iterator_t *it) { if (it->art_it.value == NULL) { if (it->saturated_forward) { - return (it->has_value = false); + return (it->pub.has_value = false); } roaring64_iterator_init_at(it->r, it, /*first=*/true); - return it->has_value; + return it->pub.has_value; } leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; - if (container_iterator_next(get_container(it->r, leaf), get_typecode(leaf), + uint8_t typecode = get_typecode(leaf); + uint16_t low16 = (uint16_t)it->pub.value; + if (container_iterator_next(get_container(it->r, leaf), typecode, &it->container_it, &low16)) { - it->value = it->high48 | low16; - return (it->has_value = true); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + if (typecode == BITSET_CONTAINER_TYPE) { + // Only reached when the cache was stale, i.e. something else moved + // the cursor. Array and run containers never have a cache, and + // priming them would cost a leaf load per value. + roaring64_iterator_prime(it); + } + return true; } - if (art_iterator_next(&it->art_it)) { - return roaring64_iterator_init_at_leaf_first(it); + return roaring64_iterator_next_leaf(it); +} + +bool roaring64_iterator_advance(roaring64_iterator_t *it) { + // Matching both the leaf and `container_it.index` is enough to know the + // cache describes where the iterator actually is: within a container the + // index determines the position. Anything that moved the cursor -- + // `previous`, `move_equalorlarger`, any of the `read` variants -- moved one + // of them, and drops through to the slow path, which rebuilds the cache. + if (it->fast_type != 0 && it->fast_art_value == it->art_it.value && + it->fast_index == it->container_it.index) { + uint64_t word = it->fast_word; + uint32_t wi = it->fast_wordindex; + while (word == 0) { + if (++wi >= BITSET_CONTAINER_SIZE_IN_WORDS) { + return roaring64_iterator_next_leaf(it); + } + word = it->fast_words[wi]; + it->fast_wordindex = wi; + } + uint32_t index = wi * 64 + (uint32_t)roaring_trailing_zeroes(word); + it->fast_word = word & (word - 1); + it->fast_index = (int32_t)index; + it->container_it.index = (int32_t)index; + it->pub.value = it->high48 | index; + return (it->pub.has_value = true); } - it->saturated_forward = true; - return (it->has_value = false); + return roaring64_iterator_advance_slow(it); } bool roaring64_iterator_previous(roaring64_iterator_t *it) { if (it->art_it.value == NULL) { if (!it->saturated_forward) { // Saturated backward. - return (it->has_value = false); + return (it->pub.has_value = false); } roaring64_iterator_init_at(it->r, it, /*first=*/false); - return it->has_value; + return it->pub.has_value; } leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; + uint16_t low16 = (uint16_t)it->pub.value; if (container_iterator_prev(get_container(it->r, leaf), get_typecode(leaf), &it->container_it, &low16)) { - it->value = it->high48 | low16; - return (it->has_value = true); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + it->fast_type = 0; + return true; } if (art_iterator_prev(&it->art_it)) { return roaring64_iterator_init_at_leaf_last(it); } it->saturated_forward = false; // Saturated backward. - return (it->has_value = false); + it->fast_type = 0; + return (it->pub.has_value = false); } bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, uint64_t val) { uint8_t val_high48[ART_KEY_BYTES]; uint16_t val_low16 = split_key(val, val_high48); - if (!it->has_value || it->high48 != (val & 0xFFFFFFFFFFFF0000)) { + if (!it->pub.has_value || it->high48 != (val & 0xFFFFFFFFFFFF0000)) { // The ART iterator is before or after the high48 bits of `val` (or // beyond the ART altogether), so we need to move to a leaf with a // key equal or greater. if (!art_iterator_lower_bound(&it->art_it, val_high48)) { // Only smaller keys found. it->saturated_forward = true; - return (it->has_value = false); + it->fast_type = 0; + return (it->pub.has_value = false); } it->high48 = combine_key(it->art_it.key, 0); // Fall through to the next if statement. @@ -20586,17 +22203,20 @@ bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, // We're at equal high bits, check if a suitable value can be found // in this container. leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; + uint16_t low16 = (uint16_t)it->pub.value; if (container_iterator_lower_bound( get_container(it->r, leaf), get_typecode(leaf), &it->container_it, &low16, val_low16)) { - it->value = it->high48 | low16; - return (it->has_value = true); + it->pub.value = it->high48 | low16; + it->pub.has_value = true; + it->fast_type = 0; + return true; } // Only smaller entries in this container, move to the next. if (!art_iterator_next(&it->art_it)) { it->saturated_forward = true; - return (it->has_value = false); + it->fast_type = 0; + return (it->pub.has_value = false); } } @@ -20608,10 +22228,10 @@ bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, uint64_t count) { uint64_t consumed = 0; - while (it->has_value && consumed < count) { + while (it->pub.has_value && consumed < count) { uint32_t container_consumed; leaf_t leaf = (leaf_t)*it->art_it.value; - uint16_t low16 = (uint16_t)it->value; + uint16_t low16 = (uint16_t)it->pub.value; uint32_t container_count = UINT32_MAX; if (count - consumed < (uint64_t)UINT32_MAX) { container_count = count - consumed; @@ -20622,21 +22242,132 @@ uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, consumed += container_consumed; buf += container_consumed; if (has_value) { - it->has_value = true; - it->value = it->high48 | low16; + it->pub.has_value = true; + it->pub.value = it->high48 | low16; + it->fast_type = 0; assert(consumed == count); return consumed; } - it->has_value = art_iterator_next(&it->art_it); - if (it->has_value) { + it->pub.has_value = art_iterator_next(&it->art_it); + if (it->pub.has_value) { roaring64_iterator_init_at_leaf_first(it); } else { it->saturated_forward = true; + it->fast_type = 0; + } + } + return consumed; +} + +uint64_t roaring64_iterator_read_backward(roaring64_iterator_t *it, + uint64_t *buf, uint64_t count) { + uint64_t consumed = 0; + while (it->pub.has_value && consumed < count) { + uint32_t container_consumed; + leaf_t leaf = *it->art_it.value; + uint16_t low16 = (uint16_t)it->pub.value; + uint32_t container_count = UINT32_MAX; + if (count - consumed < (uint64_t)UINT32_MAX) { + container_count = count - consumed; + } + bool has_value = container_iterator_read_backward_into_uint64( + get_container(it->r, leaf), get_typecode(leaf), &it->container_it, + it->high48, buf, container_count, &container_consumed, &low16); + consumed += container_consumed; + buf += container_consumed; + if (has_value) { + it->pub.has_value = true; + it->pub.value = it->high48 | low16; + assert(consumed == count); + return consumed; + } + it->pub.has_value = art_iterator_prev(&it->art_it); + if (it->pub.has_value) { + roaring64_iterator_init_at_leaf_last(it); + } else { + it->saturated_forward = false; } } return consumed; } +size_t roaring64_iterator_read_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->pub.has_value && ret < count) { + buf[ret].min = it->pub.value; + for (;;) { + uint16_t low16 = (uint16_t)it->pub.value; + leaf_t leaf = (leaf_t)*it->art_it.value; + bool container_has_more; + uint16_t run_end_low16 = container_iterator_find_run_end( + get_container(it->r, leaf), get_typecode(leaf), + &it->container_it, &low16, &container_has_more); + buf[ret].max = it->high48 | run_end_low16; + + if (container_has_more) { + it->pub.value = it->high48 | low16; + break; + } + // Move to next leaf + it->pub.has_value = art_iterator_next(&it->art_it); + if (it->pub.has_value) { + roaring64_iterator_init_at_leaf_first(it); + } else { + it->saturated_forward = true; + break; + } + // Continue merging only if the run reached the container + // boundary and the next leaf starts exactly at max+1. + if (run_end_low16 != UINT16_MAX || + it->pub.value != buf[ret].max + 1) { + break; + } + } + ret++; + } + return ret; +} + +size_t roaring64_iterator_read_prev_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count) { + size_t ret = 0; + while (it->pub.has_value && ret < count) { + buf[ret].max = it->pub.value; + for (;;) { + uint16_t low16 = (uint16_t)it->pub.value; + leaf_t leaf = (leaf_t)*it->art_it.value; + bool container_has_more; + uint16_t run_start_low16 = container_iterator_find_run_start( + get_container(it->r, leaf), get_typecode(leaf), + &it->container_it, &low16, &container_has_more); + buf[ret].min = it->high48 | run_start_low16; + + if (container_has_more) { + it->pub.value = it->high48 | low16; + break; + } + // Move to previous leaf + it->pub.has_value = art_iterator_prev(&it->art_it); + if (it->pub.has_value) { + roaring64_iterator_init_at_leaf_last(it); + } else { + it->saturated_forward = false; + break; + } + // Continue merging only if the run reached the container + // boundary and the previous leaf ends exactly at min-1. + if (run_start_low16 != 0 || it->pub.value != buf[ret].min - 1) { + break; + } + } + ret++; + } + return ret; +} + #ifdef __cplusplus } // extern "C" } // namespace roaring diff --git a/roaring.h b/roaring.h index 62ce1a8..b5aa180 100644 --- a/roaring.h +++ b/roaring.h @@ -1,5 +1,5 @@ // !!! DO NOT EDIT - THIS IS AN AUTO-GENERATED FILE !!! -// Created by amalgamation.sh on 2025-12-05T04:42:38Z +// Created by amalgamation.sh on 2026-08-22T02:17:56Z /* * The CRoaring project is under a dual license (Apache/MIT). @@ -59,10 +59,10 @@ // /include/roaring/roaring_version.h automatically generated by release.py, do not change by hand #ifndef ROARING_INCLUDE_ROARING_VERSION #define ROARING_INCLUDE_ROARING_VERSION -#define ROARING_VERSION "4.5.0" +#define ROARING_VERSION "5.1.0" enum { - ROARING_VERSION_MAJOR = 4, - ROARING_VERSION_MINOR = 5, + ROARING_VERSION_MAJOR = 5, + ROARING_VERSION_MINOR = 1, ROARING_VERSION_REVISION = 0 }; #endif // ROARING_INCLUDE_ROARING_VERSION @@ -71,12 +71,21 @@ enum { /* * portability.h * + * This header centralizes compiler-, platform-, and architecture-specific + * portability definitions used throughout CRoaring. It provides feature + * detection, calling-convention and attribute macros, intrinsic and inline + * assembly enablement, endianness helpers, alignment annotations, atomic + * reference-count support, and other low-level compatibility glue. + * + * The goal is to keep these conditional definitions in one place so the rest + * of the codebase can rely on a more uniform interface across GCC, Clang, + * MSVC, x86/x64, ARM/NEON, and other supported environments. */ /** * All macros should be prefixed with either CROARING or ROARING. * The library uses both ROARING_... - * as well as CROAIRING_ as prefixes. The ROARING_ prefix is for + * as well as CROARING_ as prefixes. The ROARING_ prefix is for * macros that are provided by the build system or that are closely * related to the format. The header macros may also use ROARING_. * The CROARING_ prefix is for internal macros that a user is unlikely @@ -125,6 +134,12 @@ enum { #ifdef __GLIBC__ #include // this should never be needed but there are some reports that it is needed. #endif +// alignas/alignof are keywords in C++ and in C23+, where is +// deprecated. Only include it for C11..C17. +#if !defined(__cplusplus) && \ + (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 202311L) +#include +#endif #ifdef __cplusplus extern "C" { // portability definitions are in global scope, not a namespace @@ -140,7 +155,14 @@ extern "C" { // portability definitions are in global scope, not a namespace #endif // __restrict__ #endif // CROARING_REGULAR_VISUAL_STUDIO -#if defined(__x86_64__) || defined(_M_X64) +#if defined(__riscv) || defined(_M_RISCV32) || defined(_M_RISCV64) +#define CROARING_IS_RISCV 1 + +#if (defined(__riscv_xlen) && (__riscv_xlen == 64)) || defined(_M_RISCV64) +#define CROARING_IS_RISCV64 1 +#endif + +#elif defined(__x86_64__) || defined(_M_X64) // we have an x64 processor #define CROARING_IS_X64 1 @@ -225,7 +247,7 @@ extern "C" { // portability definitions are in global scope, not a namespace #define CROARING_IS_E2K 1 #endif -#if !CROARING_REGULAR_VISUAL_STUDIO && !defined(CROARING_IS_E2K) +#if !CROARING_REGULAR_VISUAL_STUDIO && !defined(CROARING_IS_E2K) && !__FILC__ /* Non-Microsoft C/C++-compatible compiler, assumes that it supports inline * assembly */ #define CROARING_INLINE_ASM 1 @@ -523,6 +545,55 @@ static inline int roaring_hamming(uint64_t x) { #define croaring_be64toh(x) croaring_htobe64(x) // End of host <-> big endian conversion. +// Host <-> little-endian conversion helpers. +// +// The CRoaring "portable" serialization format (and the regular +// roaring_bitmap_serialize / Roaring64Map::write formats which build on it) +// is defined to be little-endian on the wire. Code that reads or writes +// multi-byte integers to such buffers must convert between host and +// little-endian byte order. On little-endian hosts these are no-ops; on +// big-endian hosts they swap bytes. +// +// The "frozen" format is intentionally non-portable and uses native byte +// order; it must not use these helpers. +#if CROARING_IS_BIG_ENDIAN + +static inline uint16_t croaring_bswap16(uint16_t x) { + return (uint16_t)((x << 8) | (x >> 8)); +} + +static inline uint32_t croaring_bswap32(uint32_t x) { + return ((x & 0x000000FFU) << 24) | ((x & 0x0000FF00U) << 8) | + ((x & 0x00FF0000U) >> 8) | ((x & 0xFF000000U) >> 24); +} + +static inline uint64_t croaring_bswap64(uint64_t x) { + return ((x & 0x00000000000000FFULL) << 56) | + ((x & 0x000000000000FF00ULL) << 40) | + ((x & 0x0000000000FF0000ULL) << 24) | + ((x & 0x00000000FF000000ULL) << 8) | + ((x & 0x000000FF00000000ULL) >> 8) | + ((x & 0x0000FF0000000000ULL) >> 24) | + ((x & 0x00FF000000000000ULL) >> 40) | + ((x & 0xFF00000000000000ULL) >> 56); +} + +#define croaring_htole16(x) croaring_bswap16(x) +#define croaring_htole32(x) croaring_bswap32(x) +#define croaring_htole64(x) croaring_bswap64(x) + +#else // CROARING_IS_BIG_ENDIAN + +#define croaring_htole16(x) (x) +#define croaring_htole32(x) (x) +#define croaring_htole64(x) (x) + +#endif // CROARING_IS_BIG_ENDIAN + +#define croaring_letoh16(x) croaring_htole16(x) +#define croaring_letoh32(x) croaring_htole32(x) +#define croaring_letoh64(x) croaring_htole64(x) + // Defines for the possible CROARING atomic implementations #define CROARING_ATOMIC_IMPL_NONE 1 #define CROARING_ATOMIC_IMPL_CPP 2 @@ -538,13 +609,13 @@ static inline int roaring_hamming(uint64_t x) { #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP #endif //__has_include() #else - // We lack __has_include to check: +// We lack __has_include to check: #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP #endif //__has_include #elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C #elif CROARING_REGULAR_VISUAL_STUDIO - // https://www.technetworkhub.com/c11-atomics-in-visual-studio-2022-version-17/ +// https://www.technetworkhub.com/c11-atomics-in-visual-studio-2022-version-17/ #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C_WINDOWS #endif #endif // !defined(CROARING_ATOMIC_IMPL) @@ -634,7 +705,7 @@ static inline void croaring_refcount_inc(croaring_refcount_t *val) { static inline bool croaring_refcount_dec(croaring_refcount_t *val) { assert(*val > 0); *val -= 1; - return val == 0; + return *val == 0; } static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { @@ -678,9 +749,22 @@ static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { #endif /* INCLUDE_PORTABILITY_H_ */ /* end file include/roaring/portability.h */ /* begin file include/roaring/isadetection.h */ +/* + * isadetection.h + * + * This header declares the small interface used to detect instruction-set + * capabilities relevant to CRoaring's optimized kernels on x64 platforms. It + * also defines compile-time feature macros that indicate whether the compiler + * toolchain is capable of building AVX-512 code paths. + * + * The resulting flags are used to decide whether accelerated implementations, + * such as AVX2 or AVX-512 variants, can be selected safely at runtime. + */ #ifndef ROARING_ISADETECTION_H #define ROARING_ISADETECTION_H -#if defined(__x86_64__) || defined(_M_AMD64) // x64 + + +#if CROARING_IS_X64 // x64 #ifndef CROARING_COMPILER_SUPPORTS_AVX512 #ifdef __has_include @@ -718,12 +802,18 @@ int croaring_hardware_support(void); } } // extern "C" { namespace roaring { namespace internal { #endif -#endif // x64 +#endif // CROARING_IS_X64 #endif // ROARING_ISADETECTION_H /* end file include/roaring/isadetection.h */ /* begin file include/roaring/roaring_types.h */ /* - Typedefs used by various components + Shared type definitions used across the CRoaring public API and internal + components. + + This file centralizes common typedefs and small structs that are referenced + by multiple headers, including iterator callback signatures, roaring array + metadata, statistics structures, and compatibility types used to bridge the + C and C++ builds. */ #ifndef ROARING_TYPES_H @@ -762,6 +852,8 @@ struct container_s {}; #define ROARING_FLAG_COW UINT8_C(0x1) #define ROARING_FLAG_FROZEN UINT8_C(0x2) +// 64-bit only: ART node arrays alias a frozen buffer (do not art_free). +#define ROARING_FLAG_FROZEN_ART UINT8_C(0x4) /** * Roaring arrays are array-based key-value pairs having containers as values @@ -876,17 +968,31 @@ typedef struct roaring_container_iterator_s { #endif /* ROARING_TYPES_H */ /* end file include/roaring/roaring_types.h */ /* begin file include/roaring/bitset/bitset.h */ +/* + * bitset.h + * + * This bitset is a general-purpose dynamic bitmap storing bits in a contiguous + * array of 64-bit words. The array field points to the word buffer, arraysize + * records how many words are currently in use, and capacity records how many + * words are allocated. + * + * Unlike the fixed 16-bit-domain container bitset, this structure can grow to + * cover an arbitrary number of bit positions. It is useful when callers need a + * resizable bitmap with efficient bitwise operations, scans, and shifts over a + * larger or runtime-defined domain. + */ #ifndef CROARING_CBITSET_BITSET_H #define CROARING_CBITSET_BITSET_H // For compatibility with MSVC with the use of `restrict` -#if (__STDC_VERSION__ >= 199901L) || \ +#ifdef __cplusplus +#define CROARING_CBITSET_RESTRICT +#elif (__STDC_VERSION__ >= 199901L) || \ (defined(__GNUC__) && defined(__STDC_VERSION__)) #define CROARING_CBITSET_RESTRICT restrict #else #define CROARING_CBITSET_RESTRICT -#endif // (__STDC_VERSION__ >= 199901L) || (defined(__GNUC__) && - // defined(__STDC_VERSION__ )) +#endif #include #include @@ -1275,6 +1381,18 @@ PPDerived movable_CAST_HELPER(Base **ptr_to_ptr) { #endif /* INCLUDE_CONTAINERS_CONTAINER_DEFS_H_ */ /* end file include/roaring/containers/container_defs.h */ /* begin file include/roaring/array_util.h */ +/* + * array_util.h + * + * This header provides low-level utility routines for sorted arrays of + * 16-bit integers, which are used heavily by CRoaring's array-based + * containers and set-operation kernels. It includes search helpers, counting + * helpers, and array intersection/difference primitives. + * + * Some of the routines also have SIMD-accelerated implementations on supported + * platforms, allowing efficient operations on sorted integer arrays that form + * the basis of sparse container processing. + */ #ifndef CROARING_ARRAY_UTIL_H #define CROARING_ARRAY_UTIL_H @@ -1299,29 +1417,95 @@ namespace internal { #endif /* - * Good old binary search. + * Sorted-array search. * Assumes that array is sorted, has logarithmic complexity. * if the result is x, then: * if ( x>0 ) you have array[x] = ikey * if ( x<0 ) then inserting ikey at position -x-1 in array (insuring that * array[-x-1]=ikey) keys the array sorted. + * + * Adapted from array_container_contains: a SIMD-quad block-narrowing + * search at gap=16 (Daniel Lemire, + * https://lemire.me/blog/2026/04/27/you-can-beat-the-binary-search/) + * followed by a scalar in-block scan that recovers the exact insertion + * point required by the binarySearch contract. */ inline int32_t binarySearch(const uint16_t *array, int32_t lenarray, uint16_t ikey) { - int32_t low = 0; - int32_t high = lenarray - 1; - while (low <= high) { - int32_t middleIndex = (low + high) >> 1; - uint16_t middleValue = array[middleIndex]; - if (middleValue < ikey) { - low = middleIndex + 1; - } else if (middleValue > ikey) { - high = middleIndex - 1; - } else { - return middleIndex; + const int32_t gap = 16; + if (lenarray < gap) { + for (int32_t j = 0; j < lenarray; j++) { + if (array[j] >= ikey) { + return (array[j] == ikey) ? j : -(j + 1); + } + } + return -(lenarray + 1); + } + const int32_t num_blocks = lenarray / gap; + int32_t base = 0; + int32_t n = num_blocks; + while (n > 3) { + int32_t quarter = n >> 2; + + int32_t k1 = array[(base + quarter + 1) * gap - 1]; + int32_t k2 = array[(base + 2 * quarter + 1) * gap - 1]; + int32_t k3 = array[(base + 3 * quarter + 1) * gap - 1]; + + int32_t c1 = (k1 < ikey); + int32_t c2 = (k2 < ikey); + int32_t c3 = (k3 < ikey); + + base += (c1 + c2 + c3) * quarter; + n -= 3 * quarter; + } + while (n > 1) { + int32_t half = n >> 1; + base = (array[(base + half + 1) * gap - 1] < ikey) ? base + half : base; + n -= half; + } + int32_t lo = (array[(base + 1) * gap - 1] < ikey) ? base + 1 : base; + + if (lo < num_blocks) { + const int32_t start = lo * gap; +#if defined(CROARING_IS_X64) + // SSE2: subs_epu16 yields zero where lane >= ikey. movemask of an + // epi16 compare gives 2 bits per lane; ctz>>1 = lane index. Scan + // the first 8 lanes first and exit early when they contain the + // answer; otherwise the block-narrowing invariant guarantees the + // second-half mask is non-zero. + __m128i needle = _mm_set1_epi16((short)ikey); + __m128i zero = _mm_setzero_si128(); + __m128i v0 = _mm_loadu_si128((const __m128i *)(array + start)); + __m128i ge0 = _mm_cmpeq_epi16(_mm_subs_epu16(needle, v0), zero); + unsigned m0 = (unsigned)_mm_movemask_epi8(ge0); + if (m0 != 0) { + int32_t j = start + (int32_t)(roaring_trailing_zeroes(m0) >> 1); + return (array[j] == ikey) ? j : -(j + 1); + } + __m128i v1 = _mm_loadu_si128((const __m128i *)(array + start + 8)); + __m128i ge1 = _mm_cmpeq_epi16(_mm_subs_epu16(needle, v1), zero); + unsigned m1 = (unsigned)_mm_movemask_epi8(ge1); + int32_t j = start + 8 + (int32_t)(roaring_trailing_zeroes(m1) >> 1); + return (array[j] == ikey) ? j : -(j + 1); +#else + const int32_t end = start + gap; + for (int32_t j = start; j < end; j++) { + if (array[j] >= ikey) { + return (array[j] == ikey) ? j : -(j + 1); + } } + // Unreachable: the narrowing guarantees the last element of the + // selected block is >= ikey. + return -(end + 1); +#endif } - return -(low + 1); + + for (int32_t j = num_blocks * gap; j < lenarray; j++) { + if (array[j] >= ikey) { + return (array[j] == ikey) ? j : -(j + 1); + } + } + return -(lenarray + 1); } /** @@ -1409,12 +1593,11 @@ static inline int32_t count_greater(const uint16_t *array, int32_t lenarray, * C should have capacity greater than the minimum of s_1 and s_b + 8 * where 8 is sizeof(__m128i)/sizeof(uint16_t). */ -int32_t intersect_vector16(const uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b, - uint16_t *C); +int32_t intersect_vector16(const uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b, uint16_t *C); -int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b); +int32_t intersect_vector16_inplace(uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b); /** * Take an array container and write it out to a 32-bit array, using base @@ -1429,10 +1612,8 @@ int avx512_array_container_to_uint32_array(void *vout, const uint16_t *array, /** * Compute the cardinality of the intersection using SSE4 instructions */ -int32_t intersect_vector16_cardinality(const uint16_t *__restrict__ A, - size_t s_a, - const uint16_t *__restrict__ B, - size_t s_b); +int32_t intersect_vector16_cardinality(const uint16_t *A, size_t s_a, + const uint16_t *B, size_t s_b); /* Computes the intersection between one small and one large set of uint16_t. * Stores the result into buffer and return the number of elements. */ @@ -1507,22 +1688,33 @@ size_t union_uint32(const uint32_t *set_1, size_t size_1, const uint32_t *set_2, /** * A fast SSE-based union function. */ -uint32_t union_vector16(const uint16_t *__restrict__ set_1, uint32_t size_1, - const uint16_t *__restrict__ set_2, uint32_t size_2, - uint16_t *__restrict__ buffer); +uint32_t union_vector16(const uint16_t *set_1, uint32_t size_1, + const uint16_t *set_2, uint32_t size_2, + uint16_t *buffer); + +#if CROARING_COMPILER_SUPPORTS_AVX512 +/** + * AVX-512 union of two sorted uint16 arrays, using a Batcher bitonic merge + * network over 32 lanes. Same contract as union_vector16: `buffer` must have + * room for size_1 + size_2 values. + */ +uint32_t avx512_union_uint16(const uint16_t *set_1, uint32_t size_1, + const uint16_t *set_2, uint32_t size_2, + uint16_t *buffer); +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + /** * A fast SSE-based XOR function. */ -uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, - const uint16_t *__restrict__ array2, uint32_t length2, - uint16_t *__restrict__ output); +uint32_t xor_vector16(const uint16_t *array1, uint32_t length1, + const uint16_t *array2, uint32_t length2, + uint16_t *output); /** * A fast SSE-based difference function. */ -int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, - const uint16_t *__restrict__ B, size_t s_b, - uint16_t *C); +int32_t difference_vector16(const uint16_t *A, size_t s_a, const uint16_t *B, + size_t s_b, uint16_t *C); /** * Generic union function, returns just the cardinality. @@ -1550,6 +1742,19 @@ bool memequals(const void *s1, const void *s2, size_t n); #endif /* end file include/roaring/array_util.h */ /* begin file include/roaring/bitset_util.h */ +/* + * bitset_util.h + * + * This header collects low-level utility functions for operating on raw + * bitsets represented as arrays of 64-bit words. It includes helpers for + * setting, clearing, flipping, counting, extracting, and combining bit ranges, + * along with architecture-specific SIMD implementations for performance- + * critical routines. + * + * In particular, this file contains accelerated extraction and population- + * count code paths for x64 targets using AVX2 and, when supported by the + * compiler and hardware, AVX-512. + */ #ifndef CROARING_BITSET_UTIL_H #define CROARING_BITSET_UTIL_H @@ -2271,6 +2476,14 @@ CROARING_UNTARGET_AVX512 /* * array.h * + * Array containers store a sparse set of 16-bit integers as a sorted dynamic + * array. The cardinality field tracks how many values are present, capacity + * tracks the allocated length, and array points to the sorted values. + * + * This representation is used for low-cardinality containers because it is + * compact and supports fast iteration and binary-search-based membership + * tests. When the number of stored values grows beyond DEFAULT_MAX_SIZE, + * CRoaring will typically switch to a denser container representation. */ #ifndef INCLUDE_CONTAINERS_ARRAY_H_ @@ -2599,30 +2812,72 @@ static inline bool array_container_remove(array_container_t *arr, /* Check whether x is present. */ inline bool array_container_contains(const array_container_t *arr, uint16_t pos) { - // return binarySearch(arr->array, arr->cardinality, pos) >= 0; - // binary search with fallback to linear search for short ranges - int32_t low = 0; - const uint16_t *carr = (const uint16_t *)arr->array; - int32_t high = arr->cardinality - 1; - // while (high - low >= 0) { - while (high >= low + 16) { - int32_t middleIndex = (low + high) >> 1; - uint16_t middleValue = carr[middleIndex]; - if (middleValue < pos) { - low = middleIndex + 1; - } else if (middleValue > pos) { - high = middleIndex - 1; - } else { - return true; + /** + * SIMD Quad algorithm + * Daniel Lemire, "You can beat the binary search," in Daniel Lemire's blog, + * April 27, 2026, + * https://lemire.me/blog/2026/04/27/you-can-beat-the-binary-search/. + */ + const int32_t gap = 16; + const uint16_t *carr = arr->array; + int32_t cardinality = arr->cardinality; + if (cardinality < gap) { + for (int32_t j = 0; j < cardinality; j++) { + if (carr[j] >= pos) return carr[j] == pos; } + return false; } + int32_t num_blocks = cardinality / gap; + int32_t base = 0; + int32_t n = num_blocks; + while (n > 3) { + int32_t quarter = n >> 2; - for (int i = low; i <= high; i++) { - uint16_t v = carr[i]; - if (v == pos) { - return true; + int32_t k1 = carr[(base + quarter + 1) * gap - 1]; + int32_t k2 = carr[(base + 2 * quarter + 1) * gap - 1]; + int32_t k3 = carr[(base + 3 * quarter + 1) * gap - 1]; + + int32_t c1 = (k1 < pos); + int32_t c2 = (k2 < pos); + int32_t c3 = (k3 < pos); + + base += (c1 + c2 + c3) * quarter; + n -= 3 * quarter; + } + while (n > 1) { + int32_t half = n >> 1; + base = (carr[(base + half + 1) * gap - 1] < pos) ? base + half : base; + n -= half; + } + int32_t lo = (carr[(base + 1) * gap - 1] < pos) ? base + 1 : base; + + if (lo < num_blocks) { + const uint16_t *blk = carr + lo * gap; +#ifdef CROARING_USENEON + uint16x8_t needle = vdupq_n_u16(pos); + uint16x8_t v0 = vld1q_u16(blk); + uint16x8_t v1 = vld1q_u16(blk + 8); + uint16x8_t hit = + vorrq_u16(vceqq_u16(v0, needle), vceqq_u16(v1, needle)); + return vmaxvq_u16(hit) != 0; +#elif defined(CROARING_IS_X64) + __m128i needle = _mm_set1_epi16((short)pos); + __m128i v0 = _mm_loadu_si128((const __m128i *)blk); + __m128i v1 = _mm_loadu_si128((const __m128i *)(blk + 8)); + __m128i hit = _mm_or_si128(_mm_cmpeq_epi16(v0, needle), + _mm_cmpeq_epi16(v1, needle)); + return _mm_movemask_epi8(hit) != 0; +#else + for (int32_t j = 0; j < gap; j++) { + if (blk[j] >= pos) return blk[j] == pos; } - if (v > pos) return false; + return false; +#endif + } + + for (int32_t j = num_blocks * gap; j < cardinality; j++) { + uint16_t v = carr[j]; + if (v >= pos) return (v == pos); } return false; } @@ -2790,6 +3045,15 @@ static inline void array_container_remove_range(array_container_t *array, /* * bitset.h * + * Bitset containers store a set of 16-bit integers as a fixed-size bitmap. + * The words pointer references an array of 64-bit words covering the full + * 16-bit domain, with one bit per possible value. The cardinality field tracks + * the number of set bits; when it is BITSET_UNKNOWN_CARDINALITY, the count must + * be recomputed from the bitmap contents. + * + * This representation is used for denser containers because membership tests, + * set operations, and sequential scans can be implemented efficiently with + * word-level bitwise operations. */ #ifndef INCLUDE_CONTAINERS_BITSET_H_ @@ -2831,6 +3095,10 @@ typedef struct bitset_container_s bitset_container_t; /* Create a new bitset. Return NULL in case of failure. */ bitset_container_t *bitset_container_create(void); +/* Create a bitset without zeroing the words. Caller must overwrite `words` + * before the container is used. Return NULL in case of failure. */ +bitset_container_t *bitset_container_create_uninitialized(void); + /* Free memory. */ void bitset_container_free(bitset_container_t *bitset); @@ -3301,6 +3569,14 @@ int bitset_container_index_equalorlarger(const bitset_container_t *container, /* * run.h * + * Run containers store a set of 16-bit integers as a sorted array of + * non-overlapping runs. Each run is represented by a starting value and a + * length, encoding one contiguous interval of present integers. + * + * This representation is effective when the data contains long consecutive + * ranges because it compresses many adjacent values into a small number of + * run records while still supporting search and set operations over the + * interval list. */ #ifndef INCLUDE_CONTAINERS_RUN_H_ @@ -4017,6 +4293,15 @@ static inline void run_container_remove_range(run_container_t *run, /* * convert.h * + * This header declares conversion helpers between the different Roaring + * container representations: array, bitset, and run containers. These + * routines are used when an operation produces data better represented in a + * different form, or when the library wants to switch to the most space- + * efficient container type. + * + * In addition to direct conversions, the file also provides helpers that + * choose between candidate result representations based on cardinality and + * storage efficiency. */ #ifndef INCLUDE_CONTAINERS_CONVERT_H_ @@ -4130,6 +4415,12 @@ bool run_container_equals_bitset(const run_container_t* container1, /* * mixed_subset.h * + * This header declares subset-checking routines between different Roaring + * container types. These helpers are used when two containers do not share the + * same representation and a direct type-specific subset predicate is needed. + * + * Each function answers whether all values from one container are contained in + * another, across combinations of array, bitset, and run containers. */ #ifndef CONTAINERS_MIXED_SUBSET_H_ @@ -4183,6 +4474,16 @@ bool bitset_container_is_subset_run(const bitset_container_t* container1, /* begin file include/roaring/containers/mixed_andnot.h */ /* * mixed_andnot.h + * + * This header declares mixed-container difference operations of the form + * `A \ B` (also called andnot) between Roaring container types such as array, + * bitset, and run containers. These helpers are used when the operands have + * different internal representations and the result may need to change + * representation depending on density. + * + * The file includes both allocating and inplace-oriented variants so callers + * can either materialize a fresh result or reuse storage when that is + * efficient and semantically allowed. */ #ifndef INCLUDE_CONTAINERS_MIXED_ANDNOT_H_ #define INCLUDE_CONTAINERS_MIXED_ANDNOT_H_ @@ -4364,6 +4665,15 @@ bool bitset_bitset_container_iandnot(bitset_container_t *src_1, /* * mixed_intersection.h * + * This header declares intersection operations between different Roaring + * container types, such as array, bitset, and run containers. These mixed + * routines are used when the input containers have different internal + * representations and the implementation must choose the appropriate result + * form based on the data. + * + * In addition to materializing intersections, the file also provides helpers + * for intersection cardinality, intersection predicates, and selected inplace + * variants. */ #ifndef INCLUDE_CONTAINERS_MIXED_INTERSECTION_H_ @@ -4464,6 +4774,15 @@ bool bitset_bitset_container_intersection_inplace( /* * mixed_negation.h * + * This header declares negation (complement) operations for Roaring + * containers, both over the full 16-bit container domain and over specified + * subranges. Depending on the input representation and the density of the + * complement, the result may need to switch between array, bitset, and run + * containers. + * + * The file includes both allocating and inplace-oriented variants so callers + * can choose between simple result construction and reuse of an existing + * container when that is practical. */ #ifndef INCLUDE_CONTAINERS_MIXED_NEGATION_H_ @@ -4601,8 +4920,16 @@ int run_container_negation_range_inplace(run_container_t *src, /* end file include/roaring/containers/mixed_negation.h */ /* begin file include/roaring/containers/mixed_union.h */ /* - * mixed_intersection.h + * mixed_union.h * + * This header declares union operations between different Roaring container + * types, such as array, bitset, and run containers. These mixed-operation + * helpers are used when the two input containers do not share the same + * representation and the result may need to stay in, or be converted to, a + * representation chosen according to the data. + * + * The file includes regular, lazy, and inplace variants so callers can select + * between fully maintained results and faster deferred-maintenance paths. */ #ifndef INCLUDE_CONTAINERS_MIXED_UNION_H_ @@ -4719,6 +5046,14 @@ void run_bitset_container_lazy_union(const run_container_t *src_1, /* * mixed_xor.h * + * This header declares XOR operations between different Roaring container + * types, such as array, bitset, and run containers. These "mixed" routines + * handle cases where the two inputs do not share the same representation and + * where the most appropriate output representation may depend on the data. + * + * It includes regular, lazy, and inplace variants so higher-level bitmap code + * can choose between fully normalized results and faster deferred-maintenance + * paths. */ #ifndef INCLUDE_CONTAINERS_MIXED_XOR_H_ @@ -4891,6 +5226,18 @@ int run_run_container_ixor(run_container_t *src_1, const run_container_t *src_2, #endif /* end file include/roaring/containers/mixed_xor.h */ /* begin file include/roaring/containers/containers.h */ +/* + * containers.h + * + * This header is the central internal interface for Roaring container + * operations. It ties together the concrete container types (array, bitset, + * run, and shared containers), their type codes, common helper functions, and + * the mixed-operation headers used to combine different representations. + * + * In practice, it acts as the dispatch layer that lets higher-level bitmap + * code manipulate containers through a uniform interface while still selecting + * type-specific implementations when needed. + */ #ifndef CONTAINERS_CONTAINERS_H #define CONTAINERS_CONTAINERS_H @@ -5429,13 +5776,18 @@ static inline container_t *container_remove( } /** - * Check whether a value is in a container, requires a typecode + * Check whether a value is in a container, requires a typecode */ -static inline bool container_contains( +inline bool container_contains( const container_t *c, uint16_t val, uint8_t typecode // !!! should be second argument? ) { - c = container_unwrap_shared(c, &typecode); + if (typecode == SHARED_CONTAINER_TYPE) { + typecode = const_CAST_shared(c)->typecode; + assert(typecode != SHARED_CONTAINER_TYPE); + c = const_CAST_shared(c)->container; + } + switch (typecode) { case BITSET_CONTAINER_TYPE: return bitset_container_get(const_CAST_bitset(c), val); @@ -7315,6 +7667,7 @@ roaring_container_iterator_t container_init_iterator_last(const container_t *c, * Moves the iterator to the next entry. Returns true and sets `value` if a * value is present. */ +CROARING_ALLOW_UNALIGNED inline bool container_iterator_next(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint16_t *value) { @@ -7383,6 +7736,7 @@ inline bool container_iterator_next(const container_t *c, uint8_t typecode, * Moves the iterator to the previous entry. Returns true and sets `value` if a * value is present. */ +CROARING_ALLOW_UNALIGNED inline bool container_iterator_prev(const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, uint16_t *value) { @@ -7474,6 +7828,32 @@ bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, uint32_t count, uint32_t *consumed, uint16_t *value_out); +/** + * Reads up to `count` entries backward from the container, writing them into + * `buf` as `high16 | entry` in descending order. Returns true and sets + * `value_out` if a value is present before the entries read. Sets `consumed` + * to the number of values read. `count` should be greater than zero. + * + * `value_out` must be initialized to the current value yielded by the iterator. + */ +bool container_iterator_read_backward_into_uint32( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out); + +/** + * Reads up to `count` entries backward from the container, writing them into + * `buf` as `high48 | entry` in descending order. Returns true and sets + * `value_out` if a value is present before the entries read. Sets `consumed` + * to the number of values read. `count` should be greater than zero. + * + * `value_out` must be initialized to the current value yielded by the iterator. + */ +bool container_iterator_read_backward_into_uint64( + const container_t *c, uint8_t typecode, roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, uint32_t count, uint32_t *consumed, + uint16_t *value_out); + /** * Skips the next `skip_count` entries in the container iterator. Returns true * and sets `value_out` if a value is present after skipping. Returns false if @@ -7508,6 +7888,35 @@ bool container_iterator_skip_backward(const container_t *c, uint8_t typecode, uint32_t *consumed_count, uint16_t *value_out); +/** + * Finds the end of the consecutive run starting at the current iterator + * position within a container. Returns the low16 of the last consecutive + * value. If there are more values in the container after the run, + * *has_more is set to true, the iterator is positioned at the next value, + * and *value is updated to that value. Otherwise *has_more is set to false. + * + * *value must be the low 16 bits of the current value at the iterator's + * position on entry. + */ +uint16_t container_iterator_find_run_end(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more); + +/** + * Finds the start of the consecutive run ending at the current iterator + * position within a container. Returns the low16 of the first consecutive + * value. If there are more values in the container before the run, + * *has_more is set to true, the iterator is positioned at the previous value, + * and *value is updated to that value. Otherwise *has_more is set to false. + * + * *value must be the low 16 bits of the current value at the iterator's + * position on entry. + */ +uint16_t container_iterator_find_run_start(const container_t *c, + uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value, bool *has_more); + #ifdef __cplusplus } } @@ -7517,6 +7926,20 @@ bool container_iterator_skip_backward(const container_t *c, uint8_t typecode, #endif /* end file include/roaring/containers/containers.h */ /* begin file include/roaring/roaring_array.h */ +/* + * roaring_array.h + * + * This file declares the roaring_array helper structure and the operations + * used to manage it. A roaring array is the top-level index used by a 32-bit + * Roaring bitmap: it stores sorted 16-bit high keys alongside the container + * pointers and type codes associated with each key. + * + * In effect, it is the directory that maps each populated 16-bit chunk of the + * 32-bit value space to the container holding that chunk's low 16-bit values. + * The functions in this header handle allocation, lookup, insertion, + * replacement, copying, serialization support, and structural updates on that + * directory. + */ #ifndef INCLUDE_ROARING_ARRAY_H #define INCLUDE_ROARING_ARRAY_H @@ -7821,6 +8244,17 @@ void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance); /* begin file include/roaring/roaring.h */ /* * An implementation of Roaring Bitmaps in C. + * + * This is the main public header for the 32-bit CRoaring API. A Roaring bitmap + * represents a set of unsigned 32-bit integers by partitioning the value space + * into 16-bit chunks and storing each chunk in a container chosen to match the + * local data density. Sparse chunks are typically kept as sorted arrays, + * denser chunks as bitsets, and long consecutive runs as run containers. + * + * This hybrid representation aims to keep bitmaps compact while still + * supporting fast membership tests, iteration, rank/select queries, + * serialization, and set operations such as union, intersection, difference, + * and symmetric difference. */ #ifndef ROARING_H @@ -8439,10 +8873,6 @@ size_t roaring_bitmap_shrink_to_fit(roaring_bitmap_t *r); * * Returns how many bytes written, should be `roaring_bitmap_size_in_bytes(r)`. * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident * that you are recovering the correct data. @@ -8455,27 +8885,34 @@ size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf); * (See `roaring_bitmap_portable_deserialize()` if you want a format that's * compatible with Java and Go implementations). * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The returned pointer may be NULL in case of errors. */ roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf); /** + * Load a bitmap from a serialized buffer safely (reading up to maxbytes). + * * Use with `roaring_bitmap_serialize()`. * * (See `roaring_bitmap_portable_deserialize_safe()` if you want a format that's * compatible with Java and Go implementations). * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The difference with `roaring_bitmap_deserialize()` is that this function - * checks that the input buffer is a valid bitmap. If the buffer is too small, - * NULL is returned. + * is guaranteed to not read beyond the provided buffer. If the buffer is too + * small, NULL is returned. + * + * The function itself is safe in the sense that it will not cause buffer + * overflows: it will not read beyond the scope of the provided buffer + * (buf,maxbytes). + * + * However, for correct operations, it is assumed that the bitmap + * read was once serialized from a valid bitmap (i.e., it follows the format + * specification). If you provided an incorrect input (garbage), then the bitmap + * read may not be in a valid state and following operations may not lead to + * sensible results (using it may cause crashes, or it may just give incoherent + * answers). You can call roaring_bitmap_internal_validate to check the validity + * of the bitmap if the source is untrusted. Only after calling + * roaring_bitmap_internal_validate is the bitmap considered safe for use. * * The returned pointer may be NULL in case of errors. */ @@ -8494,15 +8931,20 @@ size_t roaring_bitmap_size_in_bytes(const roaring_bitmap_t *r); * * This function is unsafe in the sense that if there is no valid serialized * bitmap at the pointer, then many bytes could be read, possibly causing a - * buffer overflow. See also roaring_bitmap_portable_deserialize_safe(). + * buffer overflow. In other words, this routine assumes that `buf` points to a + * complete, correctly formatted serialized bitmap and does not take a buffer + * length argument that would let it enforce a read bound. + * + * Use this function only when the input buffer is already trusted, for example + * because it comes from memory that was previously filled by + * `roaring_bitmap_portable_serialize()` and whose size is known by some other + * means. If the source is untrusted, truncated, or otherwise not guaranteed to + * contain a valid serialized bitmap, prefer + * `roaring_bitmap_portable_deserialize_safe()`. * * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The returned pointer may be NULL in case of errors. */ roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf); @@ -8536,10 +8978,6 @@ roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf); * corresponds to the serialized bitmap. The CRoaring library does not provide * checksumming. * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * The returned pointer may be NULL in case of errors. */ roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, @@ -8561,9 +8999,11 @@ roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * Returns NULL on a big-endian system (e.g., a mainframe IBM s390x). The + * portable format is little-endian on every host, and this function uses the + * container payloads where they sit rather than converting them, so there is + * no correct in-place view of them there. Use + * `roaring_bitmap_portable_deserialize_safe()`, which converts as it copies. * * The returned pointer may be NULL in case of errors. */ @@ -8597,10 +9037,6 @@ size_t roaring_bitmap_portable_size_in_bytes(const roaring_bitmap_t *r); * This is meant to be compatible with the Java and Go versions: * https://github.com/RoaringBitmap/RoaringFormatSpec * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident * that you are recovering the correct data. @@ -8637,7 +9073,8 @@ size_t roaring_bitmap_frozen_size_in_bytes(const roaring_bitmap_t *r); * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + *since the format imitates C memory layout * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident @@ -8658,7 +9095,8 @@ void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *r, char *buf); * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + *since the format imitates C memory layout of roaring_bitmap_t. */ const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, size_t length); @@ -8679,6 +9117,16 @@ const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, bool roaring_iterate(const roaring_bitmap_t *r, roaring_iterator iterator, void *ptr); +/** + * Like `roaring_iterate`, but the 32-bit values are widened to 64 bits by + * adding `high_bits` (shifted into the upper 32 bits) before being passed to + * the iterator. This is used to build 64-bit iteration on top of 32-bit + * bitmaps. `ptr` (can be NULL) is forwarded as the second argument of each + * call. + * + * Returns true if the iterator returned true throughout (so that all values + * were necessarily visited). + */ bool roaring_iterate64(const roaring_bitmap_t *r, roaring_iterator64 iterator, uint64_t high_bits, void *ptr); @@ -9051,6 +9499,23 @@ CROARING_DEPRECATED static inline uint32_t roaring_read_uint32_iterator( return roaring_uint32_iterator_read(it, buf, count); } +/** + * Reads previous ${count} values from iterator into user-supplied ${buf}. + * Returns the number of read elements. + * This number can be smaller than ${count}, which means that iterator is + * drained. + * + * Values are written in descending order: buf[0] is the highest (current) + * value, buf[ret-1] is the lowest value read. + * + * This function satisfies semantics of reverse iteration and can be used + * together with other iterator functions. + * - first value is copied from ${it}->current_value + * - after function returns, iterator is positioned at the previous element + */ +uint32_t roaring_uint32_iterator_read_backward(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count); + /** * Skip the next ${count} values from iterator. * Returns the number of values actually skipped. @@ -9075,6 +9540,60 @@ uint32_t roaring_uint32_iterator_skip(roaring_uint32_iterator_t *it, uint32_t roaring_uint32_iterator_skip_backward(roaring_uint32_iterator_t *it, uint32_t count); +typedef struct roaring_uint32_range_closed_s { + uint32_t min; + uint32_t max; +} roaring_uint32_range_closed_t; + +/** + * Reads next ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * This function satisfies the semantics of iteration and can be used together + * with other iterator functions. + * - first range will start with ${it}->current_value + * - after the function returns, the iterator is positioned at the next element + * after the end of the last returned range, or ${it}->has_value is false if + * the bitmap is exhausted. + */ +size_t roaring_uint32_iterator_read_ranges(roaring_uint32_iterator_t *it, + roaring_uint32_range_closed_t *buf, + size_t count); + +/** + * Reads previous ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * Ranges are returned in reverse order, e.g. the first range returned is the + * highest range (ending at the current value) + * + * This function satisfies the semantics of reverse iteration and can be used + * together with other iterator functions. + * - first range will end with ${it}->current_value + * - after the function returns, the iterator is positioned at the element + * before the beginning of the last returned range, or ${it}->has_value is + * false if the bitmap is exhausted. + */ +size_t roaring_uint32_iterator_read_prev_ranges( + roaring_uint32_iterator_t *it, roaring_uint32_range_closed_t *buf, + size_t count); + #ifdef __cplusplus } } @@ -9105,6 +9624,18 @@ using namespace ::roaring::api; // in addition to roaring.h. /* end file include/roaring/roaring.h */ /* begin file include/roaring/memory.h */ +/* + * memory.h + * + * This header defines CRoaring's memory-allocation abstraction layer. It + * declares the function pointer types and hook structure used to override the + * library's malloc/realloc/calloc/free and aligned allocation routines, along + * with the wrapper functions used throughout the codebase. + * + * This allows applications to integrate CRoaring with custom allocators, + * memory trackers, arenas, or platform-specific aligned allocation policies + * without changing the rest of the library code. + */ #ifndef INCLUDE_ROARING_MEMORY_H_ #define INCLUDE_ROARING_MEMORY_H_ @@ -9146,6 +9677,15 @@ void roaring_aligned_free(void*); #endif // INCLUDE_ROARING_MEMORY_H_ /* end file include/roaring/memory.h */ /* begin file include/roaring/roaring64.h */ +/* + * roaring64.h + * + * This file declares the 64-bit Roaring bitmap API. A roaring64 bitmap stores + * sets of 64-bit unsigned integers by partitioning the value space by high + * bits and using Roaring containers for the lower bits inside each partition. + * This keeps the structure compact while preserving fast membership tests, + * insertions, iteration, and set operations over large sparse integer sets. + */ #ifndef ROARING64_H #define ROARING64_H @@ -9160,10 +9700,25 @@ namespace roaring { namespace api { #endif +/** An opaque 64-bit Roaring bitmap. Create one with `roaring64_bitmap_create()` + * and release it with `roaring64_bitmap_free()`. */ typedef struct roaring64_bitmap_s roaring64_bitmap_t; +/** Internal leaf type, exposed only for use inside `roaring64_bulk_context_t`. + * Callers should treat it as opaque. */ typedef uint64_t roaring64_leaf_t; +/** An opaque iterator over a 64-bit bitmap. See `roaring64_iterator_create()`. + */ typedef struct roaring64_iterator_s roaring64_iterator_t; +/** The leading members of `roaring64_iterator_t`, so that + * `roaring64_iterator_value()` and `roaring64_iterator_has_value()` can be + * read without a call. The iterator itself stays opaque; do not declare one of + * these, and do not rely on the layout beyond these two members. */ +typedef struct roaring64_iterator_public_s { + uint64_t value; + bool has_value; +} roaring64_iterator_public_t; + /** * A bit of context usable with `roaring64_bitmap_*_bulk()` functions. * @@ -9194,6 +9749,18 @@ void roaring64_bitmap_free(roaring64_bitmap_t *r); */ roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r); +/** + * Copies a bitmap from src to dest. It is assumed that the pointer dest + * is to an already allocated bitmap. The content of the dest bitmap is + * freed/deleted. + * + * It might be preferable and simpler to call roaring64_bitmap_copy except + * that roaring64_bitmap_overwrite can save on memory allocations. + * + */ +void roaring64_bitmap_overwrite(roaring64_bitmap_t *dest, + const roaring64_bitmap_t *src); + /** * Creates a new bitmap of a pointer to N 64-bit integers. */ @@ -9374,6 +9941,12 @@ bool roaring64_bitmap_contains(const roaring64_bitmap_t *r, uint64_t val); bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, uint64_t max); +/** + * Returns true if all values in the range [min, max] are present. + */ +bool roaring64_bitmap_contains_range_closed(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + /** * Check if an item is present using context from a previous insert or search * for faster search. @@ -9454,6 +10027,12 @@ uint64_t roaring64_bitmap_minimum(const roaring64_bitmap_t *r); */ uint64_t roaring64_bitmap_maximum(const roaring64_bitmap_t *r); +/** + * Remove run-length encoding even when it is more space efficient. + * Return whether a change was applied. + */ +bool roaring64_bitmap_remove_run_compression(roaring64_bitmap_t *r); + /** * Returns true if the result has at least one run container. */ @@ -9650,6 +10229,38 @@ void roaring64_bitmap_flip_inplace(roaring64_bitmap_t *r, uint64_t min, */ void roaring64_bitmap_flip_closed_inplace(roaring64_bitmap_t *r, uint64_t min, uint64_t max); +/** + * Return a copy of the bitmap with all values shifted by offset. + * + * If `positive` is true, the shift is added, otherwise subtracted. Values that + * overflow or underflow uint64_t are dropped. The caller is responsible for + * freeing the returned bitmap. + */ +roaring64_bitmap_t *roaring64_bitmap_add_offset_signed( + const roaring64_bitmap_t *r, bool positive, uint64_t offset); + +/** + * Return a copy of the bitmap with all values shifted up by offset. + * + * Values that overflow or underflow uint64_t are dropped. The caller is + * responsible for freeing the returned bitmap. + */ +static inline roaring64_bitmap_t *roaring64_bitmap_add_offset( + const roaring64_bitmap_t *r, uint64_t offset) { + return roaring64_bitmap_add_offset_signed(r, true, offset); +} + +/** + * Return a copy of the bitmap with all values shifted down by offset. + * + * Values that overflow or underflow uint64_t are dropped. The caller is + * responsible for freeing the returned bitmap. + */ +static inline roaring64_bitmap_t *roaring64_bitmap_sub_offset( + const roaring64_bitmap_t *r, uint64_t offset) { + return roaring64_bitmap_add_offset_signed(r, false, offset); +} + /** * How many bytes are required to serialize this bitmap. * @@ -9668,10 +10279,6 @@ size_t roaring64_bitmap_portable_size_in_bytes(const roaring64_bitmap_t *r); * This is meant to be compatible with other languages: * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. - * * When serializing data to a file, we recommend that you also use * checksums so that, at deserialization, you can be confident * that you are recovering the correct data. @@ -9716,14 +10323,50 @@ size_t roaring64_bitmap_portable_deserialize_size(const char *buf, * We also recommend that you use checksums to check that serialized data * corresponds to the serialized bitmap. The CRoaring library does not provide * checksumming. - * - * This function is endian-sensitive. If you have a big-endian system (e.g., a - * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. */ roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe(const char *buf, size_t maxbytes); +/** + * Read a bitmap from a portable serialized buffer as a read-only view of the + * container payloads. Headers and the ART index are allocated; bitset/array/run + * payloads alias `buf` and are not copied. + * + * In case of failure, NULL is returned. The function will not read beyond + * `maxbytes`. + * + * The returned bitmap must only be used in a readonly manner. It must be + * freed with `roaring64_bitmap_free()`. The backing buffer must outlive the + * bitmap and must not be freed or modified while it backs it. Calling any + * mutating function on the result is undefined behavior: its container array + * and headers live in a single allocation, so growing it would reallocate an + * interior pointer. + * + * The function itself is safe in the sense that it will not read beyond + * (buf, maxbytes). However, as with + * `roaring64_bitmap_portable_deserialize_safe()`, a bitmap read from garbage + * may not be in a valid state, and subsequent operations on it may not lead + * to sensible results: array containers must be sorted, and run containers + * sorted and non-overlapping, which is guaranteed only when the input came + * from a real serialized bitmap. + * + * If the source is untrusted, you should call + * `roaring64_bitmap_internal_validate` on the result before using it. Only + * after that is the bitmap considered safe for use. We also recommend + * checksumming the serialized data; CRoaring does not provide checksumming. + * + * Returns NULL on a big-endian system (e.g., a mainframe IBM s390x). The + * portable format is little-endian and this function uses the payload bytes + * where they sit, so there is no correct in-place view of them there; use + * `roaring64_bitmap_portable_deserialize_safe()`, which converts as it copies. + * + * Container payloads are used where they sit in the buffer, so they may be + * unaligned. Every access path is either SIMD with unaligned loads or marked + * `CROARING_ALLOW_UNALIGNED`. + */ +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_frozen( + const char *buf, size_t maxbytes); + /** * Returns the number of bytes required to serialize this bitmap in a "frozen" * format. This is not compatible with any other serialization formats. @@ -9748,7 +10391,8 @@ size_t roaring64_bitmap_frozen_size_in_bytes(const roaring64_bitmap_t *r); * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + * since the format imitates C memory layout of roaring64_bitmap_t. */ size_t roaring64_bitmap_frozen_serialize(const roaring64_bitmap_t *r, char *buf); @@ -9766,7 +10410,8 @@ size_t roaring64_bitmap_frozen_serialize(const roaring64_bitmap_t *r, * * This function is endian-sensitive. If you have a big-endian system (e.g., a * mainframe IBM s390x), the data format is going to be big-endian and not - * compatible with little-endian systems. + * compatible with little-endian systems. This is not a bug, it is by design, + * since the format imitates C memory layout of roaring64_bitmap_t. */ roaring64_bitmap_t *roaring64_bitmap_frozen_view(const char *buf, size_t maxbytes); @@ -9847,14 +10492,22 @@ void roaring64_iterator_free(roaring64_iterator_t *it); /** * Returns true if the iterator currently points to a value. If so, calling * `roaring64_iterator_value()` returns the value. + * + * A pointer to a structure, suitably converted, points to its initial member + * (C17 6.7.2.1p15), and `roaring64_iterator_public_t` is the initial member of + * `roaring64_iterator_t`, so this reads the field directly. */ -bool roaring64_iterator_has_value(const roaring64_iterator_t *it); +inline bool roaring64_iterator_has_value(const roaring64_iterator_t *it) { + return ((const roaring64_iterator_public_t *)it)->has_value; +} /** * Returns the value the iterator currently points to. Should only be called if * `roaring64_iterator_has_value()` returns true. */ -uint64_t roaring64_iterator_value(const roaring64_iterator_t *it); +inline uint64_t roaring64_iterator_value(const roaring64_iterator_t *it) { + return ((const roaring64_iterator_public_t *)it)->value; +} /** * Advance the iterator. If there is a new value, then @@ -9898,6 +10551,75 @@ bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, uint64_t count); +/** + * Reads previous ${count} values from iterator into user-supplied ${buf}. + * Returns the number of read elements. + * This number can be smaller than ${count}, which means that iterator is + * drained. + * + * Values are written in descending order: buf[0] is the highest (current) + * value, buf[ret-1] is the lowest value read. + * + * This function satisfies semantics of reverse iteration and can be used + * together with other iterator functions. + * - first value is copied from the current iterator value + * - after function returns, iterator is positioned at the previous element + */ +uint64_t roaring64_iterator_read_backward(roaring64_iterator_t *it, + uint64_t *buf, uint64_t count); + +typedef struct roaring64_range_closed_s { + uint64_t min; + uint64_t max; +} roaring64_range_closed_t; + +/** + * Reads next ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * This function can be used together with other iterator functions. + * - first range will start with the current iterator value + * - after the function returns, the iterator is positioned at the next element + * after the end of the last returned range, or has_value is false if + * the bitmap is exhausted. + */ +size_t roaring64_iterator_read_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count); + +/** + * Reads previous ${count} ranges from iterator into user-supplied ${buf}. + * A range is defined as a maximal interval of consecutive values. + * For example, the set {1,2,3,5,6} contains two ranges: [1..3] and [5..6]. + * Each range is represented as a struct {min,max}, both endpoints included. + * Consecutive values that span internal container boundaries are merged into + * a single range. + * + * Returns the number of read ranges. + * This number can be smaller than ${count}, which means that the iterator is + * drained. + * + * Ranges are returned in reverse order, e.g. the first range returned is the + * highest range (ending at the current value). + * + * This function can be used together with other iterator functions. + * - first range will end with the current iterator value + * - after the function returns, the iterator is positioned at the element + * before the beginning of the last returned range, or has_value is false if + * the bitmap is exhausted. + */ +size_t roaring64_iterator_read_prev_ranges(roaring64_iterator_t *it, + roaring64_range_closed_t *buf, + size_t count); + #ifdef __cplusplus } // extern "C" } // namespace roaring