From 4fbfff47cc5957ce59c890b26c13eb9f773577fb Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 29 Aug 2026 15:41:42 +0200 Subject: [PATCH 1/5] perf(bitutil): add arm64 bitmap SIMD ops --- arrow/bitutil/bitmap_ops_arm64.go | 43 +++++++- arrow/bitutil/bitmap_ops_benchmark_test.go | 52 +++++++++ arrow/bitutil/bitmap_ops_neon_arm64.s | 121 +++++++++++++++++++++ arrow/bitutil/bitmaps_test.go | 49 +++++++++ 4 files changed, 260 insertions(+), 5 deletions(-) create mode 100644 arrow/bitutil/bitmap_ops_benchmark_test.go create mode 100644 arrow/bitutil/bitmap_ops_neon_arm64.s diff --git a/arrow/bitutil/bitmap_ops_arm64.go b/arrow/bitutil/bitmap_ops_arm64.go index 28d95d84a..22dafc46f 100644 --- a/arrow/bitutil/bitmap_ops_arm64.go +++ b/arrow/bitutil/bitmap_ops_arm64.go @@ -14,14 +14,47 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !noasm -// +build !noasm +//go:build !noasm && !appengine +// +build !noasm,!appengine package bitutil +import ( + "unsafe" + + "golang.org/x/sys/cpu" +) + +//go:noescape +func _bitmap_aligned_and_neon(left, right, out unsafe.Pointer, length int64) + +//go:noescape +func _bitmap_aligned_or_neon(left, right, out unsafe.Pointer, length int64) + +//go:noescape +func _bitmap_aligned_and_not_neon(left, right, out unsafe.Pointer, length int64) + +func bitmapAlignedAndNEON(left, right, out []byte) { + _bitmap_aligned_and_neon(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) +} + +func bitmapAlignedOrNEON(left, right, out []byte) { + _bitmap_aligned_or_neon(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) +} + +func bitmapAlignedAndNotNEON(left, right, out []byte) { + _bitmap_aligned_and_not_neon(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) +} + func init() { - bitAndOp.opAligned = alignedBitAndGo - bitOrOp.opAligned = alignedBitOrGo - bitAndNotOp.opAligned = alignedBitAndNotGo + if cpu.ARM64.HasASIMD { + bitAndOp.opAligned = bitmapAlignedAndNEON + bitOrOp.opAligned = bitmapAlignedOrNEON + bitAndNotOp.opAligned = bitmapAlignedAndNotNEON + } else { + bitAndOp.opAligned = alignedBitAndGo + bitOrOp.opAligned = alignedBitOrGo + bitAndNotOp.opAligned = alignedBitAndNotGo + } bitXorOp.opAligned = alignedBitXorGo } diff --git a/arrow/bitutil/bitmap_ops_benchmark_test.go b/arrow/bitutil/bitmap_ops_benchmark_test.go new file mode 100644 index 000000000..81af10c3d --- /dev/null +++ b/arrow/bitutil/bitmap_ops_benchmark_test.go @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bitutil_test + +import ( + "strconv" + "testing" + + "github.com/apache/arrow-go/v18/arrow/bitutil" +) + +func BenchmarkBitmapAlignedOps(b *testing.B) { + for _, nbytes := range []int{bufferSize * 4, bufferSize * 16} { + b.Run(strconv.Itoa(nbytes), func(b *testing.B) { + left := randomBuffer(int64(nbytes)) + right := randomBuffer(int64(nbytes)) + out := make([]byte, nbytes) + length := int64(nbytes * 8) + + for _, op := range []struct { + name string + fn noAllocFn + }{ + {name: "and", fn: bitutil.BitmapAnd}, + {name: "or", fn: bitutil.BitmapOr}, + {name: "and-not", fn: bitutil.BitmapAndNot}, + } { + b.Run(op.name, func(b *testing.B) { + b.SetBytes(int64(2 * nbytes)) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + op.fn(left, right, 0, 0, out, 0, length) + } + }) + } + }) + } +} diff --git a/arrow/bitutil/bitmap_ops_neon_arm64.s b/arrow/bitutil/bitmap_ops_neon_arm64.s new file mode 100644 index 000000000..36d85248b --- /dev/null +++ b/arrow/bitutil/bitmap_ops_neon_arm64.s @@ -0,0 +1,121 @@ +//go:build !noasm && !appengine +// +build !noasm,!appengine + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +TEXT ·_bitmap_aligned_and_neon(SB), $0-32 + MOVD left+0(FP), R0 + MOVD right+8(FP), R1 + MOVD out+16(FP), R2 + MOVD length+24(FP), R3 + +and_loop: + CMP $64, R3 + BLO and_tail + VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16] + VLD1.P 64(R1), [V4.B16, V5.B16, V6.B16, V7.B16] + VAND V0.B16, V4.B16, V0.B16 + VAND V1.B16, V5.B16, V1.B16 + VAND V2.B16, V6.B16, V2.B16 + VAND V3.B16, V7.B16, V3.B16 + VST1.P [V0.B16, V1.B16, V2.B16, V3.B16], 64(R2) + SUB $64, R3 + B and_loop + +and_tail: + CBZ R3, and_done +and_tail_loop: + MOVBU (R0), R4 + MOVBU (R1), R5 + AND R5, R4, R4 + MOVB R4, (R2) + ADD $1, R0 + ADD $1, R1 + ADD $1, R2 + SUBS $1, R3 + BNE and_tail_loop +and_done: + RET + +TEXT ·_bitmap_aligned_or_neon(SB), $0-32 + MOVD left+0(FP), R0 + MOVD right+8(FP), R1 + MOVD out+16(FP), R2 + MOVD length+24(FP), R3 + +or_loop: + CMP $64, R3 + BLO or_tail + VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16] + VLD1.P 64(R1), [V4.B16, V5.B16, V6.B16, V7.B16] + VORR V0.B16, V4.B16, V0.B16 + VORR V1.B16, V5.B16, V1.B16 + VORR V2.B16, V6.B16, V2.B16 + VORR V3.B16, V7.B16, V3.B16 + VST1.P [V0.B16, V1.B16, V2.B16, V3.B16], 64(R2) + SUB $64, R3 + B or_loop + +or_tail: + CBZ R3, or_done +or_tail_loop: + MOVBU (R0), R4 + MOVBU (R1), R5 + ORR R5, R4, R4 + MOVB R4, (R2) + ADD $1, R0 + ADD $1, R1 + ADD $1, R2 + SUBS $1, R3 + BNE or_tail_loop +or_done: + RET + +TEXT ·_bitmap_aligned_and_not_neon(SB), $0-32 + MOVD left+0(FP), R0 + MOVD right+8(FP), R1 + MOVD out+16(FP), R2 + MOVD length+24(FP), R3 + VEOR V31.B16, V31.B16, V31.B16 + +and_not_loop: + CMP $64, R3 + BLO and_not_tail + VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16] + VLD1.P 64(R1), [V4.B16, V5.B16, V6.B16, V7.B16] + VBSL V0.B16, V31.B16, V4.B16 + VBSL V1.B16, V31.B16, V5.B16 + VBSL V2.B16, V31.B16, V6.B16 + VBSL V3.B16, V31.B16, V7.B16 + VST1.P [V4.B16, V5.B16, V6.B16, V7.B16], 64(R2) + SUB $64, R3 + B and_not_loop + +and_not_tail: + CBZ R3, and_not_done +and_not_tail_loop: + MOVBU (R0), R4 + MOVBU (R1), R5 + BIC R5, R4, R4 + MOVB R4, (R2) + ADD $1, R0 + ADD $1, R1 + ADD $1, R2 + SUBS $1, R3 + BNE and_not_tail_loop +and_not_done: + RET diff --git a/arrow/bitutil/bitmaps_test.go b/arrow/bitutil/bitmaps_test.go index 3d76f1059..69d66d600 100644 --- a/arrow/bitutil/bitmaps_test.go +++ b/arrow/bitutil/bitmaps_test.go @@ -517,6 +517,24 @@ func (s *BitmapOpSuite) TestBitmapOr() { }) } +func (s *BitmapOpSuite) TestBitmapAndNot() { + op := bitmapOp{ + noAlloc: bitutil.BitmapAndNot, + alloc: bitutil.BitmapAndNotAlloc, + } + + leftBits := []int{0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1} + rightBits := []int{0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0} + resultBits := []bool{false, true, false, true, false, false, false, true, false, false, false, true, false, true} + + s.Run("aligned", func() { + s.testAligned(op, leftBits, rightBits, resultBits) + }) + s.Run("unaligned", func() { + s.testUnaligned(op, leftBits, rightBits, resultBits) + }) +} + func (s *BitmapOpSuite) TestBitmapXnor() { op := bitmapOp{ noAlloc: bitutil.BitmapXnor, @@ -555,6 +573,37 @@ func TestSmallBitmapOp(t *testing.T) { assert.Equal(t, results, out) } +func TestBitmapOpsLargeAligned(t *testing.T) { + const nbytes = 257 + + rng := rand.New(rand.NewSource(0)) + left := make([]byte, nbytes) + right := make([]byte, nbytes) + _, _ = rng.Read(left) + _, _ = rng.Read(right) + + for _, op := range []struct { + name string + fn noAllocFn + want func(byte, byte) byte + }{ + {name: "and", fn: bitutil.BitmapAnd, want: func(left, right byte) byte { return left & right }}, + {name: "or", fn: bitutil.BitmapOr, want: func(left, right byte) byte { return left | right }}, + {name: "and-not", fn: bitutil.BitmapAndNot, want: func(left, right byte) byte { return left &^ right }}, + } { + t.Run(op.name, func(t *testing.T) { + out := make([]byte, nbytes) + op.fn(left, right, 0, 0, out, 0, nbytes*8) + + expected := make([]byte, nbytes) + for i := range expected { + expected[i] = op.want(left[i], right[i]) + } + assert.Equal(t, expected, out) + }) + } +} + func createRandomBuffer(mem memory.Allocator, src *rand.Rand, nbytes int) []byte { buf := mem.Allocate(nbytes) src.Read(buf) From ea82216a162794ba6f57b16d6df6f633f21c27e9 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 30 Aug 2026 16:43:45 +0200 Subject: [PATCH 2/5] perf(bitutil): add neon xor and xnor --- arrow/bitutil/bitmap_ops_arm64.go | 19 +++++- arrow/bitutil/bitmap_ops_benchmark_test.go | 2 + arrow/bitutil/bitmap_ops_neon_arm64.s | 74 ++++++++++++++++++++++ arrow/bitutil/bitmaps_test.go | 20 ++++++ 4 files changed, 114 insertions(+), 1 deletion(-) diff --git a/arrow/bitutil/bitmap_ops_arm64.go b/arrow/bitutil/bitmap_ops_arm64.go index 22dafc46f..016628a2c 100644 --- a/arrow/bitutil/bitmap_ops_arm64.go +++ b/arrow/bitutil/bitmap_ops_arm64.go @@ -34,6 +34,12 @@ func _bitmap_aligned_or_neon(left, right, out unsafe.Pointer, length int64) //go:noescape func _bitmap_aligned_and_not_neon(left, right, out unsafe.Pointer, length int64) +//go:noescape +func _bitmap_aligned_xor_neon(left, right, out unsafe.Pointer, length int64) + +//go:noescape +func _bitmap_aligned_xnor_neon(left, right, out unsafe.Pointer, length int64) + func bitmapAlignedAndNEON(left, right, out []byte) { _bitmap_aligned_and_neon(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) } @@ -46,15 +52,26 @@ func bitmapAlignedAndNotNEON(left, right, out []byte) { _bitmap_aligned_and_not_neon(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) } +func bitmapAlignedXorNEON(left, right, out []byte) { + _bitmap_aligned_xor_neon(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) +} + +func bitmapAlignedXnorNEON(left, right, out []byte) { + _bitmap_aligned_xnor_neon(unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), int64(len(out))) +} + func init() { if cpu.ARM64.HasASIMD { bitAndOp.opAligned = bitmapAlignedAndNEON bitOrOp.opAligned = bitmapAlignedOrNEON bitAndNotOp.opAligned = bitmapAlignedAndNotNEON + bitXorOp.opAligned = bitmapAlignedXorNEON + bitXnorOp.opAligned = bitmapAlignedXnorNEON } else { bitAndOp.opAligned = alignedBitAndGo bitOrOp.opAligned = alignedBitOrGo bitAndNotOp.opAligned = alignedBitAndNotGo + bitXorOp.opAligned = alignedBitXorGo + bitXnorOp.opAligned = alignedBitXnorGo } - bitXorOp.opAligned = alignedBitXorGo } diff --git a/arrow/bitutil/bitmap_ops_benchmark_test.go b/arrow/bitutil/bitmap_ops_benchmark_test.go index 81af10c3d..658969cad 100644 --- a/arrow/bitutil/bitmap_ops_benchmark_test.go +++ b/arrow/bitutil/bitmap_ops_benchmark_test.go @@ -38,6 +38,8 @@ func BenchmarkBitmapAlignedOps(b *testing.B) { {name: "and", fn: bitutil.BitmapAnd}, {name: "or", fn: bitutil.BitmapOr}, {name: "and-not", fn: bitutil.BitmapAndNot}, + {name: "xor", fn: bitutil.BitmapXor}, + {name: "xnor", fn: bitutil.BitmapXnor}, } { b.Run(op.name, func(b *testing.B) { b.SetBytes(int64(2 * nbytes)) diff --git a/arrow/bitutil/bitmap_ops_neon_arm64.s b/arrow/bitutil/bitmap_ops_neon_arm64.s index 36d85248b..5acb3022e 100644 --- a/arrow/bitutil/bitmap_ops_neon_arm64.s +++ b/arrow/bitutil/bitmap_ops_neon_arm64.s @@ -119,3 +119,77 @@ and_not_tail_loop: BNE and_not_tail_loop and_not_done: RET + +TEXT ·_bitmap_aligned_xor_neon(SB), $0-32 + MOVD left+0(FP), R0 + MOVD right+8(FP), R1 + MOVD out+16(FP), R2 + MOVD length+24(FP), R3 + +xor_loop: + CMP $64, R3 + BLO xor_tail + VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16] + VLD1.P 64(R1), [V4.B16, V5.B16, V6.B16, V7.B16] + VEOR V0.B16, V4.B16, V0.B16 + VEOR V1.B16, V5.B16, V1.B16 + VEOR V2.B16, V6.B16, V2.B16 + VEOR V3.B16, V7.B16, V3.B16 + VST1.P [V0.B16, V1.B16, V2.B16, V3.B16], 64(R2) + SUB $64, R3 + B xor_loop + +xor_tail: + CBZ R3, xor_done +xor_tail_loop: + MOVBU (R0), R4 + MOVBU (R1), R5 + EOR R5, R4, R4 + MOVB R4, (R2) + ADD $1, R0 + ADD $1, R1 + ADD $1, R2 + SUBS $1, R3 + BNE xor_tail_loop +xor_done: + RET + +TEXT ·_bitmap_aligned_xnor_neon(SB), $0-32 + MOVD left+0(FP), R0 + MOVD right+8(FP), R1 + MOVD out+16(FP), R2 + MOVD length+24(FP), R3 + VMOVI $0xff, V31.B16 + +xnor_loop: + CMP $64, R3 + BLO xnor_tail + VLD1.P 64(R0), [V0.B16, V1.B16, V2.B16, V3.B16] + VLD1.P 64(R1), [V4.B16, V5.B16, V6.B16, V7.B16] + VEOR V0.B16, V4.B16, V0.B16 + VEOR V31.B16, V0.B16, V0.B16 + VEOR V1.B16, V5.B16, V1.B16 + VEOR V31.B16, V1.B16, V1.B16 + VEOR V2.B16, V6.B16, V2.B16 + VEOR V31.B16, V2.B16, V2.B16 + VEOR V3.B16, V7.B16, V3.B16 + VEOR V31.B16, V3.B16, V3.B16 + VST1.P [V0.B16, V1.B16, V2.B16, V3.B16], 64(R2) + SUB $64, R3 + B xnor_loop + +xnor_tail: + CBZ R3, xnor_done +xnor_tail_loop: + MOVBU (R0), R4 + MOVBU (R1), R5 + EOR R5, R4, R4 + MVN R4, R4 + MOVB R4, (R2) + ADD $1, R0 + ADD $1, R1 + ADD $1, R2 + SUBS $1, R3 + BNE xnor_tail_loop +xnor_done: + RET diff --git a/arrow/bitutil/bitmaps_test.go b/arrow/bitutil/bitmaps_test.go index 69d66d600..84b4a5ff3 100644 --- a/arrow/bitutil/bitmaps_test.go +++ b/arrow/bitutil/bitmaps_test.go @@ -535,6 +535,24 @@ func (s *BitmapOpSuite) TestBitmapAndNot() { }) } +func (s *BitmapOpSuite) TestBitmapXor() { + op := bitmapOp{ + noAlloc: bitutil.BitmapXor, + alloc: bitutil.BitmapXorAlloc, + } + + leftBits := []int{0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1} + rightBits := []int{0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0} + resultBits := []bool{false, true, false, true, true, true, false, true, true, false, true, true, true, true} + + s.Run("aligned", func() { + s.testAligned(op, leftBits, rightBits, resultBits) + }) + s.Run("unaligned", func() { + s.testUnaligned(op, leftBits, rightBits, resultBits) + }) +} + func (s *BitmapOpSuite) TestBitmapXnor() { op := bitmapOp{ noAlloc: bitutil.BitmapXnor, @@ -590,6 +608,8 @@ func TestBitmapOpsLargeAligned(t *testing.T) { {name: "and", fn: bitutil.BitmapAnd, want: func(left, right byte) byte { return left & right }}, {name: "or", fn: bitutil.BitmapOr, want: func(left, right byte) byte { return left | right }}, {name: "and-not", fn: bitutil.BitmapAndNot, want: func(left, right byte) byte { return left &^ right }}, + {name: "xor", fn: bitutil.BitmapXor, want: func(left, right byte) byte { return left ^ right }}, + {name: "xnor", fn: bitutil.BitmapXnor, want: func(left, right byte) byte { return ^(left ^ right) }}, } { t.Run(op.name, func(t *testing.T) { out := make([]byte, nbytes) From 79e57da10995975886cf7d310f38325c5102c01a Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 30 Aug 2026 22:42:12 +0200 Subject: [PATCH 3/5] test(bitutil): cover bitmap boundaries and in-place output --- arrow/bitutil/bitmaps_test.go | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/arrow/bitutil/bitmaps_test.go b/arrow/bitutil/bitmaps_test.go index 84b4a5ff3..3046dc902 100644 --- a/arrow/bitutil/bitmaps_test.go +++ b/arrow/bitutil/bitmaps_test.go @@ -795,3 +795,47 @@ func TestBitmapWriterAppendBitmapLarge(t *testing.T) { assert.Equal(t, expected, actual, "bit mismatch at position %d", i) } } + +func TestBitmapOpsBoundaries(t *testing.T) { + rng := rand.New(rand.NewSource(17)) + left, right := make([]byte, 260), make([]byte, 260) + _, _ = rng.Read(left) + _, _ = rng.Read(right) + for _, op := range []struct { + name string + fn noAllocFn + want func(bool, bool) bool + }{ + {"and", bitutil.BitmapAnd, func(l, r bool) bool { return l && r }}, + {"or", bitutil.BitmapOr, func(l, r bool) bool { return l || r }}, + {"and-not", bitutil.BitmapAndNot, func(l, r bool) bool { return l && !r }}, + {"xor", bitutil.BitmapXor, func(l, r bool) bool { return l != r }}, + {"xnor", bitutil.BitmapXnor, func(l, r bool) bool { return l == r }}, + } { + t.Run(op.name, func(t *testing.T) { + for _, length := range []int{0, 1, 7, 8, 9, 15, 16, 17, 504, 512, 520, 528, 536, 1016, 1024, 1032, 1040, 2056} { + for _, offset := range []int{0, 1, 7, 8, 15} { + for _, alias := range []string{"none", "left", "right"} { + l, r := append([]byte(nil), left...), append([]byte(nil), right...) + out := make([]byte, len(left)) + for i := range out { + out[i] = 0xa5 + } + switch alias { + case "left": + out = l + case "right": + out = r + } + expected := append([]byte(nil), out...) + for i := offset; i < offset+length; i++ { + bitutil.SetBitTo(expected, i, op.want(bitutil.BitIsSet(left, i), bitutil.BitIsSet(right, i))) + } + op.fn(l, r, int64(offset), int64(offset), out, int64(offset), int64(length)) + assert.Equal(t, expected, out, "length=%d offset=%d alias=%s", length, offset, alias) + } + } + } + }) + } +} From 7fecc4ee841aec978496e2af469038beaf46f9f0 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 30 Aug 2026 22:48:41 +0200 Subject: [PATCH 4/5] fix(bitutil): apply bitmap tail masks after byte offsets --- arrow/bitutil/bitmaps.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow/bitutil/bitmaps.go b/arrow/bitutil/bitmaps.go index eb8cf908e..c4cef33a6 100644 --- a/arrow/bitutil/bitmaps.go +++ b/arrow/bitutil/bitmaps.go @@ -532,7 +532,7 @@ func alignedBitmapOp(op bitOp, left, right []byte, lOffset, rOffset int64, out [ left = left[lOffset/8:] right = right[rOffset/8:] out = out[outOffset/8:] - endMask := (lOffset + length%8) + endMask := (lOffset + length) % 8 switch nbytes { case 0: return From cd66e1a1da5269dfcf83d4577c2b3e7b0ec8fc6d Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 30 Aug 2026 23:30:29 +0200 Subject: [PATCH 5/5] perf(bitutil): process NEON bitmap tails a word at a time --- arrow/bitutil/bitmap_ops_benchmark_test.go | 2 +- arrow/bitutil/bitmap_ops_neon_arm64.s | 66 ++++++++++++++++++++++ arrow/bitutil/bitmaps_test.go | 2 +- 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/arrow/bitutil/bitmap_ops_benchmark_test.go b/arrow/bitutil/bitmap_ops_benchmark_test.go index 658969cad..0588c2bea 100644 --- a/arrow/bitutil/bitmap_ops_benchmark_test.go +++ b/arrow/bitutil/bitmap_ops_benchmark_test.go @@ -24,7 +24,7 @@ import ( ) func BenchmarkBitmapAlignedOps(b *testing.B) { - for _, nbytes := range []int{bufferSize * 4, bufferSize * 16} { + for _, nbytes := range []int{8, 32, 64, 128, 1024, bufferSize * 4, bufferSize * 16} { b.Run(strconv.Itoa(nbytes), func(b *testing.B) { left := randomBuffer(int64(nbytes)) right := randomBuffer(int64(nbytes)) diff --git a/arrow/bitutil/bitmap_ops_neon_arm64.s b/arrow/bitutil/bitmap_ops_neon_arm64.s index 5acb3022e..20fac5869 100644 --- a/arrow/bitutil/bitmap_ops_neon_arm64.s +++ b/arrow/bitutil/bitmap_ops_neon_arm64.s @@ -37,6 +37,19 @@ and_loop: B and_loop and_tail: + CMP $8, R3 + BLO and_byte_tail + MOVD (R0), R4 + MOVD (R1), R5 + AND R5, R4, R4 + MOVD R4, (R2) + ADD $8, R0 + ADD $8, R1 + ADD $8, R2 + SUB $8, R3 + B and_tail + +and_byte_tail: CBZ R3, and_done and_tail_loop: MOVBU (R0), R4 @@ -71,6 +84,19 @@ or_loop: B or_loop or_tail: + CMP $8, R3 + BLO or_byte_tail + MOVD (R0), R4 + MOVD (R1), R5 + ORR R5, R4, R4 + MOVD R4, (R2) + ADD $8, R0 + ADD $8, R1 + ADD $8, R2 + SUB $8, R3 + B or_tail + +or_byte_tail: CBZ R3, or_done or_tail_loop: MOVBU (R0), R4 @@ -106,6 +132,19 @@ and_not_loop: B and_not_loop and_not_tail: + CMP $8, R3 + BLO and_not_byte_tail + MOVD (R0), R4 + MOVD (R1), R5 + BIC R5, R4, R4 + MOVD R4, (R2) + ADD $8, R0 + ADD $8, R1 + ADD $8, R2 + SUB $8, R3 + B and_not_tail + +and_not_byte_tail: CBZ R3, and_not_done and_not_tail_loop: MOVBU (R0), R4 @@ -140,6 +179,19 @@ xor_loop: B xor_loop xor_tail: + CMP $8, R3 + BLO xor_byte_tail + MOVD (R0), R4 + MOVD (R1), R5 + EOR R5, R4, R4 + MOVD R4, (R2) + ADD $8, R0 + ADD $8, R1 + ADD $8, R2 + SUB $8, R3 + B xor_tail + +xor_byte_tail: CBZ R3, xor_done xor_tail_loop: MOVBU (R0), R4 @@ -179,6 +231,20 @@ xnor_loop: B xnor_loop xnor_tail: + CMP $8, R3 + BLO xnor_byte_tail + MOVD (R0), R4 + MOVD (R1), R5 + EOR R5, R4, R4 + MVN R4, R4 + MOVD R4, (R2) + ADD $8, R0 + ADD $8, R1 + ADD $8, R2 + SUB $8, R3 + B xnor_tail + +xnor_byte_tail: CBZ R3, xnor_done xnor_tail_loop: MOVBU (R0), R4 diff --git a/arrow/bitutil/bitmaps_test.go b/arrow/bitutil/bitmaps_test.go index 3046dc902..0f836837c 100644 --- a/arrow/bitutil/bitmaps_test.go +++ b/arrow/bitutil/bitmaps_test.go @@ -813,7 +813,7 @@ func TestBitmapOpsBoundaries(t *testing.T) { {"xnor", bitutil.BitmapXnor, func(l, r bool) bool { return l == r }}, } { t.Run(op.name, func(t *testing.T) { - for _, length := range []int{0, 1, 7, 8, 9, 15, 16, 17, 504, 512, 520, 528, 536, 1016, 1024, 1032, 1040, 2056} { + for _, length := range []int{0, 1, 7, 8, 9, 15, 16, 17, 24, 32, 56, 64, 72, 120, 128, 136, 504, 512, 520, 528, 536, 1016, 1024, 1032, 1040, 2056} { for _, offset := range []int{0, 1, 7, 8, 15} { for _, alias := range []string{"none", "left", "right"} { l, r := append([]byte(nil), left...), append([]byte(nil), right...)