From dca0f16c440092ac56867ca4cb2785d38124ac85 Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 29 Aug 2026 17:07:19 +0200 Subject: [PATCH 1/3] perf(compute): add ARM64 NEON comparison kernels --- .../kernels/scalar_comparison_arm64.go | 132 + .../kernels/scalar_comparison_arm64.s | 2923 +++++++++++++++++ .../kernels/scalar_comparison_arm64_test.go | 169 + .../kernels/scalar_comparison_noasm.go | 2 +- 4 files changed, 3225 insertions(+), 1 deletion(-) create mode 100644 arrow/compute/internal/kernels/scalar_comparison_arm64.go create mode 100644 arrow/compute/internal/kernels/scalar_comparison_arm64.s create mode 100644 arrow/compute/internal/kernels/scalar_comparison_arm64_test.go diff --git a/arrow/compute/internal/kernels/scalar_comparison_arm64.go b/arrow/compute/internal/kernels/scalar_comparison_arm64.go new file mode 100644 index 000000000..01a8e8247 --- /dev/null +++ b/arrow/compute/internal/kernels/scalar_comparison_arm64.go @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.18 && arm64 && !noasm && !appengine + +package kernels + +import ( + "unsafe" + + "github.com/apache/arrow-go/v18/arrow" + "golang.org/x/sys/cpu" +) + +const ( + neonCompareArrayArray = iota + neonCompareArrayScalar + neonCompareScalarArray +) + +//go:noescape +func _comparison_neon(typ int, op int, shape int, left, right, out unsafe.Pointer, groups int64) + +func neonComparisonSupported(typ arrow.Type) bool { + switch typ { + case arrow.INT32, arrow.UINT32, arrow.INT64, arrow.UINT64, + arrow.FLOAT32, arrow.FLOAT64: + return true + default: + return false + } +} + +func compareNeon(typ arrow.Type, op CompareOperator, width, shape int, left, right, out []byte, offset int, fallback binaryKernel) { + n := len(left) / width + if shape == neonCompareScalarArray { + n = len(right) / width + } + if n == 0 { + return + } + + if !cpu.ARM64.HasASIMD || !neonComparisonSupported(typ) { + fallback(left, right, out, offset) + return + } + + if offset != 0 { + prefix := min(n, 8-offset) + switch shape { + case neonCompareArrayArray: + fallback(left[:prefix*width], right[:prefix*width], out, offset) + case neonCompareArrayScalar: + fallback(left[:prefix*width], right, out, offset) + case neonCompareScalarArray: + fallback(left, right[:prefix*width], out, offset) + } + if prefix == n { + return + } + + switch shape { + case neonCompareArrayArray: + left = left[prefix*width:] + right = right[prefix*width:] + case neonCompareArrayScalar: + left = left[prefix*width:] + case neonCompareScalarArray: + right = right[prefix*width:] + } + out = out[1:] + n -= prefix + } + + bulk := n &^ 7 + if bulk != 0 { + assemblyLeft, assemblyRight := left, right + switch shape { + case neonCompareArrayArray: + assemblyLeft = left[:bulk*width] + assemblyRight = right[:bulk*width] + case neonCompareArrayScalar: + assemblyLeft = left[:bulk*width] + case neonCompareScalarArray: + assemblyRight = right[:bulk*width] + } + _comparison_neon(int(typ), int(op), shape, + unsafe.Pointer(&assemblyLeft[0]), unsafe.Pointer(&assemblyRight[0]), unsafe.Pointer(&out[0]), int64(bulk/8)) + } + + if tail := n - bulk; tail != 0 { + out = out[bulk/8:] + switch shape { + case neonCompareArrayArray: + fallback(left[bulk*width:bulk*width+tail*width], right[bulk*width:bulk*width+tail*width], out, 0) + case neonCompareArrayScalar: + fallback(left[bulk*width:bulk*width+tail*width], right, out, 0) + case neonCompareScalarArray: + fallback(left, right[bulk*width:bulk*width+tail*width], out, 0) + } + } +} + +func genCompareKernel[T arrow.NumericType](op CompareOperator) *CompareData { + ty := arrow.GetType[T]() + width := int(unsafe.Sizeof(T(0))) + fallback := genGoCompareKernel(getCmpOp[T](op)) + return &CompareData{ + funcAA: func(left, right, out []byte, offset int) { + compareNeon(ty, op, width, neonCompareArrayArray, left, right, out, offset, fallback.funcAA) + }, + funcAS: func(left, right, out []byte, offset int) { + compareNeon(ty, op, width, neonCompareArrayScalar, left, right, out, offset, fallback.funcAS) + }, + funcSA: func(left, right, out []byte, offset int) { + compareNeon(ty, op, width, neonCompareScalarArray, left, right, out, offset, fallback.funcSA) + }, + } +} diff --git a/arrow/compute/internal/kernels/scalar_comparison_arm64.s b/arrow/compute/internal/kernels/scalar_comparison_arm64.s new file mode 100644 index 000000000..1bc12edb9 --- /dev/null +++ b/arrow/compute/internal/kernels/scalar_comparison_arm64.s @@ -0,0 +1,2923 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.18 && arm64 && !noasm && !appengine + +#include "textflag.h" + +// 32-bit packing weights: [1, 2, 4, 8]. +DATA LCMPNEON_WEIGHTS<>+0(SB)/8, $0x0000000200000001 +DATA LCMPNEON_WEIGHTS<>+8(SB)/8, $0x0000000800000004 +// 64-bit variable shift counts: [0, 1]. +DATA LCMPNEON_WEIGHTS<>+16(SB)/8, $0x0000000000000000 +DATA LCMPNEON_WEIGHTS<>+24(SB)/8, $0x0000000000000001 +GLOBL LCMPNEON_WEIGHTS<>(SB), (NOPTR+RODATA), $32 + +TEXT ยท_comparison_neon(SB), NOSPLIT|NOFRAME, $0-56 + MOVD typ+0(FP), R0 + MOVD op+8(FP), R1 + MOVD shape+16(FP), R2 + MOVD left+24(FP), R3 + MOVD right+32(FP), R4 + MOVD out+40(FP), R5 + MOVD groups+48(FP), R6 + + CMP $6, R0 + BEQ Lcomparison32Int + CMP $7, R0 + BEQ Lcomparison32Int + CMP $8, R0 + BEQ Lcomparison64Int + CMP $9, R0 + BEQ Lcomparison64Int + CMP $11, R0 + BEQ Lcomparison32Float + CMP $12, R0 + BEQ Lcomparison64Float + JMP LcomparisonReturn + +Lcomparison32Int: + MOVD $LCMPNEON_WEIGHTS<>(SB), R10 + VLD1 (R10), [V20.S4] + CMP $0, R2 + BEQ Lcomparison32IntAASelect + CMP $1, R2 + BEQ Lcomparison32IntASSelect + CMP $2, R2 + BEQ Lcomparison32IntSASelect + JMP LcomparisonReturn + +Lcomparison32IntAASelect: + CMP $0, R1 + BEQ Lcomparison32IntAAEQ + CMP $1, R1 + BEQ Lcomparison32IntAANE + CMP $2, R1 + BEQ Lcomparison32IntAAGTSelect + CMP $3, R1 + BEQ Lcomparison32IntAAGESelect + JMP LcomparisonReturn + +Lcomparison32IntAAGTSelect: + CMP $6, R0 + BEQ Lcomparison32IntAAUGT + JMP Lcomparison32IntAASGT + +Lcomparison32IntAAGESelect: + CMP $6, R0 + BEQ Lcomparison32IntAAUGE + JMP Lcomparison32IntAASGE + +Lcomparison32IntASSelect: + CMP $0, R1 + BEQ Lcomparison32IntASEQ + CMP $1, R1 + BEQ Lcomparison32IntASNE + CMP $2, R1 + BEQ Lcomparison32IntASGTSelect + CMP $3, R1 + BEQ Lcomparison32IntASGESelect + JMP LcomparisonReturn + +Lcomparison32IntASGTSelect: + CMP $6, R0 + BEQ Lcomparison32IntASUGT + JMP Lcomparison32IntASSGT + +Lcomparison32IntASGESelect: + CMP $6, R0 + BEQ Lcomparison32IntASUGE + JMP Lcomparison32IntASSGE + +Lcomparison32IntSASelect: + CMP $0, R1 + BEQ Lcomparison32IntSAEQ + CMP $1, R1 + BEQ Lcomparison32IntSANE + CMP $2, R1 + BEQ Lcomparison32IntSAGTSelect + CMP $3, R1 + BEQ Lcomparison32IntSAGESelect + JMP LcomparisonReturn + +Lcomparison32IntSAGTSelect: + CMP $6, R0 + BEQ Lcomparison32IntSAUGT + JMP Lcomparison32IntSASGT + +Lcomparison32IntSAGESelect: + CMP $6, R0 + BEQ Lcomparison32IntSAUGE + JMP Lcomparison32IntSASGE + +Lcomparison32IntAAEQ: +Lcomparison32IntAAEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6ea18c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6ea18c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntAAEQLoop + +Lcomparison32IntAANE: +Lcomparison32IntAANELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6ea18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6ea18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntAANELoop + +Lcomparison32IntAASGT: +Lcomparison32IntAASGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x4ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x4ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntAASGTLoop + +Lcomparison32IntAAUGT: +Lcomparison32IntAAUGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntAAUGTLoop + +Lcomparison32IntAASGE: +Lcomparison32IntAASGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x4ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x4ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntAASGELoop + +Lcomparison32IntAAUGE: +Lcomparison32IntAAUGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntAAUGELoop + +Lcomparison32IntASEQ: + VLD1R (R4), [V1.S4] +Lcomparison32IntASEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + WORD $0x6ea18c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + VLD1 (R3), [V0.S4] + WORD $0x6ea18c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntASEQLoop + +Lcomparison32IntASNE: + VLD1R (R4), [V1.S4] +Lcomparison32IntASNELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + WORD $0x6ea18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + VLD1 (R3), [V0.S4] + WORD $0x6ea18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntASNELoop + +Lcomparison32IntASSGT: + VLD1R (R4), [V1.S4] +Lcomparison32IntASSGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + WORD $0x4ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + VLD1 (R3), [V0.S4] + WORD $0x4ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntASSGTLoop + +Lcomparison32IntASUGT: + VLD1R (R4), [V1.S4] +Lcomparison32IntASUGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + WORD $0x6ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + VLD1 (R3), [V0.S4] + WORD $0x6ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntASUGTLoop + +Lcomparison32IntASSGE: + VLD1R (R4), [V1.S4] +Lcomparison32IntASSGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + WORD $0x4ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + VLD1 (R3), [V0.S4] + WORD $0x4ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntASSGELoop + +Lcomparison32IntASUGE: + VLD1R (R4), [V1.S4] +Lcomparison32IntASUGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + WORD $0x6ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + VLD1 (R3), [V0.S4] + WORD $0x6ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntASUGELoop + +Lcomparison32IntSAEQ: + VLD1R (R3), [V0.S4] +Lcomparison32IntSAEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.S4] + WORD $0x6ea18c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R4, R4 + VLD1 (R4), [V1.S4] + WORD $0x6ea18c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntSAEQLoop + +Lcomparison32IntSANE: + VLD1R (R3), [V0.S4] +Lcomparison32IntSANELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.S4] + WORD $0x6ea18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R4, R4 + VLD1 (R4), [V1.S4] + WORD $0x6ea18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntSANELoop + +Lcomparison32IntSASGT: + VLD1R (R3), [V0.S4] +Lcomparison32IntSASGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.S4] + WORD $0x4ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R4, R4 + VLD1 (R4), [V1.S4] + WORD $0x4ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntSASGTLoop + +Lcomparison32IntSAUGT: + VLD1R (R3), [V0.S4] +Lcomparison32IntSAUGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.S4] + WORD $0x6ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R4, R4 + VLD1 (R4), [V1.S4] + WORD $0x6ea13402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntSAUGTLoop + +Lcomparison32IntSASGE: + VLD1R (R3), [V0.S4] +Lcomparison32IntSASGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.S4] + WORD $0x4ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R4, R4 + VLD1 (R4), [V1.S4] + WORD $0x4ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntSASGELoop + +Lcomparison32IntSAUGE: + VLD1R (R3), [V0.S4] +Lcomparison32IntSAUGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.S4] + WORD $0x6ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R4, R4 + VLD1 (R4), [V1.S4] + WORD $0x6ea13c02 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32IntSAUGELoop + +Lcomparison64Int: + MOVD $LCMPNEON_WEIGHTS<>(SB), R10 + ADD $16, R10, R10 + VLD1 (R10), [V20.D2] + CMP $0, R2 + BEQ Lcomparison64IntAASelect + CMP $1, R2 + BEQ Lcomparison64IntASSelect + CMP $2, R2 + BEQ Lcomparison64IntSASelect + JMP LcomparisonReturn + +Lcomparison64IntAASelect: + CMP $0, R1 + BEQ Lcomparison64IntAAEQ + CMP $1, R1 + BEQ Lcomparison64IntAANE + CMP $2, R1 + BEQ Lcomparison64IntAAGTSelect + CMP $3, R1 + BEQ Lcomparison64IntAAGESelect + JMP LcomparisonReturn + +Lcomparison64IntAAGTSelect: + CMP $8, R0 + BEQ Lcomparison64IntAAUGT + JMP Lcomparison64IntAASGT + +Lcomparison64IntAAGESelect: + CMP $8, R0 + BEQ Lcomparison64IntAAUGE + JMP Lcomparison64IntAASGE + +Lcomparison64IntASSelect: + CMP $0, R1 + BEQ Lcomparison64IntASEQ + CMP $1, R1 + BEQ Lcomparison64IntASNE + CMP $2, R1 + BEQ Lcomparison64IntASGTSelect + CMP $3, R1 + BEQ Lcomparison64IntASGESelect + JMP LcomparisonReturn + +Lcomparison64IntASGTSelect: + CMP $8, R0 + BEQ Lcomparison64IntASUGT + JMP Lcomparison64IntASSGT + +Lcomparison64IntASGESelect: + CMP $8, R0 + BEQ Lcomparison64IntASUGE + JMP Lcomparison64IntASSGE + +Lcomparison64IntSASelect: + CMP $0, R1 + BEQ Lcomparison64IntSAEQ + CMP $1, R1 + BEQ Lcomparison64IntSANE + CMP $2, R1 + BEQ Lcomparison64IntSAGTSelect + CMP $3, R1 + BEQ Lcomparison64IntSAGESelect + JMP LcomparisonReturn + +Lcomparison64IntSAGTSelect: + CMP $8, R0 + BEQ Lcomparison64IntSAUGT + JMP Lcomparison64IntSASGT + +Lcomparison64IntSAGESelect: + CMP $8, R0 + BEQ Lcomparison64IntSAUGE + JMP Lcomparison64IntSASGE + +Lcomparison64IntAAEQ: +Lcomparison64IntAAEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntAAEQLoop + +Lcomparison64IntAANE: +Lcomparison64IntAANELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntAANELoop + +Lcomparison64IntAASGT: +Lcomparison64IntAASGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntAASGTLoop + +Lcomparison64IntAAUGT: +Lcomparison64IntAAUGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntAAUGTLoop + +Lcomparison64IntAASGE: +Lcomparison64IntAASGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntAASGELoop + +Lcomparison64IntAAUGE: +Lcomparison64IntAAUGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntAAUGELoop + +Lcomparison64IntASEQ: + VLD1R (R4), [V1.D2] +Lcomparison64IntASEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntASEQLoop + +Lcomparison64IntASNE: + VLD1R (R4), [V1.D2] +Lcomparison64IntASNELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntASNELoop + +Lcomparison64IntASSGT: + VLD1R (R4), [V1.D2] +Lcomparison64IntASSGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntASSGTLoop + +Lcomparison64IntASUGT: + VLD1R (R4), [V1.D2] +Lcomparison64IntASUGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntASUGTLoop + +Lcomparison64IntASSGE: + VLD1R (R4), [V1.D2] +Lcomparison64IntASSGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntASSGELoop + +Lcomparison64IntASUGE: + VLD1R (R4), [V1.D2] +Lcomparison64IntASUGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntASUGELoop + +Lcomparison64IntSAEQ: + VLD1R (R3), [V0.D2] +Lcomparison64IntSAEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntSAEQLoop + +Lcomparison64IntSANE: + VLD1R (R3), [V0.D2] +Lcomparison64IntSANELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee18c02 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntSANELoop + +Lcomparison64IntSASGT: + VLD1R (R3), [V0.D2] +Lcomparison64IntSASGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntSASGTLoop + +Lcomparison64IntSAUGT: + VLD1R (R3), [V0.D2] +Lcomparison64IntSAUGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee13402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntSAUGTLoop + +Lcomparison64IntSASGE: + VLD1R (R3), [V0.D2] +Lcomparison64IntSASGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntSASGELoop + +Lcomparison64IntSAUGE: + VLD1R (R3), [V0.D2] +Lcomparison64IntSAUGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee13c02 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64IntSAUGELoop + +Lcomparison32Float: + MOVD $LCMPNEON_WEIGHTS<>(SB), R10 + VLD1 (R10), [V20.S4] + CMP $0, R2 + BEQ Lcomparison32FloatAASelect + CMP $1, R2 + BEQ Lcomparison32FloatASSelect + CMP $2, R2 + BEQ Lcomparison32FloatSASelect + JMP LcomparisonReturn + +Lcomparison32FloatAASelect: + CMP $0, R1 + BEQ Lcomparison32FloatAAEQ + CMP $1, R1 + BEQ Lcomparison32FloatAANE + CMP $2, R1 + BEQ Lcomparison32FloatAAGTSelect + CMP $3, R1 + BEQ Lcomparison32FloatAAGESelect + JMP LcomparisonReturn + +Lcomparison32FloatAAGTSelect: + JMP Lcomparison32FloatAAFGT + +Lcomparison32FloatAAGESelect: + JMP Lcomparison32FloatAAFGE + +Lcomparison32FloatASSelect: + CMP $0, R1 + BEQ Lcomparison32FloatASEQ + CMP $1, R1 + BEQ Lcomparison32FloatASNE + CMP $2, R1 + BEQ Lcomparison32FloatASGTSelect + CMP $3, R1 + BEQ Lcomparison32FloatASGESelect + JMP LcomparisonReturn + +Lcomparison32FloatASGTSelect: + JMP Lcomparison32FloatASFGT + +Lcomparison32FloatASGESelect: + JMP Lcomparison32FloatASFGE + +Lcomparison32FloatSASelect: + CMP $0, R1 + BEQ Lcomparison32FloatSAEQ + CMP $1, R1 + BEQ Lcomparison32FloatSANE + CMP $2, R1 + BEQ Lcomparison32FloatSAGTSelect + CMP $3, R1 + BEQ Lcomparison32FloatSAGESelect + JMP LcomparisonReturn + +Lcomparison32FloatSAGTSelect: + JMP Lcomparison32FloatSAFGT + +Lcomparison32FloatSAGESelect: + JMP Lcomparison32FloatSAFGE + +Lcomparison32FloatAAEQ: +Lcomparison32FloatAAEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x4e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x4e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatAAEQLoop + +Lcomparison32FloatAANE: +Lcomparison32FloatAANELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x4e21e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x4e21e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatAANELoop + +Lcomparison32FloatAAFGT: +Lcomparison32FloatAAFGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6ea1e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6ea1e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatAAFGTLoop + +Lcomparison32FloatAAFGE: +Lcomparison32FloatAAFGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.S4] + VLD1 (R4), [V1.S4] + WORD $0x6e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatAAFGELoop + +Lcomparison32FloatASEQ: + VLD1R (R4), [V1.S4] +Lcomparison32FloatASEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + WORD $0x4e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + VLD1 (R3), [V0.S4] + WORD $0x4e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatASEQLoop + +Lcomparison32FloatASNE: + VLD1R (R4), [V1.S4] +Lcomparison32FloatASNELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + WORD $0x4e21e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + VLD1 (R3), [V0.S4] + WORD $0x4e21e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatASNELoop + +Lcomparison32FloatASFGT: + VLD1R (R4), [V1.S4] +Lcomparison32FloatASFGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + WORD $0x6ea1e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + VLD1 (R3), [V0.S4] + WORD $0x6ea1e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatASFGTLoop + +Lcomparison32FloatASFGE: + VLD1R (R4), [V1.S4] +Lcomparison32FloatASFGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.S4] + WORD $0x6e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R3, R3 + VLD1 (R3), [V0.S4] + WORD $0x6e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatASFGELoop + +Lcomparison32FloatSAEQ: + VLD1R (R3), [V0.S4] +Lcomparison32FloatSAEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.S4] + WORD $0x4e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R4, R4 + VLD1 (R4), [V1.S4] + WORD $0x4e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatSAEQLoop + +Lcomparison32FloatSANE: + VLD1R (R3), [V0.S4] +Lcomparison32FloatSANELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.S4] + WORD $0x4e21e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R4, R4 + VLD1 (R4), [V1.S4] + WORD $0x4e21e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatSANELoop + +Lcomparison32FloatSAFGT: + VLD1R (R3), [V0.S4] +Lcomparison32FloatSAFGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.S4] + WORD $0x6ea1e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R4, R4 + VLD1 (R4), [V1.S4] + WORD $0x6ea1e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatSAFGTLoop + +Lcomparison32FloatSAFGE: + VLD1R (R3), [V0.S4] +Lcomparison32FloatSAFGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.S4] + WORD $0x6e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c47 // umov result + ADD $16, R4, R4 + VLD1 (R4), [V1.S4] + WORD $0x6e21e402 + VUSHR $31, V2.S4, V2.S4 + WORD $0x4eb49c42 // mul v2.4s, v2.4s, v20.4s + VADDV V2.S4, V2 + WORD $0x0e043c48 // umov result + LSL $4, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison32FloatSAFGELoop + +Lcomparison64Float: + MOVD $LCMPNEON_WEIGHTS<>(SB), R10 + ADD $16, R10, R10 + VLD1 (R10), [V20.D2] + CMP $0, R2 + BEQ Lcomparison64FloatAASelect + CMP $1, R2 + BEQ Lcomparison64FloatASSelect + CMP $2, R2 + BEQ Lcomparison64FloatSASelect + JMP LcomparisonReturn + +Lcomparison64FloatAASelect: + CMP $0, R1 + BEQ Lcomparison64FloatAAEQ + CMP $1, R1 + BEQ Lcomparison64FloatAANE + CMP $2, R1 + BEQ Lcomparison64FloatAAGTSelect + CMP $3, R1 + BEQ Lcomparison64FloatAAGESelect + JMP LcomparisonReturn + +Lcomparison64FloatAAGTSelect: + JMP Lcomparison64FloatAAFGT + +Lcomparison64FloatAAGESelect: + JMP Lcomparison64FloatAAFGE + +Lcomparison64FloatASSelect: + CMP $0, R1 + BEQ Lcomparison64FloatASEQ + CMP $1, R1 + BEQ Lcomparison64FloatASNE + CMP $2, R1 + BEQ Lcomparison64FloatASGTSelect + CMP $3, R1 + BEQ Lcomparison64FloatASGESelect + JMP LcomparisonReturn + +Lcomparison64FloatASGTSelect: + JMP Lcomparison64FloatASFGT + +Lcomparison64FloatASGESelect: + JMP Lcomparison64FloatASFGE + +Lcomparison64FloatSASelect: + CMP $0, R1 + BEQ Lcomparison64FloatSAEQ + CMP $1, R1 + BEQ Lcomparison64FloatSANE + CMP $2, R1 + BEQ Lcomparison64FloatSAGTSelect + CMP $3, R1 + BEQ Lcomparison64FloatSAGESelect + JMP LcomparisonReturn + +Lcomparison64FloatSAGTSelect: + JMP Lcomparison64FloatSAFGT + +Lcomparison64FloatSAGESelect: + JMP Lcomparison64FloatSAFGE + +Lcomparison64FloatAAEQ: +Lcomparison64FloatAAEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatAAEQLoop + +Lcomparison64FloatAANE: +Lcomparison64FloatAANELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatAANELoop + +Lcomparison64FloatAAFGT: +Lcomparison64FloatAAFGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatAAFGTLoop + +Lcomparison64FloatAAFGE: +Lcomparison64FloatAAFGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + ADD $16, R4, R4 + VLD1 (R3), [V0.D2] + VLD1 (R4), [V1.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatAAFGELoop + +Lcomparison64FloatASEQ: + VLD1R (R4), [V1.D2] +Lcomparison64FloatASEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatASEQLoop + +Lcomparison64FloatASNE: + VLD1R (R4), [V1.D2] +Lcomparison64FloatASNELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatASNELoop + +Lcomparison64FloatASFGT: + VLD1R (R4), [V1.D2] +Lcomparison64FloatASFGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatASFGTLoop + +Lcomparison64FloatASFGE: + VLD1R (R4), [V1.D2] +Lcomparison64FloatASFGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R3), [V0.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R3, R3 + VLD1 (R3), [V0.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R3, R3 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatASFGELoop + +Lcomparison64FloatSAEQ: + VLD1R (R3), [V0.D2] +Lcomparison64FloatSAEQLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatSAEQLoop + +Lcomparison64FloatSANE: + VLD1R (R3), [V0.D2] +Lcomparison64FloatSANELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x4e61e402 + WORD $0x6e205842 // mvn v2.16b, v2.16b + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatSANELoop + +Lcomparison64FloatSAFGT: + VLD1R (R3), [V0.D2] +Lcomparison64FloatSAFGTLoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6ee1e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatSAFGTLoop + +Lcomparison64FloatSAFGE: + VLD1R (R3), [V0.D2] +Lcomparison64FloatSAFGELoop: + CMP $0, R6 + BEQ LcomparisonReturn + VLD1 (R4), [V1.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c47 // umov x7, v2.d[0] + WORD $0x4e183c48 // umov x8, v2.d[1] + LSR $1, R8, R8 + LSL $1, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $2, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $4, R8, R8 + ORR R8, R7, R7 + ADD $16, R4, R4 + VLD1 (R4), [V1.D2] + WORD $0x6e61e402 + VUSHR $63, V2.D2, V2.D2 + WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d + WORD $0x4e083c48 // umov x8, v2.d[0] + WORD $0x4e183c49 // umov x9, v2.d[1] + LSR $1, R9, R9 + LSL $1, R9, R9 + ORR R9, R8, R8 + LSL $6, R8, R8 + ORR R8, R7, R7 + MOVB R7, (R5) + ADD $16, R4, R4 + ADD $1, R5, R5 + SUB $1, R6, R6 + JMP Lcomparison64FloatSAFGELoop + +LcomparisonReturn: + RET diff --git a/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go b/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go new file mode 100644 index 000000000..eb31d31b2 --- /dev/null +++ b/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.18 && arm64 && !noasm && !appengine + +package kernels + +import ( + "bytes" + "fmt" + "math" + "testing" + "unsafe" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/bitutil" +) + +func testNeonComparison[T arrow.NumericType](t *testing.T, left, right []T, leftScalar, rightScalar T) { + t.Helper() + + operations := []struct { + name string + op CompareOperator + fn func(T, T) bool + }{ + {"equal", CmpEQ, func(l, r T) bool { return l == r }}, + {"not_equal", CmpNE, func(l, r T) bool { return l != r }}, + {"greater", CmpGT, func(l, r T) bool { return l > r }}, + {"greater_equal", CmpGE, func(l, r T) bool { return l >= r }}, + } + + leftBytes := arrow.GetBytes(left) + rightBytes := arrow.GetBytes(right) + leftScalarBytes := arrow.GetBytes([]T{leftScalar}) + rightScalarBytes := arrow.GetBytes([]T{rightScalar}) + width := int(unsafe.Sizeof(T(0))) + lengths := []int{0, 1, 2, 7, 8, 9, 15, 16, 17, 23, 24, 25} + + for _, operation := range operations { + t.Run(operation.name, func(t *testing.T) { + cmp := genCompareKernel[T](operation.op) + for _, shape := range []string{"array_array", "array_scalar", "scalar_array"} { + t.Run(shape, func(t *testing.T) { + for offset := 0; offset < 8; offset++ { + for _, length := range lengths { + t.Run(fmt.Sprintf("offset_%d/length_%d", offset, length), func(t *testing.T) { + out := bytes.Repeat([]byte{0xa5}, int(bitutil.BytesForBits(int64(offset+length)))) + expected := append([]byte(nil), out...) + for i := 0; i < length; i++ { + var result bool + switch shape { + case "array_array": + result = operation.fn(left[i], right[i]) + case "array_scalar": + result = operation.fn(left[i], rightScalar) + case "scalar_array": + result = operation.fn(leftScalar, right[i]) + } + bitutil.SetBitTo(expected, offset+i, result) + } + + switch shape { + case "array_array": + cmp.funcAA(leftBytes[:length*width], rightBytes[:length*width], out, offset) + case "array_scalar": + cmp.funcAS(leftBytes[:length*width], rightScalarBytes, out, offset) + case "scalar_array": + cmp.funcSA(leftScalarBytes, rightBytes[:length*width], out, offset) + } + + if !bytes.Equal(expected, out) { + t.Fatalf("expected %08b, got %08b", expected, out) + } + }) + } + } + }) + } + }) + } +} + +func TestNeonComparisons(t *testing.T) { + t.Run("int32", func(t *testing.T) { + left := make([]int32, 32) + right := make([]int32, 32) + for i := range left { + left[i] = int32((i*37)%17 - 8) + right[i] = int32((i*19)%13 - 6) + } + left[0], left[1] = -1<<31, 1<<31-1 + right[0], right[1] = 1<<31-1, -1<<31 + testNeonComparison(t, left, right, int32(-3), int32(4)) + }) + + t.Run("uint32", func(t *testing.T) { + left := make([]uint32, 32) + right := make([]uint32, 32) + for i := range left { + left[i] = uint32(i * 37) + right[i] = uint32(i*19 + 3) + } + left[0], left[1], left[2] = 0, 1<<31, ^uint32(0) + right[0], right[1], right[2] = ^uint32(0), 1<<31, 1 + testNeonComparison(t, left, right, uint32(1<<31), uint32(7)) + }) + + t.Run("int64", func(t *testing.T) { + left := make([]int64, 32) + right := make([]int64, 32) + for i := range left { + left[i] = int64(i*37 - 400) + right[i] = int64(i*19 - 200) + } + left[0], left[1] = -1<<63, 1<<63-1 + right[0], right[1] = 1<<63-1, -1<<63 + testNeonComparison(t, left, right, int64(-3), int64(4)) + }) + + t.Run("uint64", func(t *testing.T) { + left := make([]uint64, 32) + right := make([]uint64, 32) + for i := range left { + left[i] = uint64(i * 37) + right[i] = uint64(i*19 + 3) + } + left[0], left[1], left[2] = 0, 1<<63, ^uint64(0) + right[0], right[1], right[2] = ^uint64(0), 1<<63, 1 + testNeonComparison(t, left, right, uint64(1<<63), uint64(7)) + }) + + t.Run("float32", func(t *testing.T) { + leftPattern := []float32{float32(math.NaN()), float32(math.Inf(-1)), -1.5, float32(math.Copysign(0, -1)), 0, 1.5, float32(math.Inf(1)), float32(math.NaN())} + rightPattern := []float32{float32(math.NaN()), -float32(math.Inf(1)), -1.5, 0, float32(math.Copysign(0, -1)), 2.5, float32(math.Inf(1)), 3} + left := make([]float32, 32) + right := make([]float32, 32) + for i := range left { + left[i] = leftPattern[i%len(leftPattern)] + right[i] = rightPattern[i%len(rightPattern)] + } + testNeonComparison(t, left, right, float32(math.NaN()), float32(1.5)) + }) + + t.Run("float64", func(t *testing.T) { + leftPattern := []float64{math.NaN(), math.Inf(-1), -1.5, math.Copysign(0, -1), 0, 1.5, math.Inf(1), math.NaN()} + rightPattern := []float64{math.NaN(), -math.Inf(1), -1.5, 0, math.Copysign(0, -1), 2.5, math.Inf(1), 3} + left := make([]float64, 32) + right := make([]float64, 32) + for i := range left { + left[i] = leftPattern[i%len(leftPattern)] + right[i] = rightPattern[i%len(rightPattern)] + } + testNeonComparison(t, left, right, math.NaN(), 1.5) + }) +} diff --git a/arrow/compute/internal/kernels/scalar_comparison_noasm.go b/arrow/compute/internal/kernels/scalar_comparison_noasm.go index 619667693..c94ec5377 100644 --- a/arrow/compute/internal/kernels/scalar_comparison_noasm.go +++ b/arrow/compute/internal/kernels/scalar_comparison_noasm.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build go1.18 && (noasm || !amd64) +//go:build go1.18 && (noasm || (!amd64 && !arm64) || (appengine && arm64)) package kernels From cc7170343458d6d4e1639cee3d0297c0b8a1c76e Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sat, 29 Aug 2026 19:11:08 +0200 Subject: [PATCH 2/3] perf(compute): tighten ARM64 comparison fast path --- .../kernels/scalar_comparison_arm64.go | 11 ++- .../kernels/scalar_comparison_arm64.s | 60 ------------- .../kernels/scalar_comparison_arm64_test.go | 89 +++++++++---------- 3 files changed, 49 insertions(+), 111 deletions(-) diff --git a/arrow/compute/internal/kernels/scalar_comparison_arm64.go b/arrow/compute/internal/kernels/scalar_comparison_arm64.go index 01a8e8247..f5d996111 100644 --- a/arrow/compute/internal/kernels/scalar_comparison_arm64.go +++ b/arrow/compute/internal/kernels/scalar_comparison_arm64.go @@ -53,11 +53,6 @@ func compareNeon(typ arrow.Type, op CompareOperator, width, shape int, left, rig return } - if !cpu.ARM64.HasASIMD || !neonComparisonSupported(typ) { - fallback(left, right, out, offset) - return - } - if offset != 0 { prefix := min(n, 8-offset) switch shape { @@ -116,8 +111,12 @@ func compareNeon(typ arrow.Type, op CompareOperator, width, shape int, left, rig func genCompareKernel[T arrow.NumericType](op CompareOperator) *CompareData { ty := arrow.GetType[T]() - width := int(unsafe.Sizeof(T(0))) fallback := genGoCompareKernel(getCmpOp[T](op)) + if !cpu.ARM64.HasASIMD || !neonComparisonSupported(ty) { + return fallback + } + + width := int(unsafe.Sizeof(T(0))) return &CompareData{ funcAA: func(left, right, out []byte, offset int) { compareNeon(ty, op, width, neonCompareArrayArray, left, right, out, offset, fallback.funcAA) diff --git a/arrow/compute/internal/kernels/scalar_comparison_arm64.s b/arrow/compute/internal/kernels/scalar_comparison_arm64.s index 1bc12edb9..48711b9fb 100644 --- a/arrow/compute/internal/kernels/scalar_comparison_arm64.s +++ b/arrow/compute/internal/kernels/scalar_comparison_arm64.s @@ -701,8 +701,6 @@ Lcomparison64IntAAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 ADD $16, R4, R4 @@ -765,8 +763,6 @@ Lcomparison64IntAANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 ADD $16, R4, R4 @@ -831,8 +827,6 @@ Lcomparison64IntAASGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 ADD $16, R4, R4 @@ -894,8 +888,6 @@ Lcomparison64IntAAUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 ADD $16, R4, R4 @@ -957,8 +949,6 @@ Lcomparison64IntAASGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 ADD $16, R4, R4 @@ -1020,8 +1010,6 @@ Lcomparison64IntAAUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 ADD $16, R4, R4 @@ -1083,8 +1071,6 @@ Lcomparison64IntASEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 VLD1 (R3), [V0.D2] @@ -1140,8 +1126,6 @@ Lcomparison64IntASNELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 VLD1 (R3), [V0.D2] @@ -1199,8 +1183,6 @@ Lcomparison64IntASSGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 VLD1 (R3), [V0.D2] @@ -1255,8 +1237,6 @@ Lcomparison64IntASUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 VLD1 (R3), [V0.D2] @@ -1311,8 +1291,6 @@ Lcomparison64IntASSGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 VLD1 (R3), [V0.D2] @@ -1367,8 +1345,6 @@ Lcomparison64IntASUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 VLD1 (R3), [V0.D2] @@ -1423,8 +1399,6 @@ Lcomparison64IntSAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R4, R4 VLD1 (R4), [V1.D2] @@ -1480,8 +1454,6 @@ Lcomparison64IntSANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R4, R4 VLD1 (R4), [V1.D2] @@ -1539,8 +1511,6 @@ Lcomparison64IntSASGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R4, R4 VLD1 (R4), [V1.D2] @@ -1595,8 +1565,6 @@ Lcomparison64IntSAUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R4, R4 VLD1 (R4), [V1.D2] @@ -1651,8 +1619,6 @@ Lcomparison64IntSASGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R4, R4 VLD1 (R4), [V1.D2] @@ -1707,8 +1673,6 @@ Lcomparison64IntSAUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R4, R4 VLD1 (R4), [V1.D2] @@ -2218,8 +2182,6 @@ Lcomparison64FloatAAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 ADD $16, R4, R4 @@ -2282,8 +2244,6 @@ Lcomparison64FloatAANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 ADD $16, R4, R4 @@ -2348,8 +2308,6 @@ Lcomparison64FloatAAFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 ADD $16, R4, R4 @@ -2411,8 +2369,6 @@ Lcomparison64FloatAAFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 ADD $16, R4, R4 @@ -2474,8 +2430,6 @@ Lcomparison64FloatASEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 VLD1 (R3), [V0.D2] @@ -2531,8 +2485,6 @@ Lcomparison64FloatASNELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 VLD1 (R3), [V0.D2] @@ -2590,8 +2542,6 @@ Lcomparison64FloatASFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 VLD1 (R3), [V0.D2] @@ -2646,8 +2596,6 @@ Lcomparison64FloatASFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R3, R3 VLD1 (R3), [V0.D2] @@ -2702,8 +2650,6 @@ Lcomparison64FloatSAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R4, R4 VLD1 (R4), [V1.D2] @@ -2759,8 +2705,6 @@ Lcomparison64FloatSANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R4, R4 VLD1 (R4), [V1.D2] @@ -2818,8 +2762,6 @@ Lcomparison64FloatSAFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R4, R4 VLD1 (R4), [V1.D2] @@ -2874,8 +2816,6 @@ Lcomparison64FloatSAFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c47 // umov x7, v2.d[0] WORD $0x4e183c48 // umov x8, v2.d[1] - LSR $1, R8, R8 - LSL $1, R8, R8 ORR R8, R7, R7 ADD $16, R4, R4 VLD1 (R4), [V1.D2] diff --git a/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go b/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go index eb31d31b2..0714b2dfb 100644 --- a/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go +++ b/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go @@ -29,18 +29,19 @@ import ( "github.com/apache/arrow-go/v18/arrow/bitutil" ) +const neonComparisonTestLength = 1024 + func testNeonComparison[T arrow.NumericType](t *testing.T, left, right []T, leftScalar, rightScalar T) { t.Helper() operations := []struct { name string op CompareOperator - fn func(T, T) bool }{ - {"equal", CmpEQ, func(l, r T) bool { return l == r }}, - {"not_equal", CmpNE, func(l, r T) bool { return l != r }}, - {"greater", CmpGT, func(l, r T) bool { return l > r }}, - {"greater_equal", CmpGE, func(l, r T) bool { return l >= r }}, + {"equal", CmpEQ}, + {"not_equal", CmpNE}, + {"greater", CmpGT}, + {"greater_equal", CmpGE}, } leftBytes := arrow.GetBytes(left) @@ -48,11 +49,12 @@ func testNeonComparison[T arrow.NumericType](t *testing.T, left, right []T, left leftScalarBytes := arrow.GetBytes([]T{leftScalar}) rightScalarBytes := arrow.GetBytes([]T{rightScalar}) width := int(unsafe.Sizeof(T(0))) - lengths := []int{0, 1, 2, 7, 8, 9, 15, 16, 17, 23, 24, 25} + lengths := []int{0, 1, 2, 7, 8, 9, 15, 16, 17, 23, 24, 25, 31, 32, 33, 63, 64, 65, 127, 128, 129, neonComparisonTestLength} for _, operation := range operations { t.Run(operation.name, func(t *testing.T) { cmp := genCompareKernel[T](operation.op) + fallback := genGoCompareKernel(getCmpOp[T](operation.op)) for _, shape := range []string{"array_array", "array_scalar", "scalar_array"} { t.Run(shape, func(t *testing.T) { for offset := 0; offset < 8; offset++ { @@ -60,17 +62,14 @@ func testNeonComparison[T arrow.NumericType](t *testing.T, left, right []T, left t.Run(fmt.Sprintf("offset_%d/length_%d", offset, length), func(t *testing.T) { out := bytes.Repeat([]byte{0xa5}, int(bitutil.BytesForBits(int64(offset+length)))) expected := append([]byte(nil), out...) - for i := 0; i < length; i++ { - var result bool - switch shape { - case "array_array": - result = operation.fn(left[i], right[i]) - case "array_scalar": - result = operation.fn(left[i], rightScalar) - case "scalar_array": - result = operation.fn(leftScalar, right[i]) - } - bitutil.SetBitTo(expected, offset+i, result) + + switch shape { + case "array_array": + fallback.funcAA(leftBytes[:length*width], rightBytes[:length*width], expected, offset) + case "array_scalar": + fallback.funcAS(leftBytes[:length*width], rightScalarBytes, expected, offset) + case "scalar_array": + fallback.funcSA(leftScalarBytes, rightBytes[:length*width], expected, offset) } switch shape { @@ -96,58 +95,58 @@ func testNeonComparison[T arrow.NumericType](t *testing.T, left, right []T, left func TestNeonComparisons(t *testing.T) { t.Run("int32", func(t *testing.T) { - left := make([]int32, 32) - right := make([]int32, 32) + leftPattern := []int32{-1 << 31, -17, -1, 0, 1, 17, 1<<31 - 1, 42, -42, 3, 9, -9, 5, -5, 2, -2} + rightPattern := []int32{1<<31 - 1, -17, 0, 1, -1, -42, -1 << 31, 42, 4, -3, 9, -10, -5, 5, -2, 2} + left := make([]int32, neonComparisonTestLength) + right := make([]int32, neonComparisonTestLength) for i := range left { - left[i] = int32((i*37)%17 - 8) - right[i] = int32((i*19)%13 - 6) + left[i] = leftPattern[i%len(leftPattern)] + right[i] = rightPattern[i%len(rightPattern)] } - left[0], left[1] = -1<<31, 1<<31-1 - right[0], right[1] = 1<<31-1, -1<<31 testNeonComparison(t, left, right, int32(-3), int32(4)) }) t.Run("uint32", func(t *testing.T) { - left := make([]uint32, 32) - right := make([]uint32, 32) + leftPattern := []uint32{0, 1, 2, 1<<31 - 1, 1 << 31, ^uint32(0), 42, 7, 100, 3, 19, 5, 77, 11, 12, 13} + rightPattern := []uint32{^uint32(0), 1, 3, 1 << 31, 1<<31 - 1, 0, 42, 8, 99, 4, 19, 6, 78, 10, 13, 12} + left := make([]uint32, neonComparisonTestLength) + right := make([]uint32, neonComparisonTestLength) for i := range left { - left[i] = uint32(i * 37) - right[i] = uint32(i*19 + 3) + left[i] = leftPattern[i%len(leftPattern)] + right[i] = rightPattern[i%len(rightPattern)] } - left[0], left[1], left[2] = 0, 1<<31, ^uint32(0) - right[0], right[1], right[2] = ^uint32(0), 1<<31, 1 testNeonComparison(t, left, right, uint32(1<<31), uint32(7)) }) t.Run("int64", func(t *testing.T) { - left := make([]int64, 32) - right := make([]int64, 32) + leftPattern := []int64{-1 << 63, -17, -1, 0, 1, 17, 1<<63 - 1, 42, -42, 3, 9, -9, 5, -5, 2, -2} + rightPattern := []int64{1<<63 - 1, -17, 0, 1, -1, -42, -1 << 63, 42, 4, -3, 9, -10, -5, 5, -2, 2} + left := make([]int64, neonComparisonTestLength) + right := make([]int64, neonComparisonTestLength) for i := range left { - left[i] = int64(i*37 - 400) - right[i] = int64(i*19 - 200) + left[i] = leftPattern[i%len(leftPattern)] + right[i] = rightPattern[i%len(rightPattern)] } - left[0], left[1] = -1<<63, 1<<63-1 - right[0], right[1] = 1<<63-1, -1<<63 testNeonComparison(t, left, right, int64(-3), int64(4)) }) t.Run("uint64", func(t *testing.T) { - left := make([]uint64, 32) - right := make([]uint64, 32) + leftPattern := []uint64{0, 1, 2, 1<<63 - 1, 1 << 63, ^uint64(0), 42, 7, 100, 3, 19, 5, 77, 11, 12, 13} + rightPattern := []uint64{^uint64(0), 1, 3, 1 << 63, 1<<63 - 1, 0, 42, 8, 99, 4, 19, 6, 78, 10, 13, 12} + left := make([]uint64, neonComparisonTestLength) + right := make([]uint64, neonComparisonTestLength) for i := range left { - left[i] = uint64(i * 37) - right[i] = uint64(i*19 + 3) + left[i] = leftPattern[i%len(leftPattern)] + right[i] = rightPattern[i%len(rightPattern)] } - left[0], left[1], left[2] = 0, 1<<63, ^uint64(0) - right[0], right[1], right[2] = ^uint64(0), 1<<63, 1 testNeonComparison(t, left, right, uint64(1<<63), uint64(7)) }) t.Run("float32", func(t *testing.T) { leftPattern := []float32{float32(math.NaN()), float32(math.Inf(-1)), -1.5, float32(math.Copysign(0, -1)), 0, 1.5, float32(math.Inf(1)), float32(math.NaN())} rightPattern := []float32{float32(math.NaN()), -float32(math.Inf(1)), -1.5, 0, float32(math.Copysign(0, -1)), 2.5, float32(math.Inf(1)), 3} - left := make([]float32, 32) - right := make([]float32, 32) + left := make([]float32, neonComparisonTestLength) + right := make([]float32, neonComparisonTestLength) for i := range left { left[i] = leftPattern[i%len(leftPattern)] right[i] = rightPattern[i%len(rightPattern)] @@ -158,8 +157,8 @@ func TestNeonComparisons(t *testing.T) { t.Run("float64", func(t *testing.T) { leftPattern := []float64{math.NaN(), math.Inf(-1), -1.5, math.Copysign(0, -1), 0, 1.5, math.Inf(1), math.NaN()} rightPattern := []float64{math.NaN(), -math.Inf(1), -1.5, 0, math.Copysign(0, -1), 2.5, math.Inf(1), 3} - left := make([]float64, 32) - right := make([]float64, 32) + left := make([]float64, neonComparisonTestLength) + right := make([]float64, neonComparisonTestLength) for i := range left { left[i] = leftPattern[i%len(leftPattern)] right[i] = rightPattern[i%len(rightPattern)] From 92d4bfe69b58a54a5a0cb96ea48328dda992116b Mon Sep 17 00:00:00 2001 From: Minh Vu Date: Sun, 30 Aug 2026 22:42:12 +0200 Subject: [PATCH 3/3] perf(compute): trim redundant NEON bitmap packing shifts --- .../kernels/scalar_comparison_arm64.s | 180 ------------------ .../kernels/scalar_comparison_arm64_test.go | 42 ++++ 2 files changed, 42 insertions(+), 180 deletions(-) diff --git a/arrow/compute/internal/kernels/scalar_comparison_arm64.s b/arrow/compute/internal/kernels/scalar_comparison_arm64.s index 48711b9fb..80ffc09cb 100644 --- a/arrow/compute/internal/kernels/scalar_comparison_arm64.s +++ b/arrow/compute/internal/kernels/scalar_comparison_arm64.s @@ -711,8 +711,6 @@ Lcomparison64IntAAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -725,8 +723,6 @@ Lcomparison64IntAAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -739,8 +735,6 @@ Lcomparison64IntAAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -774,8 +768,6 @@ Lcomparison64IntAANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -789,8 +781,6 @@ Lcomparison64IntAANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -804,8 +794,6 @@ Lcomparison64IntAANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -837,8 +825,6 @@ Lcomparison64IntAASGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -851,8 +837,6 @@ Lcomparison64IntAASGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -865,8 +849,6 @@ Lcomparison64IntAASGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -898,8 +880,6 @@ Lcomparison64IntAAUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -912,8 +892,6 @@ Lcomparison64IntAAUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -926,8 +904,6 @@ Lcomparison64IntAAUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -959,8 +935,6 @@ Lcomparison64IntAASGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -973,8 +947,6 @@ Lcomparison64IntAASGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -987,8 +959,6 @@ Lcomparison64IntAASGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1020,8 +990,6 @@ Lcomparison64IntAAUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1034,8 +1002,6 @@ Lcomparison64IntAAUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1048,8 +1014,6 @@ Lcomparison64IntAAUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1079,8 +1043,6 @@ Lcomparison64IntASEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1091,8 +1053,6 @@ Lcomparison64IntASEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1103,8 +1063,6 @@ Lcomparison64IntASEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1135,8 +1093,6 @@ Lcomparison64IntASNELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1148,8 +1104,6 @@ Lcomparison64IntASNELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1161,8 +1115,6 @@ Lcomparison64IntASNELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1191,8 +1143,6 @@ Lcomparison64IntASSGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1203,8 +1153,6 @@ Lcomparison64IntASSGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1215,8 +1163,6 @@ Lcomparison64IntASSGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1245,8 +1191,6 @@ Lcomparison64IntASUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1257,8 +1201,6 @@ Lcomparison64IntASUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1269,8 +1211,6 @@ Lcomparison64IntASUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1299,8 +1239,6 @@ Lcomparison64IntASSGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1311,8 +1249,6 @@ Lcomparison64IntASSGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1323,8 +1259,6 @@ Lcomparison64IntASSGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1353,8 +1287,6 @@ Lcomparison64IntASUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1365,8 +1297,6 @@ Lcomparison64IntASUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1377,8 +1307,6 @@ Lcomparison64IntASUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1407,8 +1335,6 @@ Lcomparison64IntSAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1419,8 +1345,6 @@ Lcomparison64IntSAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1431,8 +1355,6 @@ Lcomparison64IntSAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1463,8 +1385,6 @@ Lcomparison64IntSANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1476,8 +1396,6 @@ Lcomparison64IntSANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1489,8 +1407,6 @@ Lcomparison64IntSANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1519,8 +1435,6 @@ Lcomparison64IntSASGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1531,8 +1445,6 @@ Lcomparison64IntSASGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1543,8 +1455,6 @@ Lcomparison64IntSASGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1573,8 +1483,6 @@ Lcomparison64IntSAUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1585,8 +1493,6 @@ Lcomparison64IntSAUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1597,8 +1503,6 @@ Lcomparison64IntSAUGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1627,8 +1531,6 @@ Lcomparison64IntSASGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1639,8 +1541,6 @@ Lcomparison64IntSASGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1651,8 +1551,6 @@ Lcomparison64IntSASGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -1681,8 +1579,6 @@ Lcomparison64IntSAUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -1693,8 +1589,6 @@ Lcomparison64IntSAUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -1705,8 +1599,6 @@ Lcomparison64IntSAUGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2192,8 +2084,6 @@ Lcomparison64FloatAAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2206,8 +2096,6 @@ Lcomparison64FloatAAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2220,8 +2108,6 @@ Lcomparison64FloatAAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2255,8 +2141,6 @@ Lcomparison64FloatAANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2270,8 +2154,6 @@ Lcomparison64FloatAANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2285,8 +2167,6 @@ Lcomparison64FloatAANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2318,8 +2198,6 @@ Lcomparison64FloatAAFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2332,8 +2210,6 @@ Lcomparison64FloatAAFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2346,8 +2222,6 @@ Lcomparison64FloatAAFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2379,8 +2253,6 @@ Lcomparison64FloatAAFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2393,8 +2265,6 @@ Lcomparison64FloatAAFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2407,8 +2277,6 @@ Lcomparison64FloatAAFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2438,8 +2306,6 @@ Lcomparison64FloatASEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2450,8 +2316,6 @@ Lcomparison64FloatASEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2462,8 +2326,6 @@ Lcomparison64FloatASEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2494,8 +2356,6 @@ Lcomparison64FloatASNELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2507,8 +2367,6 @@ Lcomparison64FloatASNELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2520,8 +2378,6 @@ Lcomparison64FloatASNELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2550,8 +2406,6 @@ Lcomparison64FloatASFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2562,8 +2416,6 @@ Lcomparison64FloatASFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2574,8 +2426,6 @@ Lcomparison64FloatASFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2604,8 +2454,6 @@ Lcomparison64FloatASFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2616,8 +2464,6 @@ Lcomparison64FloatASFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2628,8 +2474,6 @@ Lcomparison64FloatASFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2658,8 +2502,6 @@ Lcomparison64FloatSAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2670,8 +2512,6 @@ Lcomparison64FloatSAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2682,8 +2522,6 @@ Lcomparison64FloatSAEQLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2714,8 +2552,6 @@ Lcomparison64FloatSANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2727,8 +2563,6 @@ Lcomparison64FloatSANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2740,8 +2574,6 @@ Lcomparison64FloatSANELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2770,8 +2602,6 @@ Lcomparison64FloatSAFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2782,8 +2612,6 @@ Lcomparison64FloatSAFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2794,8 +2622,6 @@ Lcomparison64FloatSAFGTLoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 @@ -2824,8 +2650,6 @@ Lcomparison64FloatSAFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $2, R8, R8 ORR R8, R7, R7 @@ -2836,8 +2660,6 @@ Lcomparison64FloatSAFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $4, R8, R8 ORR R8, R7, R7 @@ -2848,8 +2670,6 @@ Lcomparison64FloatSAFGELoop: WORD $0x6ef44442 // ushl v2.2d, v2.2d, v20.2d WORD $0x4e083c48 // umov x8, v2.d[0] WORD $0x4e183c49 // umov x9, v2.d[1] - LSR $1, R9, R9 - LSL $1, R9, R9 ORR R9, R8, R8 LSL $6, R8, R8 ORR R8, R7, R7 diff --git a/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go b/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go index 0714b2dfb..460d8acad 100644 --- a/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go +++ b/arrow/compute/internal/kernels/scalar_comparison_arm64_test.go @@ -166,3 +166,45 @@ func TestNeonComparisons(t *testing.T) { testNeonComparison(t, left, right, math.NaN(), 1.5) }) } + +func TestNeonComparisonBitPacking(t *testing.T) { + t.Run("int32", testNeonComparisonBitPacking[int32]) + t.Run("uint32", testNeonComparisonBitPacking[uint32]) + t.Run("int64", testNeonComparisonBitPacking[int64]) + t.Run("uint64", testNeonComparisonBitPacking[uint64]) + t.Run("float32", testNeonComparisonBitPacking[float32]) + t.Run("float64", testNeonComparisonBitPacking[float64]) +} + +func testNeonComparisonBitPacking[T arrow.NumericType](t *testing.T) { + const n = 256 * 8 + left, right := make([]T, n), make([]T, n) + for mask := 0; mask < 256; mask++ { + for bit := 0; bit < 8; bit++ { + left[mask*8+bit] = T(mask >> bit & 1) + right[mask*8+bit] = T(mask >> (7 - bit) & 1) + } + } + leftBytes, rightBytes := arrow.GetBytes(left), arrow.GetBytes(right) + scalar := arrow.GetBytes([]T{0}) + for _, op := range []CompareOperator{CmpEQ, CmpNE, CmpGT, CmpGE} { + got, want := genCompareKernel[T](op), genGoCompareKernel(getCmpOp[T](op)) + for _, shape := range []struct { + name string + got, want binaryKernel + left, right []byte + }{ + {"array_array", got.funcAA, want.funcAA, leftBytes, rightBytes}, + {"array_scalar", got.funcAS, want.funcAS, leftBytes, scalar}, + {"scalar_array", got.funcSA, want.funcSA, scalar, rightBytes}, + } { + output := bytes.Repeat([]byte{0xa5}, n/8+2) + expected := bytes.Clone(output) + shape.want(shape.left, shape.right, expected[1:len(expected)-1], 0) + shape.got(shape.left, shape.right, output[1:len(output)-1], 0) + if !bytes.Equal(output, expected) { + t.Fatalf("%s %s: expected %08b, got %08b", op, shape.name, expected, output) + } + } + } +}