diff --git a/arrow/compute/internal/kernels/base_arithmetic_arm64.go b/arrow/compute/internal/kernels/base_arithmetic_arm64.go new file mode 100644 index 00000000..804875eb --- /dev/null +++ b/arrow/compute/internal/kernels/base_arithmetic_arm64.go @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.18 && arm64 && !noasm + +package kernels + +import ( + "unsafe" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/compute/exec" + "golang.org/x/exp/constraints" + "golang.org/x/sys/cpu" +) + +//go:noescape +func _arithmetic_binary_neon(typ int, op int8, inLeft, inRight, out unsafe.Pointer, len int) + +func arithmeticNeon(typ arrow.Type, op ArithmeticOp, left, right, out []byte, len int) { + if len == 0 { + return + } + _arithmetic_binary_neon(int(typ), int8(op), unsafe.Pointer(&left[0]), unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), len) +} + +//go:noescape +func _arithmetic_arr_scalar_neon(typ int, op int8, inLeft, inRight, out unsafe.Pointer, len int) + +func arithmeticArrScalarNeon(typ arrow.Type, op ArithmeticOp, left []byte, right unsafe.Pointer, out []byte, len int) { + if len == 0 { + return + } + _arithmetic_arr_scalar_neon(int(typ), int8(op), unsafe.Pointer(&left[0]), right, unsafe.Pointer(&out[0]), len) +} + +//go:noescape +func _arithmetic_scalar_arr_neon(typ int, op int8, inLeft, inRight, out unsafe.Pointer, len int) + +func arithmeticScalarArrNeon(typ arrow.Type, op ArithmeticOp, left unsafe.Pointer, right, out []byte, len int) { + if len == 0 { + return + } + _arithmetic_scalar_arr_neon(int(typ), int8(op), left, unsafe.Pointer(&right[0]), unsafe.Pointer(&out[0]), len) +} + +//go:noescape +func _arithmetic_unary_same_types_neon(typ int, op int8, input, output unsafe.Pointer, len int) + +func arithmeticUnaryNeon(typ arrow.Type, op ArithmeticOp, input, out []byte, len int) { + if len == 0 { + return + } + _arithmetic_unary_same_types_neon(int(typ), int8(op), unsafe.Pointer(&input[0]), unsafe.Pointer(&out[0]), len) +} + +func normalizeNeonArithmeticOp(op ArithmeticOp) ArithmeticOp { + switch op { + case OpAddChecked: + return OpAdd + case OpSubChecked: + return OpSub + case OpMulChecked: + return OpMul + case OpAbsoluteValueChecked: + return OpAbsoluteValue + case OpNegateChecked: + return OpNegate + default: + return op + } +} + +func neonIntegralBinarySupported(typ arrow.Type, op ArithmeticOp) bool { + switch typ { + case arrow.INT32, arrow.UINT32: + return op == OpAdd || op == OpSub || op == OpMul + case arrow.INT64, arrow.UINT64: + return op == OpAdd || op == OpSub + default: + return false + } +} + +func neonIntegralUnarySupported(typ arrow.Type) bool { + switch typ { + case arrow.INT32, arrow.UINT32, arrow.INT64, arrow.UINT64: + return true + default: + return false + } +} + +func getNeonArithmeticBinaryNumeric[T arrow.NumericType](op ArithmeticOp) binaryOps[T, T, T] { + typ := arrow.GetType[T]() + return binaryOps[T, T, T]{ + arrArr: func(_ *exec.KernelCtx, Arg0, Arg1, Out []T) error { + arithmeticNeon(typ, op, arrow.GetBytes(Arg0), arrow.GetBytes(Arg1), arrow.GetBytes(Out), len(Arg0)) + return nil + }, + arrScalar: func(_ *exec.KernelCtx, Arg0 []T, Arg1 T, Out []T) error { + arithmeticArrScalarNeon(typ, op, arrow.GetBytes(Arg0), unsafe.Pointer(&Arg1), arrow.GetBytes(Out), len(Arg0)) + return nil + }, + scalarArr: func(_ *exec.KernelCtx, Arg0 T, Arg1, Out []T) error { + arithmeticScalarArrNeon(typ, op, unsafe.Pointer(&Arg0), arrow.GetBytes(Arg1), arrow.GetBytes(Out), len(Arg1)) + return nil + }, + } +} + +func getArithmeticOpIntegral[InT, OutT arrow.UintType | arrow.IntType](op ArithmeticOp) exec.ArrayKernelExec { + typ := arrow.GetType[InT]() + if cpu.ARM64.HasASIMD && typ == arrow.GetType[OutT]() { + switch op { + case OpAdd, OpSub, OpMul: + if neonIntegralBinarySupported(typ, op) { + return ScalarBinary(getNeonArithmeticBinaryNumeric[InT](op)) + } + case OpAbsoluteValue, OpNegate: + if neonIntegralUnarySupported(typ) { + return ScalarUnary(func(_ *exec.KernelCtx, arg, out []InT) error { + arithmeticUnaryNeon(typ, op, arrow.GetBytes(arg), arrow.GetBytes(out), len(arg)) + return nil + }) + } + } + } + + // no SIMD for POWER or SQRT functions + // integral checked funcs need to use NotNull versions + return getGoArithmeticOpIntegral[InT, OutT](op) +} + +func getArithmeticOpFloating[InT, OutT constraints.Float](op ArithmeticOp) exec.ArrayKernelExec { + if cpu.ARM64.HasASIMD && arrow.GetType[InT]() == arrow.GetType[OutT]() { + typ := arrow.GetType[InT]() + switch op { + case OpAdd, OpSub, OpAddChecked, OpSubChecked, OpMul, OpMulChecked: + return ScalarBinary(getNeonArithmeticBinaryNumeric[InT](normalizeNeonArithmeticOp(op))) + case OpAbsoluteValue, OpAbsoluteValueChecked, OpNegate, OpNegateChecked: + return ScalarUnary(func(_ *exec.KernelCtx, arg, out []InT) error { + arithmeticUnaryNeon(typ, normalizeNeonArithmeticOp(op), arrow.GetBytes(arg), arrow.GetBytes(out), len(arg)) + return nil + }) + } + } + + // no SIMD for POWER or SQRT functions + return getGoArithmeticOpFloating[InT, OutT](op) +} diff --git a/arrow/compute/internal/kernels/base_arithmetic_arm64.s b/arrow/compute/internal/kernels/base_arithmetic_arm64.s new file mode 100644 index 00000000..bd0e6552 --- /dev/null +++ b/arrow/compute/internal/kernels/base_arithmetic_arm64.s @@ -0,0 +1,1430 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.18 && arm64 && !noasm && !appengine + +#include "textflag.h" + +TEXT ·_arithmetic_binary_neon(SB), NOSPLIT|NOFRAME, $0-48 + MOVD typ+0(FP), R0 + MOVB op+8(FP), R1 + MOVD inLeft+16(FP), R2 + MOVD inRight+24(FP), R3 + MOVD out+32(FP), R4 + MOVD len+40(FP), R5 + + CMP $6, R0 + BEQ LneonBinary32Int + CMP $7, R0 + BEQ LneonBinary32Int + CMP $8, R0 + BEQ LneonBinary64Int + CMP $9, R0 + BEQ LneonBinary64Int + CMP $11, R0 + BEQ LneonBinary32Float + CMP $12, R0 + BEQ LneonBinary64Float + RET + +LneonBinary32Int: + CMP $0, R1 + BEQ LneonBinary32IntAdd + CMP $1, R1 + BEQ LneonBinary32IntSub + CMP $2, R1 + BEQ LneonBinary32IntMul + RET + +LneonBinary32IntAdd: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary32IntAddVector: + CMP $4, R5 + BLT LneonBinary32IntAddTail + VLD1 (R2), [V0.S4] + VLD1 (R3), [V1.S4] + VADD V1.S4, V0.S4, V0.S4 + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonBinary32IntAddVector +LneonBinary32IntAddTail: + CMP $0, R5 + BEQ LneonBinaryReturn + MOVW (R2), R6 + MOVW (R3), R7 + ADDW R7, R6, R6 + MOVW R6, (R4) + ADD $4, R2, R2 + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary32IntAddTail + +LneonBinary32IntSub: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary32IntSubVector: + CMP $4, R5 + BLT LneonBinary32IntSubTail + VLD1 (R2), [V0.S4] + VLD1 (R3), [V1.S4] + VSUB V1.S4, V0.S4, V0.S4 + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonBinary32IntSubVector +LneonBinary32IntSubTail: + CMP $0, R5 + BEQ LneonBinaryReturn + MOVW (R2), R6 + MOVW (R3), R7 + SUBW R7, R6, R6 + MOVW R6, (R4) + ADD $4, R2, R2 + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary32IntSubTail + +LneonBinary32IntMul: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary32IntMulVector: + CMP $4, R5 + BLT LneonBinary32IntMulTail + VLD1 (R2), [V0.S4] + VLD1 (R3), [V1.S4] + WORD $0x4ea09c20 // mul v0.4s, v1.4s, v0.4s + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonBinary32IntMulVector +LneonBinary32IntMulTail: + CMP $0, R5 + BEQ LneonBinaryReturn + MOVW (R2), R6 + MOVW (R3), R7 + MULW R7, R6, R6 + MOVW R6, (R4) + ADD $4, R2, R2 + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary32IntMulTail + +LneonBinary64Int: + CMP $0, R1 + BEQ LneonBinary64IntAdd + CMP $1, R1 + BEQ LneonBinary64IntSub + RET + +LneonBinary64IntAdd: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary64IntAddVector: + CMP $2, R5 + BLT LneonBinary64IntAddTail + VLD1 (R2), [V0.D2] + VLD1 (R3), [V1.D2] + VADD V1.D2, V0.D2, V0.D2 + VST1 [V0.D2], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonBinary64IntAddVector +LneonBinary64IntAddTail: + CMP $0, R5 + BEQ LneonBinaryReturn + MOVD (R2), R6 + MOVD (R3), R7 + ADD R7, R6, R6 + MOVD R6, (R4) + ADD $8, R2, R2 + ADD $8, R3, R3 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary64IntAddTail + +LneonBinary64IntSub: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary64IntSubVector: + CMP $2, R5 + BLT LneonBinary64IntSubTail + VLD1 (R2), [V0.D2] + VLD1 (R3), [V1.D2] + VSUB V1.D2, V0.D2, V0.D2 + VST1 [V0.D2], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonBinary64IntSubVector +LneonBinary64IntSubTail: + CMP $0, R5 + BEQ LneonBinaryReturn + MOVD (R2), R6 + MOVD (R3), R7 + SUB R7, R6, R6 + MOVD R6, (R4) + ADD $8, R2, R2 + ADD $8, R3, R3 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary64IntSubTail + +LneonBinary32Float: + CMP $0, R1 + BEQ LneonBinary32FloatAdd + CMP $1, R1 + BEQ LneonBinary32FloatSub + CMP $2, R1 + BEQ LneonBinary32FloatMul + RET + +LneonBinary32FloatAdd: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary32FloatAddVector: + CMP $4, R5 + BLT LneonBinary32FloatAddTail + VLD1 (R2), [V0.S4] + VLD1 (R3), [V1.S4] + WORD $0x4e21d400 // fadd v0.4s, v0.4s, v1.4s + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonBinary32FloatAddVector +LneonBinary32FloatAddTail: + CMP $0, R5 + BEQ LneonBinaryReturn + FMOVS (R2), F0 + FMOVS (R3), F1 + FADDS F1, F0, F0 + FMOVS F0, (R4) + ADD $4, R2, R2 + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary32FloatAddTail + +LneonBinary32FloatSub: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary32FloatSubVector: + CMP $4, R5 + BLT LneonBinary32FloatSubTail + VLD1 (R2), [V0.S4] + VLD1 (R3), [V1.S4] + WORD $0x4ea1d400 // fsub v0.4s, v0.4s, v1.4s + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonBinary32FloatSubVector +LneonBinary32FloatSubTail: + CMP $0, R5 + BEQ LneonBinaryReturn + FMOVS (R2), F0 + FMOVS (R3), F1 + FSUBS F1, F0, F0 + FMOVS F0, (R4) + ADD $4, R2, R2 + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary32FloatSubTail + +LneonBinary32FloatMul: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary32FloatMulVector: + CMP $4, R5 + BLT LneonBinary32FloatMulTail + VLD1 (R2), [V0.S4] + VLD1 (R3), [V1.S4] + WORD $0x6e21dc00 // fmul v0.4s, v0.4s, v1.4s + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonBinary32FloatMulVector +LneonBinary32FloatMulTail: + CMP $0, R5 + BEQ LneonBinaryReturn + FMOVS (R2), F0 + FMOVS (R3), F1 + FMULS F1, F0, F0 + FMOVS F0, (R4) + ADD $4, R2, R2 + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary32FloatMulTail + +LneonBinary64Float: + CMP $0, R1 + BEQ LneonBinary64FloatAdd + CMP $1, R1 + BEQ LneonBinary64FloatSub + CMP $2, R1 + BEQ LneonBinary64FloatMul + RET + +LneonBinary64FloatAdd: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary64FloatAddVector: + CMP $2, R5 + BLT LneonBinary64FloatAddTail + VLD1 (R2), [V0.D2] + VLD1 (R3), [V1.D2] + WORD $0x4e61d400 // fadd v0.2d, v0.2d, v1.2d + VST1 [V0.D2], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonBinary64FloatAddVector +LneonBinary64FloatAddTail: + CMP $0, R5 + BEQ LneonBinaryReturn + FMOVD (R2), F0 + FMOVD (R3), F1 + FADDD F1, F0, F0 + FMOVD F0, (R4) + ADD $8, R2, R2 + ADD $8, R3, R3 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary64FloatAddTail + +LneonBinary64FloatSub: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary64FloatSubVector: + CMP $2, R5 + BLT LneonBinary64FloatSubTail + VLD1 (R2), [V0.D2] + VLD1 (R3), [V1.D2] + WORD $0x4ee1d400 // fsub v0.2d, v0.2d, v1.2d + VST1 [V0.D2], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonBinary64FloatSubVector +LneonBinary64FloatSubTail: + CMP $0, R5 + BEQ LneonBinaryReturn + FMOVD (R2), F0 + FMOVD (R3), F1 + FSUBD F1, F0, F0 + FMOVD F0, (R4) + ADD $8, R2, R2 + ADD $8, R3, R3 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary64FloatSubTail + +LneonBinary64FloatMul: + CMP $0, R5 + BLE LneonBinaryReturn +LneonBinary64FloatMulVector: + CMP $2, R5 + BLT LneonBinary64FloatMulTail + VLD1 (R2), [V0.D2] + VLD1 (R3), [V1.D2] + WORD $0x6e61dc00 // fmul v0.2d, v0.2d, v1.2d + VST1 [V0.D2], (R4) + ADD $16, R2, R2 + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonBinary64FloatMulVector +LneonBinary64FloatMulTail: + CMP $0, R5 + BEQ LneonBinaryReturn + FMOVD (R2), F0 + FMOVD (R3), F1 + FMULD F1, F0, F0 + FMOVD F0, (R4) + ADD $8, R2, R2 + ADD $8, R3, R3 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonBinary64FloatMulTail + +LneonBinaryReturn: + RET +TEXT ·_arithmetic_arr_scalar_neon(SB), NOSPLIT|NOFRAME, $0-48 + MOVD typ+0(FP), R0 + MOVB op+8(FP), R1 + MOVD inLeft+16(FP), R2 + MOVD inRight+24(FP), R3 + MOVD out+32(FP), R4 + MOVD len+40(FP), R5 + + CMP $6, R0 + BEQ LneonArrScalar32Int + CMP $7, R0 + BEQ LneonArrScalar32Int + CMP $8, R0 + BEQ LneonArrScalar64Int + CMP $9, R0 + BEQ LneonArrScalar64Int + CMP $11, R0 + BEQ LneonArrScalar32Float + CMP $12, R0 + BEQ LneonArrScalar64Float + RET + +LneonArrScalar32Int: + CMP $0, R1 + BEQ LneonArrScalar32IntAdd + CMP $1, R1 + BEQ LneonArrScalar32IntSub + CMP $2, R1 + BEQ LneonArrScalar32IntMul + RET + +LneonArrScalar32IntAdd: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.S4] +LneonArrScalar32IntAddVector: + CMP $4, R5 + BLT LneonArrScalar32IntAddTail + VLD1 (R2), [V0.S4] + VADD V1.S4, V0.S4, V0.S4 + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonArrScalar32IntAddVector +LneonArrScalar32IntAddTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + MOVW (R2), R6 + MOVW (R3), R7 + ADDW R7, R6, R6 + MOVW R6, (R4) + ADD $4, R2, R2 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar32IntAddTail + +LneonArrScalar32IntSub: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.S4] +LneonArrScalar32IntSubVector: + CMP $4, R5 + BLT LneonArrScalar32IntSubTail + VLD1 (R2), [V0.S4] + VSUB V1.S4, V0.S4, V0.S4 + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonArrScalar32IntSubVector +LneonArrScalar32IntSubTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + MOVW (R2), R6 + MOVW (R3), R7 + SUBW R7, R6, R6 + MOVW R6, (R4) + ADD $4, R2, R2 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar32IntSubTail + +LneonArrScalar32IntMul: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.S4] +LneonArrScalar32IntMulVector: + CMP $4, R5 + BLT LneonArrScalar32IntMulTail + VLD1 (R2), [V0.S4] + WORD $0x4ea09c20 // mul v0.4s, v1.4s, v0.4s + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonArrScalar32IntMulVector +LneonArrScalar32IntMulTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + MOVW (R2), R6 + MOVW (R3), R7 + MULW R7, R6, R6 + MOVW R6, (R4) + ADD $4, R2, R2 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar32IntMulTail + +LneonArrScalar64Int: + CMP $0, R1 + BEQ LneonArrScalar64IntAdd + CMP $1, R1 + BEQ LneonArrScalar64IntSub + RET + +LneonArrScalar64IntAdd: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.D2] +LneonArrScalar64IntAddVector: + CMP $2, R5 + BLT LneonArrScalar64IntAddTail + VLD1 (R2), [V0.D2] + VADD V1.D2, V0.D2, V0.D2 + VST1 [V0.D2], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonArrScalar64IntAddVector +LneonArrScalar64IntAddTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + MOVD (R2), R6 + MOVD (R3), R7 + ADD R7, R6, R6 + MOVD R6, (R4) + ADD $8, R2, R2 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar64IntAddTail + +LneonArrScalar64IntSub: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.D2] +LneonArrScalar64IntSubVector: + CMP $2, R5 + BLT LneonArrScalar64IntSubTail + VLD1 (R2), [V0.D2] + VSUB V1.D2, V0.D2, V0.D2 + VST1 [V0.D2], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonArrScalar64IntSubVector +LneonArrScalar64IntSubTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + MOVD (R2), R6 + MOVD (R3), R7 + SUB R7, R6, R6 + MOVD R6, (R4) + ADD $8, R2, R2 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar64IntSubTail + +LneonArrScalar32Float: + CMP $0, R1 + BEQ LneonArrScalar32FloatAdd + CMP $1, R1 + BEQ LneonArrScalar32FloatSub + CMP $2, R1 + BEQ LneonArrScalar32FloatMul + RET + +LneonArrScalar32FloatAdd: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.S4] +LneonArrScalar32FloatAddVector: + CMP $4, R5 + BLT LneonArrScalar32FloatAddTail + VLD1 (R2), [V0.S4] + WORD $0x4e21d400 // fadd v0.4s, v0.4s, v1.4s + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonArrScalar32FloatAddVector +LneonArrScalar32FloatAddTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + FMOVS (R2), F0 + FMOVS (R3), F1 + FADDS F1, F0, F0 + FMOVS F0, (R4) + ADD $4, R2, R2 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar32FloatAddTail + +LneonArrScalar32FloatSub: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.S4] +LneonArrScalar32FloatSubVector: + CMP $4, R5 + BLT LneonArrScalar32FloatSubTail + VLD1 (R2), [V0.S4] + WORD $0x4ea1d400 // fsub v0.4s, v0.4s, v1.4s + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonArrScalar32FloatSubVector +LneonArrScalar32FloatSubTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + FMOVS (R2), F0 + FMOVS (R3), F1 + FSUBS F1, F0, F0 + FMOVS F0, (R4) + ADD $4, R2, R2 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar32FloatSubTail + +LneonArrScalar32FloatMul: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.S4] +LneonArrScalar32FloatMulVector: + CMP $4, R5 + BLT LneonArrScalar32FloatMulTail + VLD1 (R2), [V0.S4] + WORD $0x6e21dc00 // fmul v0.4s, v0.4s, v1.4s + VST1 [V0.S4], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonArrScalar32FloatMulVector +LneonArrScalar32FloatMulTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + FMOVS (R2), F0 + FMOVS (R3), F1 + FMULS F1, F0, F0 + FMOVS F0, (R4) + ADD $4, R2, R2 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar32FloatMulTail + +LneonArrScalar64Float: + CMP $0, R1 + BEQ LneonArrScalar64FloatAdd + CMP $1, R1 + BEQ LneonArrScalar64FloatSub + CMP $2, R1 + BEQ LneonArrScalar64FloatMul + RET + +LneonArrScalar64FloatAdd: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.D2] +LneonArrScalar64FloatAddVector: + CMP $2, R5 + BLT LneonArrScalar64FloatAddTail + VLD1 (R2), [V0.D2] + WORD $0x4e61d400 // fadd v0.2d, v0.2d, v1.2d + VST1 [V0.D2], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonArrScalar64FloatAddVector +LneonArrScalar64FloatAddTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + FMOVD (R2), F0 + FMOVD (R3), F1 + FADDD F1, F0, F0 + FMOVD F0, (R4) + ADD $8, R2, R2 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar64FloatAddTail + +LneonArrScalar64FloatSub: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.D2] +LneonArrScalar64FloatSubVector: + CMP $2, R5 + BLT LneonArrScalar64FloatSubTail + VLD1 (R2), [V0.D2] + WORD $0x4ee1d400 // fsub v0.2d, v0.2d, v1.2d + VST1 [V0.D2], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonArrScalar64FloatSubVector +LneonArrScalar64FloatSubTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + FMOVD (R2), F0 + FMOVD (R3), F1 + FSUBD F1, F0, F0 + FMOVD F0, (R4) + ADD $8, R2, R2 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar64FloatSubTail + +LneonArrScalar64FloatMul: + CMP $0, R5 + BLE LneonArrScalarReturn + VLD1R (R3), [V1.D2] +LneonArrScalar64FloatMulVector: + CMP $2, R5 + BLT LneonArrScalar64FloatMulTail + VLD1 (R2), [V0.D2] + WORD $0x6e61dc00 // fmul v0.2d, v0.2d, v1.2d + VST1 [V0.D2], (R4) + ADD $16, R2, R2 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonArrScalar64FloatMulVector +LneonArrScalar64FloatMulTail: + CMP $0, R5 + BEQ LneonArrScalarReturn + FMOVD (R2), F0 + FMOVD (R3), F1 + FMULD F1, F0, F0 + FMOVD F0, (R4) + ADD $8, R2, R2 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonArrScalar64FloatMulTail + +LneonArrScalarReturn: + RET + +TEXT ·_arithmetic_scalar_arr_neon(SB), NOSPLIT|NOFRAME, $0-48 + MOVD typ+0(FP), R0 + MOVB op+8(FP), R1 + MOVD inLeft+16(FP), R2 + MOVD inRight+24(FP), R3 + MOVD out+32(FP), R4 + MOVD len+40(FP), R5 + + CMP $6, R0 + BEQ LneonScalarArr32Int + CMP $7, R0 + BEQ LneonScalarArr32Int + CMP $8, R0 + BEQ LneonScalarArr64Int + CMP $9, R0 + BEQ LneonScalarArr64Int + CMP $11, R0 + BEQ LneonScalarArr32Float + CMP $12, R0 + BEQ LneonScalarArr64Float + RET + +LneonScalarArr32Int: + CMP $0, R1 + BEQ LneonScalarArr32IntAdd + CMP $1, R1 + BEQ LneonScalarArr32IntSub + CMP $2, R1 + BEQ LneonScalarArr32IntMul + RET + +LneonScalarArr32IntAdd: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.S4] +LneonScalarArr32IntAddVector: + CMP $4, R5 + BLT LneonScalarArr32IntAddTail + VLD1 (R3), [V1.S4] + VADD V1.S4, V0.S4, V2.S4 + VST1 [V2.S4], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonScalarArr32IntAddVector +LneonScalarArr32IntAddTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + MOVW (R2), R6 + MOVW (R3), R7 + ADDW R7, R6, R6 + MOVW R6, (R4) + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr32IntAddTail + +LneonScalarArr32IntSub: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.S4] +LneonScalarArr32IntSubVector: + CMP $4, R5 + BLT LneonScalarArr32IntSubTail + VLD1 (R3), [V1.S4] + VSUB V1.S4, V0.S4, V2.S4 + VST1 [V2.S4], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonScalarArr32IntSubVector +LneonScalarArr32IntSubTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + MOVW (R2), R6 + MOVW (R3), R7 + SUBW R7, R6, R6 + MOVW R6, (R4) + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr32IntSubTail + +LneonScalarArr32IntMul: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.S4] +LneonScalarArr32IntMulVector: + CMP $4, R5 + BLT LneonScalarArr32IntMulTail + VLD1 (R3), [V1.S4] + WORD $0x4ea09c22 // mul v2.4s, v1.4s, v0.4s + VST1 [V2.S4], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonScalarArr32IntMulVector +LneonScalarArr32IntMulTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + MOVW (R2), R6 + MOVW (R3), R7 + MULW R7, R6, R6 + MOVW R6, (R4) + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr32IntMulTail + +LneonScalarArr64Int: + CMP $0, R1 + BEQ LneonScalarArr64IntAdd + CMP $1, R1 + BEQ LneonScalarArr64IntSub + RET + +LneonScalarArr64IntAdd: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.D2] +LneonScalarArr64IntAddVector: + CMP $2, R5 + BLT LneonScalarArr64IntAddTail + VLD1 (R3), [V1.D2] + VADD V1.D2, V0.D2, V2.D2 + VST1 [V2.D2], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonScalarArr64IntAddVector +LneonScalarArr64IntAddTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + MOVD (R2), R6 + MOVD (R3), R7 + ADD R7, R6, R6 + MOVD R6, (R4) + ADD $8, R3, R3 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr64IntAddTail + +LneonScalarArr64IntSub: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.D2] +LneonScalarArr64IntSubVector: + CMP $2, R5 + BLT LneonScalarArr64IntSubTail + VLD1 (R3), [V1.D2] + VSUB V1.D2, V0.D2, V2.D2 + VST1 [V2.D2], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonScalarArr64IntSubVector +LneonScalarArr64IntSubTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + MOVD (R2), R6 + MOVD (R3), R7 + SUB R7, R6, R6 + MOVD R6, (R4) + ADD $8, R3, R3 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr64IntSubTail + +LneonScalarArr32Float: + CMP $0, R1 + BEQ LneonScalarArr32FloatAdd + CMP $1, R1 + BEQ LneonScalarArr32FloatSub + CMP $2, R1 + BEQ LneonScalarArr32FloatMul + RET + +LneonScalarArr32FloatAdd: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.S4] +LneonScalarArr32FloatAddVector: + CMP $4, R5 + BLT LneonScalarArr32FloatAddTail + VLD1 (R3), [V1.S4] + WORD $0x4e21d402 // fadd v2.4s, v0.4s, v1.4s + VST1 [V2.S4], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonScalarArr32FloatAddVector +LneonScalarArr32FloatAddTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + FMOVS (R2), F0 + FMOVS (R3), F1 + FADDS F1, F0, F0 + FMOVS F0, (R4) + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr32FloatAddTail + +LneonScalarArr32FloatSub: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.S4] +LneonScalarArr32FloatSubVector: + CMP $4, R5 + BLT LneonScalarArr32FloatSubTail + VLD1 (R3), [V1.S4] + WORD $0x4ea1d402 // fsub v2.4s, v0.4s, v1.4s + VST1 [V2.S4], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonScalarArr32FloatSubVector +LneonScalarArr32FloatSubTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + FMOVS (R2), F0 + FMOVS (R3), F1 + FSUBS F1, F0, F0 + FMOVS F0, (R4) + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr32FloatSubTail + +LneonScalarArr32FloatMul: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.S4] +LneonScalarArr32FloatMulVector: + CMP $4, R5 + BLT LneonScalarArr32FloatMulTail + VLD1 (R3), [V1.S4] + WORD $0x6e21dc02 // fmul v2.4s, v0.4s, v1.4s + VST1 [V2.S4], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $4, R5, R5 + JMP LneonScalarArr32FloatMulVector +LneonScalarArr32FloatMulTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + FMOVS (R2), F0 + FMOVS (R3), F1 + FMULS F1, F0, F0 + FMOVS F0, (R4) + ADD $4, R3, R3 + ADD $4, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr32FloatMulTail + +LneonScalarArr64Float: + CMP $0, R1 + BEQ LneonScalarArr64FloatAdd + CMP $1, R1 + BEQ LneonScalarArr64FloatSub + CMP $2, R1 + BEQ LneonScalarArr64FloatMul + RET + +LneonScalarArr64FloatAdd: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.D2] +LneonScalarArr64FloatAddVector: + CMP $2, R5 + BLT LneonScalarArr64FloatAddTail + VLD1 (R3), [V1.D2] + WORD $0x4e61d402 // fadd v2.2d, v0.2d, v1.2d + VST1 [V2.D2], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonScalarArr64FloatAddVector +LneonScalarArr64FloatAddTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + FMOVD (R2), F0 + FMOVD (R3), F1 + FADDD F1, F0, F0 + FMOVD F0, (R4) + ADD $8, R3, R3 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr64FloatAddTail + +LneonScalarArr64FloatSub: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.D2] +LneonScalarArr64FloatSubVector: + CMP $2, R5 + BLT LneonScalarArr64FloatSubTail + VLD1 (R3), [V1.D2] + WORD $0x4ee1d402 // fsub v2.2d, v0.2d, v1.2d + VST1 [V2.D2], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonScalarArr64FloatSubVector +LneonScalarArr64FloatSubTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + FMOVD (R2), F0 + FMOVD (R3), F1 + FSUBD F1, F0, F0 + FMOVD F0, (R4) + ADD $8, R3, R3 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr64FloatSubTail + +LneonScalarArr64FloatMul: + CMP $0, R5 + BLE LneonScalarArrReturn + VLD1R (R2), [V0.D2] +LneonScalarArr64FloatMulVector: + CMP $2, R5 + BLT LneonScalarArr64FloatMulTail + VLD1 (R3), [V1.D2] + WORD $0x6e61dc02 // fmul v2.2d, v0.2d, v1.2d + VST1 [V2.D2], (R4) + ADD $16, R3, R3 + ADD $16, R4, R4 + SUB $2, R5, R5 + JMP LneonScalarArr64FloatMulVector +LneonScalarArr64FloatMulTail: + CMP $0, R5 + BEQ LneonScalarArrReturn + FMOVD (R2), F0 + FMOVD (R3), F1 + FMULD F1, F0, F0 + FMOVD F0, (R4) + ADD $8, R3, R3 + ADD $8, R4, R4 + SUB $1, R5, R5 + JMP LneonScalarArr64FloatMulTail + +LneonScalarArrReturn: + RET +TEXT ·_arithmetic_unary_same_types_neon(SB), NOSPLIT|NOFRAME, $0-40 + MOVD typ+0(FP), R0 + MOVB op+8(FP), R1 + MOVD input+16(FP), R2 + MOVD output+24(FP), R3 + MOVD len+32(FP), R5 + + CMP $7, R0 + BEQ LneonUnary32SignedInt + CMP $6, R0 + BEQ LneonUnary32UnsignedInt + CMP $9, R0 + BEQ LneonUnary64SignedInt + CMP $8, R0 + BEQ LneonUnary64UnsignedInt + CMP $11, R0 + BEQ LneonUnary32Float + CMP $12, R0 + BEQ LneonUnary64Float + RET + +LneonUnary32SignedInt: + CMP $4, R1 + BEQ LneonUnary32SignedIntAbs + CMP $5, R1 + BEQ LneonUnary32SignedIntNeg + RET + +LneonUnary32SignedIntAbs: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary32SignedIntAbsVector: + CMP $4, R5 + BLT LneonUnary32SignedIntAbsTail + VLD1 (R2), [V0.S4] + WORD $0x4ea0b800 // abs v0.4s, v0.4s + VST1 [V0.S4], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $4, R5, R5 + JMP LneonUnary32SignedIntAbsVector +LneonUnary32SignedIntAbsTail: + CMP $0, R5 + BEQ LneonUnaryReturn + MOVW (R2), R6 + CMPW $0, R6 + BGE LneonUnary32SignedIntAbsStore + NEGW R6, R6 +LneonUnary32SignedIntAbsStore: + MOVW R6, (R3) + ADD $4, R2, R2 + ADD $4, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary32SignedIntAbsTail + +LneonUnary32SignedIntNeg: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary32SignedIntNegVector: + CMP $4, R5 + BLT LneonUnary32SignedIntNegTail + VLD1 (R2), [V0.S4] + WORD $0x6ea0b800 // neg v0.4s, v0.4s + VST1 [V0.S4], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $4, R5, R5 + JMP LneonUnary32SignedIntNegVector +LneonUnary32SignedIntNegTail: + CMP $0, R5 + BEQ LneonUnaryReturn + MOVW (R2), R6 + NEGW R6, R6 + MOVW R6, (R3) + ADD $4, R2, R2 + ADD $4, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary32SignedIntNegTail + +LneonUnary32UnsignedInt: + CMP $4, R1 + BEQ LneonUnary32UnsignedIntAbs + CMP $5, R1 + BEQ LneonUnary32UnsignedIntNeg + RET + +LneonUnary32UnsignedIntAbs: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary32UnsignedIntAbsVector: + CMP $4, R5 + BLT LneonUnary32UnsignedIntAbsTail + VLD1 (R2), [V0.S4] + VST1 [V0.S4], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $4, R5, R5 + JMP LneonUnary32UnsignedIntAbsVector +LneonUnary32UnsignedIntAbsTail: + CMP $0, R5 + BEQ LneonUnaryReturn + MOVW (R2), R6 + MOVW R6, (R3) + ADD $4, R2, R2 + ADD $4, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary32UnsignedIntAbsTail + +LneonUnary32UnsignedIntNeg: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary32UnsignedIntNegVector: + CMP $4, R5 + BLT LneonUnary32UnsignedIntNegTail + VLD1 (R2), [V0.S4] + WORD $0x6ea0b800 // neg v0.4s, v0.4s + VST1 [V0.S4], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $4, R5, R5 + JMP LneonUnary32UnsignedIntNegVector +LneonUnary32UnsignedIntNegTail: + CMP $0, R5 + BEQ LneonUnaryReturn + MOVW (R2), R6 + NEGW R6, R6 + MOVW R6, (R3) + ADD $4, R2, R2 + ADD $4, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary32UnsignedIntNegTail + +LneonUnary64SignedInt: + CMP $4, R1 + BEQ LneonUnary64SignedIntAbs + CMP $5, R1 + BEQ LneonUnary64SignedIntNeg + RET + +LneonUnary64SignedIntAbs: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary64SignedIntAbsVector: + CMP $2, R5 + BLT LneonUnary64SignedIntAbsTail + VLD1 (R2), [V0.D2] + WORD $0x4ee0b800 // abs v0.2d, v0.2d + VST1 [V0.D2], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $2, R5, R5 + JMP LneonUnary64SignedIntAbsVector +LneonUnary64SignedIntAbsTail: + CMP $0, R5 + BEQ LneonUnaryReturn + MOVD (R2), R6 + CMP $0, R6 + BGE LneonUnary64SignedIntAbsStore + NEG R6, R6 +LneonUnary64SignedIntAbsStore: + MOVD R6, (R3) + ADD $8, R2, R2 + ADD $8, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary64SignedIntAbsTail + +LneonUnary64SignedIntNeg: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary64SignedIntNegVector: + CMP $2, R5 + BLT LneonUnary64SignedIntNegTail + VLD1 (R2), [V0.D2] + WORD $0x6ee0b800 // neg v0.2d, v0.2d + VST1 [V0.D2], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $2, R5, R5 + JMP LneonUnary64SignedIntNegVector +LneonUnary64SignedIntNegTail: + CMP $0, R5 + BEQ LneonUnaryReturn + MOVD (R2), R6 + NEG R6, R6 + MOVD R6, (R3) + ADD $8, R2, R2 + ADD $8, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary64SignedIntNegTail + +LneonUnary64UnsignedInt: + CMP $4, R1 + BEQ LneonUnary64UnsignedIntAbs + CMP $5, R1 + BEQ LneonUnary64UnsignedIntNeg + RET + +LneonUnary64UnsignedIntAbs: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary64UnsignedIntAbsVector: + CMP $2, R5 + BLT LneonUnary64UnsignedIntAbsTail + VLD1 (R2), [V0.D2] + VST1 [V0.D2], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $2, R5, R5 + JMP LneonUnary64UnsignedIntAbsVector +LneonUnary64UnsignedIntAbsTail: + CMP $0, R5 + BEQ LneonUnaryReturn + MOVD (R2), R6 + MOVD R6, (R3) + ADD $8, R2, R2 + ADD $8, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary64UnsignedIntAbsTail + +LneonUnary64UnsignedIntNeg: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary64UnsignedIntNegVector: + CMP $2, R5 + BLT LneonUnary64UnsignedIntNegTail + VLD1 (R2), [V0.D2] + WORD $0x6ee0b800 // neg v0.2d, v0.2d + VST1 [V0.D2], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $2, R5, R5 + JMP LneonUnary64UnsignedIntNegVector +LneonUnary64UnsignedIntNegTail: + CMP $0, R5 + BEQ LneonUnaryReturn + MOVD (R2), R6 + NEG R6, R6 + MOVD R6, (R3) + ADD $8, R2, R2 + ADD $8, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary64UnsignedIntNegTail + +LneonUnary32Float: + CMP $4, R1 + BEQ LneonUnary32FloatAbs + CMP $5, R1 + BEQ LneonUnary32FloatNeg + RET + +LneonUnary32FloatAbs: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary32FloatAbsVector: + CMP $4, R5 + BLT LneonUnary32FloatAbsTail + VLD1 (R2), [V0.S4] + WORD $0x4ea0f800 // fabs v0.4s, v0.4s + VST1 [V0.S4], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $4, R5, R5 + JMP LneonUnary32FloatAbsVector +LneonUnary32FloatAbsTail: + CMP $0, R5 + BEQ LneonUnaryReturn + FMOVS (R2), F0 + FABSS F0, F0 + FMOVS F0, (R3) + ADD $4, R2, R2 + ADD $4, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary32FloatAbsTail + +LneonUnary32FloatNeg: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary32FloatNegVector: + CMP $4, R5 + BLT LneonUnary32FloatNegTail + VLD1 (R2), [V0.S4] + WORD $0x6ea0f800 // fneg v0.4s, v0.4s + VST1 [V0.S4], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $4, R5, R5 + JMP LneonUnary32FloatNegVector +LneonUnary32FloatNegTail: + CMP $0, R5 + BEQ LneonUnaryReturn + FMOVS (R2), F0 + FNEGS F0, F0 + FMOVS F0, (R3) + ADD $4, R2, R2 + ADD $4, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary32FloatNegTail + +LneonUnary64Float: + CMP $4, R1 + BEQ LneonUnary64FloatAbs + CMP $5, R1 + BEQ LneonUnary64FloatNeg + RET + +LneonUnary64FloatAbs: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary64FloatAbsVector: + CMP $2, R5 + BLT LneonUnary64FloatAbsTail + VLD1 (R2), [V0.D2] + WORD $0x4ee0f800 // fabs v0.2d, v0.2d + VST1 [V0.D2], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $2, R5, R5 + JMP LneonUnary64FloatAbsVector +LneonUnary64FloatAbsTail: + CMP $0, R5 + BEQ LneonUnaryReturn + FMOVD (R2), F0 + FABSD F0, F0 + FMOVD F0, (R3) + ADD $8, R2, R2 + ADD $8, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary64FloatAbsTail + +LneonUnary64FloatNeg: + CMP $0, R5 + BLE LneonUnaryReturn +LneonUnary64FloatNegVector: + CMP $2, R5 + BLT LneonUnary64FloatNegTail + VLD1 (R2), [V0.D2] + WORD $0x6ee0f800 // fneg v0.2d, v0.2d + VST1 [V0.D2], (R3) + ADD $16, R2, R2 + ADD $16, R3, R3 + SUB $2, R5, R5 + JMP LneonUnary64FloatNegVector +LneonUnary64FloatNegTail: + CMP $0, R5 + BEQ LneonUnaryReturn + FMOVD (R2), F0 + FNEGD F0, F0 + FMOVD F0, (R3) + ADD $8, R2, R2 + ADD $8, R3, R3 + SUB $1, R5, R5 + JMP LneonUnary64FloatNegTail + +LneonUnaryReturn: + RET diff --git a/arrow/compute/internal/kernels/base_arithmetic_arm64_test.go b/arrow/compute/internal/kernels/base_arithmetic_arm64_test.go new file mode 100644 index 00000000..9a3b86b8 --- /dev/null +++ b/arrow/compute/internal/kernels/base_arithmetic_arm64_test.go @@ -0,0 +1,358 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build go1.18 && arm64 && !noasm && !appengine + +package kernels + +import ( + "fmt" + "math" + "reflect" + "runtime" + "testing" + "unsafe" + + "github.com/apache/arrow-go/v18/arrow" + "golang.org/x/sys/cpu" +) + +func checkNeonBinary[T arrow.NumericType](t *testing.T, typ arrow.Type, op ArithmeticOp, left, right []T, want func(T, T) T) { + t.Helper() + + got := make([]T, len(left)) + wantValues := make([]T, len(left)) + for i := range left { + wantValues[i] = want(left[i], right[i]) + } + arithmeticNeon(typ, op, arrow.GetBytes(left), arrow.GetBytes(right), arrow.GetBytes(got), len(left)) + if !reflect.DeepEqual(got, wantValues) { + t.Fatalf("array-array: got %v, want %v", got, wantValues) + } + + var scalar T + if len(right) != 0 { + scalar = right[0] + } + wantValues = make([]T, len(left)) + for i := range left { + wantValues[i] = want(left[i], scalar) + } + arithmeticArrScalarNeon(typ, op, arrow.GetBytes(left), unsafe.Pointer(&scalar), arrow.GetBytes(got), len(left)) + if !reflect.DeepEqual(got, wantValues) { + t.Fatalf("array-scalar: got %v, want %v", got, wantValues) + } + + if len(left) != 0 { + scalar = left[0] + } + wantValues = make([]T, len(right)) + for i := range right { + wantValues[i] = want(scalar, right[i]) + } + arithmeticScalarArrNeon(typ, op, unsafe.Pointer(&scalar), arrow.GetBytes(right), arrow.GetBytes(got), len(right)) + if !reflect.DeepEqual(got[:len(right)], wantValues) { + t.Fatalf("scalar-array: got %v, want %v", got[:len(right)], wantValues) + } +} + +func checkNeonUnary[T arrow.NumericType](t *testing.T, typ arrow.Type, op ArithmeticOp, input []T, want func(T) T) { + t.Helper() + + got := make([]T, len(input)) + wantValues := make([]T, len(input)) + for i, value := range input { + wantValues[i] = want(value) + } + arithmeticUnaryNeon(typ, op, arrow.GetBytes(input), arrow.GetBytes(got), len(input)) + if !reflect.DeepEqual(got, wantValues) { + t.Fatalf("got %v, want %v", got, wantValues) + } +} + +func TestNeonArithmeticBinary(t *testing.T) { + if !cpu.ARM64.HasASIMD { + t.Skip("ARM64 SIMD is not available") + } + + lengths := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33} + for _, n := range lengths { + t.Run(fmt.Sprintf("length=%d", n), func(t *testing.T) { + int32Left := make([]int32, n) + int32Right := make([]int32, n) + uint32Left := make([]uint32, n) + uint32Right := make([]uint32, n) + int64Left := make([]int64, n) + int64Right := make([]int64, n) + uint64Left := make([]uint64, n) + uint64Right := make([]uint64, n) + float32Left := make([]float32, n) + float32Right := make([]float32, n) + float64Left := make([]float64, n) + float64Right := make([]float64, n) + for i := 0; i < n; i++ { + int32Left[i], int32Right[i] = int32(i*3-10), int32(i+2) + uint32Left[i], uint32Right[i] = uint32(i*3+10), uint32(i+2) + int64Left[i], int64Right[i] = int64(i*3-10), int64(i+2) + uint64Left[i], uint64Right[i] = uint64(i*3+10), uint64(i+2) + float32Left[i], float32Right[i] = float32(i)+0.25, float32(i)*0.5+1.5 + float64Left[i], float64Right[i] = float64(i)+0.25, float64(i)*0.5+1.5 + } + + for _, op := range []ArithmeticOp{OpAdd, OpSub} { + checkNeonBinary(t, arrow.INT32, op, int32Left, int32Right, func(a, b int32) int32 { + if op == OpAdd { + return a + b + } + return a - b + }) + checkNeonBinary(t, arrow.UINT32, op, uint32Left, uint32Right, func(a, b uint32) uint32 { + if op == OpAdd { + return a + b + } + return a - b + }) + checkNeonBinary(t, arrow.INT64, op, int64Left, int64Right, func(a, b int64) int64 { + if op == OpAdd { + return a + b + } + return a - b + }) + checkNeonBinary(t, arrow.UINT64, op, uint64Left, uint64Right, func(a, b uint64) uint64 { + if op == OpAdd { + return a + b + } + return a - b + }) + checkNeonBinary(t, arrow.FLOAT32, op, float32Left, float32Right, func(a, b float32) float32 { + if op == OpAdd { + return a + b + } + return a - b + }) + checkNeonBinary(t, arrow.FLOAT64, op, float64Left, float64Right, func(a, b float64) float64 { + if op == OpAdd { + return a + b + } + return a - b + }) + } + + checkNeonBinary(t, arrow.INT32, OpMul, int32Left, int32Right, func(a, b int32) int32 { return a * b }) + checkNeonBinary(t, arrow.UINT32, OpMul, uint32Left, uint32Right, func(a, b uint32) uint32 { return a * b }) + checkNeonBinary(t, arrow.FLOAT32, OpMul, float32Left, float32Right, func(a, b float32) float32 { return a * b }) + checkNeonBinary(t, arrow.FLOAT64, OpMul, float64Left, float64Right, func(a, b float64) float64 { return a * b }) + }) + } +} + +func TestNeonArithmeticUnary(t *testing.T) { + if !cpu.ARM64.HasASIMD { + t.Skip("ARM64 SIMD is not available") + } + + lengths := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33} + for _, n := range lengths { + t.Run(fmt.Sprintf("length=%d", n), func(t *testing.T) { + int32Values := make([]int32, n) + uint32Values := make([]uint32, n) + int64Values := make([]int64, n) + uint64Values := make([]uint64, n) + float32Values := make([]float32, n) + float64Values := make([]float64, n) + for i := 0; i < n; i++ { + int32Values[i] = []int32{0, 1, -1, -1 << 31, 1<<31 - 1}[i%5] + uint32Values[i] = []uint32{0, 1, ^uint32(0), 2, 17}[i%5] + int64Values[i] = []int64{0, 1, -1, -1 << 63, 1<<63 - 1}[i%5] + uint64Values[i] = []uint64{0, 1, ^uint64(0), 2, 17}[i%5] + float32Values[i] = []float32{0, float32(math.Copysign(0, -1)), 1.5, -2.25, 3.75}[i%5] + float64Values[i] = []float64{0, math.Copysign(0, -1), 1.5, -2.25, 3.75}[i%5] + } + + checkNeonUnary(t, arrow.INT32, OpAbsoluteValue, int32Values, func(v int32) int32 { + if v < 0 { + return -v + } + return v + }) + checkNeonUnary(t, arrow.INT32, OpNegate, int32Values, func(v int32) int32 { return -v }) + checkNeonUnary(t, arrow.UINT32, OpAbsoluteValue, uint32Values, func(v uint32) uint32 { return v }) + checkNeonUnary(t, arrow.UINT32, OpNegate, uint32Values, func(v uint32) uint32 { return -v }) + checkNeonUnary(t, arrow.INT64, OpAbsoluteValue, int64Values, func(v int64) int64 { + if v < 0 { + return -v + } + return v + }) + checkNeonUnary(t, arrow.INT64, OpNegate, int64Values, func(v int64) int64 { return -v }) + checkNeonUnary(t, arrow.UINT64, OpAbsoluteValue, uint64Values, func(v uint64) uint64 { return v }) + checkNeonUnary(t, arrow.UINT64, OpNegate, uint64Values, func(v uint64) uint64 { return -v }) + checkNeonUnary(t, arrow.FLOAT32, OpAbsoluteValue, float32Values, func(v float32) float32 { + return math.Float32frombits(math.Float32bits(v) &^ (uint32(1) << 31)) + }) + checkNeonUnary(t, arrow.FLOAT32, OpNegate, float32Values, func(v float32) float32 { return -v }) + checkNeonUnary(t, arrow.FLOAT64, OpAbsoluteValue, float64Values, func(v float64) float64 { + return math.Float64frombits(math.Float64bits(v) &^ (uint64(1) << 63)) + }) + checkNeonUnary(t, arrow.FLOAT64, OpNegate, float64Values, func(v float64) float64 { return -v }) + }) + } +} + +func TestNeonArithmeticWrapping(t *testing.T) { + if !cpu.ARM64.HasASIMD { + t.Skip("ARM64 SIMD is not available") + } + + const ( + minInt32 = -1 << 31 + maxInt32 = 1<<31 - 1 + minInt64 = -1 << 63 + maxInt64 = 1<<63 - 1 + ) + + int32Left := []int32{maxInt32, minInt32, -1, 12345, -12345} + int32Right := []int32{2, -1, maxInt32, -7, 2} + uint32Left := []uint32{^uint32(0), 0, 1, 12345, 17} + uint32Right := []uint32{2, ^uint32(0), ^uint32(0), 7, 2} + int64Left := []int64{maxInt64, minInt64, -1, 12345, -12345} + int64Right := []int64{2, -1, maxInt64, -7, 2} + uint64Left := []uint64{^uint64(0), 0, 1, 12345, 17} + uint64Right := []uint64{2, ^uint64(0), ^uint64(0), 7, 2} + + for _, op := range []ArithmeticOp{OpAdd, OpSub} { + checkNeonBinary(t, arrow.INT32, op, int32Left, int32Right, func(a, b int32) int32 { + if op == OpAdd { + return a + b + } + return a - b + }) + checkNeonBinary(t, arrow.UINT32, op, uint32Left, uint32Right, func(a, b uint32) uint32 { + if op == OpAdd { + return a + b + } + return a - b + }) + checkNeonBinary(t, arrow.INT64, op, int64Left, int64Right, func(a, b int64) int64 { + if op == OpAdd { + return a + b + } + return a - b + }) + checkNeonBinary(t, arrow.UINT64, op, uint64Left, uint64Right, func(a, b uint64) uint64 { + if op == OpAdd { + return a + b + } + return a - b + }) + } + checkNeonBinary(t, arrow.INT32, OpMul, int32Left, int32Right, func(a, b int32) int32 { return a * b }) + checkNeonBinary(t, arrow.UINT32, OpMul, uint32Left, uint32Right, func(a, b uint32) uint32 { return a * b }) +} + +func BenchmarkNeonArithmetic(b *testing.B) { + if !cpu.ARM64.HasASIMD { + b.Skip("ARM64 SIMD is not available") + } + + const n = 1 << 20 + benchInt64 := func(b *testing.B, shape string, neon bool) { + left := make([]int64, n) + right := make([]int64, n) + out := make([]int64, n) + for i := range left { + left[i] = int64(i) + right[i] = int64(i + 1) + } + scalar := int64(7) + b.SetBytes(int64(n * 8)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + if neon { + switch shape { + case "array-scalar": + arithmeticArrScalarNeon(arrow.INT64, OpAdd, arrow.GetBytes(left), unsafe.Pointer(&scalar), arrow.GetBytes(out), n) + case "scalar-array": + arithmeticScalarArrNeon(arrow.INT64, OpAdd, unsafe.Pointer(&scalar), arrow.GetBytes(right), arrow.GetBytes(out), n) + default: + arithmeticNeon(arrow.INT64, OpAdd, arrow.GetBytes(left), arrow.GetBytes(right), arrow.GetBytes(out), n) + } + } else { + switch shape { + case "array-scalar": + for j, value := range left { + out[j] = value + scalar + } + case "scalar-array": + for j, value := range right { + out[j] = scalar + value + } + default: + for j, value := range left { + out[j] = value + right[j] + } + } + } + } + b.StopTimer() + runtime.KeepAlive(out) + } + + benchFloat64 := func(b *testing.B, op ArithmeticOp, neon bool) { + left := make([]float64, n) + right := make([]float64, n) + out := make([]float64, n) + for i := range left { + left[i] = float64(i) + 0.25 + right[i] = float64(i) + 1.5 + } + b.SetBytes(int64(n * 8)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + if neon { + arithmeticNeon(arrow.FLOAT64, op, arrow.GetBytes(left), arrow.GetBytes(right), arrow.GetBytes(out), n) + } else { + for j, value := range left { + switch op { + case OpAdd: + out[j] = value + right[j] + case OpMul: + out[j] = value * right[j] + } + } + } + } + b.StopTimer() + runtime.KeepAlive(out) + } + + for _, shape := range []string{"array-array", "array-scalar", "scalar-array"} { + shape := shape + b.Run("int64/add/"+shape+"/neon", func(b *testing.B) { benchInt64(b, shape, true) }) + b.Run("int64/add/"+shape+"/scalar", func(b *testing.B) { benchInt64(b, shape, false) }) + } + for _, op := range []struct { + name string + op ArithmeticOp + }{ + {"add", OpAdd}, + {"mul", OpMul}, + } { + op := op + b.Run("float64/"+op.name+"/array-array/neon", func(b *testing.B) { benchFloat64(b, op.op, true) }) + b.Run("float64/"+op.name+"/array-array/scalar", func(b *testing.B) { benchFloat64(b, op.op, false) }) + } +} diff --git a/arrow/compute/internal/kernels/basic_arithmetic_noasm.go b/arrow/compute/internal/kernels/basic_arithmetic_noasm.go index 74d64b69..dcce2d39 100644 --- a/arrow/compute/internal/kernels/basic_arithmetic_noasm.go +++ b/arrow/compute/internal/kernels/basic_arithmetic_noasm.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build go1.18 && (noasm || !amd64) +//go:build go1.18 && (noasm || (!amd64 && !arm64)) package kernels