simd: expand SIMD128 operations and runtime integration

cl/compile.go

@@ -708,7 +708,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun
 	if simdDecl {
 		b := fn.MakeBody(1)
 		if simd.op == llssa.SIMDUnimplemented {
-			b.SIMD(simd.op, b.Str(name))
+			b.SIMD(simd.op, p.prog.Void(), b.Str(name))
 			b.Unreachable()
 		} else {
 			n := sig.Params().Len()
@@ -719,7 +719,12 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun
 			for i := range args {
 				args[i] = fn.Param(i)
 			}
-			b.Return(b.SIMD(simd.op, args...))
+			result := b.SIMD(simd.op, p.simdResultType(f.Signature), args...)
+			if sig.Results().Len() == 0 {
+				b.Return()
+			} else {
+				b.Return(result)
+			}
 		}
 		b.EndBuild()
 		b.Dispose()

cl/simd.go

@@ -1,6 +1,7 @@
 package cl
 
 import (
+	"fmt"
 	"go/ast"
 	"go/types"
 
@@ -18,26 +19,107 @@ const (
 	simdExtract
 	simdInsert
 	simdUnsupported
+	simdUnary
+	simdLoad
+	simdStore
+	simdBroadcast
+	simdBitcast
+	simdCompare
+	simdToMask
+	simdTernary
+	simdMaskFromBits
+	simdMaskToBits
+	simdScalarShift
+	simdUnsignedVector
+	simdSignedVector
+	simdConvert
 )
 
 type simdOperation struct {
-	op          llssa.SIMDOp
-	signature   simdSignature
-	integerOnly bool
+	op        llssa.SIMDOp
+	signature simdSignature
+	elements  types.BasicInfo
 }
 
 // The default entry gives not-yet-implemented intrinsic declarations a defined,
 // recoverable failure. Adding an implementation replaces this fallback for that
 // operation; functions with Go bodies continue through normal compilation.
 var simdOperations = map[simdKey]simdOperation{
-	{"*", "*"}:             {llssa.SIMDUnimplemented, simdUnsupported, false},
-	{"numeric", "Add"}:     {llssa.SIMDAdd, simdBinary, false},
-	{"numeric", "Sub"}:     {llssa.SIMDSub, simdBinary, false},
-	{"numeric", "And"}:     {llssa.SIMDAnd, simdBinary, true},
-	{"numeric", "Or"}:      {llssa.SIMDOr, simdBinary, true},
-	{"numeric", "Xor"}:     {llssa.SIMDXor, simdBinary, true},
-	{"numeric", "GetElem"}: {llssa.SIMDExtractLane, simdExtract, false},
-	{"numeric", "SetElem"}: {llssa.SIMDInsertLane, simdInsert, false},
+	{"*", "*"}:                   {llssa.SIMDUnimplemented, simdUnsupported, 0},
+	{"numeric", "Add"}:           {llssa.SIMDAdd, simdBinary, 0},
+	{"numeric", "Sub"}:           {llssa.SIMDSub, simdBinary, 0},
+	{"numeric", "And"}:           {llssa.SIMDAnd, simdBinary, types.IsInteger},
+	{"numeric", "Or"}:            {llssa.SIMDOr, simdBinary, types.IsInteger},
+	{"numeric", "Xor"}:           {llssa.SIMDXor, simdBinary, types.IsInteger},
+	{"numeric", "GetElem"}:       {llssa.SIMDExtractLane, simdExtract, 0},
+	{"numeric", "SetElem"}:       {llssa.SIMDInsertLane, simdInsert, 0},
+	{"numeric", "StoreArray"}:    {llssa.SIMDStore, simdStore, 0},
+	{"numeric", "Mul"}:           {llssa.SIMDMul, simdBinary, 0},
+	{"numeric", "Div"}:           {llssa.SIMDDiv, simdBinary, types.IsFloat},
+	{"numeric", "AndNot"}:        {llssa.SIMDAndNot, simdBinary, types.IsInteger},
+	{"numeric", "OrNot"}:         {llssa.SIMDOrNot, simdBinary, types.IsInteger},
+	{"numeric", "Not"}:           {llssa.SIMDNot, simdUnary, types.IsInteger},
+	{"numeric", "Neg"}:           {llssa.SIMDNeg, simdUnary, 0},
+	{"numeric", "Abs"}:           {llssa.SIMDAbs, simdUnary, 0},
+	{"numeric", "Sqrt"}:          {llssa.SIMDSqrt, simdUnary, types.IsFloat},
+	{"numeric", "Ceil"}:          {llssa.SIMDCeil, simdUnary, types.IsFloat},
+	{"numeric", "Floor"}:         {llssa.SIMDFloor, simdUnary, types.IsFloat},
+	{"numeric", "Trunc"}:         {llssa.SIMDTrunc, simdUnary, types.IsFloat},
+	{"numeric", "Round"}:         {llssa.SIMDRound, simdUnary, types.IsFloat},
+	{"numeric", "Equal"}:         {llssa.SIMDEqual, simdCompare, 0},
+	{"numeric", "NotEqual"}:      {llssa.SIMDNotEqual, simdCompare, 0},
+	{"numeric", "Less"}:          {llssa.SIMDLess, simdCompare, 0},
+	{"numeric", "LessEqual"}:     {llssa.SIMDLessEqual, simdCompare, 0},
+	{"numeric", "Greater"}:       {llssa.SIMDGreater, simdCompare, 0},
+	{"numeric", "GreaterEqual"}:  {llssa.SIMDGreaterEqual, simdCompare, 0},
+	{"numeric", "ToMask"}:        {llssa.SIMDToMask, simdToMask, types.IsInteger},
+	{"numeric", "asMask"}:        {llssa.SIMDBitcast, simdToMask, types.IsInteger},
+	{"numeric", "bitSelect"}:     {llssa.SIMDBitSelect, simdTernary, types.IsInteger},
+	{"numeric", "BitSelect"}:     {llssa.SIMDBitSelect, simdTernary, types.IsInteger},
+	{"numeric", "bitSelectNot"}:  {llssa.SIMDBitSelectNot, simdTernary, types.IsInteger},
+	{"numeric", "blend"}:         {llssa.SIMDBlend, simdTernary, types.IsInteger},
+	{"mask", "And"}:              {llssa.SIMDAnd, simdBinary, 0},
+	{"mask", "Or"}:               {llssa.SIMDOr, simdBinary, 0},
+	{"mask", "Xor"}:              {llssa.SIMDXor, simdBinary, 0},
+	{"mask", "AndNot"}:           {llssa.SIMDAndNot, simdBinary, 0},
+	{"mask", "Not"}:              {llssa.SIMDNot, simdUnary, 0},
+	{"numeric", "ShiftAllLeft"}:  {llssa.SIMDShiftAllLeft, simdScalarShift, types.IsInteger},
+	{"numeric", "ShiftAllRight"}: {llssa.SIMDShiftAllRight, simdScalarShift, types.IsInteger},
+	{"numeric", "ShiftLeft"}:     {llssa.SIMDShiftLeft, simdUnsignedVector, types.IsInteger},
+	{"numeric", "ShiftRight"}:    {llssa.SIMDShiftRight, simdUnsignedVector, types.IsInteger},
+	{"numeric", "Shift"}:         {llssa.SIMDShift, simdSignedVector, types.IsInteger},
+	{"numeric", "AddSaturated"}:  {llssa.SIMDAddSaturated, simdBinary, types.IsInteger},
+	{"numeric", "SubSaturated"}:  {llssa.SIMDSubSaturated, simdBinary, types.IsInteger},
+	{"numeric", "Min"}:           {llssa.SIMDMin, simdBinary, 0},
+	{"numeric", "LookupOrZero"}:  {llssa.SIMDLookupOrZero, simdBinary, types.IsInteger},
+	{"numeric", "PermuteOrZero"}: {llssa.SIMDPermuteOrZero, simdSignedVector, types.IsInteger},
+	{"numeric", "Permute"}:       {llssa.SIMDPermute, simdUnsignedVector, types.IsInteger},
+	{"numeric", "Max"}:           {llssa.SIMDMax, simdBinary, 0},
+}
+
+// These registrations share lowering but retain exact declaration names and
+// signature checks. Source Go slice helpers keep their own bounds checks.
+func init() {
+	for _, name := range []string{"ConvertToInt8", "ConvertToUint8", "ConvertToInt16", "ConvertToUint16", "ConvertToInt32", "ConvertToUint32", "ConvertToInt64", "ConvertToUint64", "ConvertToFloat32", "ConvertToFloat64"} {
+		simdOperations[simdKey{"numeric", name}] = simdOperation{llssa.SIMDConvert, simdConvert, 0}
+	}
+	for _, name := range []string{"Mask8x16", "Mask16x8", "Mask32x4", "Mask64x2"} {
+		simdOperations[simdKey{"", name + "FromBits"}] = simdOperation{llssa.SIMDMaskFromBits, simdMaskFromBits, 0}
+	}
+	simdOperations[simdKey{"mask", "ToBits"}] = simdOperation{llssa.SIMDMaskToBits, simdMaskToBits, 0}
+	for _, name := range []string{"ToInt8x16", "ToInt16x8", "ToInt32x4", "ToInt64x2"} {
+		simdOperations[simdKey{"mask", name}] = simdOperation{llssa.SIMDBitcast, simdBitcast, 0}
+	}
+	for _, name := range []string{"ToBits", "BitsToInt8", "BitsToInt16", "BitsToInt32", "BitsToInt64", "BitsToFloat32", "BitsToFloat64", "ReshapeToUint8s", "ReshapeToUint16s", "ReshapeToUint32s", "ReshapeToUint64s"} {
+		simdOperations[simdKey{"numeric", name}] = simdOperation{llssa.SIMDBitcast, simdBitcast, 0}
+	}
+	for _, name := range []string{"Int8x16", "Uint8x16", "Int16x8", "Uint16x8", "Int32x4", "Uint32x4", "Int64x2", "Uint64x2", "Float32x4", "Float64x2"} {
+		simdOperations[simdKey{"", "Load" + name + "Array"}] = simdOperation{llssa.SIMDLoad, simdLoad, 0}
+		simdOperations[simdKey{"", "Broadcast" + name}] = simdOperation{llssa.SIMDBroadcast, simdBroadcast, 0}
+	}
+	for _, lanes := range []int{2, 4, 8, 16} {
+		simdOperations[simdKey{"numeric", fmt.Sprintf("broadcast1To%d", lanes)}] = simdOperation{llssa.SIMDSplatLane0, simdUnary, 0}
+	}
 }
 
 // Resolve only declared official operations, never synthetic wrapper names.
@@ -56,49 +138,127 @@ func lookupSIMD(fn *ssa.Function, arch string) (simdOperation, bool) {
 	if !types.Identical(sig, obj.Type()) {
 		return simdOperation{}, false
 	}
+	decl, isDecl := fn.Syntax().(*ast.FuncDecl)
 	fallback := func() (simdOperation, bool) {
 		// Imported declarations and synthetic wrappers are not definitions. Emit
 		// the fallback only for a source intrinsic declaration in archsimd.
-		if decl, ok := fn.Syntax().(*ast.FuncDecl); ok && decl.Body == nil {
+		if isDecl && decl.Body == nil {
 			return simdOperations[simdKey{"*", "*"}], true
 		}
 		return simdOperation{}, false
 	}
-	if decl, ok := fn.Syntax().(*ast.FuncDecl); ok && decl.Body != nil {
+	if isDecl && decl.Body != nil {
 		return simdOperation{}, false
 	}
 	key := simdKey{name: obj.Name()}
 	var vector types.Type
 	if recv := sig.Recv(); recv != nil {
 		vector = recv.Type()
-		if _, ok := llssa.SIMDNumericShape(vector); !ok {
+		if _, ok := llssa.SIMDNumericShape(vector); ok {
+			key.receiver = "numeric"
+		} else if _, ok := llssa.SIMDMaskShape(vector); ok {
+			key.receiver = "mask"
+		} else {
 			return fallback()
 		}
-		key.receiver = "numeric"
 	} else if sig.Results().Len() == 1 {
 		vector = sig.Results().At(0).Type()
 	}
 	desc, ok := simdOperations[key]
 	if !ok || !desc.matches(sig, vector) {
 		return fallback()
 	}
+	if desc.op == llssa.SIMDLookupOrZero && arch != "arm64" && arch != "wasm" {
+		return fallback()
+	}
 	return desc, true
 }
 
 func (d simdOperation) matches(sig *types.Signature, vector types.Type) bool {
-	if vector == nil || sig.Variadic() || sig.Results().Len() != 1 {
+	if vector == nil || sig.Variadic() {
 		return false
 	}
-	lanes, ok := llssa.SIMDNumericShape(vector)
+	lanes, ok := llssa.SIMDVectorShape(vector)
 	if !ok {
 		return false
 	}
-	if d.integerOnly && lanes.Elem().Underlying().(*types.Basic).Info()&types.IsInteger == 0 {
+	if d.elements != 0 && lanes.Elem().Underlying().(*types.Basic).Info()&d.elements == 0 {
+		return false
+	}
+	if (d.op == llssa.SIMDLookupOrZero || d.op == llssa.SIMDPermuteOrZero) && lanes.Len() != 16 {
 		return false
 	}
 	var params []types.Type
 	result := vector
 	switch d.signature {
+	case simdScalarShift:
+		params = []types.Type{types.Typ[types.Uint64]}
+	case simdUnsignedVector, simdSignedVector:
+		if sig.Params().Len() != 1 {
+			return false
+		}
+		counts := sig.Params().At(0).Type()
+		shape, ok := llssa.SIMDNumericShape(counts)
+		if !ok || shape.Len() != lanes.Len() {
+			return false
+		}
+		info := shape.Elem().Underlying().(*types.Basic).Info()
+		if info&types.IsInteger == 0 || (info&types.IsUnsigned != 0) != (d.signature == simdUnsignedVector) {
+			return false
+		}
+		params = []types.Type{counts}
+	case simdMaskFromBits, simdMaskToBits:
+		if _, ok := llssa.SIMDMaskShape(vector); !ok {
+			return false
+		}
+		bits := types.Typ[types.Uint8]
+		if lanes.Len() == 16 {
+			bits = types.Typ[types.Uint16]
+		}
+		if d.signature == simdMaskFromBits {
+			params = []types.Type{bits}
+		} else {
+			result = bits
+		}
+	case simdCompare, simdToMask:
+		if sig.Results().Len() != 1 {
+			return false
+		}
+		result = sig.Results().At(0).Type()
+		mask, ok := llssa.SIMDMaskShape(result)
+		if !ok || mask.Len() != lanes.Len() {
+			return false
+		}
+		if d.signature == simdCompare {
+			params = []types.Type{vector}
+		}
+	case simdTernary:
+		params = []types.Type{vector, vector}
+	case simdBitcast, simdConvert:
+		if sig.Results().Len() != 1 {
+			return false
+		}
+		result = sig.Results().At(0).Type()
+		shape, ok := llssa.SIMDNumericShape(result)
+		if !ok || d.signature == simdConvert && shape.Len() < lanes.Len() {
+			return false
+		}
+		if d.signature == simdConvert && shape.Len() != lanes.Len() {
+			fromFloat := lanes.Elem().Underlying().(*types.Basic).Info()&types.IsFloat != 0
+			toFloat := shape.Elem().Underlying().(*types.Basic).Info()&types.IsFloat != 0
+			if !fromFloat && !toFloat {
+				return false
+			}
+		}
+	case simdUnary:
+		// Receiver only.
+	case simdLoad, simdStore:
+		params = []types.Type{types.NewPointer(lanes)}
+		if d.signature == simdStore {
+			result = nil
+		}
+	case simdBroadcast:
+		params = []types.Type{lanes.Elem()}
 	case simdBinary:
 		params = []types.Type{vector}
 	case simdExtract:
@@ -108,10 +268,17 @@ func (d simdOperation) matches(sig *types.Signature, vector types.Type) bool {
 	default:
 		return false
 	}
-	if sig.Recv() == nil {
+	if sig.Recv() == nil && d.signature != simdLoad && d.signature != simdBroadcast && d.signature != simdMaskFromBits {
 		params = append([]types.Type{vector}, params...)
 	}
-	if sig.Params().Len() != len(params) || !types.Identical(sig.Results().At(0).Type(), result) {
+	if sig.Params().Len() != len(params) {
+		return false
+	}
+	if result == nil {
+		if sig.Results().Len() != 0 {
+			return false
+		}
+	} else if sig.Results().Len() != 1 || !types.Identical(sig.Results().At(0).Type(), result) {
 		return false
 	}
 	for i, typ := range params {
@@ -132,5 +299,12 @@ func (p *context) simdCall(b llssa.Builder, fn *ssa.Function, args []ssa.Value)
 	if !ok {
 		panic("invalid SIMD intrinsic")
 	}
-	return b.SIMD(desc.op, p.compileValues(b, args, fnNormal)...)
+	return b.SIMD(desc.op, p.simdResultType(fn.Signature), p.compileValues(b, args, fnNormal)...)
+}
+
+func (p *context) simdResultType(sig *types.Signature) llssa.Type {
+	if sig.Results().Len() == 0 {
+		return p.prog.Void()
+	}
+	return p.prog.Type(sig.Results().At(0).Type(), llssa.InGo)
 }

cl/simd_test.go

@@ -23,7 +23,7 @@ func TestSIMDOperationIdentity(t *testing.T) {
 	// A test-only package-function registration exercises the same signature
 	// family without adding an operation to the supported official API.
 	key := simdKey{"", "Sum"}
-	simdOperations[key] = simdOperation{llssa.SIMDAdd, simdBinary, false}
+	simdOperations[key] = simdOperation{llssa.SIMDAdd, simdBinary, 0}
 	defer delete(simdOperations, key)
 	for _, tc := range []struct {
 		name, path, arch, source string
@@ -114,3 +114,34 @@ func TestSIMDFallbackDeclarations(t *testing.T) {
 		}
 	}
 }
+
+func TestSIMDLookupOrZeroTargets(t *testing.T) {
+	const source = `package archsimd
+ type v128 struct { _ [0]func() }
+ type Int8x16 struct { tag v128; vals [16]int8 }
+ func (x Int8x16) LookupOrZero(y Int8x16) Int8x16
+ `
+	fs := token.NewFileSet()
+	file, err := parser.ParseFile(fs, "simd.go", source, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	pkg, _, err := ssautil.BuildPackage(&types.Config{}, fs, types.NewPackage("simd/archsimd", "archsimd"), []*ast.File{file}, ssa.SanityCheckFunctions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	vector := pkg.Pkg.Scope().Lookup("Int8x16").Type().(*types.Named)
+	fn := pkg.Prog.FuncValue(vector.Method(0))
+	for _, arch := range []string{"arm64", "wasm", "amd64"} {
+		t.Run(arch, func(t *testing.T) {
+			want := llssa.SIMDLookupOrZero
+			if arch == "amd64" {
+				want = llssa.SIMDUnimplemented
+			}
+			op, ok := lookupSIMD(fn, arch)
+			if !ok || op.op != want {
+				t.Fatalf("lookup = (%v, %v), want %v", op.op, ok, want)
+			}
+		})
+	}
+}

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/qiniu/x v1.18.3
 	github.com/xgo-dev/llgo/runtime v0.0.0-00010101000000-000000000000
 	github.com/xgo-dev/llvm v0.10.0
-	github.com/xgo-dev/plan9asm v0.6.1
+	github.com/xgo-dev/plan9asm v0.6.2
 	go.bug.st/serial v1.6.4
 	go.yaml.in/yaml/v3 v3.0.5
 	golang.org/x/mod v0.41.0

go.sum

@@ -26,6 +26,8 @@ github.com/xgo-dev/llvm v0.10.0 h1:o27kypAGI7DrLFckrsUVjWChXqHUBDAFtZbwMiDFs50=
 github.com/xgo-dev/llvm v0.10.0/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M=
 github.com/xgo-dev/plan9asm v0.6.1 h1:WcwHdcNKBzZnEwl/m4AQPkBQ0XO96d9kgBhanmpINuA=
 github.com/xgo-dev/plan9asm v0.6.1/go.mod h1:UTpcZz3aQXSWYruhfzOi8aK4QZTEFPIvB61YW6jHT4M=
+github.com/xgo-dev/plan9asm v0.6.2 h1:9mV7u9stDT2V3wZP+ywRP/wbWmtQwf5bb6zym5Unyww=
+github.com/xgo-dev/plan9asm v0.6.2/go.mod h1:UTpcZz3aQXSWYruhfzOi8aK4QZTEFPIvB61YW6jHT4M=
 go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A=
 go.bug.st/serial v1.6.4/go.mod h1:nofMJxTeNVny/m6+KaafC6vJGj3miwQZ6vW4BZUGJPI=
 go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=

internal/build/build.go

@@ -3229,6 +3229,7 @@ func lowerMainCExportModule(ctx *context, pkg llssa.Package, exports []cExport)
 	if err := optimizeLLVMModule(ctx, pkg.Path(), mod); err != nil {
 		return true, err
 	}
+	lowerEmscriptenSIMDCalls(ctx, mod)
 	localizeWasmStackAddresses(ctx.buildConf.Goarch, mod)
 	return true, nil
 }
@@ -3297,6 +3298,7 @@ func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbos
 	if err := optimizeLLVMModule(ctx, pkgPath, ret.Module()); err != nil {
 		return err
 	}
+	lowerEmscriptenSIMDCalls(ctx, ret.Module())
 	localizeWasmStackAddresses(ctx.buildConf.Goarch, ret.Module())
 	dropUnusedWindowsTestMain(ctx, aPkg, ret.Module())
 	emitFuncInfoEntrySites(ctx, ret)