amd64: preserve XMM/YMM/ZMM register aliasing and upper-bit semantics

amd64_ctx.go

@@ -31,8 +31,9 @@ type amd64Ctx struct {
 	usedRegs map[Reg]bool
 	regSlot  map[Reg]string // gp reg -> alloca name
 
-	usedXRegs map[int]bool
-	xRegSlot  map[int]string // xmm reg index -> alloca name (<16 x i8>)
+	zeroUpperVector bool // VEX/EVEX writes clear the upper register bits
+	usedXRegs       map[int]bool
+	xRegSlot        map[int]string // xmm reg index -> alloca name (<16 x i8>)
 
 	usedYRegs map[int]bool
 	yRegSlot  map[int]string // ymm reg index -> alloca name (<32 x i8>)
@@ -334,40 +335,41 @@ func (c *amd64Ctx) emitEntryAllocas() error {
 		fmt.Fprintf(c.b, "  store i64 %%%s, ptr %s\n", addr, spSlot)
 	}
 
-	xIdx := make([]int, 0, len(c.usedXRegs))
+	// XMM, YMM and ZMM are overlapping views of the same physical register.
+	// Allocate the widest view used by the function and share its low bytes.
+	widths := make(map[int]int)
 	for i := range c.usedXRegs {
-		xIdx = append(xIdx, i)
+		widths[i] = 16
 	}
-	sort.Ints(xIdx)
-	for _, i := range xIdx {
-		name := c.xSlotName(i)
-		c.xRegSlot[i] = name
-		fmt.Fprintf(c.b, "  %s = alloca <16 x i8>\n", name)
-		fmt.Fprintf(c.b, "  store <16 x i8> zeroinitializer, ptr %s\n", name)
-	}
-
-	yIdx := make([]int, 0, len(c.usedYRegs))
 	for i := range c.usedYRegs {
-		yIdx = append(yIdx, i)
-	}
-	sort.Ints(yIdx)
-	for _, i := range yIdx {
-		name := fmt.Sprintf("%%y%d", i)
-		c.yRegSlot[i] = name
-		fmt.Fprintf(c.b, "  %s = alloca <32 x i8>\n", name)
-		fmt.Fprintf(c.b, "  store <32 x i8> zeroinitializer, ptr %s\n", name)
+		widths[i] = 32
 	}
-
-	zIdx := make([]int, 0, len(c.usedZRegs))
 	for i := range c.usedZRegs {
-		zIdx = append(zIdx, i)
+		widths[i] = 64
+	}
+	indices := make([]int, 0, len(widths))
+	for i := range widths {
+		indices = append(indices, i)
 	}
-	sort.Ints(zIdx)
-	for _, i := range zIdx {
-		name := fmt.Sprintf("%%z%d", i)
-		c.zRegSlot[i] = name
-		fmt.Fprintf(c.b, "  %s = alloca <64 x i8>\n", name)
-		fmt.Fprintf(c.b, "  store <64 x i8> zeroinitializer, ptr %s\n", name)
+	sort.Ints(indices)
+	for _, i := range indices {
+		width := widths[i]
+		name := c.xSlotName(i)
+		if width == 32 {
+			name = fmt.Sprintf("%%y%d", i)
+		}
+		if width == 64 {
+			name = fmt.Sprintf("%%z%d", i)
+		}
+		c.xRegSlot[i] = name
+		if width >= 32 {
+			c.yRegSlot[i] = name
+		}
+		if width == 64 {
+			c.zRegSlot[i] = name
+		}
+		fmt.Fprintf(c.b, "  %s = alloca <%d x i8>\n", name, width)
+		fmt.Fprintf(c.b, "  store <%d x i8> zeroinitializer, ptr %s\n", width, name)
 	}
 
 	kIdx := make([]int, 0, len(c.usedKRegs))
@@ -998,6 +1000,21 @@ func (c *amd64Ctx) storeRegSized(r Reg, ty LLVMType, v string) error {
 	}
 }
 
+// clearVectorUpper clears bytes above the written width while leaving the
+// low view unchanged. Legacy SSE writes deliberately preserve those bytes.
+func (c *amd64Ctx) clearVectorUpper(idx, lowBytes int) {
+	slot, width := c.zRegSlot[idx], 64
+	if slot == "" {
+		slot, width = c.yRegSlot[idx], 32
+	}
+	if slot == "" || width <= lowBytes {
+		return
+	}
+	upper := c.newTmp()
+	fmt.Fprintf(c.b, "  %%%s = getelementptr i8, ptr %s, i64 %d\n", upper, slot, lowBytes)
+	fmt.Fprintf(c.b, "  store <%d x i8> zeroinitializer, ptr %%%s, align 1\n", width-lowBytes, upper)
+}
+
 func (c *amd64Ctx) loadX(r Reg) (string, error) {
 	idx, ok := amd64ParseXReg(r)
 	if !ok {
@@ -1021,6 +1038,9 @@ func (c *amd64Ctx) storeX(r Reg, v string) error {
 	if !ok {
 		return nil
 	}
+	if c.zeroUpperVector {
+		c.clearVectorUpper(idx, 16)
+	}
 	fmt.Fprintf(c.b, "  store <16 x i8> %s, ptr %s\n", v, slot)
 	return nil
 }
@@ -1048,6 +1068,9 @@ func (c *amd64Ctx) storeY(r Reg, v string) error {
 	if !ok {
 		return nil
 	}
+	if c.zeroUpperVector {
+		c.clearVectorUpper(idx, 32)
+	}
 	fmt.Fprintf(c.b, "  store <32 x i8> %s, ptr %s\n", v, slot)
 	return nil
 }

amd64_lower_vec.go

@@ -111,11 +111,16 @@ func (c *amd64Ctx) lowerVec(op Op, ins Instr) (ok bool, terminated bool, err err
 		}
 	}
 
-	if op == "VZEROUPPER" {
-		// No-op in LLVM IR. Kept for completeness.
-		return true, false, nil
-	}
-	if op == "VZEROALL" {
+	if op == "VZEROUPPER" || op == "VZEROALL" {
+		// These instructions affect registers 0-15, including their ZMM view.
+		for i := 0; i < 16; i++ {
+			c.clearVectorUpper(i, 16)
+			if op == "VZEROALL" {
+				if slot := c.xRegSlot[i]; slot != "" {
+					fmt.Fprintf(c.b, "  store <16 x i8> zeroinitializer, ptr %s\n", slot)
+				}
+			}
+		}
 		return true, false, nil
 	}
 

amd64_translate.go

@@ -241,6 +241,7 @@ func (c *amd64Ctx) lowerInstr(bi int, ii int, ins Instr, emitBr amd64EmitBr, emi
 	c.allowSPWrite = models386SPWrite(ins)
 	defer func() { c.allowSPWrite = false }()
 	op := strings.ToUpper(string(ins.Op))
+	c.zeroUpperVector = strings.HasPrefix(op, "V")
 	if c.repeatPrefix != "" && op != "MOVSB" && op != "MOVSL" && op != "STOSL" && op != "SCASB" {
 		prefix := c.repeatPrefix
 		c.repeatPrefix = ""

amd64_vector_alias_test.go

@@ -0,0 +1,58 @@
+//go:build !llgo
+
+package plan9asm
+
+import (
+	"fmt"
+	"runtime"
+	"testing"
+)
+
+func TestAMD64VectorRegisterAliases(t *testing.T) {
+	crossRosetta := runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && rosettaAvailable()
+	if runtime.GOARCH != "amd64" && !crossRosetta {
+		t.Skip("requires amd64 execution")
+	}
+	llc, clang, ok := findLlcAndClang(t)
+	if !ok {
+		t.Skip("llc/clang not found")
+	}
+	triple := testTargetTriple(runtime.GOOS, "amd64")
+	var prefix []string
+	if crossRosetta {
+		prefix = []string{"/usr/bin/arch", "-x86_64"}
+	}
+	for _, tc := range []struct{ name, instructions, expected string }{
+		{"wide_to_narrow", "VMOVDQU64 (SI), Z1\nMOVOU X1, (DI)", "i < 16 ? in[i] : 0"},
+		{"legacy_preserves_upper", "VMOVDQU64 (SI), Z1\nMOVOU 64(SI), X1\nVMOVDQU64 Z1, (DI)", "i < 16 ? in[64+i] : in[i]"},
+		{"vex128_clears_upper", "VMOVDQU64 (SI), Z1\nMOVOU 64(SI), X2\nVMOVAPS X2, X1\nVMOVDQU64 Z1, (DI)", "i < 16 ? in[64+i] : 0"},
+		{"vex256_clears_upper", "VMOVDQU64 (SI), Z1\nVMOVDQU 64(SI), Y1\nVMOVDQU64 Z1, (DI)", "i < 32 ? in[64+i] : 0"},
+		{"zero_upper", "VMOVDQU64 (SI), Z1\nVZEROUPPER\nVMOVDQU64 Z1, (DI)", "i < 16 ? in[i] : 0"},
+		{"zero_all", "VMOVDQU64 (SI), Z1\nVZEROALL\nVMOVDQU64 Z1, (DI)", "0"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			source := "TEXT probe(SB),NOSPLIT,$0-16\nMOVQ in+0(FP), SI\nMOVQ out+8(FP), DI\n" + tc.instructions + "\nRET\n"
+			file, err := Parse(ArchAMD64, source)
+			if err != nil {
+				t.Fatal(err)
+			}
+			ll, err := Translate(file, Options{TargetTriple: triple, Goarch: "amd64", Sigs: map[string]FuncSig{"probe": {Name: "probe", Args: []LLVMType{Ptr, Ptr}, Ret: Void, Frame: FrameLayout{Params: []FrameSlot{{Offset: 0, Type: Ptr, Index: 0, Field: -1}, {Offset: 8, Type: Ptr, Index: 1, Field: -1}}}}}})
+			if err != nil {
+				t.Fatal(err)
+			}
+			mainC := fmt.Sprintf(`#include <stdio.h>
+extern void probe(unsigned char *, unsigned char *);
+int main(void) {
+ unsigned char in[128], out[64] = {0};
+ for (int i = 0; i < 128; ++i) in[i] = (unsigned char)(i + 1);
+ probe(in, out);
+ for (int i = 0; i < 64; ++i) {
+  unsigned char want = %s;
+  if (out[i] != want) { fprintf(stderr, "byte %%d: got %%u, want %%u\n", i, out[i], want); return 1; }
+ }
+ return 0;
+}`, tc.expected)
+			compileAndRunRuntimeTestForTarget(t, llc, clang, "vector_alias", triple, ll, mainC, prefix)
+		})
+	}
+}

arm64_conformance_test.go

@@ -51,6 +51,15 @@ func translateARM64Conformance(t *testing.T, triple string) string {
 		ResolveSym:   func(sym string) string { return strings.TrimPrefix(sym, "·") },
 		Goarch:       "arm64",
 		Sigs: map[string]FuncSig{
+			"pairStores": {
+				Name: "pairStores",
+				Args: []LLVMType{Ptr, Ptr},
+				Ret:  Void,
+				Frame: FrameLayout{Params: []FrameSlot{
+					{Offset: 0, Type: Ptr, Index: 0, Field: -1},
+					{Offset: 8, Type: Ptr, Index: 1, Field: -1},
+				}},
+			},
 			"families": {
 				Name: "families",
 				Args: []LLVMType{Ptr, Ptr},
@@ -96,6 +105,8 @@ func TestARM64ConformanceLLVMRuntime(t *testing.T) {
 	mainC := `
 #include <stdint.h>
 extern void families(uint64_t *out, uint64_t *data);
+extern void pairStores(uint64_t *out, uint64_t *data);
+uint64_t pairStoreData[10];
 int main(void) {
     uint64_t data[8] = {0x0123456789abcdefULL, 0xfedcba9876543210ULL, 0x1122334455667788ULL, 0x8877665544332211ULL};
     uint64_t want[76] = {
@@ -121,6 +132,26 @@ int main(void) {
     for (int i = 0; i < 76; i++)
         if (got[i] != want[i])
             return i + 1;
+    uint64_t pairs[20], expected[20];
+    for (int i = 0; i < 20; i++)
+        pairs[i] = expected[i] = 0xdeadbeef;
+    expected[0] = 0;
+    expected[1] = 32;
+    expected[6] = (uint64_t)-32;
+    for (int i = 0; i < 4; i++) {
+        expected[2 + i] = expected[16 + i] = data[i];
+        expected[8 + i] = data[(i + 2) % 4];
+    }
+    for (int i = 0; i < 10; i++)
+        pairStoreData[i] = 0xdeadbeef;
+    pairStores(pairs, data);
+    for (int i = 0; i < 20; i++)
+        if (pairs[i] != expected[i])
+            return 100 + i;
+    uint64_t symbolExpected[10] = {data[0], data[1], data[2], data[3], 0xdeadbeef, data[2], data[3], data[0], data[1], 0xdeadbeef};
+    for (int i = 0; i < 10; i++)
+        if (pairStoreData[i] != symbolExpected[i])
+            return 120 + i;
     return 0;
 }
 `

arm64_lower_vec.go

@@ -93,6 +93,40 @@ func (c *arm64Ctx) lowerVec(op Op, postInc bool, ins Instr) (ok bool, terminated
 			return true, false, fmt.Errorf("arm64 FMOVQ expects memory and F register operands: %q", ins.Raw)
 		}
 
+	case "FSTPQ":
+		if len(ins.Args) != 2 || ins.Args[0].Kind != OpRegList || len(ins.Args[0].RegList) != 2 || (ins.Args[1].Kind != OpMem && ins.Args[1].Kind != OpSym) {
+			return true, false, fmt.Errorf("arm64 FSTPQ expects (Freg,Freg), mem: %q", ins.Raw)
+		}
+		for _, f := range ins.Args[0].RegList {
+			if _, ok := arm64ParseFReg(f); !ok {
+				return true, false, fmt.Errorf("arm64 FSTPQ expects FP register pair: %q", ins.Raw)
+			}
+		}
+		preInc := strings.Contains(strings.ToUpper(string(ins.Op)), ".W")
+		ptr, base, inc, update, err := c.arm64VectorMemoryPointer(ins.Args[1], preInc, postInc)
+		if err != nil {
+			return true, false, err
+		}
+		for i, f := range ins.Args[0].RegList {
+			value, err := c.loadVReg(f)
+			if err != nil {
+				return true, false, err
+			}
+			storePtr := ptr
+			if i != 0 {
+				next := c.newTmp()
+				fmt.Fprintf(c.b, "  %%%s = getelementptr i8, ptr %s, i64 16\n", next, ptr)
+				storePtr = "%" + next
+			}
+			fmt.Fprintf(c.b, "  store <16 x i8> %s, ptr %s, align 1\n", value, storePtr)
+		}
+		if update {
+			if err := c.updatePostInc(base, inc); err != nil {
+				return true, false, err
+			}
+		}
+		return true, false, nil
+
 	case "FLDPQ":
 		if len(ins.Args) != 2 || ins.Args[0].Kind != OpMem || ins.Args[1].Kind != OpRegList || len(ins.Args[1].RegList) != 2 {
 			return true, false, fmt.Errorf("arm64 FLDPQ expects mem, (Freg,Freg): %q", ins.Raw)

testdata/conformance/arm64/conformance.go

@@ -3,3 +3,7 @@
 package arm64conformance
 
 func families(out *[76]uint64, data *[8]uint64)
+
+func pairStores(out *[20]uint64, data *[8]uint64)
+
+var pairStoreData [10]uint64

testdata/conformance/arm64/conformance_arm64.s

@@ -185,3 +185,31 @@ TEXT ·families(SB), NOSPLIT, $0-16
 	FMOVD R2, F4
 	FMOVQ F4, 592(R0)
 	RET
+
+// Check full-width stores, offsets, and both writeback modes.
+TEXT ·pairStores(SB), NOSPLIT, $0-16
+	MOVD out+0(FP), R0
+	MOVD data+8(FP), R1
+	FMOVQ (R1), F0
+	FMOVQ 16(R1), F1
+
+	MOVD R0, R2
+	FSTPQ (F0, F1), 16(R2)
+	SUB R0, R2, R3
+	MOVD R3, (R0)
+
+	ADD $64, R0, R2
+	MOVD R2, R4
+	FSTPQ.P (F1, F0), 32(R2)
+	SUB R4, R2, R3
+	MOVD R3, 8(R0)
+
+	ADD $160, R0, R2
+	MOVD R2, R4
+	FSTPQ.W (F0, F1), -32(R2)
+	SUB R4, R2, R3
+	MOVD R3, 48(R0)
+
+	FSTPQ (F0, F1), ·pairStoreData(SB)
+	FSTPQ (F1, F0), ·pairStoreData+40(SB)
+	RET

testdata/conformance/arm64/conformance_test.go

@@ -102,3 +102,38 @@ func bool64(value bool) uint64 {
 	}
 	return 0
 }
+
+func TestPairStores(t *testing.T) {
+	data := [8]uint64{0x0123456789abcdef, 0xfedcba9876543210, 0x1122334455667788, 0x8877665544332211}
+	var got, want [20]uint64
+	for i := range got {
+		got[i], want[i] = 0xdeadbeef, 0xdeadbeef
+	}
+	want[0], want[1], want[6] = 0, 32, ^uint64(31)
+	copy(want[2:6], data[:4])
+	copy(want[8:10], data[2:4])
+	copy(want[10:12], data[:2])
+	copy(want[16:20], data[:4])
+	for i := range pairStoreData {
+		pairStoreData[i] = 0xdeadbeef
+	}
+	pairStores(&got, &data)
+	for i := range got {
+		if got[i] != want[i] {
+			t.Fatalf("pairStores()[%d] = %#x, want %#x", i, got[i], want[i])
+		}
+	}
+}
+
+func TestPairStoreSymbols(t *testing.T) {
+	data := [8]uint64{0x0123456789abcdef, 0xfedcba9876543210, 0x1122334455667788, 0x8877665544332211}
+	var out [20]uint64
+	for i := range pairStoreData {
+		pairStoreData[i] = 0xdeadbeef
+	}
+	pairStores(&out, &data)
+	want := [10]uint64{data[0], data[1], data[2], data[3], 0xdeadbeef, data[2], data[3], data[0], data[1], 0xdeadbeef}
+	if pairStoreData != want {
+		t.Fatalf("pairStoreData = %#x, want %#x", pairStoreData, want)
+	}
+}

testdata/conformance/manifest.json

@@ -181,7 +181,9 @@
         "FMOVD gpr,floating-register",
         "FMOVQ memory.gpr-base,floating-register",
         "FMOVQ memory.gpr-base.offset,floating-register",
-        "FMOVQ floating-register,memory.gpr-base.offset"
+        "FMOVQ floating-register,memory.gpr-base.offset",
+        "FSTPQ register-list,memory.gpr-base.offset",
+        "FSTPQ register-list,symbol"
       ]
     }
   ]