diff --git a/cranelift/assembler-x64/meta/src/dsl.rs b/cranelift/assembler-x64/meta/src/dsl.rs index 475aa7d8d237..cd6e2801d7ca 100644 --- a/cranelift/assembler-x64/meta/src/dsl.rs +++ b/cranelift/assembler-x64/meta/src/dsl.rs @@ -10,8 +10,8 @@ mod features; pub mod format; pub use custom::{Custom, Customization}; +pub use encoding::{ApxClass, Evex, Length, Vex, VexEscape, VexPrefix, evex, vex}; pub use encoding::{Encoding, ModRmKind, OpcodeMod}; -pub use encoding::{Evex, Length, Vex, VexEscape, VexPrefix, evex, vex}; pub use encoding::{ Group1Prefix, Group2Prefix, Group3Prefix, Group4Prefix, Opcodes, Prefixes, Rex, TupleType, rex, }; diff --git a/cranelift/assembler-x64/meta/src/dsl/encoding.rs b/cranelift/assembler-x64/meta/src/dsl/encoding.rs index c843e609d7eb..658c723bdc14 100644 --- a/cranelift/assembler-x64/meta/src/dsl/encoding.rs +++ b/cranelift/assembler-x64/meta/src/dsl/encoding.rs @@ -57,6 +57,9 @@ pub fn evex(length: Length, tuple_type: TupleType) -> Evex { modrm: None, imm: Imm::None, tuple_type, + apx: None, + nd: None, + nf: None, } } @@ -841,6 +844,10 @@ pub enum VexEscape { _0F, _0F3A, _0F38, + /// APX "opcode map 4"; only valid for APX legacy-GPR (Extended EVEX) + /// encodings, never for VEX. This is the map that enables promoting legacy + /// general-purpose-register instructions into the EVEX space. + _MAP4, } impl VexEscape { @@ -851,6 +858,7 @@ impl VexEscape { Self::_0F => 0b01, Self::_0F38 => 0b10, Self::_0F3A => 0b11, + Self::_MAP4 => 0b100, } } } @@ -861,6 +869,7 @@ impl fmt::Display for VexEscape { Self::_0F => write!(f, "0F"), Self::_0F3A => write!(f, "0F3A"), Self::_0F38 => write!(f, "0F38"), + Self::_MAP4 => write!(f, "MAP4"), } } } @@ -1247,6 +1256,30 @@ pub struct Evex { /// The "Tuple Type" corresponding to scaling of the 8-bit displacement /// parameter for memory operands. See [`TupleType`] for more information. pub tuple_type: TupleType, + /// The APX "Extended EVEX" class, when this instruction is an APX + /// promotion. + /// + /// Intel APX (Advanced Performance Extensions) reuses the `0x62` EVEX + /// identifier but does *not* define a single static payload layout. + /// Instead, the meaning of the payload bytes (`P0`, `P1`, `P2`) is + /// re-mapped depending on which class of instruction is being promoted (see + /// [`ApxClass`]). `None` denotes a standard (AVX-512) EVEX encoding; + /// `Some(_)` selects one of the APX layouts and enables addressing of the + /// extended general-purpose registers `R16`–`R31` ("EGPR"). + pub apx: Option, + /// The APX `ND` (New Data destination) bit. + /// + /// When set, a legacy destructive two-operand instruction is promoted into + /// a non-destructive three-operand form (the extra destination is encoded + /// in the `EVEX.vvvv` field). `None` when the bit is not part of this + /// encoding; only valid for [`ApxClass::LegacyGpr`]. + pub nd: Option, + /// The APX `NF` (No Flags) bit. + /// + /// When set, the status-flag writes that a legacy integer instruction would + /// otherwise perform are suppressed. `None` when the bit is not part of + /// this encoding; only valid for [`ApxClass::LegacyGpr`]. + pub nf: Option, } impl Evex { @@ -1310,6 +1343,27 @@ impl Evex { } } + /// Select the APX "opcode map 4", promoting a legacy general-purpose-register + /// instruction into the Extended EVEX space; equivalent to `.MAP4` in the + /// manual. + /// + /// This both sets the `mmm` map bits and marks the encoding as + /// [`ApxClass::LegacyGpr`], which in turn enables addressing of the extended + /// GPRs `R16`–`R31` ("EGPR") and permits the `ND`/`NF` bits. + /// + /// # Panics + /// + /// Panics if the map (`mmm`) or APX class has already been set. + pub fn map4(self) -> Self { + assert!(self.mmm.is_none()); + assert!(self.apx.is_none()); + Self { + mmm: Some(VexEscape::_MAP4), + apx: Some(ApxClass::LegacyGpr), + ..self + } + } + /// Set the `W` bit to `0`; equivalent to `.W0` in the manual. pub fn w0(self) -> Self { assert!(self.w.is_ignored()); @@ -1352,9 +1406,72 @@ impl Evex { } } + /// Mark this as an APX "Extended EVEX" encoding of the given [`ApxClass`]. + /// + /// This selects the APX payload layout to emit and enables addressing of + /// the extended general-purpose registers `R16`–`R31` ("EGPR"). + /// + /// # Panics + /// + /// Panics if an APX class has already been set. + pub fn apx(self, class: ApxClass) -> Self { + assert!(self.apx.is_none()); + Self { + apx: Some(class), + ..self + } + } + + /// Set the APX `ND` (New Data destination) bit; equivalent to `.ND` in the + /// manual. + /// + /// # Panics + /// + /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit + /// has already been set. + pub fn nd(self) -> Self { + assert_eq!( + self.apx, + Some(ApxClass::LegacyGpr), + "the ND bit is only valid for APX legacy-GPR promotions" + ); + assert!(self.nd.is_none()); + Self { + nd: Some(true), + ..self + } + } + + /// Set the APX `NF` (No Flags) bit; equivalent to `.NF` in the manual. + /// + /// # Panics + /// + /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit + /// has already been set. + pub fn nf(self) -> Self { + assert_eq!( + self.apx, + Some(ApxClass::LegacyGpr), + "the NF bit is only valid for APX legacy-GPR promotions" + ); + assert!(self.nf.is_none()); + Self { + nf: Some(true), + ..self + } + } + fn validate(&self, _operands: &[Operand]) { assert!(self.opcode != u8::MAX); assert!(self.mmm.is_some()); + // The `ND`/`NF` bits are only defined for APX legacy-GPR promotions. + if self.nd.is_some() || self.nf.is_some() { + assert_eq!( + self.apx, + Some(ApxClass::LegacyGpr), + "the ND/NF bits require an APX legacy-GPR encoding" + ); + } } /// Retrieve the digit extending the opcode, if available. @@ -1412,7 +1529,14 @@ impl fmt::Display for Evex { if let Some(mmmmm) = self.mmm { write!(f, ".{mmmmm}")?; } - write!(f, ".{} {:#04X}", self.w, self.opcode)?; + write!(f, ".{}", self.w)?; + if self.nd == Some(true) { + write!(f, ".ND")?; + } + if self.nf == Some(true) { + write!(f, ".NF")?; + } + write!(f, " {:#04X}", self.opcode)?; if let Some(modrm) = self.modrm { write!(f, " {modrm}")?; } @@ -1423,6 +1547,35 @@ impl fmt::Display for Evex { } } +/// The class of an APX "Extended EVEX" encoding. +/// +/// Intel APX does not define a single static Extended-EVEX layout. Although +/// every APX instruction still begins with the `0x62` EVEX identifier, the +/// meaning of the payload bytes (`P0`, `P1`, `P2`) is re-mapped depending on +/// the class of instruction being promoted. This enum selects which layout the +/// assembler should emit; all classes gain access to the extended +/// general-purpose registers `R16`–`R31` ("EGPR"). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum ApxClass { + /// Promotion of a legacy general-purpose-register (GPR) instruction using + /// extended opcode map 4. Only this class may set the `ND` and `NF` bits. + LegacyGpr, + /// Promotion of a legacy SSE / VEX vector instruction into the EVEX space. + Vector, + /// An existing AVX-512 (EVEX) instruction extended with APX register bits. + Avx512, +} + +impl fmt::Display for ApxClass { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::LegacyGpr => write!(f, "LegacyGpr"), + Self::Vector => write!(f, "Vector"), + Self::Avx512 => write!(f, "Avx512"), + } + } +} + /// Tuple Type definitions used in EVEX encodings. /// /// This enumeration corresponds to table 2-34 and 2-35 in the Intel manual. diff --git a/cranelift/assembler-x64/meta/src/dsl/features.rs b/cranelift/assembler-x64/meta/src/dsl/features.rs index 87e4172b3873..8e2840f95144 100644 --- a/cranelift/assembler-x64/meta/src/dsl/features.rs +++ b/cranelift/assembler-x64/meta/src/dsl/features.rs @@ -96,6 +96,7 @@ pub enum Feature { avx512vbmi, cmpxchg16b, fma, + apx, } /// List all CPU features. @@ -127,6 +128,7 @@ pub const ALL_FEATURES: &[Feature] = &[ Feature::avx512vbmi, Feature::cmpxchg16b, Feature::fma, + Feature::apx, ]; impl fmt::Display for Feature { diff --git a/cranelift/assembler-x64/meta/src/generate/format.rs b/cranelift/assembler-x64/meta/src/generate/format.rs index a0664463bf7d..f4d2f512e314 100644 --- a/cranelift/assembler-x64/meta/src/generate/format.rs +++ b/cranelift/assembler-x64/meta/src/generate/format.rs @@ -229,7 +229,7 @@ impl dsl::Format { fmtln!(f, "let w = {};", vex.w.as_bool()); let bits = "len, pp, mmmmm, w"; - self.generate_vex_or_evex_prefix(f, "VexPrefix", &bits, vex.is4, None, || { + self.generate_vex_or_evex_prefix(f, "VexPrefix", &bits, vex.is4, None, "two_op", "three_op", || { vex.unwrap_digit() }) } @@ -237,16 +237,36 @@ impl dsl::Format { fn generate_evex_prefix(&self, f: &mut Formatter, evex: &dsl::Evex) -> ModRmStyle { f.empty_line(); f.comment("Emit EVEX prefix."); - let ll = evex.length.evex_bits(); - fmtln!(f, "let ll = {ll:#04b};"); + + // Intel APX promotes legacy GPR instructions into "EVEX map 4" using a + // re-purposed payload layout (see `EvexPrefix::legacy` in the runtime + // assembler and section 3.1.2.3.1 of the APX spec). In that case we + // emit the ND/NF bits and select the `legacy_*` constructors instead of + // the AVX-512 ones. + let apx_legacy = matches!(evex.apx, Some(dsl::ApxClass::LegacyGpr)); + fmtln!(f, "let pp = {:#04b};", evex.pp.map_or(0b00, |pp| pp.bits())); fmtln!(f, "let mmm = {:#07b};", evex.mmm.unwrap().bits()); fmtln!(f, "let w = {};", evex.w.as_bool()); - // NB: when bcast is supported in the future the `evex_scaling` - // calculation for `Full` and `Half` below need to be updated. + + let (bits, two_op, three_op); + if apx_legacy { + fmtln!(f, "let nd = {};", evex.nd == Some(true)); + fmtln!(f, "let nf = {};", evex.nf == Some(true)); + bits = String::from("pp, mmm, w, nd, nf"); + two_op = "legacy_two_op"; + three_op = "legacy_three_op"; + } else { + let ll = evex.length.evex_bits(); + fmtln!(f, "let ll = {ll:#04b};"); + // NB: when bcast is supported in the future the `evex_scaling` + // calculation for `Full` and `Half` below need to be updated. + fmtln!(f, "let bcast = false;"); + bits = String::from("ll, pp, mmm, w, bcast"); + two_op = "two_op"; + three_op = "three_op"; + } let bcast = false; - fmtln!(f, "let bcast = {bcast};"); - let bits = format!("ll, pp, mmm, w, bcast"); let is4 = false; let length_bytes = match evex.length { @@ -289,7 +309,7 @@ impl dsl::Format { }, }); - self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, || { + self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, two_op, three_op, || { evex.unwrap_digit() }) } @@ -304,6 +324,8 @@ impl dsl::Format { bits: &str, is4: bool, evex_scaling: Option, + two_op: &str, + three_op: &str, unwrap_digit: impl Fn() -> Option, ) -> ModRmStyle { use dsl::OperandKind::{FixedReg, Imm, Mem, Reg, RegMem}; @@ -316,7 +338,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::Reg { reg: ModRmReg::Reg(*reg), @@ -333,7 +355,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::RegMem { reg: ModRmReg::Reg(*reg), @@ -348,7 +370,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::RegMemIs4 { reg: ModRmReg::Reg(*reg), @@ -368,7 +390,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::RegMem { reg: ModRmReg::Digit(digit), @@ -381,7 +403,7 @@ impl dsl::Format { let reg = reg_or_vvvv; fmtln!(f, "let reg = self.{reg}.enc();"); fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); - fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});"); + fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});"); ModRmStyle::RegMem { reg: ModRmReg::Reg(*reg), rm: *rm, @@ -399,7 +421,7 @@ impl dsl::Format { fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); fmtln!( f, - "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});" + "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});" ); ModRmStyle::Reg { reg: ModRmReg::Digit(digit), @@ -411,7 +433,7 @@ impl dsl::Format { let reg = reg_or_vvvv; fmtln!(f, "let reg = self.{reg}.enc();"); fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); - fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});"); + fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});"); ModRmStyle::Reg { reg: ModRmReg::Reg(*reg), rm: *rm, @@ -423,7 +445,7 @@ impl dsl::Format { assert!(!is4); fmtln!(f, "let reg = self.{reg}.enc();"); fmtln!(f, "let rm = self.{rm}.encode_bx_regs();"); - fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});"); + fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});"); ModRmStyle::RegMem { reg: ModRmReg::Reg(*reg), rm: *rm, diff --git a/cranelift/assembler-x64/meta/src/instructions/add.rs b/cranelift/assembler-x64/meta/src/instructions/add.rs index f5969061b053..f4b6b4fb296f 100644 --- a/cranelift/assembler-x64/meta/src/instructions/add.rs +++ b/cranelift/assembler-x64/meta/src/instructions/add.rs @@ -97,5 +97,12 @@ pub fn list() -> Vec { inst("vphaddw", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x01).r(), (_64b | compat) & avx), inst("vphaddd", fmt("B", [w(xmm1), r(xmm2), r(xmm_m128)]), vex(L128)._66()._0f38().op(0x02).r(), (_64b | compat) & avx), inst("vaddpd", fmt("C", [w(xmm1), r(xmm2), r(xmm_m128)]), evex(L128, Full)._66()._0f().w1().op(0x58).r(), (_64b | compat) & avx512vl), + // APX: NDD (new data destination) form of `ADD`, promoted into EVEX + // "map 4" via the extended-EVEX prefix. With `ND = 1` the architectural + // destination is the `vvvv`-encoded register, so the written operand is + // placed in the `V` slot (the middle operand of the `RVM` format) while + // the two sources occupy ModRM.reg and ModRM.rm. `addq` is 64-bit so + // the `W` bit is set (`.w1()`); use `.w0()` for the 32-bit `addl` form. + inst("addq", fmt("RVM", [r(r64b), w(r64a), r(rm64)]), evex(L128, Full).map4().w1().nd().op(0x01).r(), _64b & apx), ] } diff --git a/cranelift/assembler-x64/src/evex.rs b/cranelift/assembler-x64/src/evex.rs index bf89dd13bdc1..2734fae7ac22 100644 --- a/cranelift/assembler-x64/src/evex.rs +++ b/cranelift/assembler-x64/src/evex.rs @@ -92,6 +92,107 @@ impl EvexPrefix { EvexPrefix::new(reg, vvvv, (b, x), ll, pp, mmm, w, broadcast) } + // --------------------------------------------------------------------- + // Intel APX "Extended EVEX" prefix for promoted *legacy* GPR instructions + // (EVEX map 4). See the Intel APX Architecture Specification (rev 8), + // section 3.1.2.3.1 "EVEX Extension of Legacy Instructions", Figure 3.3. + // + // The four bytes still begin with `0x62`, but several payload bits are + // re-purposed relative to the AVX-512 layout above: + // + // ┌────┬────┬────┬────┬────┬────┬────┬────┐ + // Byte 1: │ R3 │ X3 │ B3 │ R4 │ B4 │ 1 │ 0 │ 0 │ (map id = 4) + // ├────┼────┼────┼────┼────┼────┼────┼────┤ + // Byte 2: │ W │ V3 │ V2 │ V1 │ V0 │ U │ p │ p │ (U = ~X4) + // ├────┼────┼────┼────┼────┼────┼────┼────┤ + // Byte 3: │ 0 │ 0 │ 0 │ ND │ V4 │ NF │ 0 │ 0 │ + // └────┴────┴────┴────┴────┴────┴────┴────┘ + // + // The "underlined" fields (`R3`, `X3`, `B3`, `R4`, the `vvvv` bits and `V4`) + // are stored inverted, exactly as in the AVX-512 layout. `B4` and `X4` are + // newly repurposed reserved bits: `B4` uses *true* polarity (fixed value 0) + // and `X4` is carried inverted in the `U` bit (`EVEX.X4 = ~EVEX.U`), so a + // register-form instruction (ModRM.Mod = 3, no index) has `U = 1`. + + /// Construct the extended-EVEX (APX map 4) prefix for a legacy GPR + /// instruction. + /// + /// `reg` is the ModRM.reg register, `vvvv` is the `V` register identifier + /// (the NDD register when `nd` is set), and `(b, x)` are the ModRM.r/m base + /// and (optional) SIB index registers. `nd`/`nf` select the New Data + /// destination and No Flags bits respectively. + pub fn legacy( + reg: u8, + vvvv: u8, + (b, x): (Option, Option), + pp: u8, + mmm: u8, + w: bool, + nd: bool, + nf: bool, + ) -> Self { + let base = b.unwrap_or(0); + let index = x.unwrap_or(0); + + // byte1 (P0) + let r3 = invert_top_bit(reg); + let x3 = invert_top_bit(index); + let b3 = invert_top_bit(base); + let r4 = invert_top_bit(reg >> 1); + let b4 = (base >> 4) & 1; // true polarity + debug_assert!(mmm <= 0b111); + let byte1 = r3 << 7 | x3 << 6 | b3 << 5 | r4 << 4 | b4 << 3 | mmm; + + // byte2 (P1) + debug_assert!(vvvv <= 0b11111); + debug_assert!(pp <= 0b11); + let vvvv_value = !vvvv & 0b1111; + // `EVEX.X4 = ~EVEX.U`; with no index register X4 = 0 so U = 1. + let x4 = (index >> 4) & 1; + let u = (!x4) & 1; + let byte2 = (w as u8) << 7 | vvvv_value << 3 | u << 2 | (pp & 0b11); + + // byte3 (P2) + let v_prime = invert_top_bit(vvvv >> 1); // V4, inverted + let byte3 = (nd as u8) << 4 | v_prime << 3 | (nf as u8) << 2; + + Self { + byte1, + byte2, + byte3, + } + } + + /// Construct the extended-EVEX (APX map 4) prefix for a two-operand legacy + /// GPR instruction (no NDD); the `V` register identifier is unused. + #[allow(dead_code, reason = "not all APX legacy forms are emitted yet")] + pub fn legacy_two_op( + reg: u8, + (b, x): (Option, Option), + pp: u8, + mmm: u8, + w: bool, + nd: bool, + nf: bool, + ) -> Self { + EvexPrefix::legacy(reg, 0, (b, x), pp, mmm, w, nd, nf) + } + + /// Construct the extended-EVEX (APX map 4) prefix for a three-operand + /// legacy GPR instruction; `vvvv` carries the NDD register. + pub fn legacy_three_op( + reg: u8, + vvvv: u8, + (b, x): (Option, Option), + pp: u8, + mmm: u8, + w: bool, + nd: bool, + nf: bool, + ) -> Self { + EvexPrefix::legacy(reg, vvvv, (b, x), pp, mmm, w, nd, nf) + } + pub(crate) fn encode(&self, sink: &mut impl CodeSink) { sink.put1(0x62); sink.put1(self.byte1); diff --git a/cranelift/assembler-x64/src/fuzz.rs b/cranelift/assembler-x64/src/fuzz.rs index 290d888f5a00..4d948a62c9be 100644 --- a/cranelift/assembler-x64/src/fuzz.rs +++ b/cranelift/assembler-x64/src/fuzz.rs @@ -412,4 +412,37 @@ mod test { roundtrip(&inst.into()); } } + + /// Byte-level encoding check for the APX NDD (new-data-destination) form of + /// `ADD`, promoted into EVEX "map 4" via the extended-EVEX prefix. + /// + /// We assert the exact bytes rather than round-tripping through Capstone + /// because the bundled disassembler does not yet understand APX. With + /// `ND = 1` the architectural destination is the `vvvv`-encoded register, so + /// the `RVM` operands are `[ModRM.reg source, vvvv destination, ModRM.rm + /// source]`. For `addq %rax, %rcx, %rdx` (destination `%rax` = 0 in `vvvv`, + /// source `%rcx` = 1 in ModRM.reg, source `%rdx` = 2 in ModRM.rm), `W = 1`, + /// `ND = 1`, `NF = 0`, the expected sequence is: + /// + /// ```text + /// 62 F4 FC 18 01 CA + /// ^^ ^^ ^^ ^^ ^^ ^^ + /// | | | | | └ ModRM: mod=11 reg=rcx r/m=rdx + /// | | | | └ opcode 0x01 (ADD r/m, reg) + /// | | | └ P2: ND=1 (bit4), V4=1 (bit3, inverted), NF=0 + /// | | └ P1: W=1, vvvv=~rax=1111, U=1, pp=00 + /// | └ P0: R3 X3 B3 R4 = 1111, B4=0, map=100 (map 4) + /// └ EVEX identifier + /// ``` + #[test] + fn apx_addq_rvm_ndd_encoding() { + use crate::inst::addq_rvm; + // Format is `RVM` = [ModRM.reg source, vvvv destination, ModRM.rm + // source], so the constructor arguments are (reg source = %rcx, + // destination = %rax, r/m source = %rdx). + let inst = + addq_rvm::::new(FuzzReg::new(1), FuzzReg::new(0), FuzzReg::new(2)); + let assembled = assemble(&inst.into()); + assert_eq!(pretty_print_hexadecimal(&assembled), "62F4FC1801CA"); + } } diff --git a/cranelift/codegen/meta/src/isa/x86.rs b/cranelift/codegen/meta/src/isa/x86.rs index b757e174035d..d37bea4a5607 100644 --- a/cranelift/codegen/meta/src/isa/x86.rs +++ b/cranelift/codegen/meta/src/isa/x86.rs @@ -83,6 +83,15 @@ pub(crate) fn define() -> TargetIsa { "AVX512F: CPUID.07H:EBX.AVX512F[bit 16]", false, ); + // Registered so `target x86_64 has_apx` parses; not yet referenced by any + // preset because no shipping microarchitecture ships APX today. Drop the + // leading underscore once a preset needs it. + let _has_apx = settings.add_bool( + "has_apx", + "Has support for APX.", + "APX_F: CPUID.(EAX=07H,ECX=1H):EDX.APX_F[bit 21]", + false, + ); let has_popcnt = settings.add_bool( "has_popcnt", "Has support for POPCNT.", diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index e1104520d6f9..3597e33f2725 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -1329,6 +1329,9 @@ (decl pure use_avx2 () bool) (extern constructor use_avx2 use_avx2) +(decl pure use_apx () bool) +(extern constructor use_apx use_apx) + (decl pure has_cmpxchg16b () bool) (extern constructor has_cmpxchg16b has_cmpxchg16b) diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index bc4f24971814..96fc2a2dc4ba 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1590,6 +1590,10 @@ impl asm::AvailableFeatures for &EmitInfo { fn avx512vbmi(&self) -> bool { self.isa_flags.has_avx512vbmi() } + + fn apx(&self) -> bool { + self.isa_flags.has_apx() + } } impl MachInstEmit for Inst { diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 8a3c511c2699..76c0e9222192 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -94,6 +94,15 @@ (rule iadd_base_case_32_or_64_lea -5 (lower (iadd (ty_32_or_64 ty) x y)) (x64_lea ty (to_amode_add (mem_flags_trusted_data) x y (zero_offset)))) +;; APX: when the `has_apx` feature is enabled, use the NDD (new-data- +;; destination) form of `add` for 64-bit adds. Unlike the legacy two-operand +;; `add`, the NDD form writes its result to a fresh register encoded in the +;; EVEX `vvvv` field, so the register allocator does not need to insert a move +;; to preserve either input. +(rule iadd_apx_ndd 1 (lower (iadd $I64 x y)) + (if-let true (use_apx)) + (x64_addq_rvm x y)) + ;; Higher-priority cases than the previous two where a load can be sunk into ;; the add instruction itself. Note that both operands are tested for ;; sink-ability since addition is commutative diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index fb464b2012a7..88f61956ee5f 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -492,6 +492,11 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> { self.backend.x64_flags.has_avx512vbmi() } + #[inline] + fn use_apx(&mut self) -> bool { + self.backend.x64_flags.has_apx() + } + #[inline] fn has_lzcnt(&mut self) -> bool { self.backend.x64_flags.has_lzcnt() diff --git a/cranelift/filetests/filetests/isa/x64/apx-add.clif b/cranelift/filetests/filetests/isa/x64/apx-add.clif new file mode 100644 index 000000000000..56582a2c848b --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/apx-add.clif @@ -0,0 +1,24 @@ +test compile precise-output +target x86_64 has_apx + +function %add_i64(i64, i64) -> i64 { +block0(v0: i64, v1: i64): + v2 = iadd v0, v1 + return v2 +} + +; VCode: +; pushq %rbp +; movq %rsp, %rbp +; block0: +; addq %rsi, %rax, %rdi +; movq %rbp, %rsp +; popq %rbp +; retq +; +; Disassembled: +; block0: ; offset 0x0 +; pushq %rbp +; movq %rsp, %rbp +; block1: ; offset 0x4 +