Apx add ndd

cranelift/assembler-x64/meta/src/dsl.rs

@@ -10,8 +10,8 @@ mod features;
 pub mod format;
 
 pub use custom::{Custom, Customization};
+pub use encoding::{ApxClass, Evex, Length, Vex, VexEscape, VexPrefix, evex, vex};
 pub use encoding::{Encoding, ModRmKind, OpcodeMod};
-pub use encoding::{Evex, Length, Vex, VexEscape, VexPrefix, evex, vex};
 pub use encoding::{
     Group1Prefix, Group2Prefix, Group3Prefix, Group4Prefix, Opcodes, Prefixes, Rex, TupleType, rex,
 };

cranelift/assembler-x64/meta/src/dsl/encoding.rs

@@ -57,6 +57,9 @@ pub fn evex(length: Length, tuple_type: TupleType) -> Evex {
         modrm: None,
         imm: Imm::None,
         tuple_type,
+        apx: None,
+        nd: None,
+        nf: None,
     }
 }
 
@@ -841,6 +844,10 @@ pub enum VexEscape {
     _0F,
     _0F3A,
     _0F38,
+    /// APX "opcode map 4"; only valid for APX legacy-GPR (Extended EVEX)
+    /// encodings, never for VEX. This is the map that enables promoting legacy
+    /// general-purpose-register instructions into the EVEX space.
+    _MAP4,
 }
 
 impl VexEscape {
@@ -851,6 +858,7 @@ impl VexEscape {
             Self::_0F => 0b01,
             Self::_0F38 => 0b10,
             Self::_0F3A => 0b11,
+            Self::_MAP4 => 0b100,
         }
     }
 }
@@ -861,6 +869,7 @@ impl fmt::Display for VexEscape {
             Self::_0F => write!(f, "0F"),
             Self::_0F3A => write!(f, "0F3A"),
             Self::_0F38 => write!(f, "0F38"),
+            Self::_MAP4 => write!(f, "MAP4"),
         }
     }
 }
@@ -1247,6 +1256,30 @@ pub struct Evex {
     /// The "Tuple Type" corresponding to scaling of the 8-bit displacement
     /// parameter for memory operands. See [`TupleType`] for more information.
     pub tuple_type: TupleType,
+    /// The APX "Extended EVEX" class, when this instruction is an APX
+    /// promotion.
+    ///
+    /// Intel APX (Advanced Performance Extensions) reuses the `0x62` EVEX
+    /// identifier but does *not* define a single static payload layout.
+    /// Instead, the meaning of the payload bytes (`P0`, `P1`, `P2`) is
+    /// re-mapped depending on which class of instruction is being promoted (see
+    /// [`ApxClass`]). `None` denotes a standard (AVX-512) EVEX encoding;
+    /// `Some(_)` selects one of the APX layouts and enables addressing of the
+    /// extended general-purpose registers `R16`–`R31` ("EGPR").
+    pub apx: Option<ApxClass>,
+    /// The APX `ND` (New Data destination) bit.
+    ///
+    /// When set, a legacy destructive two-operand instruction is promoted into
+    /// a non-destructive three-operand form (the extra destination is encoded
+    /// in the `EVEX.vvvv` field). `None` when the bit is not part of this
+    /// encoding; only valid for [`ApxClass::LegacyGpr`].
+    pub nd: Option<bool>,
+    /// The APX `NF` (No Flags) bit.
+    ///
+    /// When set, the status-flag writes that a legacy integer instruction would
+    /// otherwise perform are suppressed. `None` when the bit is not part of
+    /// this encoding; only valid for [`ApxClass::LegacyGpr`].
+    pub nf: Option<bool>,
 }
 
 impl Evex {
@@ -1310,6 +1343,27 @@ impl Evex {
         }
     }
 
+    /// Select the APX "opcode map 4", promoting a legacy general-purpose-register
+    /// instruction into the Extended EVEX space; equivalent to `.MAP4` in the
+    /// manual.
+    ///
+    /// This both sets the `mmm` map bits and marks the encoding as
+    /// [`ApxClass::LegacyGpr`], which in turn enables addressing of the extended
+    /// GPRs `R16`–`R31` ("EGPR") and permits the `ND`/`NF` bits.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the map (`mmm`) or APX class has already been set.
+    pub fn map4(self) -> Self {
+        assert!(self.mmm.is_none());
+        assert!(self.apx.is_none());
+        Self {
+            mmm: Some(VexEscape::_MAP4),
+            apx: Some(ApxClass::LegacyGpr),
+            ..self
+        }
+    }
+
     /// Set the `W` bit to `0`; equivalent to `.W0` in the manual.
     pub fn w0(self) -> Self {
         assert!(self.w.is_ignored());
@@ -1352,9 +1406,72 @@ impl Evex {
         }
     }
 
+    /// Mark this as an APX "Extended EVEX" encoding of the given [`ApxClass`].
+    ///
+    /// This selects the APX payload layout to emit and enables addressing of
+    /// the extended general-purpose registers `R16`–`R31` ("EGPR").
+    ///
+    /// # Panics
+    ///
+    /// Panics if an APX class has already been set.
+    pub fn apx(self, class: ApxClass) -> Self {
+        assert!(self.apx.is_none());
+        Self {
+            apx: Some(class),
+            ..self
+        }
+    }
+
+    /// Set the APX `ND` (New Data destination) bit; equivalent to `.ND` in the
+    /// manual.
+    ///
+    /// # Panics
+    ///
+    /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit
+    /// has already been set.
+    pub fn nd(self) -> Self {
+        assert_eq!(
+            self.apx,
+            Some(ApxClass::LegacyGpr),
+            "the ND bit is only valid for APX legacy-GPR promotions"
+        );
+        assert!(self.nd.is_none());
+        Self {
+            nd: Some(true),
+            ..self
+        }
+    }
+
+    /// Set the APX `NF` (No Flags) bit; equivalent to `.NF` in the manual.
+    ///
+    /// # Panics
+    ///
+    /// Panics if this is not an [`ApxClass::LegacyGpr`] encoding, or if the bit
+    /// has already been set.
+    pub fn nf(self) -> Self {
+        assert_eq!(
+            self.apx,
+            Some(ApxClass::LegacyGpr),
+            "the NF bit is only valid for APX legacy-GPR promotions"
+        );
+        assert!(self.nf.is_none());
+        Self {
+            nf: Some(true),
+            ..self
+        }
+    }
+
     fn validate(&self, _operands: &[Operand]) {
         assert!(self.opcode != u8::MAX);
         assert!(self.mmm.is_some());
+        // The `ND`/`NF` bits are only defined for APX legacy-GPR promotions.
+        if self.nd.is_some() || self.nf.is_some() {
+            assert_eq!(
+                self.apx,
+                Some(ApxClass::LegacyGpr),
+                "the ND/NF bits require an APX legacy-GPR encoding"
+            );
+        }
     }
 
     /// Retrieve the digit extending the opcode, if available.
@@ -1412,7 +1529,14 @@ impl fmt::Display for Evex {
         if let Some(mmmmm) = self.mmm {
             write!(f, ".{mmmmm}")?;
         }
-        write!(f, ".{} {:#04X}", self.w, self.opcode)?;
+        write!(f, ".{}", self.w)?;
+        if self.nd == Some(true) {
+            write!(f, ".ND")?;
+        }
+        if self.nf == Some(true) {
+            write!(f, ".NF")?;
+        }
+        write!(f, " {:#04X}", self.opcode)?;
         if let Some(modrm) = self.modrm {
             write!(f, " {modrm}")?;
         }
@@ -1423,6 +1547,35 @@ impl fmt::Display for Evex {
     }
 }
 
+/// The class of an APX "Extended EVEX" encoding.
+///
+/// Intel APX does not define a single static Extended-EVEX layout. Although
+/// every APX instruction still begins with the `0x62` EVEX identifier, the
+/// meaning of the payload bytes (`P0`, `P1`, `P2`) is re-mapped depending on
+/// the class of instruction being promoted. This enum selects which layout the
+/// assembler should emit; all classes gain access to the extended
+/// general-purpose registers `R16`–`R31` ("EGPR").
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum ApxClass {
+    /// Promotion of a legacy general-purpose-register (GPR) instruction using
+    /// extended opcode map 4. Only this class may set the `ND` and `NF` bits.
+    LegacyGpr,
+    /// Promotion of a legacy SSE / VEX vector instruction into the EVEX space.
+    Vector,
+    /// An existing AVX-512 (EVEX) instruction extended with APX register bits.
+    Avx512,
+}
+
+impl fmt::Display for ApxClass {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            Self::LegacyGpr => write!(f, "LegacyGpr"),
+            Self::Vector => write!(f, "Vector"),
+            Self::Avx512 => write!(f, "Avx512"),
+        }
+    }
+}
+
 /// Tuple Type definitions used in EVEX encodings.
 ///
 /// This enumeration corresponds to table 2-34 and 2-35 in the Intel manual.

cranelift/assembler-x64/meta/src/dsl/features.rs

@@ -96,6 +96,7 @@ pub enum Feature {
     avx512vbmi,
     cmpxchg16b,
     fma,
+    apx,
 }
 
 /// List all CPU features.
@@ -127,6 +128,7 @@ pub const ALL_FEATURES: &[Feature] = &[
     Feature::avx512vbmi,
     Feature::cmpxchg16b,
     Feature::fma,
+    Feature::apx,
 ];
 
 impl fmt::Display for Feature {

cranelift/assembler-x64/meta/src/generate/format.rs

@@ -229,24 +229,44 @@ impl dsl::Format {
         fmtln!(f, "let w = {};", vex.w.as_bool());
         let bits = "len, pp, mmmmm, w";
 
-        self.generate_vex_or_evex_prefix(f, "VexPrefix", &bits, vex.is4, None, || {
+        self.generate_vex_or_evex_prefix(f, "VexPrefix", &bits, vex.is4, None, "two_op", "three_op", || {
             vex.unwrap_digit()
         })
     }
 
     fn generate_evex_prefix(&self, f: &mut Formatter, evex: &dsl::Evex) -> ModRmStyle {
         f.empty_line();
         f.comment("Emit EVEX prefix.");
-        let ll = evex.length.evex_bits();
-        fmtln!(f, "let ll = {ll:#04b};");
+
+        // Intel APX promotes legacy GPR instructions into "EVEX map 4" using a
+        // re-purposed payload layout (see `EvexPrefix::legacy` in the runtime
+        // assembler and section 3.1.2.3.1 of the APX spec). In that case we
+        // emit the ND/NF bits and select the `legacy_*` constructors instead of
+        // the AVX-512 ones.
+        let apx_legacy = matches!(evex.apx, Some(dsl::ApxClass::LegacyGpr));
+
         fmtln!(f, "let pp = {:#04b};", evex.pp.map_or(0b00, |pp| pp.bits()));
         fmtln!(f, "let mmm = {:#07b};", evex.mmm.unwrap().bits());
         fmtln!(f, "let w = {};", evex.w.as_bool());
-        // NB: when bcast is supported in the future the `evex_scaling`
-        // calculation for `Full` and `Half` below need to be updated.
+
+        let (bits, two_op, three_op);
+        if apx_legacy {
+            fmtln!(f, "let nd = {};", evex.nd == Some(true));
+            fmtln!(f, "let nf = {};", evex.nf == Some(true));
+            bits = String::from("pp, mmm, w, nd, nf");
+            two_op = "legacy_two_op";
+            three_op = "legacy_three_op";
+        } else {
+            let ll = evex.length.evex_bits();
+            fmtln!(f, "let ll = {ll:#04b};");
+            // NB: when bcast is supported in the future the `evex_scaling`
+            // calculation for `Full` and `Half` below need to be updated.
+            fmtln!(f, "let bcast = false;");
+            bits = String::from("ll, pp, mmm, w, bcast");
+            two_op = "two_op";
+            three_op = "three_op";
+        }
         let bcast = false;
-        fmtln!(f, "let bcast = {bcast};");
-        let bits = format!("ll, pp, mmm, w, bcast");
         let is4 = false;
 
         let length_bytes = match evex.length {
@@ -289,7 +309,7 @@ impl dsl::Format {
             },
         });
 
-        self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, || {
+        self.generate_vex_or_evex_prefix(f, "EvexPrefix", &bits, is4, evex_scaling, two_op, three_op, || {
             evex.unwrap_digit()
         })
     }
@@ -304,6 +324,8 @@ impl dsl::Format {
         bits: &str,
         is4: bool,
         evex_scaling: Option<i8>,
+        two_op: &str,
+        three_op: &str,
         unwrap_digit: impl Fn() -> Option<u8>,
     ) -> ModRmStyle {
         use dsl::OperandKind::{FixedReg, Imm, Mem, Reg, RegMem};
@@ -316,7 +338,7 @@ impl dsl::Format {
                 fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
                 fmtln!(
                     f,
-                    "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"
+                    "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});"
                 );
                 ModRmStyle::Reg {
                     reg: ModRmReg::Reg(*reg),
@@ -333,7 +355,7 @@ impl dsl::Format {
                 fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
                 fmtln!(
                     f,
-                    "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"
+                    "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});"
                 );
                 ModRmStyle::RegMem {
                     reg: ModRmReg::Reg(*reg),
@@ -348,7 +370,7 @@ impl dsl::Format {
                 fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
                 fmtln!(
                     f,
-                    "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"
+                    "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});"
                 );
                 ModRmStyle::RegMemIs4 {
                     reg: ModRmReg::Reg(*reg),
@@ -368,7 +390,7 @@ impl dsl::Format {
                     fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
                     fmtln!(
                         f,
-                        "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"
+                        "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});"
                     );
                     ModRmStyle::RegMem {
                         reg: ModRmReg::Digit(digit),
@@ -381,7 +403,7 @@ impl dsl::Format {
                     let reg = reg_or_vvvv;
                     fmtln!(f, "let reg = self.{reg}.enc();");
                     fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
-                    fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});");
+                    fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});");
                     ModRmStyle::RegMem {
                         reg: ModRmReg::Reg(*reg),
                         rm: *rm,
@@ -399,7 +421,7 @@ impl dsl::Format {
                         fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
                         fmtln!(
                             f,
-                            "let prefix = {prefix_type}::three_op(reg, vvvv, rm, {bits});"
+                            "let prefix = {prefix_type}::{three_op}(reg, vvvv, rm, {bits});"
                         );
                         ModRmStyle::Reg {
                             reg: ModRmReg::Digit(digit),
@@ -411,7 +433,7 @@ impl dsl::Format {
                         let reg = reg_or_vvvv;
                         fmtln!(f, "let reg = self.{reg}.enc();");
                         fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
-                        fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});");
+                        fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});");
                         ModRmStyle::Reg {
                             reg: ModRmReg::Reg(*reg),
                             rm: *rm,
@@ -423,7 +445,7 @@ impl dsl::Format {
                 assert!(!is4);
                 fmtln!(f, "let reg = self.{reg}.enc();");
                 fmtln!(f, "let rm = self.{rm}.encode_bx_regs();");
-                fmtln!(f, "let prefix = {prefix_type}::two_op(reg, rm, {bits});");
+                fmtln!(f, "let prefix = {prefix_type}::{two_op}(reg, rm, {bits});");
                 ModRmStyle::RegMem {
                     reg: ModRmReg::Reg(*reg),
                     rm: *rm,