Support "presence" mode in coverage instrumentation

compiler/rustc_codegen_llvm/src/builder.rs

@@ -2017,7 +2017,8 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         kcfi_bundle
     }
 
-    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
+    /// Emits a call to `llvm.instrprof.increment`. Used by counter-style coverage
+    /// instrumentation, i.e. the default mode.
     #[instrument(level = "debug", skip(self))]
     pub(crate) fn instrprof_increment(
         &mut self,
@@ -2028,4 +2029,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
     ) {
         self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
     }
+
+    /// Emits a call to `llvm.instrprof.cover`. Used by presence-only coverage
+    /// instrumentation.
+    #[instrument(level = "debug", skip(self))]
+    pub(crate) fn instrprof_cover(
+        &mut self,
+        fn_name: &'ll Value,
+        hash: &'ll Value,
+        num_counters: &'ll Value,
+        index: &'ll Value,
+    ) {
+        self.call_intrinsic("llvm.instrprof.cover", &[], &[fn_name, hash, num_counters, index]);
+    }
 }

compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs

@@ -10,9 +10,8 @@ pub(crate) enum CounterKind {
 /// A reference to an instance of an abstract "counter" that will yield a value in a coverage
 /// report. Note that `id` has different interpretations, depending on the `kind`:
 ///   * For `CounterKind::Zero`, `id` is assumed to be `0`
-///   * For `CounterKind::CounterValueReference`,  `id` matches the `counter_id` of the injected
-///     instrumentation counter (the `index` argument to the LLVM intrinsic
-///     `instrprof.increment()`)
+///   * For `CounterKind::CounterValueReference`, `id` matches the `counter_id` of the injected
+///     instrumentation counter (the `index` argument to the LLVM coverage intrinsic)
 ///   * For `CounterKind::Expression`, `id` is the index into the coverage map's array of
 ///     counter expressions.
 ///

compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs

@@ -39,6 +39,13 @@ pub(crate) fn create_pgo_func_name_var<'ll>(
     }
 }
 
+/// Defines `__llvm_profile_raw_version` with LLVM's byte-coverage variant bit.
+/// This tells compiler-rt and llvm-profdata that the module's profile counters
+/// are one byte wide rather than the usual eight bytes.
+pub(crate) fn set_single_byte_profile_version(llmod: &llvm::Module) {
+    unsafe { llvm::LLVMRustCoverageSetSingleByteProfileVersion(llmod) }
+}
+
 pub(crate) fn write_filenames_to_buffer(filenames: &[impl AsRef<str>]) -> Vec<u8> {
     let (pointers, lengths) = filenames
         .into_iter()

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -47,6 +47,14 @@ impl<'ll, 'tcx> CguCoverageContext<'ll, 'tcx> {
 
 impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     pub(crate) fn coverageinfo_finalize(&mut self) {
+        // Single-byte counters use a different raw profile layout; downstream tools
+        // expect to read the version to interpret the profile correctly.
+        // Do this before map generation, which can return early even when the
+        // module contains coverage intrinsics (for example, when all source
+        // spans were discarded during codegen).
+        if self.tcx.sess.instrument_coverage_single_byte() {
+            llvm_cov::set_single_byte_profile_version(self.llmod);
+        }
         mapgen::finalize(self)
     }
 
@@ -65,8 +73,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     }
 
     /// For LLVM codegen, returns a function-specific `Value` for a global
-    /// string, to hold the function name passed to LLVM intrinsic
-    /// `instrprof.increment()`. The `Value` is only created once per instance.
+    /// string, to hold the function name passed to an LLVM coverage intrinsic.
+    /// The `Value` is only created once per instance.
     /// Multiple invocations with the same instance return the same `Value`.
     ///
     /// This has the side-effect of causing coverage codegen to consider this
@@ -123,11 +131,19 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
                 let hash = bx.const_u64(function_coverage_info.function_source_hash);
                 let num_counters = bx.const_u32(ids_info.num_counters);
                 let index = bx.const_u32(id.as_u32());
-                debug!(
-                    "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})",
-                    fn_name, hash, num_counters, index,
-                );
-                bx.instrprof_increment(fn_name, hash, num_counters, index);
+                if bx.tcx.sess.instrument_coverage_single_byte() {
+                    debug!(
+                        "codegen intrinsic instrprof.cover(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})",
+                        fn_name, hash, num_counters, index,
+                    );
+                    bx.instrprof_cover(fn_name, hash, num_counters, index);
+                } else {
+                    debug!(
+                        "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})",
+                        fn_name, hash, num_counters, index,
+                    );
+                    bx.instrprof_increment(fn_name, hash, num_counters, index);
+                }
             }
             // If a BCB doesn't have an associated physical counter, there's nothing to codegen.
             CoverageKind::VirtualCounter { .. } => {}

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -2193,6 +2193,7 @@ unsafe extern "C" {
         FuncName: *const c_uchar, // See "PTR_LEN_STR".
         FuncNameLen: size_t,
     ) -> &Value;
+    pub(crate) fn LLVMRustCoverageSetSingleByteProfileVersion(M: &Module);
     pub(crate) fn LLVMRustCoverageHashBytes(
         Bytes: *const c_uchar, // See "PTR_LEN_STR".
         NumBytes: size_t,

compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp

@@ -3,6 +3,8 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/Module.h"
 #include "llvm/ProfileData/Coverage/CoverageMapping.h"
 #include "llvm/ProfileData/Coverage/CoverageMappingWriter.h"
@@ -175,6 +177,28 @@ LLVMRustCoverageCreatePGOFuncNameVar(LLVMValueRef F, const char *FuncName,
   return wrap(createPGOFuncNameVar(*cast<Function>(unwrap(F)), FuncNameRef));
 }
 
+extern "C" void
+LLVMRustCoverageSetSingleByteProfileVersion(LLVMModuleRef MRef) {
+  Module &M = *unwrap(MRef);
+  const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));
+  Type *IntTy64 = Type::getInt64Ty(M.getContext());
+  uint64_t ProfileVersion = INSTR_PROF_RAW_VERSION | VARIANT_MASK_BYTE_COVERAGE;
+
+  auto *VersionVariable = new GlobalVariable(
+      M, IntTy64, true, GlobalValue::WeakAnyLinkage,
+      Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName);
+
+  VersionVariable->setVisibility(GlobalValue::HiddenVisibility);
+  Triple TT(M.getTargetTriple());
+  if (TT.isGPU())
+    VersionVariable->setVisibility(GlobalValue::ProtectedVisibility);
+  if (TT.supportsCOMDAT()) {
+    VersionVariable->setLinkage(GlobalValue::ExternalLinkage);
+    VersionVariable->setComdat(M.getOrInsertComdat(VarName));
+  }
+  VersionVariable->setDSOLocal(true);
+}
+
 extern "C" uint64_t LLVMRustCoverageHashBytes(const char *Bytes,
                                               size_t NumBytes) {
   return IndexedInstrProf::ComputeHash(StringRef(Bytes, NumBytes));

compiler/rustc_middle/src/mir/coverage.rs

@@ -87,8 +87,9 @@ pub enum CoverageKind {
     /// Marks its enclosing basic block with the ID of the coverage graph node
     /// that it was part of during the `InstrumentCoverage` MIR pass.
     ///
-    /// During codegen, this might be lowered to `llvm.instrprof.increment` or
-    /// to a no-op, depending on the outcome of counter-creation.
+    /// During codegen, this might be lowered to `llvm.instrprof.increment`,
+    /// `llvm.instrprof.cover`, or a no-op, depending on the coverage mode and
+    /// the outcome of counter-creation.
     VirtualCounter { bcb: BasicCoverageBlock },
 }
 

compiler/rustc_mir_transform/src/coverage/counters.rs

@@ -124,10 +124,34 @@ pub(crate) fn transcribe_counters(
     new
 }
 
+/// Assigns a physical counter directly to every mapped BCB that survived MIR optimization.
+///
+/// Unlike [`transcribe_counters`], this does not create derived counter expressions. This is used
+/// for presence-only coverage, where counters only record whether their BCB was ever executed and
+/// therefore cannot be combined with arithmetic expressions to recover execution counts.
+pub(crate) fn make_direct_counters(
+    bcb_needs_counter: &DenseBitSet<BasicCoverageBlock>,
+    bcbs_seen: &DenseBitSet<BasicCoverageBlock>,
+) -> CoverageCounters {
+    let mut counters = CoverageCounters::with_num_bcbs(bcb_needs_counter.domain_size());
+
+    for bcb in bcb_needs_counter.iter() {
+        let counter = if bcbs_seen.contains(bcb) {
+            counters.ensure_phys_counter(bcb)
+        } else {
+            // This BCB's code was removed by MIR opts, so its counter is always zero.
+            CovTerm::Zero
+        };
+        counters.set_node_counter(bcb, counter);
+    }
+
+    counters
+}
+
 /// Generates and stores coverage counter and coverage expression information
 /// associated with nodes in the coverage graph.
 pub(super) struct CoverageCounters {
-    /// List of places where a counter-increment statement should be injected
+    /// List of places where a physical counter statement should be injected
     /// into MIR, each with its corresponding counter ID.
     pub(crate) phys_counter_for_node: FxIndexMap<BasicCoverageBlock, CounterId>,
     pub(crate) next_counter_id: CounterId,

compiler/rustc_mir_transform/src/coverage/mod.rs

@@ -18,9 +18,8 @@ mod spans;
 #[cfg(test)]
 mod tests;
 
-/// Inserts `StatementKind::Coverage` statements that either instrument the binary with injected
-/// counters, via intrinsic `llvm.instrprof.increment`, and/or inject metadata used during codegen
-/// to construct the coverage map.
+/// Inserts `StatementKind::Coverage` statements that either instrument the binary with LLVM
+/// coverage intrinsics, and/or inject metadata used during codegen to construct the coverage map.
 pub(super) struct InstrumentCoverage;
 
 impl<'tcx> crate::MirPass<'tcx> for InstrumentCoverage {

compiler/rustc_mir_transform/src/coverage/query.rs

@@ -10,7 +10,7 @@ use rustc_span::def_id::LocalDefId;
 use tracing::trace;
 
 use crate::coverage::counters::node_flow::make_node_counters;
-use crate::coverage::counters::{CoverageCounters, transcribe_counters};
+use crate::coverage::counters::{CoverageCounters, make_direct_counters, transcribe_counters};
 
 /// Registers query/hook implementations related to coverage.
 pub(crate) fn provide(providers: &mut Providers) {
@@ -112,21 +112,27 @@ fn coverage_ids_info<'tcx>(
         }
     }
 
-    // Clone the priority list so that we can re-sort it.
-    let mut priority_list = fn_cov_info.priority_list.clone();
-    // The first ID in the priority list represents the synthetic "sink" node,
-    // and must remain first so that it _never_ gets a physical counter.
-    debug_assert_eq!(priority_list[0], priority_list.iter().copied().max().unwrap());
-    assert!(!bcbs_seen.contains(priority_list[0]));
-    // Partition the priority list, so that unreachable nodes (removed by MIR opts)
-    // are sorted later and therefore are _more_ likely to get a physical counter.
-    // This is counter-intuitive, but it means that `transcribe_counters` can
-    // easily skip those unused physical counters and replace them with zero.
-    // (The original ordering remains in effect within both partitions.)
-    priority_list[1..].sort_by_key(|&bcb| !bcbs_seen.contains(bcb));
-
-    let node_counters = make_node_counters(&fn_cov_info.node_flow_data, &priority_list);
-    let coverage_counters = transcribe_counters(&node_counters, &bcb_needs_counter, &bcbs_seen);
+    let coverage_counters = if tcx.sess.instrument_coverage_single_byte() {
+        // Boolean counters cannot preserve mappings that reconstruct execution counts through
+        // arithmetic expressions, so give every mapped BCB its own physical counter.
+        make_direct_counters(&bcb_needs_counter, &bcbs_seen)
+    } else {
+        // Clone the priority list so that we can re-sort it.
+        let mut priority_list = fn_cov_info.priority_list.clone();
+        // The first ID in the priority list represents the synthetic "sink" node,
+        // and must remain first so that it _never_ gets a physical counter.
+        debug_assert_eq!(priority_list[0], priority_list.iter().copied().max().unwrap());
+        assert!(!bcbs_seen.contains(priority_list[0]));
+        // Partition the priority list, so that unreachable nodes (removed by MIR opts)
+        // are sorted later and therefore are _more_ likely to get a physical counter.
+        // This is counter-intuitive, but it means that `transcribe_counters` can
+        // easily skip those unused physical counters and replace them with zero.
+        // (The original ordering remains in effect within both partitions.)
+        priority_list[1..].sort_by_key(|&bcb| !bcbs_seen.contains(bcb));
+
+        let node_counters = make_node_counters(&fn_cov_info.node_flow_data, &priority_list);
+        transcribe_counters(&node_counters, &bcb_needs_counter, &bcbs_seen)
+    };
 
     let CoverageCounters {
         phys_counter_for_node, next_counter_id, node_counters, expressions, ..

compiler/rustc_mir_transform/src/coverage/tests.rs

@@ -27,17 +27,57 @@
 use itertools::Itertools;
 use rustc_data_structures::graph::{DirectedGraph, Successors};
 use rustc_data_structures::thin_vec::ThinVec;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_index::{Idx, IndexVec};
+use rustc_middle::mir::coverage::{CounterId, CovTerm};
 use rustc_middle::mir::*;
 use rustc_middle::{bug, ty};
 use rustc_span::{BytePos, DUMMY_SP, Pos, Span};
 
+use super::counters::make_direct_counters;
 use super::graph::{self, BasicCoverageBlock};
 
 fn bcb(index: u32) -> BasicCoverageBlock {
     BasicCoverageBlock::from_u32(index)
 }
 
+fn bcb_set(domain_size: usize, indices: &[u32]) -> DenseBitSet<BasicCoverageBlock> {
+    let mut set = DenseBitSet::new_empty(domain_size);
+    for &index in indices {
+        set.insert(bcb(index));
+    }
+    set
+}
+
+#[test]
+fn direct_counters_use_no_expressions() {
+    let bcb_needs_counter = bcb_set(5, &[0, 2, 3, 4]);
+    let bcbs_seen = bcb_set(5, &[0, 1, 2, 4]);
+
+    let counters = make_direct_counters(&bcb_needs_counter, &bcbs_seen);
+
+    assert_eq!(counters.next_counter_id.as_u32(), 3);
+    assert_eq!(
+        counters
+            .phys_counter_for_node
+            .iter()
+            .map(|(&bcb, &counter)| (bcb.as_u32(), counter.as_u32()))
+            .collect::<Vec<_>>(),
+        [(0, 0), (2, 1), (4, 2)]
+    );
+    assert_eq!(
+        counters.node_counters.raw,
+        [
+            Some(CovTerm::Counter(CounterId::from_u32(0))),
+            None,
+            Some(CovTerm::Counter(CounterId::from_u32(1))),
+            Some(CovTerm::Zero),
+            Some(CovTerm::Counter(CounterId::from_u32(2))),
+        ]
+    );
+    assert!(counters.expressions.is_empty());
+}
+
 // All `TEMP_BLOCK` targets should be replaced before calling `to_body() -> mir::Body`.
 const TEMP_BLOCK: BasicBlock = BasicBlock::MAX;
 

compiler/rustc_session/src/config.rs

@@ -146,8 +146,11 @@ pub enum LtoCli {
 pub enum InstrumentCoverage {
     /// `-C instrument-coverage=no` (or `off`, `false` etc.)
     No,
-    /// `-C instrument-coverage` or `-C instrument-coverage=yes`
+    /// `-C instrument-coverage`, `-C instrument-coverage=yes`, or
+    /// `-C instrument-coverage=counter`
     Yes,
+    /// `-C instrument-coverage=presence-only`
+    SingleByte,
 }
 
 /// Individual flag values controlled by `-Zcoverage-options`.
@@ -2554,6 +2557,20 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
     }
 
     if cg.instrument_coverage != InstrumentCoverage::No {
+        if cg.instrument_coverage == InstrumentCoverage::SingleByte {
+            if !unstable_opts.unstable_options {
+                early_dcx.early_fatal(
+                    "`-C instrument-coverage=presence-only` requires `-Z unstable-options`",
+                );
+            }
+            if unstable_opts.coverage_options.level >= CoverageLevel::Branch {
+                early_dcx.early_fatal(
+                    "`-C instrument-coverage=presence-only` is not compatible with branch or \
+                     condition coverage",
+                );
+            }
+        }
+
         if cg.profile_generate.enabled() || cg.profile_use.is_some() {
             early_dcx.early_fatal(
                 "option `-C instrument-coverage` is not compatible with either `-C profile-use` \

compiler/rustc_session/src/options.rs

@@ -800,7 +800,7 @@ mod desc {
     pub(crate) const parse_strip: &str = "either `none`, `debuginfo`, or `symbols`";
     pub(crate) const parse_linker_flavor: &str = ::rustc_target::spec::LinkerFlavorCli::one_of();
     pub(crate) const parse_dump_mono_stats: &str = "`markdown` (default) or `json`";
-    pub(crate) const parse_instrument_coverage: &str = parse_bool;
+    pub(crate) const parse_instrument_coverage: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), `counter`, or (with `-Z unstable-options`) `presence-only`";
     pub(crate) const parse_coverage_options: &str = "`block` | `branch` | `condition`";
     pub(crate) const parse_codegen_retag_options: &str =
         "either no value or a comma-separated list of settings: `no-precise-im`, `no-precise-pin`";
@@ -1535,11 +1535,13 @@ pub mod parse {
             return true;
         };
 
-        // Parse values that have historically been accepted by stable compilers,
-        // even though they're currently just aliases for boolean values.
+        // `all` and `0` have historically been accepted by stable compilers,
+        // even though they are currently just aliases for boolean values.
+        // `counter` is an explicit name for the default counter-based mode.
         *slot = match v {
-            "all" => InstrumentCoverage::Yes,
+            "all" | "counter" => InstrumentCoverage::Yes,
             "0" => InstrumentCoverage::No,
+            "presence-only" => InstrumentCoverage::SingleByte,
             _ => return false,
         };
         true
@@ -2141,6 +2143,7 @@ options! {
     instrument_coverage: InstrumentCoverage = (InstrumentCoverage::No, parse_instrument_coverage, [TRACKED],
         "instrument the generated code to support LLVM source-based code coverage reports \
         (note, the compiler build config must include `profiler = true`); \
+        `presence-only` mode requires `-Z unstable-options`; \
         implies `-C symbol-mangling-version=v0`"),
     jump_tables: bool = (true, parse_bool, [TRACKED],
         "allow jump table and lookup table generation from switch case lowering (default: yes)"),

compiler/rustc_session/src/session.rs

@@ -331,6 +331,10 @@ impl Session {
         self.opts.cg.instrument_coverage() != InstrumentCoverage::No
     }
 
+    pub fn instrument_coverage_single_byte(&self) -> bool {
+        self.opts.cg.instrument_coverage() == InstrumentCoverage::SingleByte
+    }
+
     pub fn instrument_coverage_branch(&self) -> bool {
         self.instrument_coverage()
             && self.opts.unstable_opts.coverage_options.level >= CoverageLevel::Branch