diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index acfec99438059..e9c50be20a80c 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -2017,7 +2017,8 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { kcfi_bundle } - /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation. + /// Emits a call to `llvm.instrprof.increment`. Used by counter-style coverage + /// instrumentation, i.e. the default mode. #[instrument(level = "debug", skip(self))] pub(crate) fn instrprof_increment( &mut self, @@ -2028,4 +2029,17 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { ) { self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]); } + + /// Emits a call to `llvm.instrprof.cover`. Used by presence-only coverage + /// instrumentation. + #[instrument(level = "debug", skip(self))] + pub(crate) fn instrprof_cover( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + num_counters: &'ll Value, + index: &'ll Value, + ) { + self.call_intrinsic("llvm.instrprof.cover", &[], &[fn_name, hash, num_counters, index]); + } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs index caa36c5dd4992..2bba0c97f6564 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs @@ -10,9 +10,8 @@ pub(crate) enum CounterKind { /// A reference to an instance of an abstract "counter" that will yield a value in a coverage /// report. Note that `id` has different interpretations, depending on the `kind`: /// * For `CounterKind::Zero`, `id` is assumed to be `0` -/// * For `CounterKind::CounterValueReference`, `id` matches the `counter_id` of the injected -/// instrumentation counter (the `index` argument to the LLVM intrinsic -/// `instrprof.increment()`) +/// * For `CounterKind::CounterValueReference`, `id` matches the `counter_id` of the injected +/// instrumentation counter (the `index` argument to the LLVM coverage intrinsic) /// * For `CounterKind::Expression`, `id` is the index into the coverage map's array of /// counter expressions. /// diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs index a58202834cfa5..36948d6a25f19 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/llvm_cov.rs @@ -39,6 +39,13 @@ pub(crate) fn create_pgo_func_name_var<'ll>( } } +/// Defines `__llvm_profile_raw_version` with LLVM's byte-coverage variant bit. +/// This tells compiler-rt and llvm-profdata that the module's profile counters +/// are one byte wide rather than the usual eight bytes. +pub(crate) fn set_single_byte_profile_version(llmod: &llvm::Module) { + unsafe { llvm::LLVMRustCoverageSetSingleByteProfileVersion(llmod) } +} + pub(crate) fn write_filenames_to_buffer(filenames: &[impl AsRef]) -> Vec { let (pointers, lengths) = filenames .into_iter() diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 6a58f495c9d8f..d5588a24ccf95 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -47,6 +47,14 @@ impl<'ll, 'tcx> CguCoverageContext<'ll, 'tcx> { impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { pub(crate) fn coverageinfo_finalize(&mut self) { + // Single-byte counters use a different raw profile layout; downstream tools + // expect to read the version to interpret the profile correctly. + // Do this before map generation, which can return early even when the + // module contains coverage intrinsics (for example, when all source + // spans were discarded during codegen). + if self.tcx.sess.instrument_coverage_single_byte() { + llvm_cov::set_single_byte_profile_version(self.llmod); + } mapgen::finalize(self) } @@ -65,8 +73,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { } /// For LLVM codegen, returns a function-specific `Value` for a global - /// string, to hold the function name passed to LLVM intrinsic - /// `instrprof.increment()`. The `Value` is only created once per instance. + /// string, to hold the function name passed to an LLVM coverage intrinsic. + /// The `Value` is only created once per instance. /// Multiple invocations with the same instance return the same `Value`. /// /// This has the side-effect of causing coverage codegen to consider this @@ -123,11 +131,19 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { let hash = bx.const_u64(function_coverage_info.function_source_hash); let num_counters = bx.const_u32(ids_info.num_counters); let index = bx.const_u32(id.as_u32()); - debug!( - "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", - fn_name, hash, num_counters, index, - ); - bx.instrprof_increment(fn_name, hash, num_counters, index); + if bx.tcx.sess.instrument_coverage_single_byte() { + debug!( + "codegen intrinsic instrprof.cover(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", + fn_name, hash, num_counters, index, + ); + bx.instrprof_cover(fn_name, hash, num_counters, index); + } else { + debug!( + "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})", + fn_name, hash, num_counters, index, + ); + bx.instrprof_increment(fn_name, hash, num_counters, index); + } } // If a BCB doesn't have an associated physical counter, there's nothing to codegen. CoverageKind::VirtualCounter { .. } => {} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index b14a6206ae682..25737147359f4 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2193,6 +2193,7 @@ unsafe extern "C" { FuncName: *const c_uchar, // See "PTR_LEN_STR". FuncNameLen: size_t, ) -> &Value; + pub(crate) fn LLVMRustCoverageSetSingleByteProfileVersion(M: &Module); pub(crate) fn LLVMRustCoverageHashBytes( Bytes: *const c_uchar, // See "PTR_LEN_STR". NumBytes: size_t, diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp index 22e7c7a160ff5..6a4f5128a559b 100644 --- a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp @@ -3,6 +3,8 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Module.h" #include "llvm/ProfileData/Coverage/CoverageMapping.h" #include "llvm/ProfileData/Coverage/CoverageMappingWriter.h" @@ -175,6 +177,28 @@ LLVMRustCoverageCreatePGOFuncNameVar(LLVMValueRef F, const char *FuncName, return wrap(createPGOFuncNameVar(*cast(unwrap(F)), FuncNameRef)); } +extern "C" void +LLVMRustCoverageSetSingleByteProfileVersion(LLVMModuleRef MRef) { + Module &M = *unwrap(MRef); + const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)); + Type *IntTy64 = Type::getInt64Ty(M.getContext()); + uint64_t ProfileVersion = INSTR_PROF_RAW_VERSION | VARIANT_MASK_BYTE_COVERAGE; + + auto *VersionVariable = new GlobalVariable( + M, IntTy64, true, GlobalValue::WeakAnyLinkage, + Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)), VarName); + + VersionVariable->setVisibility(GlobalValue::HiddenVisibility); + Triple TT(M.getTargetTriple()); + if (TT.isGPU()) + VersionVariable->setVisibility(GlobalValue::ProtectedVisibility); + if (TT.supportsCOMDAT()) { + VersionVariable->setLinkage(GlobalValue::ExternalLinkage); + VersionVariable->setComdat(M.getOrInsertComdat(VarName)); + } + VersionVariable->setDSOLocal(true); +} + extern "C" uint64_t LLVMRustCoverageHashBytes(const char *Bytes, size_t NumBytes) { return IndexedInstrProf::ComputeHash(StringRef(Bytes, NumBytes)); diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs index cdb373cf01de5..82e0c6cd24670 100644 --- a/compiler/rustc_middle/src/mir/coverage.rs +++ b/compiler/rustc_middle/src/mir/coverage.rs @@ -87,8 +87,9 @@ pub enum CoverageKind { /// Marks its enclosing basic block with the ID of the coverage graph node /// that it was part of during the `InstrumentCoverage` MIR pass. /// - /// During codegen, this might be lowered to `llvm.instrprof.increment` or - /// to a no-op, depending on the outcome of counter-creation. + /// During codegen, this might be lowered to `llvm.instrprof.increment`, + /// `llvm.instrprof.cover`, or a no-op, depending on the coverage mode and + /// the outcome of counter-creation. VirtualCounter { bcb: BasicCoverageBlock }, } diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index 879a20e771d66..787bb02cf1e8c 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -124,10 +124,34 @@ pub(crate) fn transcribe_counters( new } +/// Assigns a physical counter directly to every mapped BCB that survived MIR optimization. +/// +/// Unlike [`transcribe_counters`], this does not create derived counter expressions. This is used +/// for presence-only coverage, where counters only record whether their BCB was ever executed and +/// therefore cannot be combined with arithmetic expressions to recover execution counts. +pub(crate) fn make_direct_counters( + bcb_needs_counter: &DenseBitSet, + bcbs_seen: &DenseBitSet, +) -> CoverageCounters { + let mut counters = CoverageCounters::with_num_bcbs(bcb_needs_counter.domain_size()); + + for bcb in bcb_needs_counter.iter() { + let counter = if bcbs_seen.contains(bcb) { + counters.ensure_phys_counter(bcb) + } else { + // This BCB's code was removed by MIR opts, so its counter is always zero. + CovTerm::Zero + }; + counters.set_node_counter(bcb, counter); + } + + counters +} + /// Generates and stores coverage counter and coverage expression information /// associated with nodes in the coverage graph. pub(super) struct CoverageCounters { - /// List of places where a counter-increment statement should be injected + /// List of places where a physical counter statement should be injected /// into MIR, each with its corresponding counter ID. pub(crate) phys_counter_for_node: FxIndexMap, pub(crate) next_counter_id: CounterId, diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index be8b02f61133d..3e2d733355cd7 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -18,9 +18,8 @@ mod spans; #[cfg(test)] mod tests; -/// Inserts `StatementKind::Coverage` statements that either instrument the binary with injected -/// counters, via intrinsic `llvm.instrprof.increment`, and/or inject metadata used during codegen -/// to construct the coverage map. +/// Inserts `StatementKind::Coverage` statements that either instrument the binary with LLVM +/// coverage intrinsics, and/or inject metadata used during codegen to construct the coverage map. pub(super) struct InstrumentCoverage; impl<'tcx> crate::MirPass<'tcx> for InstrumentCoverage { diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index 39e0769373ede..c58a13588a548 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -10,7 +10,7 @@ use rustc_span::def_id::LocalDefId; use tracing::trace; use crate::coverage::counters::node_flow::make_node_counters; -use crate::coverage::counters::{CoverageCounters, transcribe_counters}; +use crate::coverage::counters::{CoverageCounters, make_direct_counters, transcribe_counters}; /// Registers query/hook implementations related to coverage. pub(crate) fn provide(providers: &mut Providers) { @@ -112,21 +112,27 @@ fn coverage_ids_info<'tcx>( } } - // Clone the priority list so that we can re-sort it. - let mut priority_list = fn_cov_info.priority_list.clone(); - // The first ID in the priority list represents the synthetic "sink" node, - // and must remain first so that it _never_ gets a physical counter. - debug_assert_eq!(priority_list[0], priority_list.iter().copied().max().unwrap()); - assert!(!bcbs_seen.contains(priority_list[0])); - // Partition the priority list, so that unreachable nodes (removed by MIR opts) - // are sorted later and therefore are _more_ likely to get a physical counter. - // This is counter-intuitive, but it means that `transcribe_counters` can - // easily skip those unused physical counters and replace them with zero. - // (The original ordering remains in effect within both partitions.) - priority_list[1..].sort_by_key(|&bcb| !bcbs_seen.contains(bcb)); - - let node_counters = make_node_counters(&fn_cov_info.node_flow_data, &priority_list); - let coverage_counters = transcribe_counters(&node_counters, &bcb_needs_counter, &bcbs_seen); + let coverage_counters = if tcx.sess.instrument_coverage_single_byte() { + // Boolean counters cannot preserve mappings that reconstruct execution counts through + // arithmetic expressions, so give every mapped BCB its own physical counter. + make_direct_counters(&bcb_needs_counter, &bcbs_seen) + } else { + // Clone the priority list so that we can re-sort it. + let mut priority_list = fn_cov_info.priority_list.clone(); + // The first ID in the priority list represents the synthetic "sink" node, + // and must remain first so that it _never_ gets a physical counter. + debug_assert_eq!(priority_list[0], priority_list.iter().copied().max().unwrap()); + assert!(!bcbs_seen.contains(priority_list[0])); + // Partition the priority list, so that unreachable nodes (removed by MIR opts) + // are sorted later and therefore are _more_ likely to get a physical counter. + // This is counter-intuitive, but it means that `transcribe_counters` can + // easily skip those unused physical counters and replace them with zero. + // (The original ordering remains in effect within both partitions.) + priority_list[1..].sort_by_key(|&bcb| !bcbs_seen.contains(bcb)); + + let node_counters = make_node_counters(&fn_cov_info.node_flow_data, &priority_list); + transcribe_counters(&node_counters, &bcb_needs_counter, &bcbs_seen) + }; let CoverageCounters { phys_counter_for_node, next_counter_id, node_counters, expressions, .. diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index da2a60fafe828..d377a3bf94fe8 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -27,17 +27,57 @@ use itertools::Itertools; use rustc_data_structures::graph::{DirectedGraph, Successors}; use rustc_data_structures::thin_vec::ThinVec; +use rustc_index::bit_set::DenseBitSet; use rustc_index::{Idx, IndexVec}; +use rustc_middle::mir::coverage::{CounterId, CovTerm}; use rustc_middle::mir::*; use rustc_middle::{bug, ty}; use rustc_span::{BytePos, DUMMY_SP, Pos, Span}; +use super::counters::make_direct_counters; use super::graph::{self, BasicCoverageBlock}; fn bcb(index: u32) -> BasicCoverageBlock { BasicCoverageBlock::from_u32(index) } +fn bcb_set(domain_size: usize, indices: &[u32]) -> DenseBitSet { + let mut set = DenseBitSet::new_empty(domain_size); + for &index in indices { + set.insert(bcb(index)); + } + set +} + +#[test] +fn direct_counters_use_no_expressions() { + let bcb_needs_counter = bcb_set(5, &[0, 2, 3, 4]); + let bcbs_seen = bcb_set(5, &[0, 1, 2, 4]); + + let counters = make_direct_counters(&bcb_needs_counter, &bcbs_seen); + + assert_eq!(counters.next_counter_id.as_u32(), 3); + assert_eq!( + counters + .phys_counter_for_node + .iter() + .map(|(&bcb, &counter)| (bcb.as_u32(), counter.as_u32())) + .collect::>(), + [(0, 0), (2, 1), (4, 2)] + ); + assert_eq!( + counters.node_counters.raw, + [ + Some(CovTerm::Counter(CounterId::from_u32(0))), + None, + Some(CovTerm::Counter(CounterId::from_u32(1))), + Some(CovTerm::Zero), + Some(CovTerm::Counter(CounterId::from_u32(2))), + ] + ); + assert!(counters.expressions.is_empty()); +} + // All `TEMP_BLOCK` targets should be replaced before calling `to_body() -> mir::Body`. const TEMP_BLOCK: BasicBlock = BasicBlock::MAX; diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 089a9322e6a3c..936188d179210 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -146,8 +146,11 @@ pub enum LtoCli { pub enum InstrumentCoverage { /// `-C instrument-coverage=no` (or `off`, `false` etc.) No, - /// `-C instrument-coverage` or `-C instrument-coverage=yes` + /// `-C instrument-coverage`, `-C instrument-coverage=yes`, or + /// `-C instrument-coverage=counter` Yes, + /// `-C instrument-coverage=presence-only` + SingleByte, } /// Individual flag values controlled by `-Zcoverage-options`. @@ -2554,6 +2557,20 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M } if cg.instrument_coverage != InstrumentCoverage::No { + if cg.instrument_coverage == InstrumentCoverage::SingleByte { + if !unstable_opts.unstable_options { + early_dcx.early_fatal( + "`-C instrument-coverage=presence-only` requires `-Z unstable-options`", + ); + } + if unstable_opts.coverage_options.level >= CoverageLevel::Branch { + early_dcx.early_fatal( + "`-C instrument-coverage=presence-only` is not compatible with branch or \ + condition coverage", + ); + } + } + if cg.profile_generate.enabled() || cg.profile_use.is_some() { early_dcx.early_fatal( "option `-C instrument-coverage` is not compatible with either `-C profile-use` \ diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 897aa91f7dbda..a45458a40c48a 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -800,7 +800,7 @@ mod desc { pub(crate) const parse_strip: &str = "either `none`, `debuginfo`, or `symbols`"; pub(crate) const parse_linker_flavor: &str = ::rustc_target::spec::LinkerFlavorCli::one_of(); pub(crate) const parse_dump_mono_stats: &str = "`markdown` (default) or `json`"; - pub(crate) const parse_instrument_coverage: &str = parse_bool; + pub(crate) const parse_instrument_coverage: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), `counter`, or (with `-Z unstable-options`) `presence-only`"; pub(crate) const parse_coverage_options: &str = "`block` | `branch` | `condition`"; pub(crate) const parse_codegen_retag_options: &str = "either no value or a comma-separated list of settings: `no-precise-im`, `no-precise-pin`"; @@ -1535,11 +1535,13 @@ pub mod parse { return true; }; - // Parse values that have historically been accepted by stable compilers, - // even though they're currently just aliases for boolean values. + // `all` and `0` have historically been accepted by stable compilers, + // even though they are currently just aliases for boolean values. + // `counter` is an explicit name for the default counter-based mode. *slot = match v { - "all" => InstrumentCoverage::Yes, + "all" | "counter" => InstrumentCoverage::Yes, "0" => InstrumentCoverage::No, + "presence-only" => InstrumentCoverage::SingleByte, _ => return false, }; true @@ -2141,6 +2143,7 @@ options! { instrument_coverage: InstrumentCoverage = (InstrumentCoverage::No, parse_instrument_coverage, [TRACKED], "instrument the generated code to support LLVM source-based code coverage reports \ (note, the compiler build config must include `profiler = true`); \ + `presence-only` mode requires `-Z unstable-options`; \ implies `-C symbol-mangling-version=v0`"), jump_tables: bool = (true, parse_bool, [TRACKED], "allow jump table and lookup table generation from switch case lowering (default: yes)"), diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs index 15a87771056c4..2166f38e3ff6a 100644 --- a/compiler/rustc_session/src/session.rs +++ b/compiler/rustc_session/src/session.rs @@ -331,6 +331,10 @@ impl Session { self.opts.cg.instrument_coverage() != InstrumentCoverage::No } + pub fn instrument_coverage_single_byte(&self) -> bool { + self.opts.cg.instrument_coverage() == InstrumentCoverage::SingleByte + } + pub fn instrument_coverage_branch(&self) -> bool { self.instrument_coverage() && self.opts.unstable_opts.coverage_options.level >= CoverageLevel::Branch diff --git a/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md b/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md index 8e94928c8a1a0..0f03b111d81c2 100644 --- a/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md +++ b/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md @@ -4,15 +4,18 @@ with a command line option (`-C instrument-coverage`) that instruments Rust libraries and binaries with additional instructions and data, at compile time. -The coverage instrumentation injects calls to the LLVM intrinsic instruction -[`llvm.instrprof.increment`][llvm-instrprof-increment] at code branches +The coverage instrumentation injects calls to LLVM intrinsic instructions +([`llvm.instrprof.increment`][llvm-instrprof-increment], or +[`llvm.instrprof.cover`][llvm-instrprof-cover] in presence-only mode) at code branches (based on a MIR-based control flow analysis), and LLVM converts these to -instructions that increment static counters, when executed. +instructions that update static counters when executed. The LLVM coverage instrumentation also requires a [Coverage Map] that encodes source metadata, mapping counter IDs--directly and indirectly--to the file locations (with start and end line and column). -Rust libraries, with or without coverage instrumentation, can be linked into instrumented binaries. +Rust libraries, with or without coverage instrumentation, can be linked into +instrumented binaries. However, all coverage-instrumented objects in a linked +image must use the same counter mode. When the program is executed and cleanly terminates, LLVM libraries write the final counter values to a file (`default.profraw` or a custom file set through environment variable `LLVM_PROFILE_FILE`). @@ -27,6 +30,7 @@ Detailed instructions and examples are documented in the [rustc book][rustc-book-instrument-coverage]. [llvm-instrprof-increment]: https://llvm.org/docs/LangRef.html#llvm-instrprof-increment-intrinsic +[llvm-instrprof-cover]: https://llvm.org/docs/LangRef.html#llvm-instrprof-cover-intrinsic [coverage map]: https://llvm.org/docs/CoverageMappingFormat.html [rustc-book-instrument-coverage]: https://doc.rust-lang.org/nightly/rustc/instrument-coverage.html diff --git a/src/doc/rustc/src/codegen-options/index.md b/src/doc/rustc/src/codegen-options/index.md index 3c7577b9135e6..8936d0f4664bd 100644 --- a/src/doc/rustc/src/codegen-options/index.md +++ b/src/doc/rustc/src/codegen-options/index.md @@ -195,6 +195,12 @@ Using incremental compilation inhibits certain optimizations (for example by inc This option enables instrumentation-based code coverage support. See the chapter on [instrumentation-based code coverage] for more information. +The value `counter` is an alias for the default counter-based coverage mode. +The unstable value `presence-only`, which requires `-Z unstable-options`, +records whether each covered region executed instead of counting executions. +It cannot be combined with branch or condition coverage, or mixed with ordinary +coverage or PGO objects in the same linked image. + Note that while the `-C instrument-coverage` option is stable, the profile data format produced by the resulting instrumentation may change, and may not work with coverage tools other than those built and shipped with the compiler. diff --git a/src/doc/rustc/src/instrument-coverage.md b/src/doc/rustc/src/instrument-coverage.md index 57679f82f48d2..6746936ef46b4 100644 --- a/src/doc/rustc/src/instrument-coverage.md +++ b/src/doc/rustc/src/instrument-coverage.md @@ -9,12 +9,13 @@ via the `-C instrument-coverage` compiler flag. When `-C instrument-coverage` is enabled, the Rust compiler enhances rust-based libraries and binaries by: -- Automatically injecting calls to an LLVM intrinsic ([`llvm.instrprof.increment`]), at functions and branches in compiled code, to increment counters when conditional sections of code are executed. +- Automatically injecting calls to LLVM coverage intrinsics ([`llvm.instrprof.increment`] and, in presence-only mode, [`llvm.instrprof.cover`]), at functions and branches in compiled code, to update counters when conditional sections of code are executed. - Embedding additional information in the data section of each library and binary (using the [LLVM Code Coverage Mapping Format] _Version 5_, if compiling with LLVM 12, or _Version 6_, if compiling with LLVM 13 or higher), to define the code regions (start and end positions in the source code) being counted. When running a coverage-instrumented program, the counter values are written to a `profraw` file at program termination. LLVM bundles tools that read the counter results, combine those results with the coverage map (embedded in the program binary), and generate coverage reports in multiple formats. [`llvm.instrprof.increment`]: https://llvm.org/docs/LangRef.html#llvm-instrprof-increment-intrinsic +[`llvm.instrprof.cover`]: https://llvm.org/docs/LangRef.html#llvm-instrprof-cover-intrinsic [llvm code coverage mapping format]: https://llvm.org/docs/CoverageMappingFormat.html > **Note**: `-C instrument-coverage` also automatically enables `-C symbol-mangling-version=v0` (tracking issue [#60705]). The `v0` symbol mangler is strongly recommended. The `v0` demangler can be overridden by explicitly adding `-Z unstable-options -C symbol-mangling-version=legacy`. @@ -331,10 +332,22 @@ $ llvm-cov report \ ### Other values +- `-C instrument-coverage=counter`: + An alias for the default counter-based coverage mode (`yes`). - `-C instrument-coverage=all`: Currently an alias for `yes`, but may behave differently in the future if more fine-grained coverage options are added. Using this value is currently not recommended. +- `-C instrument-coverage=presence-only`: + Records whether each covered region executed, rather than its execution + count, using one-byte coverage counters. This mode requires + `-Z unstable-options` and cannot be combined with branch or condition + coverage. LLVM lowers these probes to idempotent, non-atomic byte stores, + matching Clang's single-byte coverage representation. + + All coverage-instrumented objects linked into the same executable or shared + library must use the same counter mode. Mixing presence-only objects with + ordinary coverage or PGO objects in one linked image is not supported. ## `-Z coverage-options=` diff --git a/src/doc/unstable-book/src/compiler-flags/coverage-options.md b/src/doc/unstable-book/src/compiler-flags/coverage-options.md index 693a0a58b49ef..aad06c62807b9 100644 --- a/src/doc/unstable-book/src/compiler-flags/coverage-options.md +++ b/src/doc/unstable-book/src/compiler-flags/coverage-options.md @@ -15,3 +15,6 @@ Multiple options can be passed, separated by commas. Valid options are: - `condition`: In addition to branch coverage, also instruments some boolean expressions as branches, even if they are not directly used as branch conditions. + +The `branch` and `condition` levels cannot be combined with +`-C instrument-coverage=presence-only`. diff --git a/tests/codegen-llvm/instrument-coverage/instrument-coverage.rs b/tests/codegen-llvm/instrument-coverage/instrument-coverage.rs index 23d23651c72ff..781b0d9a93aa9 100644 --- a/tests/codegen-llvm/instrument-coverage/instrument-coverage.rs +++ b/tests/codegen-llvm/instrument-coverage/instrument-coverage.rs @@ -1,13 +1,14 @@ // Test that `-Cinstrument-coverage` creates expected __llvm_profile_filename symbol in LLVM IR. //@ compile-flags: -Zno-profiler-runtime -//@ revisions: default y yes on true_ all +//@ revisions: default y yes on true_ all counter //@ [default] compile-flags: -Cinstrument-coverage //@ [y] compile-flags: -Cinstrument-coverage=y //@ [yes] compile-flags: -Cinstrument-coverage=yes //@ [on] compile-flags: -Cinstrument-coverage=on //@ [true_] compile-flags: -Cinstrument-coverage=true //@ [all] compile-flags: -Cinstrument-coverage=all +//@ [counter] compile-flags: -Cinstrument-coverage=counter // CHECK-DAG: @__llvm_coverage_mapping // CHECK-DAG: @__llvm_profile_filename = {{.*}}"default_%m_%p.profraw\00"{{.*}} diff --git a/tests/codegen-llvm/instrument-coverage/single-byte.rs b/tests/codegen-llvm/instrument-coverage/single-byte.rs new file mode 100644 index 0000000000000..7bf837ebc96da --- /dev/null +++ b/tests/codegen-llvm/instrument-coverage/single-byte.rs @@ -0,0 +1,26 @@ +//@ compile-flags: -Zno-profiler-runtime -Zunstable-options +//@ compile-flags: -Cinstrument-coverage=presence-only -Copt-level=0 + +// Verify that presence-only coverage selects LLVM's byte-counter ABI and that +// the instrumentation pass lowers each probe to a one-byte covered store. + +pub fn covered() -> bool { + true +} + +fn main() { + covered(); +} + +// CHECK: @__llvm_profile_raw_version = {{.*}}constant i64 1152921504606846986 + +// CHECK: @[[COUNTER:__profc_.*single_byte7covered]] = {{private|internal}} global +// CHECK-SAME: [{{[0-9]+}} x i8] c"\FF +// CHECK-SAME: align 1 + +// CHECK: define{{.*}} @_R{{[a-zA-Z0-9_]+}}single_byte7covered +// CHECK-NOT: atomicrmw add +// CHECK: store i8 0, ptr @[[COUNTER]], align 1 + +// CHECK: declare void @llvm.instrprof.cover(ptr, i64, i32, i32) +// CHECK-NOT: declare void @llvm.instrprof.increment(ptr, i64, i32, i32) diff --git a/tests/coverage/single_byte.cov-map b/tests/coverage/single_byte.cov-map new file mode 100644 index 0000000000000..8f3bb85d09a65 --- /dev/null +++ b/tests/coverage/single_byte.cov-map @@ -0,0 +1,82 @@ +Function name: single_byte::concurrent_hit +Raw bytes (24): 0x[01, 01, 00, 04, 01, 12, 01, 00, 14, 01, 01, 05, 00, 0e, 01, 00, 0f, 00, 11, 01, 01, 01, 00, 02] +Number of files: 1 +- file 0 => $DIR/single_byte.rs +Number of expressions: 0 +Number of file 0 mappings: 4 +- Code(Counter(0)) at (prev + 18, 1) to (start + 0, 20) +- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 14) +- Code(Counter(0)) at (prev + 0, 15) to (start + 0, 17) +- Code(Counter(0)) at (prev + 1, 1) to (start + 0, 2) +Highest counter ID seen: c0 + +Function name: single_byte::main +Raw bytes (69): 0x[01, 01, 00, 0d, 01, 16, 01, 00, 0a, 01, 01, 05, 00, 12, 01, 01, 05, 00, 0f, 01, 00, 10, 00, 1f, 05, 02, 09, 00, 10, 05, 00, 14, 00, 18, 05, 01, 0a, 00, 0d, 05, 07, 0a, 00, 11, 09, 01, 09, 00, 0f, 05, 00, 13, 00, 1a, 09, 00, 1b, 02, 06, 09, 01, 09, 00, 0f, 0d, 02, 01, 00, 02] +Number of files: 1 +- file 0 => $DIR/single_byte.rs +Number of expressions: 0 +Number of file 0 mappings: 13 +- Code(Counter(0)) at (prev + 22, 1) to (start + 0, 10) +- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 18) +- Code(Counter(0)) at (prev + 1, 5) to (start + 0, 15) +- Code(Counter(0)) at (prev + 0, 16) to (start + 0, 31) +- Code(Counter(1)) at (prev + 2, 9) to (start + 0, 16) +- Code(Counter(1)) at (prev + 0, 20) to (start + 0, 24) +- Code(Counter(1)) at (prev + 1, 10) to (start + 0, 13) +- Code(Counter(1)) at (prev + 7, 10) to (start + 0, 17) +- Code(Counter(2)) at (prev + 1, 9) to (start + 0, 15) +- Code(Counter(1)) at (prev + 0, 19) to (start + 0, 26) +- Code(Counter(2)) at (prev + 0, 27) to (start + 2, 6) +- Code(Counter(2)) at (prev + 1, 9) to (start + 0, 15) +- Code(Counter(3)) at (prev + 2, 1) to (start + 0, 2) +Highest counter ID seen: c3 + +Function name: single_byte::main::{closure#0} +Raw bytes (19): 0x[01, 01, 00, 03, 01, 1b, 12, 00, 13, 01, 01, 0d, 00, 1f, 01, 05, 09, 00, 0a] +Number of files: 1 +- file 0 => $DIR/single_byte.rs +Number of expressions: 0 +Number of file 0 mappings: 3 +- Code(Counter(0)) at (prev + 27, 18) to (start + 0, 19) +- Code(Counter(0)) at (prev + 1, 13) to (start + 0, 31) +- Code(Counter(0)) at (prev + 5, 9) to (start + 0, 10) +Highest counter ID seen: c0 + +Function name: single_byte::main::{closure#0}::{closure#0} +Raw bytes (24): 0x[01, 01, 00, 04, 01, 1c, 23, 00, 24, 01, 01, 1a, 00, 20, 05, 01, 15, 00, 23, 09, 02, 0d, 00, 0e] +Number of files: 1 +- file 0 => $DIR/single_byte.rs +Number of expressions: 0 +Number of file 0 mappings: 4 +- Code(Counter(0)) at (prev + 28, 35) to (start + 0, 36) +- Code(Counter(0)) at (prev + 1, 26) to (start + 0, 32) +- Code(Counter(1)) at (prev + 1, 21) to (start + 0, 35) +- Code(Counter(2)) at (prev + 2, 13) to (start + 0, 14) +Highest counter ID seen: c2 + +Function name: single_byte::repeated_hits +Raw bytes (29): 0x[01, 01, 00, 05, 01, 06, 01, 00, 13, 05, 01, 09, 00, 0e, 01, 00, 12, 00, 18, 05, 00, 19, 02, 06, 09, 03, 01, 00, 02] +Number of files: 1 +- file 0 => $DIR/single_byte.rs +Number of expressions: 0 +Number of file 0 mappings: 5 +- Code(Counter(0)) at (prev + 6, 1) to (start + 0, 19) +- Code(Counter(1)) at (prev + 1, 9) to (start + 0, 14) +- Code(Counter(0)) at (prev + 0, 18) to (start + 0, 24) +- Code(Counter(1)) at (prev + 0, 25) to (start + 2, 6) +- Code(Counter(2)) at (prev + 3, 1) to (start + 0, 2) +Highest counter ID seen: c2 + +Function name: single_byte::selected_branch +Raw bytes (29): 0x[01, 01, 00, 05, 01, 0d, 01, 00, 2b, 01, 01, 08, 00, 11, 05, 00, 14, 00, 15, 0d, 00, 1f, 00, 20, 09, 01, 01, 00, 02] +Number of files: 1 +- file 0 => $DIR/single_byte.rs +Number of expressions: 0 +Number of file 0 mappings: 5 +- Code(Counter(0)) at (prev + 13, 1) to (start + 0, 43) +- Code(Counter(0)) at (prev + 1, 8) to (start + 0, 17) +- Code(Counter(1)) at (prev + 0, 20) to (start + 0, 21) +- Code(Counter(3)) at (prev + 0, 31) to (start + 0, 32) +- Code(Counter(2)) at (prev + 1, 1) to (start + 0, 2) +Highest counter ID seen: c3 + diff --git a/tests/coverage/single_byte.coverage b/tests/coverage/single_byte.coverage new file mode 100644 index 0000000000000..f612e0cd9783f --- /dev/null +++ b/tests/coverage/single_byte.coverage @@ -0,0 +1,40 @@ + LL| |//@ compile-flags: -Cinstrument-coverage=presence-only -Zunstable-options -Copt-level=0 + LL| | + LL| |use std::hint::black_box; + LL| | + LL| |#[inline(never)] + LL| 1|fn repeated_hits() { + LL| 1| for value in 0..100 { + LL| 1| black_box(value); + LL| 1| } + LL| 1|} + LL| | + LL| |#[inline(never)] + LL| 1|fn selected_branch(take_true: bool) -> u32 { + LL| 1| if take_true { 1 } else { 2 } + ^0 + LL| 1|} + LL| | + LL| |#[inline(never)] + LL| 1|fn concurrent_hit() { + LL| 1| black_box(()); + LL| 1|} + LL| | + LL| 1|fn main() { + LL| 1| repeated_hits(); + LL| 1| assert_eq!(selected_branch(true), 1); + LL| | + LL| 1| let threads = (0..4) + LL| 1| .map(|_| { + LL| 1| std::thread::spawn(|| { + LL| 1| for _ in 0..100 { + LL| 1| concurrent_hit() + LL| | } + LL| 1| }) + LL| 1| }) + LL| 1| .collect::>(); + LL| 1| for thread in threads { + LL| 1| thread.join().unwrap(); + LL| 1| } + LL| 1|} + diff --git a/tests/coverage/single_byte.rs b/tests/coverage/single_byte.rs new file mode 100644 index 0000000000000..417196e98aa34 --- /dev/null +++ b/tests/coverage/single_byte.rs @@ -0,0 +1,38 @@ +//@ compile-flags: -Cinstrument-coverage=presence-only -Zunstable-options -Copt-level=0 + +use std::hint::black_box; + +#[inline(never)] +fn repeated_hits() { + for value in 0..100 { + black_box(value); + } +} + +#[inline(never)] +fn selected_branch(take_true: bool) -> u32 { + if take_true { 1 } else { 2 } +} + +#[inline(never)] +fn concurrent_hit() { + black_box(()); +} + +fn main() { + repeated_hits(); + assert_eq!(selected_branch(true), 1); + + let threads = (0..4) + .map(|_| { + std::thread::spawn(|| { + for _ in 0..100 { + concurrent_hit() + } + }) + }) + .collect::>(); + for thread in threads { + thread.join().unwrap(); + } +} diff --git a/tests/ui/instrument-coverage/bad-value.bad.stderr b/tests/ui/instrument-coverage/bad-value.bad.stderr index d3415fb35d062..6c7669ea02d68 100644 --- a/tests/ui/instrument-coverage/bad-value.bad.stderr +++ b/tests/ui/instrument-coverage/bad-value.bad.stderr @@ -1,2 +1,2 @@ -error: incorrect value `bad-value` for codegen option `instrument-coverage` - one of: `y`, `yes`, `on`, `true`, `n`, `no`, `off` or `false` was expected +error: incorrect value `bad-value` for codegen option `instrument-coverage` - either a boolean (`yes`, `no`, `on`, `off`, etc), `counter`, or (with `-Z unstable-options`) `presence-only` was expected diff --git a/tests/ui/instrument-coverage/bad-value.blank.stderr b/tests/ui/instrument-coverage/bad-value.blank.stderr index 04c0eab28489d..782a2580e38c3 100644 --- a/tests/ui/instrument-coverage/bad-value.blank.stderr +++ b/tests/ui/instrument-coverage/bad-value.blank.stderr @@ -1,2 +1,2 @@ -error: incorrect value `` for codegen option `instrument-coverage` - one of: `y`, `yes`, `on`, `true`, `n`, `no`, `off` or `false` was expected +error: incorrect value `` for codegen option `instrument-coverage` - either a boolean (`yes`, `no`, `on`, `off`, etc), `counter`, or (with `-Z unstable-options`) `presence-only` was expected diff --git a/tests/ui/instrument-coverage/on-values.rs b/tests/ui/instrument-coverage/on-values.rs index 53b149fd39cce..1a65ab03555d7 100644 --- a/tests/ui/instrument-coverage/on-values.rs +++ b/tests/ui/instrument-coverage/on-values.rs @@ -1,11 +1,12 @@ //@ check-pass //@ compile-flags: -Zno-profiler-runtime -//@ revisions: default y yes on true_ all +//@ revisions: default y yes on true_ all counter //@ [default] compile-flags: -Cinstrument-coverage //@ [y] compile-flags: -Cinstrument-coverage=y //@ [yes] compile-flags: -Cinstrument-coverage=yes //@ [on] compile-flags: -Cinstrument-coverage=on //@ [true_] compile-flags: -Cinstrument-coverage=true //@ [all] compile-flags: -Cinstrument-coverage=all +//@ [counter] compile-flags: -Cinstrument-coverage=counter fn main() {} diff --git a/tests/ui/instrument-coverage/presence-only.branch.stderr b/tests/ui/instrument-coverage/presence-only.branch.stderr new file mode 100644 index 0000000000000..5e0bda9cf59dc --- /dev/null +++ b/tests/ui/instrument-coverage/presence-only.branch.stderr @@ -0,0 +1,2 @@ +error: `-C instrument-coverage=presence-only` is not compatible with branch or condition coverage + diff --git a/tests/ui/instrument-coverage/presence-only.condition.stderr b/tests/ui/instrument-coverage/presence-only.condition.stderr new file mode 100644 index 0000000000000..5e0bda9cf59dc --- /dev/null +++ b/tests/ui/instrument-coverage/presence-only.condition.stderr @@ -0,0 +1,2 @@ +error: `-C instrument-coverage=presence-only` is not compatible with branch or condition coverage + diff --git a/tests/ui/instrument-coverage/presence-only.no_unstable.stderr b/tests/ui/instrument-coverage/presence-only.no_unstable.stderr new file mode 100644 index 0000000000000..51c8574750cab --- /dev/null +++ b/tests/ui/instrument-coverage/presence-only.no_unstable.stderr @@ -0,0 +1,2 @@ +error: `-C instrument-coverage=presence-only` requires `-Z unstable-options` + diff --git a/tests/ui/instrument-coverage/presence-only.rs b/tests/ui/instrument-coverage/presence-only.rs new file mode 100644 index 0000000000000..ca7d428bcb728 --- /dev/null +++ b/tests/ui/instrument-coverage/presence-only.rs @@ -0,0 +1,23 @@ +//@ revisions: no_unstable block branch condition +//@ compile-flags: -Zno-profiler-runtime + +//@ [no_unstable] check-fail +//@ [no_unstable] compile-flags: -Cinstrument-coverage=presence-only + +//@ [block] check-pass +//@ [block] compile-flags: -Cinstrument-coverage=presence-only -Zunstable-options +//@ [block] compile-flags: -Zcoverage-options=block + +//@ [branch] check-fail +//@ [branch] compile-flags: -Cinstrument-coverage=presence-only -Zunstable-options +//@ [branch] compile-flags: -Zcoverage-options=branch + +//@ [condition] check-fail +//@ [condition] compile-flags: -Cinstrument-coverage=presence-only -Zunstable-options +//@ [condition] compile-flags: -Zcoverage-options=condition + +fn main() {} + +//[no_unstable]~? ERROR `-C instrument-coverage=presence-only` requires `-Z unstable-options` +//[branch]~? ERROR `-C instrument-coverage=presence-only` is not compatible with branch or condition coverage +//[condition]~? ERROR `-C instrument-coverage=presence-only` is not compatible with branch or condition coverage