From 514bd98459165bbd8b5657c352786f81f0de785d Mon Sep 17 00:00:00 2001 From: swananan Date: Sun, 12 Jul 2026 15:30:32 +0800 Subject: [PATCH] feat: accelerate DWARF parsing with native indexes Use embedded .gdb_index and .debug_names to select compilation units for lazy fast-parser materialization instead of scanning every unit's debug info at startup. Validate native indexes before selection and fall back to a full DWARF scan when no usable index is available. Report index selection in CLI and TUI loading flows. --- docs/configuration.md | 7 + docs/zh/configuration.md | 6 + e2e-tests/tests/dwarf_index_regressions.rs | 500 +++++++++++ ghostscope-dwarf/src/analyzer/mod.rs | 14 +- ghostscope-dwarf/src/analyzer/types.rs | 27 + .../src/index/lightweight_file_index.rs | 18 + .../src/index/lightweight_index.rs | 90 +- ghostscope-dwarf/src/index/line_mapping.rs | 21 + ghostscope-dwarf/src/index/mod.rs | 2 + .../src/index/native_gdb_index.rs | 826 ++++++++++++++++++ ghostscope-dwarf/src/index/type_index.rs | 13 + ghostscope-dwarf/src/lib.rs | 8 +- ghostscope-dwarf/src/loader.rs | 4 +- .../src/objfile/function_lookup.rs | 182 +++- ghostscope-dwarf/src/objfile/globals.rs | 81 +- ghostscope-dwarf/src/objfile/loaded.rs | 263 +++++- ghostscope-dwarf/src/objfile/loading.rs | 191 +++- .../src/objfile/source_location.rs | 66 +- ghostscope-dwarf/src/objfile/type_context.rs | 27 +- ghostscope-dwarf/src/objfile/variables.rs | 44 +- .../src/parser/detailed_parser.rs | 3 - .../src/parser/fast_parser/mod.rs | 485 ++++++++-- .../src/components/app/runtime_status.rs | 4 + .../src/components/loading/progress.rs | 10 + ghostscope-ui/src/components/loading/state.rs | 4 + ghostscope-ui/src/components/loading/ui.rs | 53 ++ ghostscope-ui/src/events/runtime.rs | 2 + ghostscope/src/cli/loading_reporter.rs | 45 +- ghostscope/src/tui/dwarf_loader.rs | 5 + 29 files changed, 2796 insertions(+), 205 deletions(-) create mode 100644 ghostscope-dwarf/src/index/native_gdb_index.rs diff --git a/docs/configuration.md b/docs/configuration.md index 95270081..38f20326 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -229,6 +229,13 @@ Behavior: - `--all --force` removes all `pid-starttime` directories, including live ones. - Legacy numeric directories are ignored. +### Native DWARF Indexes + +When the ELF file containing DWARF also contains `.debug_names` or +`.gdb_index`, GhostScope uses that native index to select compilation units for +on-demand parsing. Otherwise, it performs its normal full DWARF scan. An +unusable index is reported in CLI/TUI startup status before falling back. + ### Complete Command Reference | Option | Short | Description | Default | diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index b23e4a16..fd33f5c6 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -228,6 +228,12 @@ ghostscope bpffs prune --dry-run --json - `--all --force` 会删除所有 `pid-starttime` 目录，包括活实例。 - legacy 的纯数字目录会被忽略。 +### 原生 DWARF 索引 + +当包含 DWARF 的 ELF 文件同时带有 `.debug_names` 或 `.gdb_index` 时， +GhostScope 会使用该原生索引选择 CU，并按需调用 fast parser。否则执行原有 +的完整 DWARF 扫描。索引不可用时，CLI/TUI 启动状态会显示回退原因。 + ### 完整命令参考 | 选项 | 简写 | 说明 | 默认值 | diff --git a/e2e-tests/tests/dwarf_index_regressions.rs b/e2e-tests/tests/dwarf_index_regressions.rs index 54c748c9..97d6e13e 100644 --- a/e2e-tests/tests/dwarf_index_regressions.rs +++ b/e2e-tests/tests/dwarf_index_regressions.rs @@ -78,6 +78,37 @@ fn find_symbol_address(binary_path: &std::path::Path, symbol_name: &str) -> anyh }) } +fn assert_native_index_queries( + analyzer: &ghostscope_dwarf::DwarfAnalyzer, + target: &Path, + index_name: &str, +) -> anyhow::Result<()> { + let expected_address = find_symbol_address(target, "calculate_something")?; + let addresses = analyzer.lookup_function_addresses("calculate_something"); + assert!( + addresses.iter().any(|address| { + address.module_path == target + && address.address >= expected_address + && address.address <= expected_address.saturating_add(32) + }), + "native {index_name} function lookup did not resolve calculate_something: {addresses:?}" + ); + assert!( + analyzer + .resolve_struct_type_shallow_by_name("DataRecord") + .is_some(), + "native {index_name} type lookup did not resolve DataRecord" + ); + assert!( + analyzer + .find_global_variables_by_name("call_counter") + .iter() + .any(|(_, global)| global.link_address.is_some()), + "native {index_name} global lookup did not materialize the address of call_counter" + ); + Ok(()) +} + async fn spawn_inline_callsite_program( binary_path: &Path, ) -> anyhow::Result { @@ -113,6 +144,144 @@ fn read_uleb128(input: &[u8], offset: &mut usize) -> anyhow::Result { } } +fn skip_leb128(input: &[u8], offset: &mut usize) -> anyhow::Result<()> { + loop { + let byte = *input + .get(*offset) + .context("Unexpected EOF while skipping LEB128")?; + *offset += 1; + if byte & 0x80 == 0 { + return Ok(()); + } + } +} + +fn first_dwarf_contribution_len(data: &[u8], endian: object::Endianness) -> anyhow::Result { + let initial = data + .get(..4) + .context("DWARF contribution is missing its initial length")?; + let length = match endian { + object::Endianness::Little => u32::from_le_bytes(initial.try_into()?), + object::Endianness::Big => u32::from_be_bytes(initial.try_into()?), + }; + if length == u32::MAX { + let extended = data + .get(4..12) + .context("DWARF64 contribution is missing its extended length")?; + let length = match endian { + object::Endianness::Little => u64::from_le_bytes(extended.try_into()?), + object::Endianness::Big => u64::from_be_bytes(extended.try_into()?), + }; + usize::try_from(length)? + .checked_add(12) + .context("DWARF64 contribution length overflow") + } else { + usize::try_from(length)? + .checked_add(4) + .context("DWARF32 contribution length overflow") + } +} + +fn keep_first_debug_aranges_contribution(binary: &Path, scratch_dir: &Path) -> anyhow::Result<()> { + let bytes = fs::read(binary)?; + let object = object::File::parse(&*bytes)?; + let section = object + .section_by_name(".debug_aranges") + .context("binary has no .debug_aranges section")?; + let data = section.uncompressed_data()?.into_owned(); + let first_len = first_dwarf_contribution_len(&data, object.endianness())?; + anyhow::ensure!( + first_len < data.len(), + ".debug_aranges does not contain multiple contributions" + ); + let replacement = scratch_dir.join("partial-debug-aranges.bin"); + fs::write(&replacement, &data[..first_len])?; + run_command( + StdCommand::new("objcopy") + .arg("--update-section") + .arg(format!(".debug_aranges={}", replacement.display())) + .arg(binary), + "objcopy partial .debug_aranges update", + ) +} + +fn debug_aranges_cover_address(binary: &Path, address: u64) -> anyhow::Result { + let dwarf = load_dwarf_from_binary(binary)?; + let mut headers = dwarf.debug_aranges.headers(); + while let Some(header) = headers.next()? { + let mut entries = header.entries(); + while let Some(entry) = entries.next()? { + if entry.address() <= address && address < entry.range().end { + return Ok(true); + } + } + } + Ok(false) +} + +fn patch_compile_unit_range_attributes(abbrev: &mut [u8]) -> anyhow::Result { + let mut offset = 0; + let mut patched = 0; + while offset < abbrev.len() { + let code = read_uleb128(abbrev, &mut offset)?; + if code == 0 { + continue; + } + let tag = read_uleb128(abbrev, &mut offset)?; + offset = offset + .checked_add(1) + .filter(|next| *next <= abbrev.len()) + .context("abbreviation is missing its children byte")?; + + loop { + let name_offset = offset; + let name = read_uleb128(abbrev, &mut offset)?; + let name_end = offset; + let form = read_uleb128(abbrev, &mut offset)?; + if name == 0 && form == 0 { + break; + } + let is_cu_range = name == u64::from(ghostscope_dwarf::constants::DW_AT_low_pc.0) + || name == u64::from(ghostscope_dwarf::constants::DW_AT_high_pc.0) + || name == u64::from(ghostscope_dwarf::constants::DW_AT_ranges.0); + if tag == u64::from(ghostscope_dwarf::constants::DW_TAG_compile_unit.0) && is_cu_range { + anyhow::ensure!( + name_end == name_offset + 1, + "CU range attribute does not use a one-byte ULEB128" + ); + abbrev[name_offset] = 0x7f; + patched += 1; + } + if form == u64::from(ghostscope_dwarf::constants::DW_FORM_implicit_const.0) { + skip_leb128(abbrev, &mut offset)?; + } + } + } + Ok(patched) +} + +fn remove_all_cu_address_ranges(binary: &Path, scratch_dir: &Path) -> anyhow::Result<()> { + let bytes = fs::read(binary)?; + let object = object::File::parse(&*bytes)?; + let section = object + .section_by_name(".debug_abbrev") + .context("binary has no .debug_abbrev section")?; + let mut abbrev = section.uncompressed_data()?.into_owned(); + let patched = patch_compile_unit_range_attributes(&mut abbrev)?; + anyhow::ensure!(patched > 0, "no CU root range attributes were patched"); + let replacement = scratch_dir.join("no-cu-ranges-debug-abbrev.bin"); + fs::write(&replacement, abbrev)?; + run_command( + StdCommand::new("objcopy") + .arg("--update-section") + .arg(format!(".debug_abbrev={}", replacement.display())) + .arg("--remove-section") + .arg(".debug_aranges") + .arg(binary), + "objcopy CU range removal", + ) +} + #[test] fn test_read_uleb128_rejects_values_that_overflow_u64() { let overflow = [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x02]; @@ -124,6 +293,323 @@ fn test_read_uleb128_rejects_values_that_overflow_u64() { ); } +#[tokio::test] +#[serial_test::serial] +async fn test_gdb_index_resolves_function_type_and_global_lazily() -> anyhow::Result<()> { + init(); + + let source = FIXTURES.get_test_binary("sample_program")?; + let temp_dir = tempfile::tempdir()?; + let indexed = temp_dir.path().join("sample_program.gdb-indexed"); + fs::copy(&source, &indexed)?; + + let output = match StdCommand::new("gdb-add-index").arg(&indexed).output() { + Ok(output) => output, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + eprintln!("Skipping .gdb_index integration test because gdb-add-index is unavailable"); + return Ok(()); + } + Err(error) => return Err(error.into()), + }; + anyhow::ensure!( + output.status.success(), + "gdb-add-index failed with status {:?}\nstdout:\n{}\nstderr:\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let indexed_bytes = fs::read(&indexed)?; + let indexed_object = object::File::parse(&*indexed_bytes)?; + anyhow::ensure!( + indexed_object.section_by_name(".gdb_index").is_some(), + "gdb-add-index did not add .gdb_index" + ); + + let embedded_analyzer = ghostscope_dwarf::DwarfAnalyzer::from_exec_path(&indexed).await?; + assert!( + matches!( + embedded_analyzer.dwarf_index_status_for_module(&indexed), + Some(ghostscope_dwarf::DwarfIndexStatus::GdbIndex { version: 7..=9 }) + ), + "GhostScope did not select the embedded .gdb_index: {:?}", + embedded_analyzer.dwarf_index_status_for_module(&indexed) + ); + assert_native_index_queries(&embedded_analyzer, &indexed, ".gdb_index")?; + + let (index_file_offset, index_size) = indexed_object + .section_by_name(".gdb_index") + .and_then(|section| section.file_range()) + .context("embedded .gdb_index has no file range")?; + let index_start = usize::try_from(index_file_offset)?; + let index_end = index_start + .checked_add(usize::try_from(index_size)?) + .context("embedded .gdb_index range overflow")?; + let index_data = indexed_bytes + .get(index_start..index_end) + .context("embedded .gdb_index range is out of bounds")?; + let read_index_u32 = |word: usize| -> anyhow::Result { + let start = word + .checked_mul(std::mem::size_of::()) + .context(".gdb_index header offset overflow")?; + let bytes = index_data + .get(start..start + std::mem::size_of::()) + .context("truncated .gdb_index header")?; + Ok(u32::from_le_bytes(bytes.try_into()?)) + }; + let version = read_index_u32(0)?; + let symbol_table = usize::try_from(read_index_u32(4)?)?; + let (shortcut_table, constant_pool) = if version == 9 { + ( + usize::try_from(read_index_u32(5)?)?, + usize::try_from(read_index_u32(6)?)?, + ) + } else { + let constant_pool = usize::try_from(read_index_u32(5)?)?; + (constant_pool, constant_pool) + }; + let mut vector_offset = None; + for slot in index_data[symbol_table..shortcut_table].chunks_exact(8) { + let name = u32::from_le_bytes(slot[..4].try_into()?); + let vector = u32::from_le_bytes(slot[4..].try_into()?); + if name != 0 || vector != 0 { + vector_offset = Some(usize::try_from(vector)?); + break; + } + } + let vector_offset = vector_offset.context(".gdb_index has no populated symbol slots")?; + let vector_count_offset = index_start + .checked_add(constant_pool) + .and_then(|offset| offset.checked_add(vector_offset)) + .context(".gdb_index vector offset overflow")?; + let mut corrupted_bytes = indexed_bytes.clone(); + corrupted_bytes + .get_mut(vector_count_offset..vector_count_offset + 4) + .context(".gdb_index vector count is out of bounds")? + .copy_from_slice(&u32::MAX.to_le_bytes()); + let corrupted = temp_dir.path().join("sample_program.corrupt-gdb-index"); + fs::write(&corrupted, corrupted_bytes)?; + let fallback_analyzer = ghostscope_dwarf::DwarfAnalyzer::from_exec_path(&corrupted).await?; + match fallback_analyzer.dwarf_index_status_for_module(&corrupted) { + Some(ghostscope_dwarf::DwarfIndexStatus::Rejected { reason }) => { + assert!( + reason.contains("CU vector is truncated"), + "unexpected .gdb_index rejection: {reason}" + ); + } + status => panic!("malformed .gdb_index did not fall back: {status:?}"), + } + assert_native_index_queries( + &fallback_analyzer, + &corrupted, + "rejected .gdb_index fallback", + )?; + + let debug_names_indexed = temp_dir.path().join("sample_program.debug-names-indexed"); + fs::copy(&source, &debug_names_indexed)?; + let debug_names_output = StdCommand::new("gdb-add-index") + .arg("-dwarf-5") + .arg(&debug_names_indexed) + .output()?; + anyhow::ensure!( + debug_names_output.status.success(), + "gdb-add-index -dwarf-5 failed with status {:?}\nstdout:\n{}\nstderr:\n{}", + debug_names_output.status.code(), + String::from_utf8_lossy(&debug_names_output.stdout), + String::from_utf8_lossy(&debug_names_output.stderr) + ); + let debug_names_bytes = fs::read(&debug_names_indexed)?; + let debug_names_object = object::File::parse(&*debug_names_bytes)?; + anyhow::ensure!( + debug_names_object.section_by_name(".debug_names").is_some(), + "gdb-add-index -dwarf-5 did not add .debug_names" + ); + let debug_names_analyzer = + ghostscope_dwarf::DwarfAnalyzer::from_exec_path(&debug_names_indexed).await?; + assert_eq!( + debug_names_analyzer.dwarf_index_status_for_module(&debug_names_indexed), + Some(&ghostscope_dwarf::DwarfIndexStatus::DebugNames), + "GhostScope did not select GDB2 .debug_names" + ); + assert_native_index_queries(&debug_names_analyzer, &debug_names_indexed, ".debug_names")?; + + if command_available("clang") { + let standard_debug_names = temp_dir.path().join("sample_program.clang-debug-names"); + let fixture_dir = source + .parent() + .context("sample_program fixture has no parent directory")?; + run_command( + StdCommand::new("clang") + .arg("-gdwarf-5") + .arg("-gpubnames") + .arg("-O0") + .arg(fixture_dir.join("sample_program.c")) + .arg(fixture_dir.join("sample_lib.c")) + .arg("-o") + .arg(&standard_debug_names), + "clang standard .debug_names build", + )?; + let standard_bytes = fs::read(&standard_debug_names)?; + let standard_object = object::File::parse(&*standard_bytes)?; + anyhow::ensure!( + standard_object.section_by_name(".debug_names").is_some(), + "clang -gpubnames did not add .debug_names" + ); + let standard_analyzer = + ghostscope_dwarf::DwarfAnalyzer::from_exec_path(&standard_debug_names).await?; + assert_eq!( + standard_analyzer.dwarf_index_status_for_module(&standard_debug_names), + Some(&ghostscope_dwarf::DwarfIndexStatus::DebugNames), + "GhostScope did not select standard .debug_names" + ); + assert_native_index_queries( + &standard_analyzer, + &standard_debug_names, + "standard .debug_names", + )?; + + if command_available("clang++") && command_available("objcopy") { + let cpp_source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/cpp_complex_program/main.cpp"); + let debug_names_type_units = temp_dir.path().join("cpp-complex.debug-names-type-units"); + run_command( + StdCommand::new("clang++") + .arg("-gdwarf-5") + .arg("-gpubnames") + .arg("-fdebug-types-section") + .arg("-O0") + .arg(&cpp_source) + .arg("-o") + .arg(&debug_names_type_units), + "clang++ .debug_names type-unit build", + )?; + anyhow::ensure!( + dwarf_has_type_unit(&debug_names_type_units)?, + "clang++ did not emit a DWARF5 type unit" + ); + + let debug_names_type_analyzer = + ghostscope_dwarf::DwarfAnalyzer::from_exec_path(&debug_names_type_units).await?; + assert_eq!( + debug_names_type_analyzer.dwarf_index_status_for_module(&debug_names_type_units), + Some(&ghostscope_dwarf::DwarfIndexStatus::DebugNames), + "GhostScope did not select type-unit .debug_names" + ); + let outer_type = debug_names_type_analyzer + .resolve_struct_type_shallow_by_name("Outer") + .context(".debug_names did not resolve Outer from its type unit")?; + assert_eq!(outer_type.size(), 16, "unexpected Outer size"); + + let gdb_type_units = temp_dir.path().join("cpp-complex.gdb-type-units"); + fs::copy(&debug_names_type_units, &gdb_type_units)?; + run_command( + StdCommand::new("objcopy") + .arg("--remove-section") + .arg(".debug_names") + .arg(&gdb_type_units), + "objcopy type-unit .debug_names removal", + )?; + run_command( + StdCommand::new("gdb-add-index").arg(&gdb_type_units), + "gdb-add-index type-unit build", + )?; + let gdb_type_analyzer = + ghostscope_dwarf::DwarfAnalyzer::from_exec_path(&gdb_type_units).await?; + assert!( + matches!( + gdb_type_analyzer.dwarf_index_status_for_module(&gdb_type_units), + Some(ghostscope_dwarf::DwarfIndexStatus::GdbIndex { version: 8..=9 }) + ), + "GhostScope did not select type-unit .gdb_index: {:?}", + gdb_type_analyzer.dwarf_index_status_for_module(&gdb_type_units) + ); + let outer_type = gdb_type_analyzer + .resolve_struct_type_shallow_by_name("Outer") + .context(".gdb_index did not resolve Outer from its type unit")?; + assert_eq!(outer_type.size(), 16, "unexpected indexed Outer size"); + } + } + + if command_available("gcc") && command_available("objcopy") { + let fixture_dir = source + .parent() + .context("sample_program fixture has no parent directory")?; + let gcc_debug_names = temp_dir.path().join("sample-program.gcc-debug-names"); + run_command( + StdCommand::new("gcc") + .arg("-gdwarf-5") + .arg("-gpubnames") + .arg("-O0") + .arg(fixture_dir.join("sample_program.c")) + .arg(fixture_dir.join("sample_lib.c")) + .arg("-o") + .arg(&gcc_debug_names), + "gcc partial-aranges fixture build", + )?; + run_command( + StdCommand::new("gdb-add-index") + .arg("-dwarf-5") + .arg(&gcc_debug_names), + "gdb-add-index partial-aranges fixture build", + )?; + + let rejected_ranges = temp_dir.path().join("sample-program.no-cu-ranges"); + fs::copy(&gcc_debug_names, &rejected_ranges)?; + + keep_first_debug_aranges_contribution(&gcc_debug_names, temp_dir.path())?; + let mut uncovered_function = None; + for name in ["calculate_something", "add_numbers", "multiply_numbers"] { + let address = find_symbol_address(&gcc_debug_names, name)?; + if !debug_aranges_cover_address(&gcc_debug_names, address)? { + uncovered_function = Some((name, address)); + break; + } + } + let (function_name, function_address) = uncovered_function + .context("partial .debug_aranges still covers functions from every CU")?; + let partial_analyzer = + ghostscope_dwarf::DwarfAnalyzer::from_exec_path(&gcc_debug_names).await?; + assert_eq!( + partial_analyzer.dwarf_index_status_for_module(&gcc_debug_names), + Some(&ghostscope_dwarf::DwarfIndexStatus::DebugNames), + "GhostScope did not select partial-aranges .debug_names" + ); + let context = partial_analyzer.resolve_pc(&ghostscope_dwarf::ModuleAddress::new( + gcc_debug_names.clone(), + function_address, + ))?; + assert_eq!( + context.function_name.as_deref(), + Some(function_name), + "CU root ranges did not supplement partial .debug_aranges" + ); + assert!( + context.line.is_some(), + "partial-aranges PC did not resolve source information: {context:?}" + ); + + remove_all_cu_address_ranges(&rejected_ranges, temp_dir.path())?; + let rejected_analyzer = + ghostscope_dwarf::DwarfAnalyzer::from_exec_path(&rejected_ranges).await?; + match rejected_analyzer.dwarf_index_status_for_module(&rejected_ranges) { + Some(ghostscope_dwarf::DwarfIndexStatus::Rejected { reason }) => { + assert!( + reason.contains("no usable address-to-CU ranges"), + "unexpected .debug_names rejection: {reason}" + ); + } + status => panic!("range-less .debug_names was not rejected: {status:?}"), + } + assert_native_index_queries( + &rejected_analyzer, + &rejected_ranges, + "rejected range-less .debug_names fallback", + )?; + } + + Ok(()) +} + fn patch_inlined_subroutine_low_pc_to_entry_pc(abbrev: &mut [u8]) -> anyhow::Result { let mut offset = 0; let mut patched = 0; @@ -233,6 +719,20 @@ fn load_dwarf_from_binary(path: &Path) -> anyhow::Result anyhow::Result { + let dwarf = load_dwarf_from_binary(path)?; + let mut units = dwarf.units(); + while let Some(header) = units.next()? { + if matches!( + header.type_(), + gimli::UnitType::Type { .. } | gimli::UnitType::SplitType { .. } + ) { + return Ok(true); + } + } + Ok(false) +} + fn duplicate_line_row_address_for_source_line( binary_path: &Path, source_basename: &str, diff --git a/ghostscope-dwarf/src/analyzer/mod.rs b/ghostscope-dwarf/src/analyzer/mod.rs index 5504a880..bd3996b9 100644 --- a/ghostscope-dwarf/src/analyzer/mod.rs +++ b/ghostscope-dwarf/src/analyzer/mod.rs @@ -44,6 +44,17 @@ pub struct DwarfAnalyzer { } impl DwarfAnalyzer { + pub fn dwarf_index_status_for_module>( + &self, + module_path: P, + ) -> Option<&DwarfIndexStatus> { + let module_path = module_path.as_ref(); + self.modules + .iter() + .find(|(path, _)| Self::module_paths_equivalent(path, module_path)) + .map(|(_, module)| module.dwarf_index_status()) + } + fn build_address_query_result( &self, module_address: &ModuleAddress, @@ -703,7 +714,7 @@ impl DwarfAnalyzer { .await { Ok(module_data) => { - let (functions, variables, types) = module_data.get_lightweight_index().get_stats(); + let (functions, variables, types) = module_data.get_index_stats(); let (parse_time_ms, index_time_ms, module_total_time_ms) = module_data.get_load_timing_ms(); progress_callback(ModuleLoadingEvent::LoadingCompleted { @@ -713,6 +724,7 @@ impl DwarfAnalyzer { variables, types, debug_info_source: module_data.get_debug_info_source().clone(), + dwarf_index_status: module_data.dwarf_index_status().clone(), load_time_ms: start_time.elapsed().as_millis() as u64, parse_time_ms, index_time_ms, diff --git a/ghostscope-dwarf/src/analyzer/types.rs b/ghostscope-dwarf/src/analyzer/types.rs index 959baf7b..683ac492 100644 --- a/ghostscope-dwarf/src/analyzer/types.rs +++ b/ghostscope-dwarf/src/analyzer/types.rs @@ -1,6 +1,32 @@ use crate::{core::DebugInfoSource, semantics::VisibleVariable}; use std::path::PathBuf; +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DwarfIndexStatus { + FullScan, + DebugNames, + GdbIndex { version: u32 }, + Rejected { reason: String }, +} + +impl DwarfIndexStatus { + pub fn display_label(&self) -> String { + match self { + Self::FullScan => "full-scan".to_string(), + Self::DebugNames => ".debug_names".to_string(), + Self::GdbIndex { version } => format!(".gdb_index-v{version}"), + Self::Rejected { .. } => "full-scan (index rejected)".to_string(), + } + } + + pub fn rejection_reason(&self) -> Option<&str> { + match self { + Self::Rejected { reason } => Some(reason), + _ => None, + } + } +} + /// Events emitted during module loading process #[derive(Debug, Clone)] pub enum ModuleLoadingEvent { @@ -39,6 +65,7 @@ pub struct ModuleLoadingStats { pub variables: usize, pub types: usize, pub debug_info_source: DebugInfoSource, + pub dwarf_index_status: DwarfIndexStatus, pub load_time_ms: u64, pub parse_time_ms: u64, pub index_time_ms: u64, diff --git a/ghostscope-dwarf/src/index/lightweight_file_index.rs b/ghostscope-dwarf/src/index/lightweight_file_index.rs index 0560fe42..72361f3c 100644 --- a/ghostscope-dwarf/src/index/lightweight_file_index.rs +++ b/ghostscope-dwarf/src/index/lightweight_file_index.rs @@ -200,6 +200,24 @@ impl ScopedFileIndexManager { self.total_compilation_units += 1; } + pub(crate) fn extend(&mut self, other: Self) { + for (name, file_index) in other.cu_file_indices { + if self + .cu_file_indices + .insert(name.clone(), file_index) + .is_none() + { + self.total_compilation_units += 1; + } + self.cu_name_pool.entry(name.to_string()).or_insert(name); + } + self.total_files = self + .cu_file_indices + .values() + .map(|index| index.total_files) + .sum(); + } + /// Lookup file by scoped index (primary method, conflict-free) /// /// This is equivalent to the old FileIndexManager::lookup_by_scoped_index diff --git a/ghostscope-dwarf/src/index/lightweight_index.rs b/ghostscope-dwarf/src/index/lightweight_index.rs index df93bd72..ff81434e 100644 --- a/ghostscope-dwarf/src/index/lightweight_index.rs +++ b/ghostscope-dwarf/src/index/lightweight_index.rs @@ -161,15 +161,22 @@ impl LightweightIndex { /// Build cooked index directly from per-CU shards. pub(crate) fn from_shards(shards: Vec) -> Self { debug!("Building lightweight index from parsed data"); + let mut index = Self::new(); + index.append_shards(shards); + index + } - let total_entry_capacity: usize = shards.iter().map(|shard| shard.entries.len()).sum(); - let mut entries = Vec::with_capacity(total_entry_capacity); - let mut name_shards = Vec::with_capacity(shards.len()); + /// Append newly parsed CU shards without rebuilding entries already loaded. + pub(crate) fn append_shards(&mut self, shards: Vec) { + let added_entries: usize = shards.iter().map(|shard| shard.entries.len()).sum(); + self.entries.reserve(added_entries); + self.name_shards.reserve(shards.len()); + let first_new_entry = self.entries.len(); for shard in shards { - let entry_base = entries.len(); - entries.extend(shard.entries); - name_shards.push(NameIndexShard { + let entry_base = self.entries.len(); + self.entries.extend(shard.entries); + self.name_shards.push(NameIndexShard { entry_base, function_map: shard.function_map, function_fragment_map: shard.function_fragment_map, @@ -179,51 +186,41 @@ impl LightweightIndex { }); } - let mut address_map = BTreeMap::new(); - let mut total_functions = 0; - let mut total_variables = 0; - let mut func_indices_by_cu: HashMap> = HashMap::new(); - for (idx, entry) in entries.iter().enumerate() { + for (idx, entry) in self.entries.iter().enumerate().skip(first_new_entry) { match entry.tag { gimli::constants::DW_TAG_subprogram | gimli::constants::DW_TAG_inlined_subroutine => { - total_functions += 1; - func_indices_by_cu + self.total_functions += 1; + self.func_indices_by_cu .entry(entry.unit_offset) .or_default() .push(idx); } gimli::constants::DW_TAG_variable => { - total_variables += 1; + self.total_variables += 1; } _ => {} } if let Some(address) = entry.representative_addr.or(entry.entry_pc) { - address_map.insert(address, idx); + self.address_map.insert(address, idx); + if self.cu_maps_built && Self::is_function_tag(entry.tag) { + self.func_addr_by_cu + .entry(entry.unit_offset) + .or_default() + .insert(address, idx); + } } } debug!( - "Built lightweight index: {} function entries, {} variable entries, {} total entries, {} shards, {} with addresses", - total_functions, - total_variables, - entries.len(), - name_shards.len(), - address_map.len() + "Extended lightweight index: {} function entries, {} variable entries, {} total entries, {} shards, {} with addresses", + self.total_functions, + self.total_variables, + self.entries.len(), + self.name_shards.len(), + self.address_map.len() ); - - Self { - entries, - name_shards, - address_map, - total_functions, - total_variables, - cu_range_map: BTreeMap::new(), - func_addr_by_cu: HashMap::new(), - func_indices_by_cu, - cu_maps_built: false, - } } fn is_function_tag(tag: gimli::DwTag) -> bool { @@ -289,7 +286,7 @@ impl LightweightIndex { ) -> Vec<&'a String> { let mut names = Vec::new(); let mut seen: HashSet<&str> = HashSet::new(); - for shard in &self.name_shards { + for shard in self.name_shards.iter().rev() { for name in map_of(shard).keys() { if seen.insert(name.as_str()) { names.push(name); @@ -305,7 +302,7 @@ impl LightweightIndex { map_of: impl Fn(&'a NameIndexShard) -> &'a HashMap>, ) -> Vec<&'a IndexEntry> { let mut matches = Vec::new(); - for shard in &self.name_shards { + for shard in self.name_shards.iter().rev() { if let Some(indices) = map_of(shard).get(name) { matches.extend( indices @@ -358,7 +355,10 @@ impl LightweightIndex { } let mut ordered: Vec = candidates.into_iter().collect(); - ordered.sort_unstable(); + // Materialized CU shards are appended after native-index seeds. Visit + // newer entries first so callers that deduplicate by DIE retain the + // complete fast-parser entry. + ordered.sort_unstable_by(|left, right| right.cmp(left)); ordered } @@ -368,7 +368,7 @@ impl LightweightIndex { map_of: impl Fn(&'a NameIndexShard) -> &'a HashMap>, ) -> HashSet { let mut indices = HashSet::new(); - for shard in &self.name_shards { + for shard in self.name_shards.iter().rev() { if let Some(local_indices) = map_of(shard).get(fragment) { for &local_idx in local_indices { indices.insert(shard.entry_base + local_idx); @@ -398,7 +398,7 @@ impl LightweightIndex { /// Internal: visit type-map entries across all shards. pub(crate) fn for_each_type_map_entry(&self, mut visit: impl FnMut(&String, usize, &[usize])) { - for shard in &self.name_shards { + for shard in self.name_shards.iter().rev() { for (name, indices) in &shard.type_map { visit(name, shard.entry_base, indices); } @@ -558,6 +558,20 @@ impl LightweightIndex { added_any } + /// Build only CU root ranges without walking every function DIE as a fallback. + pub(crate) fn build_cu_maps_from_roots(&mut self, dwarf: &gimli::Dwarf) -> bool { + let compilation_units = self.func_indices_by_cu.keys().copied().collect::>(); + let mut added_any = false; + for cu in compilation_units { + for (start, end) in Self::resolve_cu_root_ranges(dwarf, cu).unwrap_or_default() { + Self::insert_cu_range(&mut self.cu_range_map, start, end, cu); + added_any = true; + } + } + self.cu_maps_built = true; + added_any + } + /// Find compilation unit by address using CU range map pub fn find_cu_by_address(&self, address: u64) -> Option { if let Some((_, (end, cu))) = self.cu_range_map.range(..=address).next_back() { diff --git a/ghostscope-dwarf/src/index/line_mapping.rs b/ghostscope-dwarf/src/index/line_mapping.rs index 92a030be..1077b2b4 100644 --- a/ghostscope-dwarf/src/index/line_mapping.rs +++ b/ghostscope-dwarf/src/index/line_mapping.rs @@ -79,6 +79,27 @@ impl LineMappingTable { } } + pub(crate) fn extend(&mut self, other: Self) { + for (address, mut entries) in other.address_to_line_map { + self.address_to_line_map + .entry(address) + .or_default() + .append(&mut entries); + } + for (key, mut addresses) in other.path_line_to_addresses { + let current = self.path_line_to_addresses.entry(key).or_default(); + current.append(&mut addresses); + current.sort_unstable(); + current.dedup(); + } + for (basename, paths) in other.basename_to_paths { + self.basename_to_paths + .entry(basename) + .or_default() + .extend(paths); + } + } + fn representative_entry(entries: &[LineEntry]) -> Option<&LineEntry> { entries.last() } diff --git a/ghostscope-dwarf/src/index/mod.rs b/ghostscope-dwarf/src/index/mod.rs index 6e63410f..31da693f 100644 --- a/ghostscope-dwarf/src/index/mod.rs +++ b/ghostscope-dwarf/src/index/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod cfi_index; pub(crate) mod lightweight_file_index; pub(crate) mod lightweight_index; pub(crate) mod line_mapping; +pub(crate) mod native_gdb_index; pub(crate) mod path; pub(crate) mod type_index; @@ -13,5 +14,6 @@ pub(crate) use cfi_index::CfiIndex; pub(crate) use lightweight_file_index::{LightweightFileIndex, ScopedFileIndexManager}; pub(crate) use lightweight_index::{LightweightIndex, LightweightIndexShard}; pub(crate) use line_mapping::LineMappingTable; +pub(crate) use native_gdb_index::{GdbIndex, GdbSymbolKind}; pub(crate) use path::{directory_from_index, resolve_file_path}; pub(crate) use type_index::TypeNameIndex; diff --git a/ghostscope-dwarf/src/index/native_gdb_index.rs b/ghostscope-dwarf/src/index/native_gdb_index.rs new file mode 100644 index 00000000..6a03ebbf --- /dev/null +++ b/ghostscope-dwarf/src/index/native_gdb_index.rs @@ -0,0 +1,826 @@ +use crate::{ + binary::DwarfReader, + core::{normalize_demangled_signature, symbol_name_matches_query, Result}, +}; +use gimli::Reader; +use std::sync::OnceLock; + +const VERSION_9_HEADER_SIZE: usize = 7 * std::mem::size_of::(); +const LEGACY_HEADER_SIZE: usize = 6 * std::mem::size_of::(); +const CU_RECORD_SIZE: usize = 2 * std::mem::size_of::(); +const TYPE_CU_RECORD_SIZE: usize = 3 * std::mem::size_of::(); +const ADDRESS_RECORD_SIZE: usize = 2 * std::mem::size_of::() + std::mem::size_of::(); +const SYMBOL_SLOT_SIZE: usize = 2 * std::mem::size_of::(); +const CU_INDEX_MASK: u32 = 0x00ff_ffff; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) enum GdbSymbolKind { + Type, + Variable, + Function, + Other, +} + +impl GdbSymbolKind { + fn from_attributes(attributes: u32) -> Option { + match (attributes >> 28) & 0x7 { + 1 => Some(Self::Type), + 2 => Some(Self::Variable), + 3 => Some(Self::Function), + 4 => Some(Self::Other), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) struct GdbSymbol { + pub(crate) cu_offset: gimli::DebugInfoOffset, + pub(crate) kind: GdbSymbolKind, + pub(crate) is_static: bool, +} + +#[derive(Debug, Clone, Copy)] +struct GdbIndexLayout { + cu_list: usize, + type_cu_list: usize, + address_area: usize, + symbol_table: usize, + shortcut_table: usize, + constant_pool: usize, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum GdbUnitKind { + Compilation, + Type, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +struct GdbUnit { + offset: gimli::DebugInfoOffset, + kind: GdbUnitKind, +} + +#[derive(Debug, Default)] +struct GdbSymbolNames { + functions: Vec, + variables: Vec, + types: Vec, + others: Vec, +} + +impl GdbSymbolNames { + fn get(&self, kind: GdbSymbolKind) -> &[String] { + match kind { + GdbSymbolKind::Function => &self.functions, + GdbSymbolKind::Variable => &self.variables, + GdbSymbolKind::Type => &self.types, + GdbSymbolKind::Other => &self.others, + } + } + + fn get_mut(&mut self, kind: GdbSymbolKind) -> &mut Vec { + match kind { + GdbSymbolKind::Function => &mut self.functions, + GdbSymbolKind::Variable => &mut self.variables, + GdbSymbolKind::Type => &mut self.types, + GdbSymbolKind::Other => &mut self.others, + } + } + + fn sort_and_deduplicate(&mut self) { + for names in [ + &mut self.functions, + &mut self.variables, + &mut self.types, + &mut self.others, + ] { + names.sort_unstable(); + names.dedup(); + } + } +} + +/// Read-only view of a GDB mapped DWARF index. +/// +/// The backing reader points directly at the ELF section. All offsets and +/// vector lengths are checked before use so an unusable index can be rejected +/// and the caller can fall back to ordinary DWARF parsing. +#[derive(Debug)] +pub(crate) struct GdbIndex { + data: DwarfReader, + version: u32, + layout: GdbIndexLayout, + compilation_unit_count: usize, + type_unit_count: usize, + units: Vec, + address_count: usize, + symbol_slot_count: usize, + symbol_names: OnceLock, +} + +impl GdbIndex { + pub(crate) fn parse(data: DwarfReader) -> Result { + let length = data.len(); + if length < LEGACY_HEADER_SIZE { + anyhow::bail!(".gdb_index header is truncated"); + } + + let version = Self::read_u32(&data, 0)?; + if !(7..=9).contains(&version) { + anyhow::bail!("unsupported .gdb_index version {version}; expected 7 through 9"); + } + + let header_size = if version == 9 { + VERSION_9_HEADER_SIZE + } else { + LEGACY_HEADER_SIZE + }; + if length < header_size { + anyhow::bail!(".gdb_index version {version} header is truncated"); + } + + let cu_list = Self::read_offset(&data, 1)?; + let type_cu_list = Self::read_offset(&data, 2)?; + let address_area = Self::read_offset(&data, 3)?; + let symbol_table = Self::read_offset(&data, 4)?; + let (shortcut_table, constant_pool) = if version == 9 { + (Self::read_offset(&data, 5)?, Self::read_offset(&data, 6)?) + } else { + let constant_pool = Self::read_offset(&data, 5)?; + (constant_pool, constant_pool) + }; + let layout = GdbIndexLayout { + cu_list, + type_cu_list, + address_area, + symbol_table, + shortcut_table, + constant_pool, + }; + Self::validate_layout(length, header_size, version, layout)?; + + let compilation_unit_count = (type_cu_list - cu_list) / CU_RECORD_SIZE; + let type_unit_count = (address_area - type_cu_list) / TYPE_CU_RECORD_SIZE; + let unit_count = compilation_unit_count + .checked_add(type_unit_count) + .ok_or_else(|| anyhow::anyhow!(".gdb_index CU count overflow"))?; + let mut units = Vec::with_capacity(unit_count); + + // DWARF5 type and compile units share .debug_info. GDB assigns the + // unified CU indices in section-offset order before serializing them + // into separate CU and type-CU lists. + for index in 0..compilation_unit_count { + let record = cu_list + index * CU_RECORD_SIZE; + let offset = usize::try_from(Self::read_u64(&data, record)?)?; + units.push(GdbUnit { + offset: gimli::DebugInfoOffset(offset), + kind: GdbUnitKind::Compilation, + }); + } + for index in 0..type_unit_count { + let record = type_cu_list + index * TYPE_CU_RECORD_SIZE; + let offset = usize::try_from(Self::read_u64(&data, record)?)?; + units.push(GdbUnit { + offset: gimli::DebugInfoOffset(offset), + kind: GdbUnitKind::Type, + }); + } + units.sort_unstable_by_key(|unit| unit.offset.0); + if units + .windows(2) + .any(|pair| pair[0].offset == pair[1].offset) + { + anyhow::bail!(".gdb_index contains duplicate CU offsets"); + } + let address_count = (symbol_table - address_area) / ADDRESS_RECORD_SIZE; + let symbol_slot_count = (shortcut_table - symbol_table) / SYMBOL_SLOT_SIZE; + if symbol_slot_count != 0 && !symbol_slot_count.is_power_of_two() { + anyhow::bail!(".gdb_index symbol table size is not a power of two"); + } + + Ok(Self { + data, + version, + layout, + compilation_unit_count, + type_unit_count, + units, + address_count, + symbol_slot_count, + symbol_names: OnceLock::new(), + }) + } + + pub(crate) fn version(&self) -> u32 { + self.version + } + + pub(crate) fn compilation_unit_count(&self) -> usize { + self.compilation_unit_count + } + + pub(crate) fn validate_debug_info_size(&self, debug_info_size: usize) -> Result<()> { + for index in 0..self.compilation_unit_count { + let record = self.layout.cu_list + index * CU_RECORD_SIZE; + let offset = usize::try_from(Self::read_u64(&self.data, record)?)?; + let length = usize::try_from(Self::read_u64( + &self.data, + record + std::mem::size_of::(), + )?)?; + let end = offset + .checked_add(length) + .ok_or_else(|| anyhow::anyhow!(".gdb_index CU range overflow"))?; + if end > debug_info_size { + anyhow::bail!( + ".gdb_index CU {index} range 0x{offset:x}..0x{end:x} exceeds .debug_info size 0x{debug_info_size:x}" + ); + } + } + for index in 0..self.type_unit_count { + let record = self.layout.type_cu_list + index * TYPE_CU_RECORD_SIZE; + let cu_offset = usize::try_from(Self::read_u64(&self.data, record)?)?; + let type_offset = usize::try_from(Self::read_u64( + &self.data, + record + std::mem::size_of::(), + )?)?; + let type_die_offset = cu_offset + .checked_add(type_offset) + .ok_or_else(|| anyhow::anyhow!(".gdb_index type CU offset overflow"))?; + if cu_offset >= debug_info_size || type_die_offset >= debug_info_size { + anyhow::bail!( + ".gdb_index type CU {index} offsets 0x{cu_offset:x}/0x{type_offset:x} exceed .debug_info size 0x{debug_info_size:x}" + ); + } + } + for index in 0..self.address_count { + let (_, _, cu_index) = self.read_address(index)?; + let Some(unit) = self.units.get(cu_index) else { + anyhow::bail!( + ".gdb_index address record {index} references invalid CU index {cu_index}" + ); + }; + if unit.kind != GdbUnitKind::Compilation { + anyhow::bail!( + ".gdb_index address record {index} references type CU index {cu_index}" + ); + } + } + Ok(()) + } + + pub(crate) fn validate_unit_headers(&self, dwarf: &gimli::Dwarf) -> Result<()> { + for (index, unit) in self.units.iter().enumerate() { + let header = dwarf.unit_header(unit.offset).map_err(|error| { + anyhow::anyhow!( + ".gdb_index CU {index} at 0x{:x} has an invalid DWARF header: {error}", + unit.offset.0 + ) + })?; + let actual_kind = match header.type_() { + gimli::UnitType::Type { .. } | gimli::UnitType::SplitType { .. } => { + GdbUnitKind::Type + } + _ => GdbUnitKind::Compilation, + }; + if actual_kind != unit.kind { + anyhow::bail!( + ".gdb_index CU {index} at 0x{:x} has the wrong DWARF unit type", + unit.offset.0 + ); + } + } + Ok(()) + } + + pub(crate) fn validate_symbol_data(&self) -> Result<()> { + let unit_count = self.units.len(); + for slot in 0..self.symbol_slot_count { + let (name_offset, vector_offset) = self.read_symbol_slot(slot)?; + if name_offset == 0 && vector_offset == 0 { + continue; + } + + self.read_pool_string(name_offset)?; + let (values_start, count) = self.symbol_vector(slot)?; + for index in 0..count { + let offset = values_start + .checked_add(index * std::mem::size_of::()) + .ok_or_else(|| anyhow::anyhow!(".gdb_index CU vector offset overflow"))?; + let attributes = Self::read_u32(&self.data, offset)?; + let cu_index = usize::try_from(attributes & CU_INDEX_MASK)?; + if cu_index >= unit_count { + anyhow::bail!( + ".gdb_index symbol slot {slot} references invalid CU index {cu_index}" + ); + } + } + } + Ok(()) + } + + pub(crate) fn lookup_symbol(&self, name: &str, kind: GdbSymbolKind) -> Result> { + let Some(slot) = self.find_symbol_slot(name)? else { + return Ok(Vec::new()); + }; + self.symbols_in_slot(slot, Some(kind)) + } + + pub(crate) fn lookup_matching_symbols( + &self, + query: &str, + kind: GdbSymbolKind, + ) -> Result> { + let normalized_query = normalize_demangled_signature(query); + let mut symbols = Vec::new(); + for slot in 0..self.symbol_slot_count { + let (name_offset, vector_offset) = self.read_symbol_slot(slot)?; + if name_offset == 0 && vector_offset == 0 { + continue; + } + let candidate = self.read_pool_string(name_offset)?; + if symbol_name_matches_query(query, normalized_query.as_deref(), &candidate, None) { + symbols.extend(self.symbols_in_slot(slot, Some(kind))?); + } + } + symbols.sort_unstable_by_key(|symbol| symbol.cu_offset.0); + symbols.dedup(); + Ok(symbols) + } + + pub(crate) fn symbol_names(&self, kind: GdbSymbolKind) -> Result<&[String]> { + if self.symbol_names.get().is_none() { + let names = self.collect_symbol_names()?; + let _ = self.symbol_names.set(names); + } + Ok(self + .symbol_names + .get() + .expect("GDB symbol names must be initialized") + .get(kind)) + } + + pub(crate) fn find_cu_by_address( + &self, + address: u64, + ) -> Result> { + let mut left = 0usize; + let mut right = self.address_count; + while left < right { + let middle = left + (right - left) / 2; + let (low, _, _) = self.read_address(middle)?; + if low <= address { + left = middle + 1; + } else { + right = middle; + } + } + + for index in (0..left).rev() { + let (low, high, cu_index) = self.read_address(index)?; + if low <= address && address < high { + return self.unit_offset(cu_index).map(Some); + } + if high <= address { + break; + } + } + Ok(None) + } + + fn validate_layout( + length: usize, + header_size: usize, + version: u32, + layout: GdbIndexLayout, + ) -> Result<()> { + let offsets = [ + layout.cu_list, + layout.type_cu_list, + layout.address_area, + layout.symbol_table, + layout.shortcut_table, + layout.constant_pool, + length, + ]; + if layout.cu_list < header_size { + anyhow::bail!(".gdb_index CU list overlaps its header"); + } + if offsets.windows(2).any(|pair| pair[0] > pair[1]) { + anyhow::bail!(".gdb_index areas are not ordered"); + } + if layout.constant_pool > length { + anyhow::bail!(".gdb_index constant pool is outside the section"); + } + if (layout.type_cu_list - layout.cu_list) % CU_RECORD_SIZE != 0 { + anyhow::bail!(".gdb_index CU list has a partial record"); + } + if (layout.address_area - layout.type_cu_list) % TYPE_CU_RECORD_SIZE != 0 { + anyhow::bail!(".gdb_index type CU list has a partial record"); + } + if (layout.symbol_table - layout.address_area) % ADDRESS_RECORD_SIZE != 0 { + anyhow::bail!(".gdb_index address area has a partial record"); + } + if (layout.shortcut_table - layout.symbol_table) % SYMBOL_SLOT_SIZE != 0 { + anyhow::bail!(".gdb_index symbol table has a partial slot"); + } + if version == 9 && layout.constant_pool - layout.shortcut_table < SYMBOL_SLOT_SIZE { + anyhow::bail!(".gdb_index shortcut table is truncated"); + } + Ok(()) + } + + fn find_symbol_slot(&self, name: &str) -> Result> { + if self.symbol_slot_count == 0 { + return Ok(None); + } + + let hash = self.symbol_hash(name.as_bytes()); + let mask = self.symbol_slot_count - 1; + let mut slot = hash as usize & mask; + let step = ((hash as usize).wrapping_mul(17) & mask) | 1; + for _ in 0..self.symbol_slot_count { + let (name_offset, vector_offset) = self.read_symbol_slot(slot)?; + if name_offset == 0 && vector_offset == 0 { + return Ok(None); + } + if self.read_pool_string(name_offset)? == name { + return Ok(Some(slot)); + } + slot = slot.wrapping_add(step) & mask; + } + Ok(None) + } + + fn symbol_hash(&self, name: &[u8]) -> u32 { + name.iter().fold(0u32, |hash, byte| { + let byte = if self.version >= 5 { + byte.to_ascii_lowercase() + } else { + *byte + }; + hash.wrapping_mul(67) + .wrapping_add(u32::from(byte)) + .wrapping_sub(113) + }) + } + + fn collect_symbol_names(&self) -> Result { + let mut names = GdbSymbolNames::default(); + for slot in 0..self.symbol_slot_count { + let (name_offset, vector_offset) = self.read_symbol_slot(slot)?; + if name_offset == 0 && vector_offset == 0 { + continue; + } + let name = self.read_pool_string(name_offset)?; + for symbol in self.symbols_in_slot(slot, None)? { + names.get_mut(symbol.kind).push(name.clone()); + } + } + names.sort_and_deduplicate(); + Ok(names) + } + + fn symbols_in_slot(&self, slot: usize, kind: Option) -> Result> { + let (values_start, count) = self.symbol_vector(slot)?; + + let mut symbols = Vec::new(); + for index in 0..count { + let offset = values_start + .checked_add(index * std::mem::size_of::()) + .ok_or_else(|| anyhow::anyhow!(".gdb_index CU vector offset overflow"))?; + let attributes = Self::read_u32(&self.data, offset)?; + let Some(symbol_kind) = GdbSymbolKind::from_attributes(attributes) else { + continue; + }; + if kind.is_some_and(|expected| expected != symbol_kind) { + continue; + } + let cu_index = usize::try_from(attributes & CU_INDEX_MASK)?; + symbols.push(GdbSymbol { + cu_offset: self.unit_offset(cu_index)?, + kind: symbol_kind, + is_static: attributes & (1 << 31) != 0, + }); + } + symbols.sort_unstable_by_key(|symbol| symbol.cu_offset.0); + symbols.dedup(); + Ok(symbols) + } + + fn symbol_vector(&self, slot: usize) -> Result<(usize, usize)> { + let (_, vector_offset) = self.read_symbol_slot(slot)?; + let vector = self.pool_offset(vector_offset)?; + let count = usize::try_from(Self::read_u32(&self.data, vector)?)?; + let values_start = vector + .checked_add(std::mem::size_of::()) + .ok_or_else(|| anyhow::anyhow!(".gdb_index CU vector offset overflow"))?; + let available = self + .data + .len() + .checked_sub(values_start) + .ok_or_else(|| anyhow::anyhow!(".gdb_index CU vector is truncated"))?; + if count > available / std::mem::size_of::() { + anyhow::bail!(".gdb_index CU vector is truncated"); + } + Ok((values_start, count)) + } + + fn read_address(&self, index: usize) -> Result<(u64, u64, usize)> { + if index >= self.address_count { + anyhow::bail!(".gdb_index address index is out of bounds"); + } + let offset = self.layout.address_area + index * ADDRESS_RECORD_SIZE; + let low = Self::read_u64(&self.data, offset)?; + let high = Self::read_u64(&self.data, offset + std::mem::size_of::())?; + let cu_index = usize::try_from(Self::read_u32( + &self.data, + offset + 2 * std::mem::size_of::(), + )?)?; + Ok((low, high, cu_index)) + } + + fn unit_offset(&self, index: usize) -> Result { + self.units + .get(index) + .map(|unit| unit.offset) + .ok_or_else(|| anyhow::anyhow!(".gdb_index CU index {index} is out of bounds")) + } + + fn read_symbol_slot(&self, slot: usize) -> Result<(u32, u32)> { + if slot >= self.symbol_slot_count { + anyhow::bail!(".gdb_index symbol slot is out of bounds"); + } + let offset = self.layout.symbol_table + slot * SYMBOL_SLOT_SIZE; + Ok(( + Self::read_u32(&self.data, offset)?, + Self::read_u32(&self.data, offset + std::mem::size_of::())?, + )) + } + + fn read_pool_string(&self, relative_offset: u32) -> Result { + let offset = self.pool_offset(relative_offset)?; + let mut reader = self.reader_at(offset)?; + let string = reader.read_null_terminated_slice()?; + Ok(string.to_string_lossy()?.into_owned()) + } + + fn pool_offset(&self, relative_offset: u32) -> Result { + let offset = self + .layout + .constant_pool + .checked_add(usize::try_from(relative_offset)?) + .ok_or_else(|| anyhow::anyhow!(".gdb_index constant-pool offset overflow"))?; + if offset >= self.data.len() { + anyhow::bail!(".gdb_index constant-pool offset is out of bounds"); + } + Ok(offset) + } + + fn read_offset(data: &DwarfReader, word_index: usize) -> Result { + usize::try_from(Self::read_u32( + data, + word_index * std::mem::size_of::(), + )?) + .map_err(Into::into) + } + + fn read_u32(data: &DwarfReader, offset: usize) -> Result { + let mut reader = Self::reader_at_data(data, offset)?; + Ok(reader.read_u32()?) + } + + fn read_u64(data: &DwarfReader, offset: usize) -> Result { + let mut reader = Self::reader_at_data(data, offset)?; + Ok(reader.read_u64()?) + } + + fn reader_at(&self, offset: usize) -> Result { + Self::reader_at_data(&self.data, offset) + } + + fn reader_at_data(data: &DwarfReader, offset: usize) -> Result { + if offset > data.len() { + anyhow::bail!(".gdb_index read offset is out of bounds"); + } + let mut reader = data.clone(); + reader.skip(offset)?; + Ok(reader) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::binary::dwarf_reader_from_arc_with_endian; + use std::sync::Arc; + + fn write_u32(bytes: &mut [u8], offset: usize, value: u32) { + bytes[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); + } + + fn push_u32(bytes: &mut Vec, value: u32) { + bytes.extend_from_slice(&value.to_le_bytes()); + } + + fn push_u64(bytes: &mut Vec, value: u64) { + bytes.extend_from_slice(&value.to_le_bytes()); + } + + fn fixture_with_symbol( + symbol_name: &str, + attributes: u32, + type_unit: Option<(u64, u64, u64)>, + ) -> Vec { + let mut bytes = vec![0; VERSION_9_HEADER_SIZE]; + let cu_list = bytes.len(); + push_u64(&mut bytes, 0x120); + push_u64(&mut bytes, 0x80); + let type_cu_list = bytes.len(); + if let Some((cu_offset, type_offset, signature)) = type_unit { + push_u64(&mut bytes, cu_offset); + push_u64(&mut bytes, type_offset); + push_u64(&mut bytes, signature); + } + let address_area = bytes.len(); + push_u64(&mut bytes, 0x4000); + push_u64(&mut bytes, 0x4100); + push_u32(&mut bytes, 0); + let symbol_table = bytes.len(); + let slot_count = 8; + bytes.resize(bytes.len() + slot_count * SYMBOL_SLOT_SIZE, 0); + let shortcut_table = bytes.len(); + push_u32(&mut bytes, 0); + push_u32(&mut bytes, 0); + let constant_pool = bytes.len(); + push_u32(&mut bytes, 1); + push_u32(&mut bytes, attributes); + let name_offset = u32::try_from(bytes.len() - constant_pool).unwrap(); + bytes.extend_from_slice(symbol_name.as_bytes()); + bytes.push(0); + + write_u32(&mut bytes, 0, 9); + for (word, value) in [ + cu_list, + type_cu_list, + address_area, + symbol_table, + shortcut_table, + constant_pool, + ] + .into_iter() + .enumerate() + { + write_u32(&mut bytes, (word + 1) * 4, u32::try_from(value).unwrap()); + } + + let reader = dwarf_reader_from_arc_with_endian( + Arc::from(bytes.clone()), + gimli::RunTimeEndian::Little, + ); + let index = GdbIndex::parse(reader).unwrap(); + let hash = index.symbol_hash(symbol_name.as_bytes()); + let slot = hash as usize & (slot_count - 1); + write_u32( + &mut bytes, + symbol_table + slot * SYMBOL_SLOT_SIZE, + name_offset, + ); + write_u32(&mut bytes, symbol_table + slot * SYMBOL_SLOT_SIZE + 4, 0); + bytes + } + + fn fixture() -> Vec { + fixture_with_symbol("target_function", 3 << 28, None) + } + + fn parse(bytes: Vec) -> Result { + GdbIndex::parse(dwarf_reader_from_arc_with_endian( + Arc::from(bytes), + gimli::RunTimeEndian::Little, + )) + } + + #[test] + fn parses_and_queries_version_9_index() { + let index = parse(fixture()).unwrap(); + assert_eq!(index.version(), 9); + assert_eq!(index.compilation_unit_count(), 1); + assert_eq!( + index + .lookup_symbol("target_function", GdbSymbolKind::Function) + .unwrap(), + vec![GdbSymbol { + cu_offset: gimli::DebugInfoOffset(0x120), + kind: GdbSymbolKind::Function, + is_static: false, + }] + ); + assert!(index + .lookup_symbol("missing", GdbSymbolKind::Function) + .unwrap() + .is_empty()); + assert_eq!( + index.find_cu_by_address(0x4080).unwrap(), + Some(gimli::DebugInfoOffset(0x120)) + ); + assert_eq!(index.find_cu_by_address(0x4100).unwrap(), None); + assert_eq!( + index.symbol_names(GdbSymbolKind::Function).unwrap(), + &["target_function".to_string()] + ); + } + + #[test] + fn resolves_symbols_owned_by_type_units() { + let index = parse(fixture_with_symbol( + "target_type", + (1 << 28) | 1, + Some((0x220, 0x30, 0x1234_5678_9abc_def0)), + )) + .unwrap(); + index.validate_debug_info_size(0x251).unwrap(); + assert_eq!( + index + .lookup_symbol("target_type", GdbSymbolKind::Type) + .unwrap(), + vec![GdbSymbol { + cu_offset: gimli::DebugInfoOffset(0x220), + kind: GdbSymbolKind::Type, + is_static: false, + }] + ); + } + + #[test] + fn rejects_type_unit_offsets_outside_debug_info() { + let index = parse(fixture_with_symbol( + "target_type", + (1 << 28) | 1, + Some((0x220, 0x30, 0x1234_5678_9abc_def0)), + )) + .unwrap(); + let error = index + .validate_debug_info_size(0x250) + .unwrap_err() + .to_string(); + assert!( + error.contains("type CU 0 offsets"), + "unexpected error: {error}" + ); + } + + #[test] + fn rejects_out_of_order_areas() { + let mut bytes = fixture(); + write_u32(&mut bytes, 3 * 4, 4); + let error = parse(bytes).unwrap_err().to_string(); + assert!(error.contains("not ordered"), "unexpected error: {error}"); + } + + #[test] + fn rejects_truncated_cu_vectors_before_allocating() { + let mut bytes = fixture(); + let constant_pool = u32::from_le_bytes(bytes[24..28].try_into().unwrap()) as usize; + write_u32(&mut bytes, constant_pool, u32::MAX); + let index = parse(bytes).unwrap(); + let error = index + .lookup_symbol("target_function", GdbSymbolKind::Function) + .unwrap_err() + .to_string(); + assert!(error.contains("truncated"), "unexpected error: {error}"); + } + + #[test] + fn validates_symbol_vectors_before_lazy_selection() { + let mut bytes = fixture(); + let constant_pool = u32::from_le_bytes(bytes[24..28].try_into().unwrap()) as usize; + write_u32(&mut bytes, constant_pool, u32::MAX); + let index = parse(bytes).unwrap(); + let error = index.validate_symbol_data().unwrap_err().to_string(); + assert!(error.contains("truncated"), "unexpected error: {error}"); + } + + #[test] + fn validates_symbol_strings_before_lazy_selection() { + let mut bytes = fixture(); + *bytes.last_mut().unwrap() = b'x'; + let index = parse(bytes).unwrap(); + assert!(index.validate_symbol_data().is_err()); + } + + #[test] + fn validates_compilation_units_against_debug_info_size() { + let index = parse(fixture()).unwrap(); + index.validate_debug_info_size(0x1a0).unwrap(); + let error = index + .validate_debug_info_size(0x19f) + .unwrap_err() + .to_string(); + assert!( + error.contains("exceeds .debug_info size"), + "unexpected error: {error}" + ); + } +} diff --git a/ghostscope-dwarf/src/index/type_index.rs b/ghostscope-dwarf/src/index/type_index.rs index 2ac0ca66..d943b025 100644 --- a/ghostscope-dwarf/src/index/type_index.rs +++ b/ghostscope-dwarf/src/index/type_index.rs @@ -56,6 +56,19 @@ impl TypeNameIndex { Self { by_name } } + pub(crate) fn unit_offsets_for_name(&self, name: &str) -> Vec { + let mut offsets = self + .by_name + .get(name) + .into_iter() + .flatten() + .map(|location| location.cu_offset) + .collect::>(); + offsets.sort_unstable_by_key(|offset| offset.0); + offsets.dedup(); + offsets + } + /// Find an aggregate definition by name and tag, preferring non-declarations pub fn find_aggregate_definition(&self, name: &str, tag: DwTag) -> Option { let cands = match self.by_name.get(name) { diff --git a/ghostscope-dwarf/src/lib.rs b/ghostscope-dwarf/src/lib.rs index ad981f86..496f61da 100644 --- a/ghostscope-dwarf/src/lib.rs +++ b/ghostscope-dwarf/src/lib.rs @@ -22,10 +22,10 @@ pub(crate) mod analyzer; // Re-export main public API only pub use analyzer::{ - AddressQueryResult, AnalyzerStats, DwarfAnalyzer, ExecutableFileInfo, FunctionQueryResult, - LoadedModuleRuntimeInfo, MainExecutableInfo, ModuleDefaultPolicy, ModuleLoadingEvent, - ModuleLoadingStats, ModuleStats, SectionInfo, SharedLibraryInfo, SimpleFileInfo, - SourceLineAddressSearch, SourceLineQuerySearch, TypeLookupAmbiguity, + AddressQueryResult, AnalyzerStats, DwarfAnalyzer, DwarfIndexStatus, ExecutableFileInfo, + FunctionQueryResult, LoadedModuleRuntimeInfo, MainExecutableInfo, ModuleDefaultPolicy, + ModuleLoadingEvent, ModuleLoadingStats, ModuleStats, SectionInfo, SharedLibraryInfo, + SimpleFileInfo, SourceLineAddressSearch, SourceLineQuerySearch, TypeLookupAmbiguity, }; pub use loader::ExplicitDebugFile; diff --git a/ghostscope-dwarf/src/loader.rs b/ghostscope-dwarf/src/loader.rs index 0cbf6206..7f6b44e6 100644 --- a/ghostscope-dwarf/src/loader.rs +++ b/ghostscope-dwarf/src/loader.rs @@ -204,8 +204,7 @@ impl ModuleLoader { match result { Ok(module) => { // Extract stats for progress reporting - let (functions, variables, types) = - module.get_lightweight_index().get_stats(); + let (functions, variables, types) = module.get_index_stats(); let (parse_time_ms, index_time_ms, module_total_time_ms) = module.get_load_timing_ms(); let stats = ModuleLoadingStats { @@ -213,6 +212,7 @@ impl ModuleLoader { variables, types, debug_info_source: module.get_debug_info_source().clone(), + dwarf_index_status: module.dwarf_index_status().clone(), load_time_ms, parse_time_ms, index_time_ms, diff --git a/ghostscope-dwarf/src/objfile/function_lookup.rs b/ghostscope-dwarf/src/objfile/function_lookup.rs index 8070f842..0c4f4354 100644 --- a/ghostscope-dwarf/src/objfile/function_lookup.rs +++ b/ghostscope-dwarf/src/objfile/function_lookup.rs @@ -3,7 +3,7 @@ use crate::{ binary::DwarfReader, core::{demangled_name, normalize_demangled_signature, symbol_name_matches_query, Result}, dwarf_expr::{errors as expr_errors, modes::DwarfExprMode}, - index::LightweightIndex, + index::{GdbSymbolKind, LightweightIndex}, parser::RangeExtractor, semantics::{range_contains_pc, resolve_attr_with_unit_origins, resolve_origin_entry}, }; @@ -18,10 +18,42 @@ impl LoadedObjfile { pub(crate) fn lookup_function_addresses(&self, name: &str) -> Vec { tracing::debug!("LoadedObjfile: looking up function '{}'", name); - let entries = self.lightweight_index.find_dies_by_function_name(name); + if let Err(error) = self.ensure_debug_info_for_symbol(name, GdbSymbolKind::Function, false) + { + tracing::warn!( + "Failed to load indexed DWARF for function '{}' in {}: {}", + name, + self.module_path().display(), + error + ); + } + let seed_entries = self + .lightweight_index + .read() + .expect("lightweight index lock poisoned") + .find_dies_by_function_name(name) + .into_iter() + .cloned() + .collect::>(); + if let Err(error) = self.ensure_debug_info_for_entries(&seed_entries) { + tracing::warn!( + "Failed to materialize indexed DWARF for function '{}' in {}: {}", + name, + self.module_path().display(), + error + ); + } + let entries = self + .lightweight_index + .read() + .expect("lightweight index lock poisoned") + .find_dies_by_function_name(name) + .into_iter() + .cloned() + .collect::>(); let mut addresses = Vec::new(); - for entry in entries { + for entry in &entries { addresses.extend(self.compute_addresses_for_entry(entry)); } @@ -72,7 +104,7 @@ impl LoadedObjfile { } fn matching_fragment_candidate_indices( - &self, + lightweight_index: &LightweightIndex, name: &str, candidate_indices: Vec, tag_filter: impl Fn(gimli::DwTag) -> bool, @@ -81,7 +113,7 @@ impl LoadedObjfile { candidate_indices .into_iter() .filter(|&idx| { - let Some(entry) = self.lightweight_index.entry(idx) else { + let Some(entry) = lightweight_index.entry(idx) else { return false; }; tag_filter(entry.tag) @@ -98,7 +130,7 @@ impl LoadedObjfile { let normalized_query = normalize_demangled_signature(name); let mut matches = Vec::new(); - for idx in 0..lightweight_index.entry_count() { + for idx in (0..lightweight_index.entry_count()).rev() { let Some(entry) = lightweight_index.entry(idx) else { continue; }; @@ -112,11 +144,11 @@ impl LoadedObjfile { matches } - fn matching_function_candidate_indices(&self, name: &str) -> Vec { - let fragment_matches = self.matching_fragment_candidate_indices( + fn function_candidate_indices(lightweight_index: &LightweightIndex, name: &str) -> Vec { + let fragment_matches = Self::matching_fragment_candidate_indices( + lightweight_index, name, - self.lightweight_index - .function_candidate_indices_by_fragment(name), + lightweight_index.function_candidate_indices_by_fragment(name), |tag| { matches!( tag, @@ -130,7 +162,7 @@ impl LoadedObjfile { return fragment_matches; } - Self::scan_matching_candidate_indices(&self.lightweight_index, name, |tag| { + Self::scan_matching_candidate_indices(lightweight_index, name, |tag| { matches!( tag, gimli::constants::DW_TAG_subprogram | gimli::constants::DW_TAG_inlined_subroutine @@ -138,11 +170,11 @@ impl LoadedObjfile { }) } - pub(super) fn matching_variable_candidate_indices(&self, name: &str) -> Vec { - let fragment_matches = self.matching_fragment_candidate_indices( + fn variable_candidate_indices(lightweight_index: &LightweightIndex, name: &str) -> Vec { + let fragment_matches = Self::matching_fragment_candidate_indices( + lightweight_index, name, - self.lightweight_index - .variable_candidate_indices_by_fragment(name), + lightweight_index.variable_candidate_indices_by_fragment(name), |tag| tag == gimli::constants::DW_TAG_variable, ); @@ -150,11 +182,83 @@ impl LoadedObjfile { return fragment_matches; } - Self::scan_matching_candidate_indices(&self.lightweight_index, name, |tag| { + Self::scan_matching_candidate_indices(lightweight_index, name, |tag| { tag == gimli::constants::DW_TAG_variable }) } + fn matching_function_candidate_indices(&self, name: &str) -> Vec { + if let Err(error) = self.ensure_debug_info_for_symbol(name, GdbSymbolKind::Function, true) { + tracing::warn!( + "Failed to load matching indexed DWARF for function '{}' in {}: {}", + name, + self.module_path().display(), + error + ); + } + let seed_entries = { + let lightweight_index = self + .lightweight_index + .read() + .expect("lightweight index lock poisoned"); + Self::function_candidate_indices(&lightweight_index, name) + .into_iter() + .filter_map(|index| lightweight_index.entry(index).cloned()) + .collect::>() + }; + if let Err(error) = self.ensure_debug_info_for_entries(&seed_entries) { + tracing::warn!( + "Failed to materialize matching DWARF for function '{}' in {}: {}", + name, + self.module_path().display(), + error + ); + } + Self::function_candidate_indices( + &self + .lightweight_index + .read() + .expect("lightweight index lock poisoned"), + name, + ) + } + + pub(super) fn matching_variable_candidate_indices(&self, name: &str) -> Vec { + if let Err(error) = self.ensure_debug_info_for_symbol(name, GdbSymbolKind::Variable, true) { + tracing::warn!( + "Failed to load matching indexed DWARF for variable '{}' in {}: {}", + name, + self.module_path().display(), + error + ); + } + let seed_entries = { + let lightweight_index = self + .lightweight_index + .read() + .expect("lightweight index lock poisoned"); + Self::variable_candidate_indices(&lightweight_index, name) + .into_iter() + .filter_map(|index| lightweight_index.entry(index).cloned()) + .collect::>() + }; + if let Err(error) = self.ensure_debug_info_for_entries(&seed_entries) { + tracing::warn!( + "Failed to materialize matching DWARF for variable '{}' in {}: {}", + name, + self.module_path().display(), + error + ); + } + Self::variable_candidate_indices( + &self + .lightweight_index + .read() + .expect("lightweight index lock poisoned"), + name, + ) + } + fn resolve_function_ranges(&self, entry: &crate::core::IndexEntry) -> Result> { if !matches!( entry.tag, @@ -196,13 +300,32 @@ impl LoadedObjfile { pub(super) fn find_function_index_entry_by_address( &self, address: u64, - ) -> Option<&crate::core::IndexEntry> { + ) -> Option { + if let Err(error) = self.ensure_debug_info_for_address(address) { + tracing::warn!( + "Failed to load indexed DWARF for address 0x{:x} in {}: {}", + address, + self.module_path().display(), + error + ); + } self.lightweight_index + .read() + .expect("lightweight index lock poisoned") .find_function_by_address(address, |entry| self.resolve_function_ranges(entry).ok()) + .cloned() } fn compute_addresses_for_entry(&self, entry: &crate::core::IndexEntry) -> Vec { let mut out = Vec::new(); + if let Err(error) = self.ensure_line_info_for_unit(entry.unit_offset) { + tracing::warn!( + "Failed to load line information for function '{}' in {}: {}", + entry.name, + self.module_path().display(), + error + ); + } let ranges = match self.resolve_function_ranges(entry) { Ok(ranges) => ranges, Err(err) => { @@ -246,6 +369,8 @@ impl LoadedObjfile { let entry_pc_has_line_entry = entry.entry_pc.is_some_and(|pc| { self.line_mapping + .read() + .expect("line mapping lock poisoned") .get_entries_in_range(pc, pc) .next() .is_some() @@ -268,7 +393,11 @@ impl LoadedObjfile { let nranges = Self::selected_non_inline_ranges(entry, &ranges, preferred_start); for (start, end) in &nranges { let candidate = { - let first_exec = self.line_mapping.find_first_executable_address(*start); + let first_exec = self + .line_mapping + .read() + .expect("line mapping lock poisoned") + .find_first_executable_address(*start); Self::selected_non_inline_probe_address(*start, *end, first_exec) }; let prefer_entry = self @@ -663,10 +792,19 @@ impl LoadedObjfile { } let mut out = Vec::new(); - for idx in self.matching_function_candidate_indices(name) { - if let Some(entry) = self.lightweight_index.entry(idx) { - out.extend(self.compute_addresses_for_entry(entry)); - } + let candidate_indices = self.matching_function_candidate_indices(name); + let entries = { + let lightweight_index = self + .lightweight_index + .read() + .expect("lightweight index lock poisoned"); + candidate_indices + .into_iter() + .filter_map(|idx| lightweight_index.entry(idx).cloned()) + .collect::>() + }; + for entry in &entries { + out.extend(self.compute_addresses_for_entry(entry)); } out.sort_unstable(); out.dedup(); diff --git a/ghostscope-dwarf/src/objfile/globals.rs b/ghostscope-dwarf/src/objfile/globals.rs index 3b766480..13cfc03b 100644 --- a/ghostscope-dwarf/src/objfile/globals.rs +++ b/ghostscope-dwarf/src/objfile/globals.rs @@ -1,5 +1,6 @@ use super::LoadedObjfile; use crate::core::{GlobalVariableInfo, SectionType}; +use crate::index::GdbSymbolKind; use object::{Object, ObjectSection}; use std::collections::HashSet; @@ -23,36 +24,76 @@ impl LoadedObjfile { let mut out = Vec::new(); let mut seen_offsets: HashSet<(u64, u64)> = HashSet::new(); - for idx in candidate_indices { - if let Some(entry) = self.lightweight_index.entry(idx) { - let key = (entry.unit_offset.0 as u64, entry.die_offset.0 as u64); - if !seen_offsets.insert(key) { - continue; - } - let link_address = entry.representative_addr; - let section = link_address.and_then(|addr| self.classify_section(&obj, addr)); - out.push(GlobalVariableInfo { - name: name.to_string(), - link_address, - section, - die_offset: entry.die_offset, - unit_offset: entry.unit_offset, - }); + let entries = { + let lightweight_index = self + .lightweight_index + .read() + .expect("lightweight index lock poisoned"); + candidate_indices + .into_iter() + .filter_map(|idx| lightweight_index.entry(idx).cloned()) + .collect::>() + }; + for entry in entries { + let key = (entry.unit_offset.0 as u64, entry.die_offset.0 as u64); + if !seen_offsets.insert(key) { + continue; } + let link_address = entry.representative_addr; + let section = link_address.and_then(|addr| self.classify_section(&obj, addr)); + out.push(GlobalVariableInfo { + name: name.to_string(), + link_address, + section, + die_offset: entry.die_offset, + unit_offset: entry.unit_offset, + }); } out } pub(crate) fn find_global_variables_by_name(&self, name: &str) -> Vec { + if let Err(error) = self.ensure_debug_info_for_symbol(name, GdbSymbolKind::Variable, false) + { + tracing::warn!( + "Failed to load indexed DWARF for global '{}' in {}: {}", + name, + self.module_path().display(), + error + ); + } let mut out = Vec::new(); - let entries = self.lightweight_index.find_variables_by_name(name); + let seed_entries = self + .lightweight_index + .read() + .expect("lightweight index lock poisoned") + .find_variables_by_name(name) + .into_iter() + .cloned() + .collect::>(); + if let Err(error) = self.ensure_debug_info_for_entries(&seed_entries) { + tracing::warn!( + "Failed to materialize indexed DWARF for global '{}' in {}: {}", + name, + self.module_path().display(), + error + ); + } + let entries = self + .lightweight_index + .read() + .expect("lightweight index lock poisoned") + .find_variables_by_name(name) + .into_iter() + .cloned() + .collect::>(); let mut seen_offsets: HashSet<(u64, u64)> = HashSet::new(); let obj = match object::File::parse(&self._binary_mapped_file.data[..]) { Ok(f) => f, Err(_) => { - for e in entries { + for e in &entries { let key = (e.unit_offset.0 as u64, e.die_offset.0 as u64); if !seen_offsets.insert(key) { continue; @@ -70,7 +111,7 @@ impl LoadedObjfile { } }; - for e in entries { + for e in &entries { let key = (e.unit_offset.0 as u64, e.die_offset.0 as u64); if !seen_offsets.insert(key) { continue; @@ -135,8 +176,8 @@ impl LoadedObjfile { } }; - for name in self.lightweight_index.get_variable_names() { - for info in self.find_global_variables_by_name(name) { + for name in self.get_variable_names() { + for info in self.find_global_variables_by_name(&name) { out.push(info); } } diff --git a/ghostscope-dwarf/src/objfile/loaded.rs b/ghostscope-dwarf/src/objfile/loaded.rs index fe98da7e..0ff16190 100644 --- a/ghostscope-dwarf/src/objfile/loaded.rs +++ b/ghostscope-dwarf/src/objfile/loaded.rs @@ -4,16 +4,17 @@ use crate::{ binary::{DwarfReader, MappedFile}, core::{mapping::ModuleMapping, DebugInfoSource, Result}, index::{ - BlockIndex, LightweightIndex, LineMappingTable, ScopedFileIndexManager, TypeNameIndex, + BlockIndex, GdbIndex, GdbSymbolKind, LightweightIndex, LineMappingTable, + ScopedFileIndexManager, TypeNameIndex, }, objfile::ModuleUnwindInfo, - parser::{CompilationUnit, DetailedParser}, + parser::{CompilationUnit, DetailedParser, DwarfParser}, }; use object::{Object, ObjectSegment}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, path::PathBuf, - sync::{Arc, RwLock}, + sync::{Arc, Mutex, RwLock}, }; type FunctionRangeCacheKey = (u64, u64); @@ -22,10 +23,12 @@ type FunctionRangeCacheKey = (u64, u64); #[derive(Debug)] pub(crate) struct LoadedObjfile { pub(super) module_mapping: ModuleMapping, - pub(super) lightweight_index: LightweightIndex, - pub(super) line_mapping: LineMappingTable, - pub(super) scoped_file_manager: ScopedFileIndexManager, - pub(super) compilation_units: HashMap, + pub(super) lightweight_index: RwLock, + pub(super) line_mapping: RwLock, + pub(super) scoped_file_manager: RwLock, + pub(super) compilation_units: RwLock>, + pub(super) line_source_unit_offsets: HashMap>, + pub(super) indexed_line_cus: Mutex>, pub(super) unwind_info: ModuleUnwindInfo, pub(super) dwarf: gimli::Dwarf, pub(super) detailed_parser: DetailedParser, @@ -36,7 +39,11 @@ pub(crate) struct LoadedObjfile { pub(super) text_symbol_starts_by_name: HashMap>, pub(super) function_ranges_cache: RwLock>>, pub(super) block_index: RwLock, - pub(super) type_name_index: Arc, + pub(super) type_name_index: RwLock, + pub(super) gdb_index: Option, + pub(super) dwarf_index_status: crate::DwarfIndexStatus, + pub(super) lazy_debug_info: bool, + pub(super) indexed_debug_info_cus: Mutex>, pub(super) load_parse_ms: u64, pub(super) load_index_ms: u64, pub(super) load_total_ms: u64, @@ -66,20 +73,75 @@ impl LoadedObjfile { self.entry_address } - pub(crate) fn get_function_names(&self) -> Vec<&String> { - self.lightweight_index.get_function_names() + pub(crate) fn get_function_names(&self) -> Vec { + if let Some(index) = &self.gdb_index { + match index.symbol_names(GdbSymbolKind::Function) { + Ok(names) => return names.to_vec(), + Err(error) => tracing::warn!( + "Failed to read .gdb_index function names for {}: {}", + self.module_path().display(), + error + ), + } + } + self.lightweight_index + .read() + .expect("lightweight index lock poisoned") + .get_function_names() + .into_iter() + .cloned() + .collect() } - pub(crate) fn get_variable_names(&self) -> Vec<&String> { - self.lightweight_index.get_variable_names() + pub(crate) fn get_variable_names(&self) -> Vec { + if let Some(index) = &self.gdb_index { + match index.symbol_names(GdbSymbolKind::Variable) { + Ok(names) => return names.to_vec(), + Err(error) => tracing::warn!( + "Failed to read .gdb_index variable names for {}: {}", + self.module_path().display(), + error + ), + } + } + self.lightweight_index + .read() + .expect("lightweight index lock poisoned") + .get_variable_names() + .into_iter() + .cloned() + .collect() } - pub(crate) fn get_lightweight_index(&self) -> &LightweightIndex { - &self.lightweight_index + pub(crate) fn get_index_stats(&self) -> (usize, usize, usize) { + if let Some(index) = &self.gdb_index { + let functions = index + .symbol_names(GdbSymbolKind::Function) + .map_or(0, <[String]>::len); + let variables = index + .symbol_names(GdbSymbolKind::Variable) + .map_or(0, <[String]>::len); + let types = index + .symbol_names(GdbSymbolKind::Type) + .map_or(0, <[String]>::len); + return (functions, variables, functions + variables + types); + } + self.lightweight_index + .read() + .expect("lightweight index lock poisoned") + .get_stats() + } + + pub(crate) fn dwarf_index_status(&self) -> &crate::DwarfIndexStatus { + &self.dwarf_index_status } pub(crate) fn get_line_header_count(&self) -> usize { - self.scoped_file_manager.get_stats().1 + self.scoped_file_manager + .read() + .expect("scoped file index lock poisoned") + .get_stats() + .1 } pub(crate) fn has_dwarf_info(&self) -> bool { @@ -105,6 +167,175 @@ impl LoadedObjfile { &self.dwarf } + pub(super) fn ensure_debug_info_for_symbol( + &self, + name: &str, + kind: GdbSymbolKind, + match_variants: bool, + ) -> Result<()> { + let Some(index) = &self.gdb_index else { + return Ok(()); + }; + let symbols = if match_variants { + index.lookup_matching_symbols(name, kind)? + } else { + index.lookup_symbol(name, kind)? + }; + self.ensure_debug_info_cus(symbols.into_iter().map(|symbol| symbol.cu_offset)) + } + + pub(super) fn ensure_debug_info_for_address(&self, address: u64) -> Result<()> { + if !self.lazy_debug_info { + return Ok(()); + } + let gdb_cu = self + .gdb_index + .as_ref() + .map(|index| index.find_cu_by_address(address)) + .transpose()? + .flatten(); + let cu_offset = gdb_cu.or_else(|| { + self.lightweight_index + .read() + .expect("lightweight index lock poisoned") + .find_cu_by_address(address) + }); + let Some(cu_offset) = cu_offset else { + return Ok(()); + }; + self.ensure_debug_info_cus(std::iter::once(cu_offset)) + } + + pub(super) fn ensure_debug_info_for_entries( + &self, + entries: &[crate::core::IndexEntry], + ) -> Result<()> { + if !self.lazy_debug_info { + return Ok(()); + } + self.ensure_debug_info_cus(entries.iter().map(|entry| entry.unit_offset)) + } + + pub(super) fn ensure_debug_info_for_type_name(&self, name: &str) -> Result<()> { + self.ensure_debug_info_for_symbol(name, GdbSymbolKind::Type, true)?; + if !self.lazy_debug_info { + return Ok(()); + } + let unit_offsets = self + .type_name_index + .read() + .expect("type name index lock poisoned") + .unit_offsets_for_name(name); + self.ensure_debug_info_cus(unit_offsets) + } + + pub(super) fn ensure_line_info_for_unit( + &self, + unit_offset: gimli::DebugInfoOffset, + ) -> Result<()> { + self.ensure_line_info_cus(std::iter::once(unit_offset)) + } + + pub(super) fn ensure_line_info_for_address(&self, address: u64) -> Result<()> { + let gdb_cu = self + .gdb_index + .as_ref() + .map(|index| index.find_cu_by_address(address)) + .transpose()? + .flatten(); + let unit_offset = gdb_cu.or_else(|| { + self.lightweight_index + .read() + .expect("lightweight index lock poisoned") + .find_cu_by_address(address) + }); + if let Some(unit_offset) = unit_offset { + self.ensure_line_info_for_unit(unit_offset)?; + } + Ok(()) + } + + pub(super) fn ensure_line_info_for_source(&self, file_path: &str) -> Result<()> { + let unit_offsets = self + .line_source_unit_offsets + .iter() + .filter(|(indexed_path, _)| { + crate::path_match::source_path_matches(indexed_path, file_path) + }) + .flat_map(|(_, offsets)| offsets.iter().copied()) + .collect::>(); + self.ensure_line_info_cus(unit_offsets) + } + + fn ensure_debug_info_cus( + &self, + unit_offsets: impl IntoIterator, + ) -> Result<()> { + let mut loaded = self + .indexed_debug_info_cus + .lock() + .expect("indexed CU lock poisoned"); + let mut missing = unit_offsets + .into_iter() + .filter(|unit_offset| !loaded.contains(unit_offset)) + .collect::>(); + missing.sort_unstable_by_key(|offset| offset.0); + missing.dedup(); + if missing.is_empty() { + return Ok(()); + } + + let shards = DwarfParser::new(&self.dwarf).parse_debug_info_cus(&missing)?; + let mut lightweight_index = self + .lightweight_index + .write() + .expect("lightweight index lock poisoned"); + lightweight_index.append_shards(shards); + *self + .type_name_index + .write() + .expect("type name index lock poisoned") = + TypeNameIndex::build_from_lightweight(&lightweight_index); + loaded.extend(missing); + Ok(()) + } + + fn ensure_line_info_cus( + &self, + unit_offsets: impl IntoIterator, + ) -> Result<()> { + let mut loaded = self + .indexed_line_cus + .lock() + .expect("indexed line CU lock poisoned"); + let mut missing = unit_offsets + .into_iter() + .filter(|unit_offset| !loaded.contains(unit_offset)) + .collect::>(); + missing.sort_unstable_by_key(|offset| offset.0); + missing.dedup(); + if missing.is_empty() { + return Ok(()); + } + + let module_path = self.module_path().to_string_lossy(); + let result = DwarfParser::new(&self.dwarf).parse_line_info_cus(&module_path, &missing)?; + self.line_mapping + .write() + .expect("line mapping lock poisoned") + .extend(result.line_mapping); + self.scoped_file_manager + .write() + .expect("scoped file index lock poisoned") + .extend(result.scoped_file_manager); + self.compilation_units + .write() + .expect("compilation unit lock poisoned") + .extend(result.compilation_units); + loaded.extend(missing); + Ok(()) + } + pub(crate) fn get_load_timing_ms(&self) -> (u64, u64, u64) { (self.load_parse_ms, self.load_index_ms, self.load_total_ms) } diff --git a/ghostscope-dwarf/src/objfile/loading.rs b/ghostscope-dwarf/src/objfile/loading.rs index ef1b45a1..e4b1a4e1 100644 --- a/ghostscope-dwarf/src/objfile/loading.rs +++ b/ghostscope-dwarf/src/objfile/loading.rs @@ -1,18 +1,33 @@ use super::LoadedObjfile; use crate::{ + analyzer::DwarfIndexStatus, binary::{ dwarf_endian_from_object, dwarf_reader_from_arc_with_endian, empty_dwarf_reader_with_endian, load_explicit_debug_file, try_load_debug_file, DwarfData, MappedFile, }, core::{mapping::ModuleMapping, DebugInfoSource, Result}, - index::{BlockIndex, TypeNameIndex}, + index::{BlockIndex, GdbIndex, TypeNameIndex}, objfile::ModuleUnwindInfo, parser::DetailedParser, }; use ghostscope_debuginfod::{build_id_to_hex, DebuginfodClient}; +use gimli::{Reader, Section}; use object::{Object, ObjectSection, ObjectSymbol, SymbolKind}; -use std::{borrow::Cow, collections::HashMap, path::Path, path::PathBuf, sync::Arc, time::Instant}; +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + path::Path, + path::PathBuf, + sync::{Arc, Mutex, RwLock}, + time::Instant, +}; + +enum InitialIndexSelection { + FullScan { rejection: Option }, + DebugNames, + GdbIndex, +} impl LoadedObjfile { /// Parallel loading: debug_info || debug_line || CFI simultaneously. @@ -166,6 +181,28 @@ impl LoadedObjfile { }; let mapped_file = mapped_file_for_dwarf; + let (gdb_index, gdb_index_rejection) = match Self::load_gdb_index(&mapped_file, &dwarf) { + Ok(Some(index)) => { + tracing::info!( + "Found .gdb_index v{} with {} compilation units in {}", + index.version(), + index.compilation_unit_count(), + mapped_file.path.display() + ); + (Some(index), None) + } + Ok(None) => (None, None), + Err(error) => { + tracing::warn!( + "Ignoring unusable .gdb_index in {}: {}", + mapped_file.path.display(), + error + ); + (None, Some(error.to_string())) + } + }; + let has_gdb_index = gdb_index.is_some(); + let initial_rejection = gdb_index_rejection.clone(); tracing::debug!( "Starting parallel DWARF parsing with true debug_line || debug_info parallelism..." @@ -175,19 +212,78 @@ impl LoadedObjfile { tokio::task::spawn_blocking({ let dwarf = Arc::clone(&dwarf); let module_path = module_mapping.path.to_string_lossy().to_string(); - move || -> Result<(crate::parser::LineParseResult, crate::parser::DebugParseResult)> { + move || -> Result<( + crate::parser::LineParseResult, + crate::parser::DebugParseResult, + InitialIndexSelection, + )> { let (line_res, info_res) = rayon::join( || { let parser = crate::parser::DwarfParser::new(&dwarf); - parser.parse_line_info(&module_path) + parser.parse_line_headers(&module_path) }, || { let parser = crate::parser::DwarfParser::new(&dwarf); - parser.parse_debug_info(&module_path) + match parser.parse_debug_names(&module_path) { + Ok(Some(index)) => Ok((index, InitialIndexSelection::DebugNames)), + Ok(None) if has_gdb_index => { + Ok(( + parser.initialize_lazy_debug_info(), + InitialIndexSelection::GdbIndex, + )) + } + Ok(None) => parser + .parse_debug_info(&module_path) + .map(|index| { + ( + index, + InitialIndexSelection::FullScan { + rejection: initial_rejection, + }, + ) + }), + Err(error) => { + tracing::warn!( + "Ignoring unusable .debug_names for {}: {}", + module_path, + error + ); + if has_gdb_index { + Ok(( + parser.initialize_lazy_debug_info(), + InitialIndexSelection::GdbIndex, + )) + } else { + let debug_names_rejection = format!( + ".debug_names could not be used: {error}" + ); + let rejection = initial_rejection.map_or( + debug_names_rejection.clone(), + |gdb_rejection| { + format!( + ".gdb_index could not be used: {gdb_rejection}; {debug_names_rejection}" + ) + }, + ); + parser + .parse_debug_info(&module_path) + .map(|index| { + ( + index, + InitialIndexSelection::FullScan { + rejection: Some(rejection), + }, + ) + }) + } + } + } }, ); match (line_res, info_res) { - (Ok(l), Ok(i)) => Ok((l, i)), + (Ok(line), Ok((info, selection))) => { + Ok((line, info, selection)) + } (Err(e), _) => Err(e), (_, Err(e)) => Err(e), } @@ -200,7 +296,24 @@ impl LoadedObjfile { }) )?; - let (line_result, info_result) = pair_result?; + let (line_result, info_result, index_selection) = pair_result?; + let lazy_debug_info = !matches!(index_selection, InitialIndexSelection::FullScan { .. }); + let (gdb_index, dwarf_index_status) = match index_selection { + InitialIndexSelection::FullScan { rejection } => { + let status = rejection.map_or(DwarfIndexStatus::FullScan, |reason| { + DwarfIndexStatus::Rejected { reason } + }); + (None, status) + } + InitialIndexSelection::DebugNames => (None, DwarfIndexStatus::DebugNames), + InitialIndexSelection::GdbIndex => { + let index = gdb_index.expect("selected GDB index must be loaded"); + let status = DwarfIndexStatus::GdbIndex { + version: index.version(), + }; + (Some(index), status) + } + }; let parse_result = crate::parser::DwarfParser::combine_parallel_results( line_result, @@ -226,6 +339,7 @@ impl LoadedObjfile { line_mapping, scoped_file_manager, compilation_units, + line_source_unit_offsets, stats, } = parse_result; let index_started_at = Instant::now(); @@ -234,11 +348,9 @@ impl LoadedObjfile { (lightweight_index, type_name_index) }) .await?; - let type_name_index = Arc::new(type_name_index); let dwarf = Arc::try_unwrap(dwarf).map_err(|_| anyhow::anyhow!("Failed to unwrap DWARF Arc"))?; - let mut detailed_parser = DetailedParser::new(); - detailed_parser.set_type_name_index(Arc::clone(&type_name_index)); + let detailed_parser = DetailedParser::new(); let entry_address = Self::read_entry_address(&binary_mapped); let text_symbol_starts_by_name = Self::collect_text_symbol_starts(&binary_mapped); @@ -267,15 +379,21 @@ impl LoadedObjfile { let module = Self { module_mapping: module_mapping.clone(), - lightweight_index, - line_mapping, - scoped_file_manager, - compilation_units, + lightweight_index: RwLock::new(lightweight_index), + line_mapping: RwLock::new(line_mapping), + scoped_file_manager: RwLock::new(scoped_file_manager), + compilation_units: RwLock::new(compilation_units), + line_source_unit_offsets, + indexed_line_cus: Mutex::new(HashSet::new()), unwind_info, dwarf, detailed_parser, block_index: std::sync::RwLock::new(BlockIndex::new()), - type_name_index, + type_name_index: RwLock::new(type_name_index), + gdb_index, + dwarf_index_status, + lazy_debug_info, + indexed_debug_info_cus: Mutex::new(HashSet::new()), _dwarf_mapped_file: mapped_file, _binary_mapped_file: binary_mapped, debug_info_source, @@ -287,11 +405,18 @@ impl LoadedObjfile { load_total_ms, }; + let (function_count, variable_count, _) = module.get_index_stats(); + tracing::debug!( + "Initial DWARF index materialized {} functions and {} variables for {}", + stats.total_functions, + stats.total_variables, + module.module_mapping.path.display() + ); tracing::info!( "True parallel loading completed for {}: {} functions, {} variables, {} line entries, {} files (state: {}, parse_ms: {}, index_ms: {}, total_ms: {})", module.module_mapping.path.display(), - stats.total_functions, - stats.total_variables, + function_count, + variable_count, stats.total_line_entries, stats.total_files, state_label, @@ -572,6 +697,38 @@ impl LoadedObjfile { by_name } + fn load_gdb_index(file_data: &Arc, dwarf: &DwarfData) -> Result> { + let object = file_data.parse_object()?; + let reader = if let Some(section) = object.section_by_name(".gdb_index") { + let compressed_range = section.compressed_file_range()?; + if compressed_range.format != object::read::CompressionFormat::None { + let data = section.uncompressed_data()?; + let bytes: Arc<[u8]> = match data { + Cow::Borrowed(bytes) => Arc::from(bytes), + Cow::Owned(bytes) => Arc::from(bytes), + }; + dwarf_reader_from_arc_with_endian(bytes, gimli::RunTimeEndian::Little) + } else if let Some((start, size)) = section.file_range() { + MappedFile::dwarf_reader_range( + Arc::clone(file_data), + start, + size, + gimli::RunTimeEndian::Little, + ) + .ok_or_else(|| anyhow::anyhow!("invalid .gdb_index section range"))? + } else { + return Ok(None); + } + } else { + return Ok(None); + }; + let index = GdbIndex::parse(reader)?; + index.validate_debug_info_size(dwarf.debug_info.reader().len())?; + index.validate_unit_headers(dwarf)?; + index.validate_symbol_data()?; + Ok(Some(index)) + } + fn load_dwarf_sections(file_data: &Arc) -> Result { let object = file_data.parse_object()?; ghostscope_process::ensure_supported_target_object(&object, &file_data.path)?; diff --git a/ghostscope-dwarf/src/objfile/source_location.rs b/ghostscope-dwarf/src/objfile/source_location.rs index bc5d8720..1fa02b52 100644 --- a/ghostscope-dwarf/src/objfile/source_location.rs +++ b/ghostscope-dwarf/src/objfile/source_location.rs @@ -13,10 +13,20 @@ mod file_selection_scoring { impl LoadedObjfile { pub(crate) fn lookup_source_location(&self, address: u64) -> Option { - let all_line_entries = self.line_mapping.lookup_all_lines_at_address(address); + if let Err(error) = self.ensure_line_info_for_address(address) { + tracing::warn!( + "Failed to load line information for address 0x{address:x} in {}: {error}", + self.module_path().display() + ); + } + let line_mapping = self + .line_mapping + .read() + .expect("line mapping lock poisoned"); + let all_line_entries = line_mapping.lookup_all_lines_at_address(address); if all_line_entries.is_empty() { - if let Some(line_entry) = self.line_mapping.lookup_line(address) { + if let Some(line_entry) = line_mapping.lookup_line(address) { return self.create_source_location_from_entry(line_entry); } return None; @@ -24,7 +34,8 @@ impl LoadedObjfile { let best_entry = if all_line_entries.len() == 1 { let entry = all_line_entries[0]; - self.find_alternative_source_file(entry).unwrap_or(entry) + self.find_alternative_source_file(&line_mapping, entry) + .unwrap_or(entry) } else { self.select_best_line_entry(&all_line_entries) }; @@ -38,7 +49,19 @@ impl LoadedObjfile { file_path: &str, line_number: u32, ) -> Option { - let all_line_entries = self.line_mapping.lookup_all_lines_at_address(address); + if let Err(error) = self.ensure_line_info_for_source(file_path) { + tracing::warn!( + "Failed to load line information for {} in {}: {}", + file_path, + self.module_path().display(), + error + ); + } + let line_mapping = self + .line_mapping + .read() + .expect("line mapping lock poisoned"); + let all_line_entries = line_mapping.lookup_all_lines_at_address(address); let matching_entries: Vec<_> = all_line_entries .iter() .copied() @@ -49,6 +72,7 @@ impl LoadedObjfile { .collect(); if matching_entries.is_empty() { + drop(line_mapping); return self.lookup_source_location(address); } @@ -69,6 +93,7 @@ impl LoadedObjfile { fn find_alternative_source_file<'a>( &'a self, + line_mapping: &'a crate::index::LineMappingTable, entry: &'a crate::core::LineEntry, ) -> Option<&'a crate::core::LineEntry> { let current_file_path = self.get_file_path_for_entry(entry)?; @@ -92,8 +117,7 @@ impl LoadedObjfile { let start_addr = entry.address.saturating_sub(search_range); let end_addr = entry.address.saturating_add(search_range); - for (addr, candidate_entry) in self.line_mapping.get_entries_in_range(start_addr, end_addr) - { + for (addr, candidate_entry) in line_mapping.get_entries_in_range(start_addr, end_addr) { if candidate_entry.compilation_unit != entry.compilation_unit { continue; } @@ -117,6 +141,8 @@ impl LoadedObjfile { if let Some(cu_file_index) = self .scoped_file_manager + .read() + .expect("scoped file index lock poisoned") .get_cu_file_index(&entry.compilation_unit) { for file_entry in cu_file_index.file_entries() { @@ -235,6 +261,8 @@ impl LoadedObjfile { if let Some(full_path) = self .scoped_file_manager + .read() + .expect("scoped file index lock poisoned") .lookup_by_scoped_index(&entry.compilation_unit, entry.file_index) { return Some(full_path); @@ -260,6 +288,8 @@ impl LoadedObjfile { { if let Some(resolved_full_path) = self .scoped_file_manager + .read() + .expect("scoped file index lock poisoned") .lookup_by_scoped_index(&line_entry.compilation_unit, line_entry.file_index) { if self.is_path_like(&resolved_full_path) { @@ -298,6 +328,8 @@ impl LoadedObjfile { let preferred_file_path = { let current_path = self .scoped_file_manager + .read() + .expect("scoped file index lock poisoned") .lookup_by_scoped_index(&line_entry.compilation_unit, line_entry.file_index) .unwrap_or_else(|| line_entry.file_path.clone()); @@ -365,7 +397,11 @@ impl LoadedObjfile { } fn find_main_source_file_in_cu(&self, compilation_unit: &str) -> Option { - if let Some(cu_file_index) = self.scoped_file_manager.get_cu_file_index(compilation_unit) { + let scoped_file_manager = self + .scoped_file_manager + .read() + .expect("scoped file index lock poisoned"); + if let Some(cu_file_index) = scoped_file_manager.get_cu_file_index(compilation_unit) { tracing::debug!( "find_main_source_file_in_cu: searching for main source file in CU '{}'", compilation_unit @@ -410,8 +446,18 @@ impl LoadedObjfile { file_path: &str, line_number: u32, ) -> Vec { + if let Err(error) = self.ensure_line_info_for_source(file_path) { + tracing::warn!( + "Failed to load line information for {} in {}: {}", + file_path, + self.module_path().display(), + error + ); + } let addresses = self .line_mapping + .read() + .expect("line mapping lock poisoned") .lookup_addresses_by_path(file_path, line_number as u64); if !addresses.is_empty() { @@ -438,7 +484,11 @@ impl LoadedObjfile { let mut source_files = Vec::new(); let mut seen_paths = HashSet::new(); - for cu in self.compilation_units.values() { + let compilation_units = self + .compilation_units + .read() + .expect("compilation unit lock poisoned"); + for cu in compilation_units.values() { for file in &cu.files { if seen_paths.insert(file.full_path.clone()) { source_files.push(file.clone()); diff --git a/ghostscope-dwarf/src/objfile/type_context.rs b/ghostscope-dwarf/src/objfile/type_context.rs index ed8b8c8e..1f04cd57 100644 --- a/ghostscope-dwarf/src/objfile/type_context.rs +++ b/ghostscope-dwarf/src/objfile/type_context.rs @@ -250,19 +250,30 @@ impl LoadedObjfile { current: TypeId, segment: &VariableAccessSegment, ) -> Result> { - projected_type_loc( - self.dwarf(), - &self.type_name_index, - type_loc(current)?, - segment, - ) - .map(|loc| loc.map(|loc| type_id_from_loc(current.module, loc))) + let type_name_index = self + .type_name_index + .read() + .expect("type name index lock poisoned"); + projected_type_loc(self.dwarf(), &type_name_index, type_loc(current)?, segment) + .map(|loc| loc.map(|loc| type_id_from_loc(current.module, loc))) } pub(crate) fn aggregate_type_id_by_name(&self, module: ModuleId, name: &str) -> Option { + if let Err(error) = self.ensure_debug_info_for_type_name(name) { + tracing::warn!( + "Failed to load indexed DWARF for type '{}' in {}: {}", + name, + self.module_path().display(), + error + ); + } + let type_name_index = self + .type_name_index + .read() + .expect("type name index lock poisoned"); [gimli::DW_TAG_structure_type, gimli::DW_TAG_class_type] .into_iter() - .find_map(|tag| self.type_name_index.find_aggregate_definition(name, tag)) + .find_map(|tag| type_name_index.find_aggregate_definition(name, tag)) .map(|loc| { type_id_from_loc( module, diff --git a/ghostscope-dwarf/src/objfile/variables.rs b/ghostscope-dwarf/src/objfile/variables.rs index 625d345c..ce3c932f 100644 --- a/ghostscope-dwarf/src/objfile/variables.rs +++ b/ghostscope-dwarf/src/objfile/variables.rs @@ -141,7 +141,12 @@ impl LoadedObjfile { { self.add_block_index_functions_if_missing(address, vec![fb]); } - } else if let Some(cu_off) = self.lightweight_index.find_cu_by_address(address) { + } else if let Some(cu_off) = self + .lightweight_index + .read() + .expect("lightweight index lock poisoned") + .find_cu_by_address(address) + { if let Some(funcs) = builder.build_for_unit(cu_off) { self.add_block_index_functions_if_missing(address, funcs); } @@ -202,6 +207,8 @@ impl LoadedObjfile { .and_then(|file_index| { cu_name.as_deref().and_then(|cu_name| { self.scoped_file_manager + .read() + .expect("scoped file index lock poisoned") .lookup_by_scoped_index(cu_name, file_index) }) }) @@ -369,13 +376,31 @@ impl LoadedObjfile { name: &str, tags: &[gimli::DwTag], ) -> Option { + if let Err(error) = self.ensure_debug_info_for_type_name(name) { + tracing::warn!( + "Failed to load indexed DWARF for type '{}' in {}: {}", + name, + self.module_path().display(), + error + ); + } for &tag in tags { - if let Some(loc) = self.type_name_index.find_aggregate_definition(name, tag) { + let loc = self + .type_name_index + .read() + .expect("type name index lock poisoned") + .find_aggregate_definition(name, tag); + if let Some(loc) = loc { return self.detailed_shallow_type(loc.cu_offset, loc.die_offset); } } - if let Some(td) = self.type_name_index.find_typedef(name) { + let typedef = self + .type_name_index + .read() + .expect("type name index lock poisoned") + .find_typedef(name); + if let Some(td) = typedef { let dwarf = self.dwarf(); if let Ok(header) = dwarf.unit_header(td.cu_offset) { if let Ok(unit) = dwarf.unit(header) { @@ -705,9 +730,16 @@ impl LoadedObjfile { let header = dwarf.unit_header(cu_off).ok()?; let unit = dwarf.unit(header).ok()?; let entry = unit.entry(die_off).ok()?; - if let Some((def_cu_off, def_die_off)) = - complete_aggregate_declaration_entry(dwarf, &self.type_name_index, &unit, &entry) - { + let definition = complete_aggregate_declaration_entry( + dwarf, + &self + .type_name_index + .read() + .expect("type name index lock poisoned"), + &unit, + &entry, + ); + if let Some((def_cu_off, def_die_off)) = definition { let def_header = dwarf.unit_header(def_cu_off).ok()?; let def_unit = dwarf.unit(def_header).ok()?; return crate::parser::DetailedParser::resolve_type_shallow_at_offset( diff --git a/ghostscope-dwarf/src/parser/detailed_parser.rs b/ghostscope-dwarf/src/parser/detailed_parser.rs index 85367dd5..8f71cc4f 100644 --- a/ghostscope-dwarf/src/parser/detailed_parser.rs +++ b/ghostscope-dwarf/src/parser/detailed_parser.rs @@ -75,9 +75,6 @@ impl DetailedParser { Self {} } - /// Attach a cross-CU type name index for faster completion - pub fn set_type_name_index(&mut self, _index: std::sync::Arc) {} - // Full type resolution intentionally removed; only shallow type resolution is supported. /// Shallow type resolution (no recursive member expansion) diff --git a/ghostscope-dwarf/src/parser/fast_parser/mod.rs b/ghostscope-dwarf/src/parser/fast_parser/mod.rs index 0fa73475..a11f1d11 100644 --- a/ghostscope-dwarf/src/parser/fast_parser/mod.rs +++ b/ghostscope-dwarf/src/parser/fast_parser/mod.rs @@ -1,14 +1,14 @@ //! Unified DWARF parser - true single-pass parsing use crate::{ - binary::DwarfReader, + binary::{dwarf_reader_from_arc_with_endian, DwarfReader}, core::{FunctionDieKind, IndexEntry, LineEntry, Result}, index::{ directory_from_index, resolve_file_path, LightweightFileIndex, LightweightIndex, LightweightIndexShard, LineMappingTable, ScopedFileIndexManager, }, }; -use gimli::Reader; +use gimli::{Reader, Section}; use rayon::prelude::*; use std::{ collections::{HashMap, HashSet}, @@ -16,6 +16,129 @@ use std::{ }; use tracing::debug; +fn is_gdb2_name_index(header: &gimli::NameIndexHeader) -> Result { + let Some(augmentation) = header.augmentation_string() else { + return Ok(false); + }; + Ok(augmentation.to_slice()?.starts_with(b"GDB2")) +} + +fn checked_debug_names_table_size(count: u32, width: usize) -> Result { + usize::try_from(count)? + .checked_mul(width) + .ok_or_else(|| anyhow::anyhow!(".debug_names table size overflow")) +} + +fn gdb2_abbreviation_range( + header: &gimli::NameIndexHeader, +) -> Result> { + let word_size = usize::from(header.format().word_size()); + let initial_length_size = match header.format() { + gimli::Format::Dwarf32 => 4, + gimli::Format::Dwarf64 => 12, + }; + let augmentation_size = header.augmentation_string().map_or(0, gimli::Reader::len); + let padded_augmentation_size = augmentation_size + .checked_add(3) + .ok_or_else(|| anyhow::anyhow!(".debug_names augmentation size overflow"))? + & !3; + + let mut start = header + .offset() + .0 + .checked_add(initial_length_size + 2 + 2 + 7 * std::mem::size_of::()) + .and_then(|offset| offset.checked_add(padded_augmentation_size)) + .ok_or_else(|| anyhow::anyhow!(".debug_names header size overflow"))?; + for size in [ + checked_debug_names_table_size(header.compile_unit_count(), word_size)?, + checked_debug_names_table_size(header.local_type_unit_count(), word_size)?, + checked_debug_names_table_size( + header.foreign_type_unit_count(), + std::mem::size_of::(), + )?, + checked_debug_names_table_size(header.bucket_count(), std::mem::size_of::())?, + if header.bucket_count() == 0 { + 0 + } else { + checked_debug_names_table_size(header.name_count(), std::mem::size_of::())? + }, + checked_debug_names_table_size(header.name_count(), word_size)?, + checked_debug_names_table_size(header.name_count(), word_size)?, + ] { + start = start + .checked_add(size) + .ok_or_else(|| anyhow::anyhow!(".debug_names table offset overflow"))?; + } + let end = start + .checked_add(usize::try_from(header.abbrev_table_size())?) + .ok_or_else(|| anyhow::anyhow!(".debug_names abbreviation size overflow"))?; + Ok(start..end) +} + +fn read_debug_names_uleb128(bytes: &[u8], offset: &mut usize, end: usize) -> Result { + let mut value = 0_u64; + let mut shift = 0_u32; + loop { + if *offset >= end { + anyhow::bail!("truncated .debug_names abbreviation table"); + } + let byte = bytes[*offset]; + *offset += 1; + let low_bits = u64::from(byte & 0x7f); + if shift >= 64 || (shift == 63 && low_bits > 1) { + anyhow::bail!("overflowing ULEB128 in .debug_names abbreviation table"); + } + value |= low_bits << shift; + if byte & 0x80 == 0 { + return Ok(value); + } + shift += 7; + } +} + +fn patch_gdb2_abbreviations( + bytes: &mut [u8], + range: std::ops::Range, + format: gimli::Format, +) -> Result<()> { + if range.end > bytes.len() { + anyhow::bail!(".debug_names abbreviation table is out of bounds"); + } + let replacement = match format { + gimli::Format::Dwarf32 => gimli::constants::DW_FORM_ref4.0, + gimli::Format::Dwarf64 => gimli::constants::DW_FORM_ref8.0, + }; + let mut offset = range.start; + while offset < range.end { + let code = read_debug_names_uleb128(bytes, &mut offset, range.end)?; + if code == 0 { + break; + } + read_debug_names_uleb128(bytes, &mut offset, range.end)?; + + loop { + let name = read_debug_names_uleb128(bytes, &mut offset, range.end)?; + let form_offset = offset; + let form = read_debug_names_uleb128(bytes, &mut offset, range.end)?; + if name == 0 && form == 0 { + break; + } + if name == 0 || form == 0 { + anyhow::bail!("invalid .debug_names abbreviation attribute"); + } + if name == u64::from(gimli::constants::DW_IDX_die_offset.0) + && form == u64::from(gimli::constants::DW_FORM_ref_addr.0) + { + if offset != form_offset + 1 { + anyhow::bail!("unsupported GDB2 DW_FORM_ref_addr encoding"); + } + bytes[form_offset] = u8::try_from(replacement)?; + } + } + } + Ok(()) +} + #[derive(Clone, Default)] struct FunctionMetadata { name: Option>, @@ -96,6 +219,7 @@ type InfoShard = LightweightIndexShard; struct LineShard { line_entries: Vec, compilation_units: HashMap, + source_unit_offsets: Vec<(String, gimli::DebugInfoOffset)>, file_indices: Vec<(String, LightweightFileIndex)>, files_count: usize, } @@ -106,6 +230,7 @@ pub(crate) struct DwarfParseResult { pub line_mapping: LineMappingTable, pub scoped_file_manager: ScopedFileIndexManager, pub compilation_units: HashMap, + pub line_source_unit_offsets: HashMap>, pub stats: DwarfParseStats, } @@ -114,6 +239,7 @@ pub(crate) struct LineParseResult { pub line_mapping: LineMappingTable, pub scoped_file_manager: ScopedFileIndexManager, pub compilation_units: HashMap, + pub source_unit_offsets: HashMap>, pub line_entries_count: usize, pub files_count: usize, } @@ -1022,15 +1148,46 @@ impl<'a> DwarfParser<'a> { } /// Parse debug_line sections (for parallel processing) + #[cfg(test)] pub fn parse_line_info(&self, module_path: &str) -> Result { + self.parse_line_info_units(module_path, None, true) + } + + pub(crate) fn parse_line_headers(&self, module_path: &str) -> Result { + self.parse_line_info_units(module_path, None, false) + } + + pub(crate) fn parse_line_info_cus( + &self, + module_path: &str, + unit_offsets: &[gimli::DebugInfoOffset], + ) -> Result { + self.parse_line_info_units(module_path, Some(unit_offsets), true) + } + + fn parse_line_info_units( + &self, + module_path: &str, + unit_offsets: Option<&[gimli::DebugInfoOffset]>, + include_rows: bool, + ) -> Result { debug!("Starting debug_line-only parsing for: {}", module_path); // Collect CU headers once - let mut headers: Vec> = Vec::new(); - let mut units = self.dwarf.units(); - while let Some(header) = units.next()? { - headers.push(header); - } + let mut headers: Vec> = + if let Some(unit_offsets) = unit_offsets { + unit_offsets + .iter() + .map(|unit_offset| self.dwarf.unit_header(*unit_offset).map_err(Into::into)) + .collect::>()? + } else { + let mut headers = Vec::new(); + let mut units = self.dwarf.units(); + while let Some(header) = units.next()? { + headers.push(header); + } + headers + }; // Sort headers by size (descending) for better load balance under work-stealing headers.sort_by_key(|h| std::cmp::Reverse(h.length_including_self() as u64)); @@ -1083,71 +1240,84 @@ impl<'a> DwarfParser<'a> { &unit, line_program, )?; + if let Some(unit_offset) = unit.header.debug_info_offset() { + for file in &compilation_unit.files { + for source_path in [&file.full_path, &file.filename] { + if !source_path.is_empty() { + shard + .source_unit_offsets + .push((source_path.clone(), unit_offset)); + } + } + } + } shard.files_count += compilation_unit.files.len(); shard .compilation_units .insert(cu_name.clone(), compilation_unit); shard.file_indices.push((cu_name.clone(), file_index)); - // Extract line rows for this CU - let (line_program, sequences) = line_program.clone().sequences()?; - for seq in sequences { - let mut rows = line_program.resume_from(&seq); - let mut pending_entries: Vec = Vec::new(); - let mut pending_address: Option = None; - while let Some((_, line_row)) = rows.next_row()? { - let row_address = line_row.address(); - if line_row.end_sequence() { - Self::flush_pending_line_entries( - &mut shard.line_entries, - &mut pending_entries, - Some(row_address), - ); - pending_address = None; - continue; - } - - if let Some(address) = pending_address { - if row_address > address { + // Extract line rows for this CU only when a query needs them. + if include_rows { + let (line_program, sequences) = line_program.clone().sequences()?; + for seq in sequences { + let mut rows = line_program.resume_from(&seq); + let mut pending_entries: Vec = Vec::new(); + let mut pending_address: Option = None; + while let Some((_, line_row)) = rows.next_row()? { + let row_address = line_row.address(); + if line_row.end_sequence() { Self::flush_pending_line_entries( &mut shard.line_entries, &mut pending_entries, Some(row_address), ); - pending_address = Some(row_address); - } else if row_address < address { - Self::flush_pending_line_entries( - &mut shard.line_entries, - &mut pending_entries, - None, - ); + pending_address = None; + continue; + } + + if let Some(address) = pending_address { + if row_address > address { + Self::flush_pending_line_entries( + &mut shard.line_entries, + &mut pending_entries, + Some(row_address), + ); + pending_address = Some(row_address); + } else if row_address < address { + Self::flush_pending_line_entries( + &mut shard.line_entries, + &mut pending_entries, + None, + ); + pending_address = Some(row_address); + } + } else { pending_address = Some(row_address); } - } else { - pending_address = Some(row_address); - } - let column = match line_row.column() { - gimli::ColumnType::LeftEdge => 0, - gimli::ColumnType::Column(x) => x.get(), - }; - pending_entries.push(LineEntry { - address: row_address, - end_address: None, - file_path: String::new(), - file_index: line_row.file_index(), - compilation_unit: std::sync::Arc::from(cu_name.as_str()), - line: line_row.line().map(|l| l.get()).unwrap_or(0), - column, - is_stmt: line_row.is_stmt(), - prologue_end: line_row.prologue_end(), - }); + let column = match line_row.column() { + gimli::ColumnType::LeftEdge => 0, + gimli::ColumnType::Column(x) => x.get(), + }; + pending_entries.push(LineEntry { + address: row_address, + end_address: None, + file_path: String::new(), + file_index: line_row.file_index(), + compilation_unit: std::sync::Arc::from(cu_name.as_str()), + line: line_row.line().map(|l| l.get()).unwrap_or(0), + column, + is_stmt: line_row.is_stmt(), + prologue_end: line_row.prologue_end(), + }); + } + Self::flush_pending_line_entries( + &mut shard.line_entries, + &mut pending_entries, + None, + ); } - Self::flush_pending_line_entries( - &mut shard.line_entries, - &mut pending_entries, - None, - ); } } Ok(shard) @@ -1158,6 +1328,7 @@ impl<'a> DwarfParser<'a> { let mut scoped_file_manager = ScopedFileIndexManager::new(); let mut line_entries = Vec::new(); let mut compilation_units: HashMap = HashMap::new(); + let mut source_unit_offsets: HashMap> = HashMap::new(); let mut total_files = 0usize; for sr in shard_results { @@ -1167,10 +1338,20 @@ impl<'a> DwarfParser<'a> { for (cu, cuinfo) in shard.compilation_units { compilation_units.insert(cu, cuinfo); } + for (source_path, unit_offset) in shard.source_unit_offsets { + source_unit_offsets + .entry(source_path) + .or_default() + .push(unit_offset); + } for (cu, fi) in shard.file_indices { scoped_file_manager.add_compilation_unit(cu, fi); } } + for offsets in source_unit_offsets.values_mut() { + offsets.sort_unstable_by_key(|offset| offset.0); + offsets.dedup(); + } // Build final line mapping let total_line_entries = line_entries.len(); @@ -1189,11 +1370,199 @@ impl<'a> DwarfParser<'a> { line_mapping, scoped_file_manager, compilation_units, + source_unit_offsets, line_entries_count: total_line_entries, files_count: total_files, }) } + fn compatible_debug_names(&self) -> Result> { + let mut patches = Vec::new(); + let mut headers = self.dwarf.debug_names.headers(); + while let Some(header) = headers.next()? { + if is_gdb2_name_index(&header)? { + patches.push((gdb2_abbreviation_range(&header)?, header.format())); + } + } + if patches.is_empty() { + return Ok(self.dwarf.debug_names.clone()); + } + + let reader = self.dwarf.debug_names.reader(); + let endian = reader.endian(); + let mut bytes = reader.to_slice()?.into_owned(); + for (range, format) in patches { + patch_gdb2_abbreviations(&mut bytes, range, format)?; + } + Ok(gimli::DebugNames::from(dwarf_reader_from_arc_with_endian( + Arc::from(bytes), + endian, + ))) + } + + fn debug_names_die_offset( + &self, + unit_offset: gimli::DebugInfoOffset, + die_offset: gimli::UnitOffset, + gdb2: bool, + ) -> Result { + if !gdb2 { + return Ok(die_offset); + } + + let relative = die_offset.0.checked_sub(unit_offset.0).ok_or_else(|| { + anyhow::anyhow!( + "GDB2 .debug_names DIE offset 0x{:x} precedes CU 0x{:x}", + die_offset.0, + unit_offset.0 + ) + })?; + let header = self.dwarf.unit_header(unit_offset)?; + if relative >= header.length_including_self() { + anyhow::bail!( + "GDB2 .debug_names DIE offset 0x{:x} is outside CU 0x{:x}", + die_offset.0, + unit_offset.0 + ); + } + Ok(gimli::UnitOffset(relative)) + } + + /// Parse debug_info sections (for parallel processing) + pub(crate) fn parse_debug_names(&self, module_path: &str) -> Result> { + debug!("Starting .debug_names parsing for: {}", module_path); + let debug_names = self.compatible_debug_names()?; + let mut headers = debug_names.headers(); + let mut shards = Vec::new(); + let mut saw_index = false; + + while let Some(header) = headers.next()? { + saw_index = true; + let gdb2 = is_gdb2_name_index(&header)?; + let names = header.index()?; + let mut shard = LightweightIndexShard::default(); + + for name_index in names.names() { + let name = names + .name_string(name_index, &self.dwarf.debug_str)? + .to_string_lossy()? + .into_owned(); + let mut entries = names.name_entries(name_index)?; + while let Some(entry) = entries.next()? { + let unit_offset = match entry.type_unit(&names)? { + Some(gimli::NameTypeUnit::Local(unit)) => unit, + Some(gimli::NameTypeUnit::Foreign(_)) => continue, + None => { + let Some(cu) = entry + .compile_unit(&names)? + .or(names.default_compile_unit()?) + else { + continue; + }; + cu + } + }; + let Some(die_offset) = entry.die_offset()? else { + continue; + }; + let die_offset = self.debug_names_die_offset(unit_offset, die_offset, gdb2)?; + let tag = entry.tag; + let function_kind = match tag { + gimli::constants::DW_TAG_subprogram => FunctionDieKind::ConcreteSubprogram, + gimli::constants::DW_TAG_inlined_subroutine => { + FunctionDieKind::InlineInstance + } + _ => FunctionDieKind::NotFunction, + }; + let index_entry = IndexEntry { + name: Arc::from(name.as_str()), + die_offset, + unit_offset, + tag, + flags: crate::core::IndexFlags { + is_main: name == "main" || name == "_main", + ..Default::default() + }, + language: None, + representative_addr: None, + entry_pc: None, + function_kind, + }; + + match tag { + gimli::constants::DW_TAG_subprogram + | gimli::constants::DW_TAG_inlined_subroutine => { + shard.push_function_entry(name.clone(), index_entry); + } + gimli::constants::DW_TAG_variable => { + shard.push_variable_entry(name.clone(), index_entry); + } + gimli::constants::DW_TAG_structure_type + | gimli::constants::DW_TAG_class_type + | gimli::constants::DW_TAG_union_type + | gimli::constants::DW_TAG_enumeration_type + | gimli::constants::DW_TAG_typedef => { + shard.push_type_entry(name.clone(), index_entry); + } + _ => {} + } + } + } + shards.push(shard); + } + + if !saw_index { + return Ok(None); + } + + let mut lightweight_index = LightweightIndex::from_shards(shards); + let functions_count = lightweight_index.get_stats().0; + let variables_count = lightweight_index.get_stats().1; + let has_aranges = lightweight_index.build_cu_maps_from_aranges(self.dwarf); + let has_root_ranges = lightweight_index.build_cu_maps_from_roots(self.dwarf); + let has_cu_ranges = has_aranges || has_root_ranges; + if !has_cu_ranges && functions_count > 0 { + anyhow::bail!( + ".debug_names for {module_path} has functions but no usable address-to-CU ranges", + ); + } + + debug!( + "Completed .debug_names parsing for {}: {} functions, {} variables", + module_path, functions_count, variables_count + ); + Ok(Some(DebugParseResult { + lightweight_index, + functions_count, + variables_count, + })) + } + + pub(crate) fn parse_debug_info_cus( + &self, + unit_offsets: &[gimli::DebugInfoOffset], + ) -> Result> { + unit_offsets + .par_iter() + .map(|unit_offset| { + let header = self.dwarf.unit_header(*unit_offset)?; + let unit = self.dwarf.unit(header)?; + let cu_lang = self.extract_language(self.dwarf, &unit); + self.process_unit_shard(&unit, *unit_offset, cu_lang) + }) + .collect() + } + + pub(crate) fn initialize_lazy_debug_info(&self) -> DebugParseResult { + let mut lightweight_index = LightweightIndex::new(); + lightweight_index.build_cu_maps_from_aranges(self.dwarf); + DebugParseResult { + lightweight_index, + functions_count: 0, + variables_count: 0, + } + } + /// Parse debug_info sections (for parallel processing) pub fn parse_debug_info(&self, module_path: &str) -> Result { debug!("Starting debug_info-only parsing for: {}", module_path); @@ -1256,6 +1625,7 @@ impl<'a> DwarfParser<'a> { line_mapping, scoped_file_manager, compilation_units, + source_unit_offsets, line_entries_count, files_count, } = line_result; @@ -1272,6 +1642,7 @@ impl<'a> DwarfParser<'a> { line_mapping, scoped_file_manager, compilation_units, + line_source_unit_offsets: source_unit_offsets, stats, } } diff --git a/ghostscope-ui/src/components/app/runtime_status.rs b/ghostscope-ui/src/components/app/runtime_status.rs index 92f6199d..8048cf38 100644 --- a/ghostscope-ui/src/components/app/runtime_status.rs +++ b/ghostscope-ui/src/components/app/runtime_status.rs @@ -30,6 +30,8 @@ impl App { types, debug_source: "unknown".to_string(), debug_source_path: None, + dwarf_index: "unknown".to_string(), + dwarf_index_warning: None, }; self.state .loading_ui @@ -218,6 +220,8 @@ impl App { types: stats.types, debug_source: stats.debug_source.clone(), debug_source_path: stats.debug_source_path.clone(), + dwarf_index: stats.dwarf_index.clone(), + dwarf_index_warning: stats.dwarf_index_warning.clone(), }; self.state .loading_ui diff --git a/ghostscope-ui/src/components/loading/progress.rs b/ghostscope-ui/src/components/loading/progress.rs index c422460b..1fd6547a 100644 --- a/ghostscope-ui/src/components/loading/progress.rs +++ b/ghostscope-ui/src/components/loading/progress.rs @@ -60,6 +60,16 @@ impl ProgressRenderer { Span::styled(" debug: ", Style::default().fg(Color::Gray)), ]; push_module_debug_source_spans(&mut spans, stats); + spans.push(Span::styled(" | index: ", Style::default().fg(Color::Gray))); + let index_color = if stats.dwarf_index_warning.is_some() { + Color::Yellow + } else { + Color::Cyan + }; + spans.push(Span::styled( + stats.dwarf_index.clone(), + Style::default().fg(index_color), + )); spans.push(Span::styled( format!( " | Functions: {} | Variables: {} | Types: {} | Time: {:.1}s", diff --git a/ghostscope-ui/src/components/loading/state.rs b/ghostscope-ui/src/components/loading/state.rs index cd1013a1..85135b61 100644 --- a/ghostscope-ui/src/components/loading/state.rs +++ b/ghostscope-ui/src/components/loading/state.rs @@ -75,6 +75,8 @@ pub struct ModuleStats { pub types: usize, pub debug_source: String, pub debug_source_path: Option, + pub dwarf_index: String, + pub dwarf_index_warning: Option, } #[derive(Debug, Clone, Default)] @@ -259,6 +261,8 @@ impl LoadingProgress { types: 0, debug_source: "summary".to_string(), debug_source_path: None, + dwarf_index: "summary".to_string(), + dwarf_index_warning: None, }; for module in &self.modules { diff --git a/ghostscope-ui/src/components/loading/ui.rs b/ghostscope-ui/src/components/loading/ui.rs index 10d6b372..8c5cba0e 100644 --- a/ghostscope-ui/src/components/loading/ui.rs +++ b/ghostscope-ui/src/components/loading/ui.rs @@ -260,6 +260,7 @@ impl LoadingUI { lines.push(Line::from(source_spans)); append_debug_source_details(&mut lines, &self.progress); } + append_dwarf_index_details(&mut lines, &self.progress); // Empty line lines.push(Line::from("")); @@ -418,6 +419,53 @@ impl LoadingUI { } } +fn append_dwarf_index_details(lines: &mut Vec>, progress: &LoadingProgress) { + let mut counts = std::collections::BTreeMap::<&str, usize>::new(); + let mut rejected = Vec::new(); + for module in progress + .modules + .iter() + .filter(|module| matches!(module.state, ModuleState::Completed)) + { + let Some(stats) = module.stats.as_ref() else { + continue; + }; + *counts.entry(stats.dwarf_index.as_str()).or_default() += 1; + if let Some(reason) = stats.dwarf_index_warning.as_deref() { + rejected.push((module, reason)); + } + } + + if !counts.is_empty() { + let summary = counts + .into_iter() + .map(|(label, count)| format!("{label}:{count}")) + .collect::>() + .join(" "); + lines.push(Line::from(vec![ + Span::styled("• DWARF indexes: ", Style::default().fg(Color::White)), + Span::styled(summary, Style::default().fg(Color::Cyan)), + ])); + } + + if !rejected.is_empty() { + lines.push(Line::from(Span::styled( + "• DWARF index fallback:", + Style::default().fg(Color::Yellow), + ))); + for (module, reason) in rejected.iter().take(MAX_WELCOME_DEBUG_SOURCE_DETAILS) { + lines.push(Line::from(Span::styled( + format!( + " {} {}", + shorten_middle(&module_file_name(&module.path), 34), + shorten_middle(reason, 120) + ), + Style::default().fg(Color::Yellow), + ))); + } + } +} + fn append_debug_source_details(lines: &mut Vec>, progress: &LoadingProgress) { let debug_source_modules: Vec<&ModuleLoadStatus> = progress .modules @@ -583,6 +631,8 @@ mod tests { debug_source_path: Some( "/usr/local/openresty/luajit/lib/libluajit-5.1.so.2.1.ROLLING".to_string(), ), + dwarf_index: ".gdb_index-v9".to_string(), + dwarf_index_warning: None, }, ); @@ -597,12 +647,15 @@ mod tests { types: 0, debug_source: "missing".to_string(), debug_source_path: None, + dwarf_index: "full-scan".to_string(), + dwarf_index_warning: None, }, ); let text = plain_text(&loading_ui.create_welcome_message(1.25)); assert!(text.contains("Debug sources: embedded:1")); + assert!(text.contains("DWARF indexes: .gdb_index-v9:1")); assert!(text.contains("missing:1")); assert!(text.contains("Debug source files:")); assert!(text.contains("embedded")); diff --git a/ghostscope-ui/src/events/runtime.rs b/ghostscope-ui/src/events/runtime.rs index 6b44054e..53af6d74 100644 --- a/ghostscope-ui/src/events/runtime.rs +++ b/ghostscope-ui/src/events/runtime.rs @@ -367,6 +367,8 @@ pub struct ModuleLoadingStats { pub types: usize, pub debug_source: String, pub debug_source_path: Option, + pub dwarf_index: String, + pub dwarf_index_warning: Option, pub load_time_ms: u64, } diff --git a/ghostscope/src/cli/loading_reporter.rs b/ghostscope/src/cli/loading_reporter.rs index 3a8aaeb5..d4dbca1c 100644 --- a/ghostscope/src/cli/loading_reporter.rs +++ b/ghostscope/src/cli/loading_reporter.rs @@ -1,4 +1,4 @@ -use ghostscope_dwarf::{DebugInfoSource, ModuleLoadingEvent}; +use ghostscope_dwarf::{DebugInfoSource, DwarfIndexStatus, ModuleLoadingEvent}; use std::io::{self, IsTerminal, Write}; use std::path::Path; use std::time::{Duration, Instant}; @@ -196,6 +196,7 @@ impl LoadingProgress { self.module_reports.push(ModuleLoadReport { path: module_path.clone(), source: stats.debug_info_source, + dwarf_index_status: stats.dwarf_index_status, functions: stats.functions, variables: stats.variables, types: stats.types, @@ -312,8 +313,9 @@ impl LoadingProgress { lines.push(" module details:".to_string()); for module in debuggable_modules.iter().take(MAX_MODULE_REPORT_LINES) { lines.push(format!( - " {} {:>6} funcs {:>6} vars {:>6} types {:>5}ms {}", + " {} {:<28} {:>6} funcs {:>6} vars {:>6} types {:>5}ms {}", format_source_column(&module.source, colors), + module.dwarf_index_status.display_label(), module.functions, module.variables, module.types, @@ -338,6 +340,33 @@ impl LoadingProgress { )); } + let rejected_indexes = self + .module_reports + .iter() + .filter(|module| module.dwarf_index_status.rejection_reason().is_some()) + .collect::>(); + if !rejected_indexes.is_empty() { + lines.push(format!(" {}", colors.yellow("DWARF index fallback:"))); + for module in rejected_indexes.iter().take(MAX_MODULE_REPORT_LINES) { + let reason = module + .dwarf_index_status + .rejection_reason() + .expect("filtered rejected index must have a reason"); + lines.push(format!( + " {} {:<64} {}", + colors.yellow("rejected"), + shorten_path(&module.path, 64), + shorten_middle(reason, 120) + )); + } + if rejected_indexes.len() > MAX_MODULE_REPORT_LINES { + lines.push(format!( + " ... {} more rejected index entries omitted", + rejected_indexes.len() - MAX_MODULE_REPORT_LINES + )); + } + } + if !self.module_failures.is_empty() { lines.push(" module failures:".to_string()); for failure in self.module_failures.iter().take(MAX_MODULE_REPORT_LINES) { @@ -437,6 +466,7 @@ impl LoadingProgress { struct ModuleLoadReport { path: String, source: DebugInfoSource, + dwarf_index_status: DwarfIndexStatus, functions: usize, variables: usize, types: usize, @@ -634,7 +664,9 @@ mod tests { CliLoadingReporter, DebugSourceCounts, }; use crate::config::CliColorMode; - use ghostscope_dwarf::{DebugInfoSource, ModuleLoadingEvent, ModuleLoadingStats}; + use ghostscope_dwarf::{ + DebugInfoSource, DwarfIndexStatus, ModuleLoadingEvent, ModuleLoadingStats, + }; use std::time::Duration; #[test] @@ -686,6 +718,7 @@ mod tests { debug_info_source: DebugInfoSource::Embedded { path: "/usr/bin/app".to_string(), }, + dwarf_index_status: DwarfIndexStatus::GdbIndex { version: 9 }, load_time_ms: 42, parse_time_ms: 30, index_time_ms: 8, @@ -713,6 +746,7 @@ mod tests { assert!(report.contains("Startup load report:")); assert!(report.contains("module details:")); assert!(report.contains("embedded")); + assert!(report.contains(".gdb_index-v9")); assert!(report.contains("/usr/bin/app")); } @@ -730,6 +764,9 @@ mod tests { variables: 0, types: 0, debug_info_source: DebugInfoSource::Missing, + dwarf_index_status: DwarfIndexStatus::Rejected { + reason: "truncated .gdb_index".to_string(), + }, load_time_ms: 5, parse_time_ms: 0, index_time_ms: 0, @@ -746,6 +783,8 @@ mod tests { assert!(report.contains("missing DWARF: 1 module")); assert!(report.contains("libmissing.so")); + assert!(report.contains("DWARF index fallback:")); + assert!(report.contains("truncated .gdb_index")); assert!(!report.contains("module details:")); assert!(!report.contains("missing 0 funcs")); } diff --git a/ghostscope/src/tui/dwarf_loader.rs b/ghostscope/src/tui/dwarf_loader.rs index c16d3d3c..f0f8bce8 100644 --- a/ghostscope/src/tui/dwarf_loader.rs +++ b/ghostscope/src/tui/dwarf_loader.rs @@ -37,6 +37,11 @@ fn convert_loading_event_to_runtime_status(event: ModuleLoadingEvent) -> Runtime types: stats.types, debug_source: stats.debug_info_source.kind_label().to_string(), debug_source_path: stats.debug_info_source.display_path().map(str::to_string), + dwarf_index: stats.dwarf_index_status.display_label(), + dwarf_index_warning: stats + .dwarf_index_status + .rejection_reason() + .map(str::to_string), load_time_ms: stats.load_time_ms, }; RuntimeStatus::DwarfModuleLoadingCompleted {