diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace.rs deleted file mode 100644 index 6f9932fc..00000000 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace.rs +++ /dev/null @@ -1,4776 +0,0 @@ -use super::backtrace_plan::{ - BacktraceEmitMode, BacktraceInstructionPlan, BPF_INLINE_BACKTRACE_FRAME_LIMIT, -}; -use super::*; -use crate::script::BacktraceStatement; -use aya_ebpf_bindings::bindings::bpf_func_id::{BPF_FUNC_map_lookup_elem, BPF_FUNC_tail_call}; -use ghostscope_dwarf::{CompactUnwindRow, MemoryAccessSize, ModuleAddress}; -use inkwell::basic_block::BasicBlock; -use inkwell::values::BasicMetadataValueEnum; -use std::path::PathBuf; - -const X86_64_DWARF_RIP: u16 = 16; -const X86_64_DWARF_RBP: u16 = 6; -const X86_64_DWARF_RSP: u16 = 7; -const BPF_BACKTRACE_FRAMES_PER_TAIL_CALL: u8 = 4; -const BPF_BACKTRACE_MAX_STEP_INVOCATIONS: u8 = 32; -const BPF_BACKTRACE_STEP_PROG_INDEX: u32 = 0; - -struct RuntimeBtUnwindRow<'ctx> { - found: IntValue<'ctx>, - cfa_register: IntValue<'ctx>, - cfa_offset: IntValue<'ctx>, - ra_kind: IntValue<'ctx>, - ra_register: IntValue<'ctx>, - ra_offset: IntValue<'ctx>, - rbp_kind: IntValue<'ctx>, - rbp_register: IntValue<'ctx>, - rbp_offset: IntValue<'ctx>, -} - -struct BtFrameModule<'ctx> { - cookie: IntValue<'ctx>, - bias: IntValue<'ctx>, - found: IntValue<'ctx>, -} - -struct BtRowBounds<'ctx> { - start: IntValue<'ctx>, - end: IntValue<'ctx>, -} - -struct BtModuleRangeMeta<'ctx> { - found: IntValue<'ctx>, - active_slot: IntValue<'ctx>, - count: IntValue<'ctx>, -} - -struct BtModuleRangeValue<'ctx> { - found: IntValue<'ctx>, - base: IntValue<'ctx>, - end: IntValue<'ctx>, - text: IntValue<'ctx>, - cookie: IntValue<'ctx>, -} - -struct RuntimeBtRowScratch<'ctx> { - found_ptr: PointerValue<'ctx>, - cfa_register_ptr: PointerValue<'ctx>, - cfa_offset_ptr: PointerValue<'ctx>, - ra_kind_ptr: PointerValue<'ctx>, - ra_register_ptr: PointerValue<'ctx>, - ra_offset_ptr: PointerValue<'ctx>, - rbp_kind_ptr: PointerValue<'ctx>, - rbp_register_ptr: PointerValue<'ctx>, - rbp_offset_ptr: PointerValue<'ctx>, -} - -struct BtScratch<'ctx> { - row: RuntimeBtRowScratch<'ctx>, - next_rbp_ptr: PointerValue<'ctx>, - next_error_code_ptr: PointerValue<'ctx>, -} - -#[derive(Clone, Copy)] -struct BtRegisterState<'ctx> { - ip: IntValue<'ctx>, - rsp: IntValue<'ctx>, - rbp: IntValue<'ctx>, -} - -#[derive(Clone, Copy)] -struct BtNextFrame<'ctx> { - ip: IntValue<'ctx>, - rsp: IntValue<'ctx>, - rbp: IntValue<'ctx>, - error_code: IntValue<'ctx>, -} - -struct BtFrameValidation<'ctx> { - valid: IntValue<'ctx>, - complete: IntValue<'ctx>, - error_code: IntValue<'ctx>, -} - -enum BacktraceUnwindRowForPc { - Usable(ghostscope_protocol::BacktraceUnwindRow), - Missing, - Unsupported, -} - -impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { - pub fn generate_backtrace_instruction(&mut self, stmt: &BacktraceStatement) -> Result<()> { - let plan = self.plan_backtrace_instruction(stmt); - if matches!(plan.mode, BacktraceEmitMode::TailCall) { - return self.generate_tail_call_backtrace_instruction(&plan); - } - - self.generate_inline_backtrace_instruction(&plan) - } - - /// Generate a DWARF-backed Backtrace instruction. - /// - /// eBPF records `(module_cookie, normalized_pc, raw_ip)` frames and advances - /// the unwind state from compact DWARF CFI rows. Userspace owns final source - /// line and inline-chain symbolization. - fn generate_inline_backtrace_instruction( - &mut self, - plan: &BacktraceInstructionPlan, - ) -> Result<()> { - let depth = plan.depth; - let flags = plan.flags; - info!("Generating Backtrace instruction: depth={}", depth); - - let payload_size = plan.payload_size; - let instruction_size = plan.instruction_size; - let inst_buffer = self - .reserve_instruction_region_or_return_zero(instruction_size as u64)? - .into_value_after_runtime_returns(); - - self.store_u8_const( - inst_buffer, - std::mem::offset_of!(InstructionHeader, inst_type), - InstructionType::Backtrace as u8, - "bt_inst_type", - )?; - self.store_u16_const( - inst_buffer, - std::mem::offset_of!(InstructionHeader, data_length), - payload_size as u16, - "bt_data_length", - )?; - - let data_base = INSTRUCTION_HEADER_SIZE; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_REQUESTED_DEPTH_OFFSET, - depth, - "bt_requested_depth", - )?; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, - 1, - "bt_frame_count", - )?; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_FLAGS_OFFSET, - flags, - "bt_flags", - )?; - self.store_u16_const( - inst_buffer, - data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, - 0, - "bt_error_code", - )?; - - let Some(compile_ctx) = self.current_compile_time_context.clone() else { - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, - 0, - "bt_frame_count_no_context", - )?; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - BacktraceStatus::DwarfUnavailable as u8, - "bt_status_no_context", - )?; - return Ok(()); - }; - - let module_cookie = self.cookie_for_module_or_fallback(&compile_ctx.module_path); - let module_cookie_value = self.context.i64_type().const_int(module_cookie, false); - let pt_regs = self.get_pt_regs_parameter()?; - let raw_ip = self.load_dwarf_register_i64(X86_64_DWARF_RIP, pt_regs)?; - let (module_bias, offsets_found) = self.generate_runtime_address_from_offsets( - self.context.i64_type().const_zero(), - 0, - module_cookie, - )?; - let normalized_pc = self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found)?; - let caller_fallback_found = self.backtrace_module_fallback_found(offsets_found); - - self.store_backtrace_frame( - inst_buffer, - 0, - module_cookie_value, - normalized_pc, - raw_ip, - 0, - )?; - - if depth == 1 { - let status = - self.status_or_offsets_unavailable(BacktraceStatus::Truncated, offsets_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_status_depth_one", - )?; - return Ok(()); - } - - let row = self - .usable_backtrace_unwind_row_for_pc(&compile_ctx.module_path, compile_ctx.pc_address); - let initial_status = self.status_for_backtrace_unwind_row_for_pc(&row); - let status = self.status_or_offsets_unavailable(initial_status, offsets_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_initial_status", - )?; - - let BacktraceUnwindRowForPc::Usable(row) = row else { - return Ok(()); - }; - - let i64_type = self.context.i64_type(); - let ip_ptr = self.build_entry_alloca(i64_type, "bt_state_ip")?; - let rsp_ptr = self.build_entry_alloca(i64_type, "bt_state_rsp")?; - let rbp_ptr = self.build_entry_alloca(i64_type, "bt_state_rbp")?; - let module_bias_ptr = self.build_entry_alloca(i64_type, "bt_state_module_bias")?; - let module_cookie_ptr = self.build_entry_alloca(i64_type, "bt_state_module_cookie")?; - let module_found_ptr = - self.build_entry_alloca(self.context.bool_type(), "bt_state_module_found")?; - let scratch = self.allocate_backtrace_scratch()?; - self.builder - .build_store(ip_ptr, raw_ip) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let initial_rsp = self.load_dwarf_register_i64(X86_64_DWARF_RSP, pt_regs)?; - let initial_rbp = self.load_dwarf_register_i64(X86_64_DWARF_RBP, pt_regs)?; - self.builder - .build_store(rsp_ptr, initial_rsp) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(rbp_ptr, initial_rbp) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_bias_ptr, module_bias) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_cookie_ptr, module_cookie_value) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_found_ptr, offsets_found) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - let current_fn = self.current_function("generate DWARF backtrace")?; - let done = self.context.append_basic_block(current_fn, "bt_done"); - - let runtime_row = self.runtime_row_from_static(row); - let state = BtRegisterState { - ip: self.load_i64(ip_ptr, "bt_initial_current_ip")?, - rsp: self.load_i64(rsp_ptr, "bt_initial_current_rsp")?, - rbp: self.load_i64(rbp_ptr, "bt_initial_current_rbp")?, - }; - let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; - let validation = self.validate_backtrace_next_frame(state, next)?; - let initial_store_block = self - .context - .append_basic_block(current_fn, "bt_initial_store_frame"); - let initial_stop_block = self - .context - .append_basic_block(current_fn, "bt_initial_stop"); - self.builder - .build_conditional_branch(validation.valid, initial_store_block, initial_stop_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(initial_stop_block); - let stop_status = self.status_for_backtrace_stop( - validation.complete, - validation.error_code, - offsets_found, - )?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - stop_status, - "bt_initial_stop_status", - )?; - self.store_u16_value( - inst_buffer, - data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, - validation.error_code, - "bt_initial_stop_error_code", - )?; - self.builder - .build_unconditional_branch(done) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(initial_store_block); - let frame_module = self.resolve_backtrace_frame_module( - next.ip, - module_cookie_value, - module_bias, - caller_fallback_found, - "bt_initial_frame_module", - )?; - let next_pc = - self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; - self.store_backtrace_frame(inst_buffer, 1, frame_module.cookie, next_pc, next.ip, 0)?; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, - 2, - "bt_initial_frame_count", - )?; - let status = if depth == 2 { - BacktraceStatus::Truncated - } else { - BacktraceStatus::ReadError - }; - let status = self.status_or_offsets_unavailable(status, offsets_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_initial_status_after_frame", - )?; - self.builder - .build_store(ip_ptr, next.ip) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(rsp_ptr, next.rsp) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(rbp_ptr, next.rbp) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_bias_ptr, frame_module.bias) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_cookie_ptr, frame_module.cookie) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_found_ptr, frame_module.found) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - if depth == 2 { - self.builder - .build_unconditional_branch(done) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - } else if self.backtrace_unwind_rows.is_empty() { - let status = self - .status_or_offsets_unavailable(BacktraceStatus::NoUnwindRowsForPc, offsets_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_status_no_rows", - )?; - self.builder - .build_unconditional_branch(done) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - } else { - let inline_depth = depth.min(BPF_INLINE_BACKTRACE_FRAME_LIMIT); - for frame_index in 2..inline_depth { - let current_ip = self.load_i64(ip_ptr, "bt_lookup_ip")?; - let current_module_bias = - self.load_i64(module_bias_ptr, "bt_lookup_module_bias")?; - let current_module_cookie = - self.load_i64(module_cookie_ptr, "bt_lookup_module_cookie")?; - let current_module_found = - self.load_bool(module_found_ptr, "bt_lookup_module_found")?; - let lookup_raw = self.add_signed_offset(current_ip, -1, "bt_lookup_raw")?; - let lookup_pc = self.backtrace_lookup_pc_from_raw( - lookup_raw, - current_module_bias, - current_module_found, - )?; - let runtime_row = self.lookup_backtrace_unwind_row( - lookup_pc, - current_module_cookie, - &scratch.row, - &format!("bt_frame_{frame_index}_row"), - )?; - let found_block = self.context.append_basic_block(current_fn, "bt_row_found"); - let missing_block = self - .context - .append_basic_block(current_fn, "bt_row_missing"); - self.builder - .build_conditional_branch(runtime_row.found, found_block, missing_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(missing_block); - let status = self.status_or_offsets_unavailable( - BacktraceStatus::NoUnwindRowsForPc, - current_module_found, - )?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_status_missing_row", - )?; - self.builder - .build_unconditional_branch(done) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(found_block); - let state = BtRegisterState { - ip: self.load_i64(ip_ptr, "bt_current_ip")?, - rsp: self.load_i64(rsp_ptr, "bt_current_rsp")?, - rbp: self.load_i64(rbp_ptr, "bt_current_rbp")?, - }; - let next = - self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; - let validation = self.validate_backtrace_next_frame(state, next)?; - let store_block = self - .context - .append_basic_block(current_fn, "bt_store_frame"); - let stop_block = self.context.append_basic_block(current_fn, "bt_stop"); - self.builder - .build_conditional_branch(validation.valid, store_block, stop_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(stop_block); - let stop_status = self.status_for_backtrace_stop( - validation.complete, - validation.error_code, - current_module_found, - )?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - stop_status, - "bt_status_stop", - )?; - self.store_u16_value( - inst_buffer, - data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, - validation.error_code, - "bt_error_code_stop", - )?; - self.builder - .build_unconditional_branch(done) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(store_block); - let frame_module = self.resolve_backtrace_frame_module( - next.ip, - current_module_cookie, - current_module_bias, - self.backtrace_module_fallback_found(current_module_found), - &format!("bt_frame_{frame_index}_module"), - )?; - let next_pc = - self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; - self.store_backtrace_frame( - inst_buffer, - frame_index as usize, - frame_module.cookie, - next_pc, - next.ip, - 0, - )?; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, - frame_index + 1, - "bt_frame_count", - )?; - let next_status = if frame_index + 1 == inline_depth { - BacktraceStatus::Truncated - } else { - BacktraceStatus::ReadError - }; - let next_status = - self.status_or_offsets_unavailable(next_status, current_module_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - next_status, - "bt_status_after_frame", - )?; - self.builder - .build_store(ip_ptr, next.ip) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(rsp_ptr, next.rsp) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(rbp_ptr, next.rbp) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_bias_ptr, frame_module.bias) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_cookie_ptr, frame_module.cookie) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_found_ptr, frame_module.found) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - if frame_index + 1 == inline_depth { - self.builder - .build_unconditional_branch(done) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - } - } - } - - self.builder.position_at_end(done); - Ok(()) - } - - fn generate_tail_call_backtrace_instruction( - &mut self, - plan: &BacktraceInstructionPlan, - ) -> Result<()> { - let depth = plan.depth; - let flags = plan.flags; - info!( - "Generating tail-call Backtrace instruction: depth={}", - depth - ); - - let payload_size = plan.payload_size; - let instruction_size = plan.instruction_size; - let offset_ptr = self.event_offset_alloca.ok_or_else(|| { - CodeGenError::LLVMError("event_offset not allocated in entry block".to_string()) - })?; - let inst_offset = self - .builder - .build_load(self.context.i32_type(), offset_ptr, "bt_tail_inst_offset") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let inst_buffer = self - .reserve_instruction_region_or_return_zero(instruction_size as u64)? - .into_value_after_runtime_returns(); - - self.store_u8_const( - inst_buffer, - std::mem::offset_of!(InstructionHeader, inst_type), - InstructionType::Backtrace as u8, - "bt_inst_type", - )?; - self.store_u16_const( - inst_buffer, - std::mem::offset_of!(InstructionHeader, data_length), - payload_size as u16, - "bt_data_length", - )?; - - let data_base = INSTRUCTION_HEADER_SIZE; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_REQUESTED_DEPTH_OFFSET, - depth, - "bt_requested_depth", - )?; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, - 1, - "bt_frame_count", - )?; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_FLAGS_OFFSET, - flags, - "bt_flags", - )?; - self.store_u16_const( - inst_buffer, - data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, - 0, - "bt_error_code", - )?; - - let Some(compile_ctx) = self.current_compile_time_context.clone() else { - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, - 0, - "bt_frame_count_no_context", - )?; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - BacktraceStatus::DwarfUnavailable as u8, - "bt_status_no_context", - )?; - return Ok(()); - }; - - let module_cookie = self.cookie_for_module_or_fallback(&compile_ctx.module_path); - let module_cookie_value = self.context.i64_type().const_int(module_cookie, false); - let pt_regs = self.get_pt_regs_parameter()?; - let raw_ip = self.load_dwarf_register_i64(X86_64_DWARF_RIP, pt_regs)?; - let initial_rsp = self.load_dwarf_register_i64(X86_64_DWARF_RSP, pt_regs)?; - let initial_rbp = self.load_dwarf_register_i64(X86_64_DWARF_RBP, pt_regs)?; - let (module_bias, offsets_found) = self.generate_runtime_address_from_offsets( - self.context.i64_type().const_zero(), - 0, - module_cookie, - )?; - let normalized_pc = self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found)?; - let caller_fallback_found = self.backtrace_module_fallback_found(offsets_found); - - self.store_backtrace_frame( - inst_buffer, - 0, - module_cookie_value, - normalized_pc, - raw_ip, - 0, - )?; - - if depth == 1 { - let status = - self.status_or_offsets_unavailable(BacktraceStatus::Truncated, offsets_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_status_depth_one", - )?; - return Ok(()); - } - - if self.backtrace_unwind_rows.is_empty() { - let status = self - .status_or_offsets_unavailable(BacktraceStatus::NoUnwindRowsForPc, offsets_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_status_no_rows", - )?; - return Ok(()); - } - - let row = match self - .usable_backtrace_unwind_row_for_pc(&compile_ctx.module_path, compile_ctx.pc_address) - { - BacktraceUnwindRowForPc::Usable(row) => row, - row_status => { - let initial_status = self.status_for_backtrace_unwind_row_for_pc(&row_status); - let status = self.status_or_offsets_unavailable(initial_status, offsets_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_status_no_initial_row", - )?; - return Ok(()); - } - }; - - let status = - self.status_or_offsets_unavailable(BacktraceStatus::ReadError, offsets_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_tail_initial_status", - )?; - - let scratch = self.allocate_backtrace_scratch()?; - let current_fn = self.current_function("initialize bt tail-call state")?; - let done_block = self - .context - .append_basic_block(current_fn, "bt_tail_state_done"); - let i64_type = self.context.i64_type(); - let ip_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_ip")?; - let rsp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rsp")?; - let rbp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rbp")?; - let module_bias_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_module_bias")?; - let module_cookie_ptr = - self.build_entry_alloca(i64_type, "bt_tail_prefix_module_cookie")?; - let module_found_ptr = - self.build_entry_alloca(self.context.bool_type(), "bt_tail_prefix_module_found")?; - self.builder - .build_store(module_bias_ptr, module_bias) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_cookie_ptr, module_cookie_value) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_found_ptr, offsets_found) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - let runtime_row = self.runtime_row_from_static(row); - let state = BtRegisterState { - ip: raw_ip, - rsp: initial_rsp, - rbp: initial_rbp, - }; - let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; - let validation = self.validate_backtrace_next_frame(state, next)?; - let initial_store_block = self - .context - .append_basic_block(current_fn, "bt_tail_initial_store_frame"); - let initial_stop_block = self - .context - .append_basic_block(current_fn, "bt_tail_initial_stop"); - self.builder - .build_conditional_branch(validation.valid, initial_store_block, initial_stop_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(initial_stop_block); - let stop_status = self.status_for_backtrace_stop( - validation.complete, - validation.error_code, - offsets_found, - )?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - stop_status, - "bt_tail_initial_stop_status", - )?; - self.store_u16_value( - inst_buffer, - data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, - validation.error_code, - "bt_tail_initial_stop_error_code", - )?; - self.builder - .build_unconditional_branch(done_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(initial_store_block); - let frame_module = self.resolve_backtrace_frame_module( - next.ip, - module_cookie_value, - module_bias, - caller_fallback_found, - "bt_tail_initial_frame_module", - )?; - let next_pc = - self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; - self.store_backtrace_frame(inst_buffer, 1, frame_module.cookie, next_pc, next.ip, 0)?; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, - 2, - "bt_tail_initial_frame_count", - )?; - let status = if depth == 2 { - BacktraceStatus::Truncated - } else { - BacktraceStatus::ReadError - }; - let status = self.status_or_offsets_unavailable(status, offsets_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_tail_initial_status_after_frame", - )?; - self.builder - .build_store(ip_ptr, next.ip) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(rsp_ptr, next.rsp) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(rbp_ptr, next.rbp) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_bias_ptr, frame_module.bias) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_cookie_ptr, frame_module.cookie) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_found_ptr, frame_module.found) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - if depth == 2 { - self.builder - .build_unconditional_branch(done_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder.position_at_end(done_block); - return Ok(()); - } - - let prefix_depth = depth.min(BPF_INLINE_BACKTRACE_FRAME_LIMIT); - for frame_index in 2..prefix_depth { - let current_ip = self.load_i64(ip_ptr, "bt_tail_prefix_lookup_ip")?; - let current_module_bias = - self.load_i64(module_bias_ptr, "bt_tail_prefix_lookup_module_bias")?; - let current_module_cookie = - self.load_i64(module_cookie_ptr, "bt_tail_prefix_lookup_module_cookie")?; - let current_module_found = - self.load_bool(module_found_ptr, "bt_tail_prefix_lookup_module_found")?; - let lookup_raw = self.add_signed_offset(current_ip, -1, "bt_tail_prefix_lookup_raw")?; - let lookup_pc = self.backtrace_lookup_pc_from_raw( - lookup_raw, - current_module_bias, - current_module_found, - )?; - let runtime_row = self.lookup_backtrace_unwind_row( - lookup_pc, - current_module_cookie, - &scratch.row, - &format!("bt_tail_prefix_frame_{frame_index}_row"), - )?; - let found_block = self - .context - .append_basic_block(current_fn, "bt_tail_prefix_row_found"); - let missing_block = self - .context - .append_basic_block(current_fn, "bt_tail_prefix_row_missing"); - self.builder - .build_conditional_branch(runtime_row.found, found_block, missing_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(missing_block); - let status = self.status_or_offsets_unavailable( - BacktraceStatus::NoUnwindRowsForPc, - current_module_found, - )?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_tail_prefix_status_missing_row", - )?; - self.builder - .build_unconditional_branch(done_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(found_block); - let state = BtRegisterState { - ip: self.load_i64(ip_ptr, "bt_tail_prefix_current_ip")?, - rsp: self.load_i64(rsp_ptr, "bt_tail_prefix_current_rsp")?, - rbp: self.load_i64(rbp_ptr, "bt_tail_prefix_current_rbp")?, - }; - let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; - let validation = self.validate_backtrace_next_frame(state, next)?; - let store_block = self - .context - .append_basic_block(current_fn, "bt_tail_prefix_store_frame"); - let stop_block = self - .context - .append_basic_block(current_fn, "bt_tail_prefix_stop"); - self.builder - .build_conditional_branch(validation.valid, store_block, stop_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(stop_block); - let stop_status = self.status_for_backtrace_stop( - validation.complete, - validation.error_code, - current_module_found, - )?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - stop_status, - "bt_tail_prefix_stop_status", - )?; - self.store_u16_value( - inst_buffer, - data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, - validation.error_code, - "bt_tail_prefix_stop_error_code", - )?; - self.builder - .build_unconditional_branch(done_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(store_block); - let frame_module = self.resolve_backtrace_frame_module( - next.ip, - current_module_cookie, - current_module_bias, - self.backtrace_module_fallback_found(current_module_found), - &format!("bt_tail_prefix_frame_{frame_index}_module"), - )?; - let next_pc = - self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; - self.store_backtrace_frame( - inst_buffer, - frame_index as usize, - frame_module.cookie, - next_pc, - next.ip, - 0, - )?; - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, - frame_index + 1, - "bt_tail_prefix_frame_count", - )?; - let status = if frame_index + 1 == depth { - BacktraceStatus::Truncated - } else { - BacktraceStatus::ReadError - }; - let status = self.status_or_offsets_unavailable(status, current_module_found)?; - self.store_u8_value( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - status, - "bt_tail_prefix_status_after_frame", - )?; - self.builder - .build_store(ip_ptr, next.ip) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(rsp_ptr, next.rsp) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(rbp_ptr, next.rbp) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_bias_ptr, frame_module.bias) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_cookie_ptr, frame_module.cookie) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(module_found_ptr, frame_module.found) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - if frame_index + 1 == depth { - self.builder - .build_unconditional_branch(done_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - } - } - - if prefix_depth == depth { - self.builder.position_at_end(done_block); - return Ok(()); - } - - let tail_slot = self.next_backtrace_tail_call_slot; - self.next_backtrace_tail_call_slot = self.next_backtrace_tail_call_slot.saturating_add(1); - if self.pending_backtrace_tail_call.is_none() { - let step_program_name = format!( - "{}_bt_step", - self.current_function("name bt tail-call step")? - .get_name() - .to_string_lossy() - ); - self.pending_backtrace_tail_call = - Some(crate::ebpf::context::PendingBacktraceTailCall { - step_program_name, - depth, - instruction_size, - }); - } - - let tail_enabled_ptr = self.get_or_create_backtrace_tail_enabled_flag()?; - - let state_ptr = self.lookup_bt_state_ptr(tail_slot as u32)?; - let state_is_null = self - .builder - .build_is_null(state_ptr, "bt_state_is_null") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let init_block = self - .context - .append_basic_block(current_fn, "bt_tail_state_init"); - let null_block = self - .context - .append_basic_block(current_fn, "bt_tail_state_null"); - self.builder - .build_conditional_branch(state_is_null, null_block, init_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(null_block); - self.store_u8_const( - inst_buffer, - data_base + BACKTRACE_DATA_STATUS_OFFSET, - BacktraceStatus::InternalError as u8, - "bt_status_state_null", - )?; - self.builder - .build_store(tail_enabled_ptr, self.context.i8_type().const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_unconditional_branch(done_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(init_block); - let tail_ip = self.load_i64(ip_ptr, "bt_tail_state_prefix_ip")?; - let tail_rsp = self.load_i64(rsp_ptr, "bt_tail_state_prefix_rsp")?; - let tail_rbp = self.load_i64(rbp_ptr, "bt_tail_state_prefix_rbp")?; - let tail_module_bias = self.load_i64(module_bias_ptr, "bt_tail_state_module_bias")?; - let tail_module_cookie = self.load_i64(module_cookie_ptr, "bt_tail_state_module_cookie")?; - let tail_module_found = self.load_bool(module_found_ptr, "bt_tail_state_module_found")?; - self.store_state_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, - tail_ip, - "bt_state_ip", - )?; - self.store_state_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, - tail_rsp, - "bt_state_rsp", - )?; - self.store_state_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, - tail_rbp, - "bt_state_rbp", - )?; - self.store_state_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, - tail_module_bias, - "bt_state_module_bias", - )?; - self.store_u64_value( - state_ptr, - crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, - tail_module_cookie, - "bt_state_module_cookie", - )?; - self.store_state_i32( - state_ptr, - crate::BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, - inst_offset, - "bt_state_inst_offset", - )?; - self.store_state_i32( - state_ptr, - crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, - self.context.i32_type().const_zero(), - "bt_state_event_size", - )?; - self.store_u8_const( - state_ptr, - crate::BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, - prefix_depth, - "bt_state_frame_count", - )?; - self.store_u8_const( - state_ptr, - crate::BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, - depth, - "bt_state_requested_depth", - )?; - let offsets_found_u8 = self.bool_to_u8(tail_module_found, "bt_offsets_found_u8")?; - self.store_u8_value( - state_ptr, - crate::BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, - offsets_found_u8, - "bt_state_offsets_found", - )?; - self.store_u8_const( - state_ptr, - crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, - 1, - "bt_state_tail_calls", - )?; - self.store_u8_const( - state_ptr, - crate::BACKTRACE_TAIL_STATE_FLAGS_OFFSET, - flags, - "bt_state_flags", - )?; - self.store_u8_const( - state_ptr, - crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, - tail_slot, - "bt_state_active_slot", - )?; - self.store_u16_const( - state_ptr, - crate::BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, - BACKTRACE_ERROR_NONE, - "bt_state_error_code", - )?; - self.store_u8_const( - state_ptr, - crate::BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, - crate::BACKTRACE_TAIL_NO_NEXT_SLOT, - "bt_state_next_slot", - )?; - self.link_backtrace_tail_slot(tail_slot, offsets_found_u8, done_block)?; - - self.builder.position_at_end(done_block); - Ok(()) - } - - pub(crate) fn finish_event_after_instructions(&mut self) -> Result<()> { - let Some(plan) = self.pending_backtrace_tail_call.clone() else { - return self.emit_accumulated_event_output_from_stack_offset(); - }; - - let main_block = self.current_insert_block("finish bt tail-call event")?; - let main_pm_key_alloca = self.pm_key_alloca; - self.generate_backtrace_tail_call_step_program(&plan)?; - self.pm_key_alloca = main_pm_key_alloca; - self.builder.position_at_end(main_block); - - let tail_enabled_ptr = self.get_or_create_backtrace_tail_enabled_flag()?; - let enabled_value = self - .builder - .build_load( - self.context.i8_type(), - tail_enabled_ptr, - "bt_tail_enabled_value", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let enabled = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - enabled_value, - self.context.i8_type().const_zero(), - "bt_tail_enabled", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let current_fn = self.current_function("finish bt tail-call event")?; - let tail_block = self - .context - .append_basic_block(current_fn, "bt_tail_dispatch"); - let output_block = self - .context - .append_basic_block(current_fn, "bt_tail_fallback_output"); - self.builder - .build_conditional_branch(enabled, tail_block, output_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(tail_block); - let state0_ptr = self.lookup_bt_state_ptr(0)?; - let state0_is_null = self - .builder - .build_is_null(state0_ptr, "bt_tail_dispatch_state_null") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let state_ok_block = self - .context - .append_basic_block(current_fn, "bt_tail_dispatch_state_ok"); - self.builder - .build_conditional_branch(state0_is_null, output_block, state_ok_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(state_ok_block); - let active_slot = self.load_row_i8( - state0_ptr, - crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, - "bt_tail_dispatch_active_slot", - )?; - let state_ptr = self.lookup_bt_state_ptr_dynamic(active_slot)?; - let state_is_null = self - .builder - .build_is_null(state_ptr, "bt_tail_dispatch_active_state_null") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let active_state_ok_block = self - .context - .append_basic_block(current_fn, "bt_tail_dispatch_active_state_ok"); - self.builder - .build_conditional_branch(state_is_null, output_block, active_state_ok_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(active_state_ok_block); - let event_size = self - .builder - .build_load( - self.context.i32_type(), - self.event_offset_alloca.ok_or_else(|| { - CodeGenError::LLVMError("event_offset not allocated in entry block".to_string()) - })?, - "bt_tail_event_size", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - self.store_state_i32( - state_ptr, - crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, - event_size, - "bt_tail_state_event_size", - )?; - self.emit_bpf_tail_call(BPF_BACKTRACE_STEP_PROG_INDEX)?; - self.builder - .build_unconditional_branch(output_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(output_block); - self.emit_accumulated_event_output_from_stack_offset() - } - - fn generate_backtrace_tail_call_step_program( - &mut self, - plan: &crate::ebpf::context::PendingBacktraceTailCall, - ) -> Result<()> { - self.create_tail_call_function(&plan.step_program_name)?; - let current_fn = self.current_function("generate bt tail-call step")?; - let return_block = self - .context - .append_basic_block(current_fn, "bt_step_return"); - let state_ok_block = self - .context - .append_basic_block(current_fn, "bt_step_state_ok"); - let accum_ok_block = self - .context - .append_basic_block(current_fn, "bt_step_accum_ok"); - let bounds_ok_block = self - .context - .append_basic_block(current_fn, "bt_step_bounds_ok"); - let inst_bounds_ok_block = self - .context - .append_basic_block(current_fn, "bt_step_inst_bounds_ok"); - let finalize_block = self - .context - .append_basic_block(current_fn, "bt_step_finalize"); - - let state0_ptr = self.lookup_bt_state_ptr(0)?; - let state_is_null = self - .builder - .build_is_null(state0_ptr, "bt_step_state_null") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_conditional_branch(state_is_null, return_block, state_ok_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(state_ok_block); - let active_slot = self.load_row_i8( - state0_ptr, - crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, - "bt_step_active_slot", - )?; - let state_ptr = self.lookup_bt_state_ptr_dynamic(active_slot)?; - let active_state_is_null = self - .builder - .build_is_null(state_ptr, "bt_step_active_state_null") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let active_state_ok_block = self - .context - .append_basic_block(current_fn, "bt_step_active_state_ok"); - self.builder - .build_conditional_branch(active_state_is_null, return_block, active_state_ok_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(active_state_ok_block); - let accum_buffer = self.lookup_percpu_value_ptr("event_accum_buffer", 0)?; - let accum_is_null = self - .builder - .build_is_null(accum_buffer, "bt_step_accum_null") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_conditional_branch(accum_is_null, return_block, accum_ok_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(accum_ok_block); - let inst_offset = self.load_state_i32( - state_ptr, - crate::BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, - "bt_step_inst_offset", - )?; - let event_size = self.load_state_i32( - state_ptr, - crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, - "bt_step_event_size", - )?; - let max_event_size = self - .context - .i32_type() - .const_int(self.compile_options.max_trace_event_size as u64, false); - let max_inst_offset = self.context.i32_type().const_int( - self.compile_options - .max_trace_event_size - .saturating_sub(plan.instruction_size as u32) as u64, - false, - ); - let inst_in_bounds = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULE, - inst_offset, - max_inst_offset, - "bt_step_inst_offset_in_bounds", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_conditional_branch(inst_in_bounds, inst_bounds_ok_block, return_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(inst_bounds_ok_block); - let event_in_bounds = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULE, - event_size, - max_event_size, - "bt_step_event_size_in_bounds", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_conditional_branch(event_in_bounds, bounds_ok_block, return_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(bounds_ok_block); - let inst_offset_i64 = self - .builder - .build_int_z_extend( - inst_offset, - self.context.i64_type(), - "bt_step_inst_offset_i64", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let inst_buffer = - self.dynamic_byte_gep(accum_buffer, inst_offset_i64, "bt_step_inst_buffer")?; - let scratch = self.allocate_backtrace_scratch()?; - for _ in 0..BPF_BACKTRACE_FRAMES_PER_TAIL_CALL { - self.generate_backtrace_tail_call_step_iteration( - plan.depth, - state_ptr, - inst_buffer, - &scratch, - finalize_block, - )?; - } - - let tail_calls = self.load_row_i8( - state_ptr, - crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, - "bt_step_tail_calls", - )?; - let can_tail_call = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULT, - tail_calls, - self.context - .i8_type() - .const_int(BPF_BACKTRACE_MAX_STEP_INVOCATIONS as u64, false), - "bt_step_can_tail_call", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let self_tail_block = self - .context - .append_basic_block(current_fn, "bt_step_self_tail"); - let tail_budget_done = self - .context - .append_basic_block(current_fn, "bt_step_tail_budget_done"); - self.builder - .build_conditional_branch(can_tail_call, self_tail_block, tail_budget_done) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(tail_budget_done); - self.store_tail_backtrace_status( - inst_buffer, - BacktraceStatus::Truncated, - BACKTRACE_ERROR_NONE, - )?; - self.builder - .build_unconditional_branch(finalize_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(self_tail_block); - let next_tail_calls = self - .builder - .build_int_add( - tail_calls, - self.context.i8_type().const_int(1, false), - "bt_step_next_tail_calls", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.store_u8_value( - state_ptr, - crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, - next_tail_calls, - "bt_step_store_tail_calls", - )?; - self.emit_bpf_tail_call(BPF_BACKTRACE_STEP_PROG_INDEX)?; - self.store_tail_backtrace_status( - inst_buffer, - BacktraceStatus::InternalError, - BACKTRACE_ERROR_NONE, - )?; - self.builder - .build_unconditional_branch(finalize_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(finalize_block); - let next_slot = self.load_row_i8( - state_ptr, - crate::BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, - "bt_final_next_slot", - )?; - let has_next_slot = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - next_slot, - self.context - .i8_type() - .const_int(crate::BACKTRACE_TAIL_NO_NEXT_SLOT as u64, false), - "bt_final_has_next_slot", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let tail_calls = self.load_row_i8( - state_ptr, - crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, - "bt_final_tail_calls", - )?; - let can_tail_call_next = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULT, - tail_calls, - self.context - .i8_type() - .const_int(BPF_BACKTRACE_MAX_STEP_INVOCATIONS as u64, false), - "bt_final_can_tail_call_next", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let should_continue_next = self - .builder - .build_and( - has_next_slot, - can_tail_call_next, - "bt_final_should_continue_next_slot", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let next_slot_block = self - .context - .append_basic_block(current_fn, "bt_final_next_slot"); - let emit_block = self.context.append_basic_block(current_fn, "bt_final_emit"); - self.builder - .build_conditional_branch(should_continue_next, next_slot_block, emit_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(next_slot_block); - self.store_u8_value( - state0_ptr, - crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, - next_slot, - "bt_store_active_slot", - )?; - let next_state_ptr = self.lookup_bt_state_ptr_dynamic(next_slot)?; - let next_state_is_null = self - .builder - .build_is_null(next_state_ptr, "bt_next_state_null") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let next_state_ok_block = self - .context - .append_basic_block(current_fn, "bt_next_state_ok"); - self.builder - .build_conditional_branch(next_state_is_null, emit_block, next_state_ok_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(next_state_ok_block); - self.store_state_i32( - next_state_ptr, - crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, - event_size, - "bt_next_slot_event_size", - )?; - let next_tail_calls = self - .builder - .build_int_add( - tail_calls, - self.context.i8_type().const_int(1, false), - "bt_next_slot_tail_calls", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.store_u8_value( - next_state_ptr, - crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, - next_tail_calls, - "bt_next_slot_store_tail_calls", - )?; - self.emit_bpf_tail_call(BPF_BACKTRACE_STEP_PROG_INDEX)?; - self.builder - .build_unconditional_branch(emit_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(emit_block); - self.emit_tail_final_event(state_ptr, accum_buffer)?; - self.builder - .build_unconditional_branch(return_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(return_block); - self.build_return_zero() - } - - fn generate_backtrace_tail_call_step_iteration( - &mut self, - depth: u8, - state_ptr: PointerValue<'ctx>, - inst_buffer: PointerValue<'ctx>, - scratch: &BtScratch<'ctx>, - finalize_block: BasicBlock<'ctx>, - ) -> Result<()> { - let current_fn = self.current_function("generate bt tail-call step iteration")?; - let depth_block = self - .context - .append_basic_block(current_fn, "bt_step_depth_done"); - let unwind_block = self - .context - .append_basic_block(current_fn, "bt_step_unwind"); - let frame_count = self.load_row_i8( - state_ptr, - crate::BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, - "bt_step_frame_count", - )?; - let depth_value = self.context.i8_type().const_int(depth as u64, false); - let at_depth = self - .builder - .build_int_compare( - inkwell::IntPredicate::UGE, - frame_count, - depth_value, - "bt_step_at_depth", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_conditional_branch(at_depth, depth_block, unwind_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(depth_block); - self.store_tail_backtrace_status( - inst_buffer, - BacktraceStatus::Truncated, - BACKTRACE_ERROR_NONE, - )?; - self.builder - .build_unconditional_branch(finalize_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(unwind_block); - let current_ip = self.load_row_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, - "bt_step_current_ip", - )?; - let current_rsp = self.load_row_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, - "bt_step_current_rsp", - )?; - let current_rbp = self.load_row_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, - "bt_step_current_rbp", - )?; - let module_bias = self.load_row_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, - "bt_step_module_bias", - )?; - let module_cookie = self.load_row_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, - "bt_step_module_cookie", - )?; - let offsets_found_u8 = self.load_row_i8( - state_ptr, - crate::BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, - "bt_step_offsets_found_u8", - )?; - let offsets_found = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - offsets_found_u8, - self.context.i8_type().const_zero(), - "bt_step_offsets_found", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let caller_fallback_found = self.backtrace_module_fallback_found(offsets_found); - let is_first_unwind = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - frame_count, - self.context.i8_type().const_int(1, false), - "bt_step_first_unwind", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let caller_lookup_ip = - self.add_signed_offset(current_ip, -1, "bt_step_caller_lookup_ip")?; - let lookup_raw = self - .builder - .build_select::, _>( - is_first_unwind, - current_ip.into(), - caller_lookup_ip.into(), - "bt_step_lookup_raw", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let lookup_pc = - self.backtrace_lookup_pc_from_raw(lookup_raw, module_bias, offsets_found)?; - let runtime_row = self.lookup_backtrace_unwind_row( - lookup_pc, - module_cookie, - &scratch.row, - "bt_step_row", - )?; - let row_found_block = self - .context - .append_basic_block(current_fn, "bt_step_row_found"); - let row_missing_block = self - .context - .append_basic_block(current_fn, "bt_step_row_missing"); - self.builder - .build_conditional_branch(runtime_row.found, row_found_block, row_missing_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(row_missing_block); - self.store_tail_backtrace_status( - inst_buffer, - BacktraceStatus::NoUnwindRowsForPc, - BACKTRACE_ERROR_NONE, - )?; - self.builder - .build_unconditional_branch(finalize_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(row_found_block); - let state = BtRegisterState { - ip: current_ip, - rsp: current_rsp, - rbp: current_rbp, - }; - let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, scratch)?; - let validation = self.validate_backtrace_next_frame(state, next)?; - let store_block = self - .context - .append_basic_block(current_fn, "bt_step_store_frame"); - let stop_block = self.context.append_basic_block(current_fn, "bt_step_stop"); - self.builder - .build_conditional_branch(validation.valid, store_block, stop_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(stop_block); - let stop_status = self.status_for_backtrace_stop( - validation.complete, - validation.error_code, - offsets_found, - )?; - self.store_u8_value( - inst_buffer, - INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_STATUS_OFFSET, - stop_status, - "bt_step_stop_status", - )?; - self.store_u16_value( - inst_buffer, - INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_ERROR_CODE_OFFSET, - validation.error_code, - "bt_step_stop_error", - )?; - self.builder - .build_unconditional_branch(finalize_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(store_block); - let frame_module = self.resolve_backtrace_frame_module( - next.ip, - module_cookie, - module_bias, - caller_fallback_found, - "bt_step_frame_module", - )?; - let next_pc = - self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; - self.store_backtrace_frame_dynamic( - inst_buffer, - frame_count, - depth.saturating_sub(1), - frame_module.cookie, - next_pc, - next.ip, - )?; - let next_count = self - .builder - .build_int_add( - frame_count, - self.context.i8_type().const_int(1, false), - "bt_step_next_frame_count", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.store_u8_value( - state_ptr, - crate::BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, - next_count, - "bt_step_state_frame_count", - )?; - self.store_u8_value( - inst_buffer, - INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_FRAME_COUNT_OFFSET, - next_count, - "bt_step_inst_frame_count", - )?; - self.store_state_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, - next.ip, - "bt_step_state_next_ip", - )?; - self.store_state_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, - next.rsp, - "bt_step_state_next_rsp", - )?; - self.store_state_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, - next.rbp, - "bt_step_state_next_rbp", - )?; - self.store_state_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, - frame_module.bias, - "bt_step_state_next_module_bias", - )?; - self.store_state_i64( - state_ptr, - crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, - frame_module.cookie, - "bt_step_state_next_module_cookie", - )?; - let next_offsets_found_u8 = - self.bool_to_u8(frame_module.found, "bt_step_next_offsets_found_u8")?; - self.store_u8_value( - state_ptr, - crate::BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, - next_offsets_found_u8, - "bt_step_state_next_offsets_found", - )?; - - let reached_depth = self - .builder - .build_int_compare( - inkwell::IntPredicate::UGE, - next_count, - depth_value, - "bt_step_reached_depth", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let reached_depth_block = self - .context - .append_basic_block(current_fn, "bt_step_reached_depth"); - let continue_block = self - .context - .append_basic_block(current_fn, "bt_step_continue"); - self.builder - .build_conditional_branch(reached_depth, reached_depth_block, continue_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(reached_depth_block); - self.store_tail_backtrace_status( - inst_buffer, - BacktraceStatus::Truncated, - BACKTRACE_ERROR_NONE, - )?; - self.builder - .build_unconditional_branch(finalize_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(continue_block); - Ok(()) - } - - fn emit_bpf_tail_call(&mut self, index: u32) -> Result<()> { - let ctx = self.get_pt_regs_parameter()?; - let prog_array = self.lookup_bt_prog_array_ptr()?; - let args = [ - ctx.into(), - prog_array.into(), - self.context - .i32_type() - .const_int(index as u64, false) - .into(), - ]; - let _ = self.create_bpf_helper_call( - BPF_FUNC_tail_call as u64, - &args, - self.context.i64_type().into(), - "bt_bpf_tail_call", - )?; - Ok(()) - } - - fn store_tail_backtrace_status( - &self, - inst_buffer: PointerValue<'ctx>, - status: BacktraceStatus, - error_code: u16, - ) -> Result<()> { - self.store_u8_const( - inst_buffer, - INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_STATUS_OFFSET, - status as u8, - "bt_tail_status", - )?; - self.store_u16_const( - inst_buffer, - INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_ERROR_CODE_OFFSET, - error_code, - "bt_tail_error_code", - ) - } - - fn emit_tail_final_event( - &mut self, - state_ptr: PointerValue<'ctx>, - accum_buffer: PointerValue<'ctx>, - ) -> Result<()> { - let event_size = self.load_state_i32( - state_ptr, - crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, - "bt_final_event_size", - )?; - self.emit_accumulated_event_output(accum_buffer, event_size) - } - - fn build_return_zero(&mut self) -> Result<()> { - self.builder - .build_return(Some(&self.context.i32_type().const_zero())) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(()) - } - - fn compact_unwind_row_for_backtrace( - &self, - module_path: &str, - pc: u64, - ) -> Option { - let analyzer = self.process_analyzer?; - let module_address = ModuleAddress::new(PathBuf::from(module_path), pc); - let ctx = analyzer.resolve_pc(&module_address).ok()?; - analyzer.compact_unwind_row_for_context(&ctx).ok().flatten() - } - - fn usable_backtrace_unwind_row_for_pc( - &self, - module_path: &str, - pc: u64, - ) -> BacktraceUnwindRowForPc { - let Some(row) = self.compact_unwind_row_for_backtrace(module_path, pc) else { - return BacktraceUnwindRowForPc::Missing; - }; - match crate::backtrace_unwind_row_from_compact(&row) { - Some(row) => BacktraceUnwindRowForPc::Usable(row), - None => BacktraceUnwindRowForPc::Unsupported, - } - } - - fn status_for_backtrace_unwind_row_for_pc( - &self, - row: &BacktraceUnwindRowForPc, - ) -> BacktraceStatus { - match row { - BacktraceUnwindRowForPc::Usable(_) => BacktraceStatus::ReadError, - BacktraceUnwindRowForPc::Missing if self.process_analyzer.is_some() => { - BacktraceStatus::NoUnwindRowsForPc - } - BacktraceUnwindRowForPc::Unsupported if self.process_analyzer.is_some() => { - BacktraceStatus::UnsupportedCfi - } - BacktraceUnwindRowForPc::Missing | BacktraceUnwindRowForPc::Unsupported => { - BacktraceStatus::DwarfUnavailable - } - } - } - - fn runtime_row_from_static( - &self, - row: ghostscope_protocol::BacktraceUnwindRow, - ) -> RuntimeBtUnwindRow<'ctx> { - let i8_type = self.context.i8_type(); - let i16_type = self.context.i16_type(); - let i64_type = self.context.i64_type(); - RuntimeBtUnwindRow { - found: self.context.bool_type().const_int(1, false), - cfa_register: i16_type.const_int(row.cfa_register as u64, false), - cfa_offset: i64_type.const_int(row.cfa_offset as u64, true), - ra_kind: i8_type.const_int(row.ra_kind as u64, false), - ra_register: i16_type.const_int(row.ra_register as u64, false), - ra_offset: i64_type.const_int(row.ra_offset as u64, true), - rbp_kind: i8_type.const_int(row.rbp_kind as u64, false), - rbp_register: i16_type.const_int(row.rbp_register as u64, false), - rbp_offset: i64_type.const_int(row.rbp_offset as u64, true), - } - } - - fn allocate_backtrace_scratch(&self) -> Result> { - let i16_type = self.context.i16_type(); - let i32_type = self.context.i32_type(); - let i64_type = self.context.i64_type(); - - Ok(BtScratch { - row: RuntimeBtRowScratch { - found_ptr: self.build_entry_alloca(i32_type, "bt_row_found")?, - cfa_register_ptr: self.build_entry_alloca(i16_type, "bt_row_cfa_register")?, - cfa_offset_ptr: self.build_entry_alloca(i64_type, "bt_row_cfa_offset")?, - ra_kind_ptr: self.build_entry_alloca(self.context.i8_type(), "bt_row_ra_kind")?, - ra_register_ptr: self.build_entry_alloca(i16_type, "bt_row_ra_register")?, - ra_offset_ptr: self.build_entry_alloca(i64_type, "bt_row_ra_offset")?, - rbp_kind_ptr: self.build_entry_alloca(self.context.i8_type(), "bt_row_rbp_kind")?, - rbp_register_ptr: self.build_entry_alloca(i16_type, "bt_row_rbp_register")?, - rbp_offset_ptr: self.build_entry_alloca(i64_type, "bt_row_rbp_offset")?, - }, - next_rbp_ptr: self.build_entry_alloca(i64_type, "bt_next_rbp")?, - next_error_code_ptr: self.build_entry_alloca(i16_type, "bt_next_error_code")?, - }) - } - - fn lookup_backtrace_unwind_row( - &mut self, - normalized_pc: IntValue<'ctx>, - module_cookie: IntValue<'ctx>, - scratch: &RuntimeBtRowScratch<'ctx>, - name_prefix: &str, - ) -> Result> { - let bounds = self.backtrace_unwind_row_bounds_for_module(module_cookie, name_prefix)?; - self.lookup_backtrace_unwind_row_in_range(normalized_pc, bounds.start, bounds.end, scratch) - } - - fn backtrace_unwind_row_bounds_for_module( - &mut self, - module_cookie: IntValue<'ctx>, - name_prefix: &str, - ) -> Result> { - let i32_type = self.context.i32_type(); - if self.backtrace_module_row_ranges.is_empty() { - return Ok(BtRowBounds { - start: i32_type.const_zero(), - end: i32_type.const_int(self.backtrace_unwind_rows.len() as u64, false), - }); - } - - let i64_type = self.context.i64_type(); - let ptr_type = self.context.ptr_type(AddressSpace::default()); - let map_global = self - .module - .get_global("bt_module_row_ranges") - .ok_or_else(|| { - CodeGenError::LLVMError("bt_module_row_ranges map not found".to_string()) - })?; - let map_ptr = self - .builder - .build_bit_cast( - map_global.as_pointer_value(), - ptr_type, - &format!("{name_prefix}_row_ranges_map_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let key_alloca = - self.build_entry_alloca(i64_type, &format!("{name_prefix}_row_range_key"))?; - self.builder - .build_store(key_alloca, module_cookie) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let key_ptr = self - .builder - .build_bit_cast( - key_alloca, - ptr_type, - &format!("{name_prefix}_row_range_key_void"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let result = self.create_bpf_helper_call( - BPF_FUNC_map_lookup_elem as u64, - &[map_ptr, key_ptr], - ptr_type.into(), - &format!("{name_prefix}_row_range_lookup"), - )?; - let range_ptr = match result { - BasicValueEnum::PointerValue(ptr) => ptr, - _ => { - return Err(CodeGenError::LLVMError( - "bt_module_row_ranges lookup did not return pointer".to_string(), - )) - } - }; - let is_null = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - self.builder - .build_ptr_to_int( - range_ptr, - i64_type, - &format!("{name_prefix}_row_range_ptr_i64"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?, - i64_type.const_zero(), - &format!("{name_prefix}_row_range_is_null"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let current_fn = self.current_function("lookup bt row range")?; - let found_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_found_row_range")); - let miss_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_miss_row_range")); - let cont_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_cont_row_range")); - self.builder - .build_conditional_branch(is_null, miss_block, found_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(found_block); - let load_range_field = |offset: usize, - field_name: &str, - ctx: &mut EbpfContext<'ctx, 'dw>| - -> Result> { - let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); - // SAFETY: range_ptr is a non-null BacktraceModuleRowRange pointer - // returned by bpf_map_lookup_elem, and offsets are shared ABI - // constants from ghostscope-protocol. - let field_ptr = unsafe { - ctx.builder - .build_gep(ctx.context.i8_type(), range_ptr, &[offset_i32], field_name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - Ok(ctx - .builder - .build_load(ctx.context.i32_type(), field_ptr, field_name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value()) - }; - let found_start = load_range_field( - ghostscope_protocol::BACKTRACE_MODULE_ROW_RANGE_ROW_START_OFFSET, - &format!("{name_prefix}_row_range_start"), - self, - )?; - let found_end = load_range_field( - ghostscope_protocol::BACKTRACE_MODULE_ROW_RANGE_ROW_END_OFFSET, - &format!("{name_prefix}_row_range_end"), - self, - )?; - self.builder - .build_unconditional_branch(cont_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let found_end_block = self.current_insert_block("finish bt row range found block")?; - - self.builder.position_at_end(miss_block); - self.builder - .build_unconditional_branch(cont_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let miss_end_block = self.current_insert_block("finish bt row range miss block")?; - - self.builder.position_at_end(cont_block); - let start_phi = self - .builder - .build_phi(i32_type, &format!("{name_prefix}_row_start_phi")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - start_phi.add_incoming(&[ - (&found_start, found_end_block), - (&i32_type.const_zero(), miss_end_block), - ]); - let end_phi = self - .builder - .build_phi(i32_type, &format!("{name_prefix}_row_end_phi")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - end_phi.add_incoming(&[ - (&found_end, found_end_block), - (&i32_type.const_zero(), miss_end_block), - ]); - - Ok(BtRowBounds { - start: start_phi.as_basic_value().into_int_value(), - end: end_phi.as_basic_value().into_int_value(), - }) - } - - fn lookup_backtrace_unwind_row_in_range( - &mut self, - normalized_pc: IntValue<'ctx>, - row_start: IntValue<'ctx>, - row_end: IntValue<'ctx>, - scratch: &RuntimeBtRowScratch<'ctx>, - ) -> Result> { - let row_count = self.backtrace_unwind_row_map_entries() as usize; - let i16_type = self.context.i16_type(); - let i32_type = self.context.i32_type(); - let i64_type = self.context.i64_type(); - let i8_type = self.context.i8_type(); - let sentinel = i32_type.const_int(row_count as u64, false); - - self.builder - .build_store(scratch.found_ptr, sentinel) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.cfa_register_ptr, i16_type.const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.cfa_offset_ptr, i64_type.const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.ra_kind_ptr, i8_type.const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.ra_register_ptr, i16_type.const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.ra_offset_ptr, i64_type.const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.rbp_kind_ptr, i8_type.const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.rbp_register_ptr, i16_type.const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.rbp_offset_ptr, i64_type.const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - let current_fn = self.current_function("lookup bt unwind row")?; - let return_block = self - .context - .append_basic_block(current_fn, "bt_row_lookup_return"); - if row_count == 0 { - self.builder - .build_unconditional_branch(return_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - } else { - let lo_ptr = self.build_entry_alloca(i32_type, "bt_row_lo")?; - let hi_ptr = self.build_entry_alloca(i32_type, "bt_row_hi")?; - self.builder - .build_store(lo_ptr, row_start) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(hi_ptr, row_end) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.emit_backtrace_row_runtime_binary_search( - normalized_pc, - scratch, - lo_ptr, - hi_ptr, - row_count, - return_block, - )?; - } - self.builder.position_at_end(return_block); - let final_found_idx = self.load_i32(scratch.found_ptr, "bt_final_found_idx")?; - let found = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - final_found_idx, - sentinel, - "bt_final_row_found", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(RuntimeBtUnwindRow { - found, - cfa_register: self.load_i16(scratch.cfa_register_ptr, "bt_final_cfa_reg")?, - cfa_offset: self.load_i64(scratch.cfa_offset_ptr, "bt_final_cfa_off")?, - ra_kind: self.load_i8(scratch.ra_kind_ptr, "bt_final_ra_kind")?, - ra_register: self.load_i16(scratch.ra_register_ptr, "bt_final_ra_reg")?, - ra_offset: self.load_i64(scratch.ra_offset_ptr, "bt_final_ra_off")?, - rbp_kind: self.load_i8(scratch.rbp_kind_ptr, "bt_final_rbp_kind")?, - rbp_register: self.load_i16(scratch.rbp_register_ptr, "bt_final_rbp_reg")?, - rbp_offset: self.load_i64(scratch.rbp_offset_ptr, "bt_final_rbp_off")?, - }) - } - - fn emit_backtrace_row_runtime_binary_search( - &mut self, - normalized_pc: IntValue<'ctx>, - scratch: &RuntimeBtRowScratch<'ctx>, - lo_ptr: PointerValue<'ctx>, - hi_ptr: PointerValue<'ctx>, - row_count: usize, - return_block: BasicBlock<'ctx>, - ) -> Result<()> { - let current_fn = self.current_function("emit bt row lookup tree")?; - let i32_type = self.context.i32_type(); - let sentinel = i32_type.const_int(row_count as u64, false); - let max_steps = backtrace_row_binary_search_steps(row_count); - - for _ in 0..max_steps { - let found_idx = self.load_i32(scratch.found_ptr, "bt_lookup_found_idx")?; - let not_found = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - found_idx, - sentinel, - "bt_lookup_not_found", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let lo = self.load_i32(lo_ptr, "bt_lookup_lo")?; - let hi = self.load_i32(hi_ptr, "bt_lookup_hi")?; - let range_active = self - .builder - .build_int_compare(inkwell::IntPredicate::ULT, lo, hi, "bt_lookup_range_active") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let should_search = self - .builder - .build_and(not_found, range_active, "bt_lookup_should_search") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - let search_block = self - .context - .append_basic_block(current_fn, "bt_lookup_search"); - let skip_block = self - .context - .append_basic_block(current_fn, "bt_lookup_skip"); - let after_block = self - .context - .append_basic_block(current_fn, "bt_lookup_after"); - self.builder - .build_conditional_branch(should_search, search_block, skip_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(skip_block); - self.builder - .build_unconditional_branch(after_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(search_block); - let lo_plus_hi = self - .builder - .build_int_add(lo, hi, "bt_lookup_lo_plus_hi") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let mid = self - .builder - .build_right_shift( - lo_plus_hi, - i32_type.const_int(1, false), - false, - "bt_lookup_mid", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let row_ptr = self.lookup_bt_unwind_row_ptr(mid)?; - let row_is_null = self - .builder - .build_is_null(row_ptr, "bt_lookup_row_is_null") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let row_null_block = self - .context - .append_basic_block(current_fn, "bt_lookup_row_null"); - let row_load_block = self - .context - .append_basic_block(current_fn, "bt_lookup_row_load"); - self.builder - .build_conditional_branch(row_is_null, row_null_block, row_load_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(row_null_block); - self.builder - .build_store(lo_ptr, hi) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_unconditional_branch(after_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(row_load_block); - let pc_start = self.load_row_i64( - row_ptr, - crate::BACKTRACE_UNWIND_ROW_PC_START_OFFSET, - "bt_lookup_row_pc_start", - )?; - let pc_end = self.load_row_i64( - row_ptr, - crate::BACKTRACE_UNWIND_ROW_PC_END_OFFSET, - "bt_lookup_row_pc_end", - )?; - let before = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULT, - normalized_pc, - pc_start, - "bt_lookup_pc_before", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let before_block = self - .context - .append_basic_block(current_fn, "bt_lookup_before"); - let not_before_block = self - .context - .append_basic_block(current_fn, "bt_lookup_not_before"); - self.builder - .build_conditional_branch(before, before_block, not_before_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(before_block); - self.builder - .build_store(hi_ptr, mid) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_unconditional_branch(after_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(not_before_block); - let after = self - .builder - .build_int_compare( - inkwell::IntPredicate::UGE, - normalized_pc, - pc_end, - "bt_lookup_pc_after", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let after_range_block = self - .context - .append_basic_block(current_fn, "bt_lookup_after_range"); - let match_block = self - .context - .append_basic_block(current_fn, "bt_lookup_match"); - self.builder - .build_conditional_branch(after, after_range_block, match_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(after_range_block); - let mid_plus_one = self - .builder - .build_int_add(mid, i32_type.const_int(1, false), "bt_lookup_mid_plus_one") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(lo_ptr, mid_plus_one) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_unconditional_branch(after_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(match_block); - self.store_backtrace_unwind_row_from_ptr(row_ptr, mid, scratch)?; - self.builder - .build_unconditional_branch(after_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(after_block); - } - - self.builder - .build_unconditional_branch(return_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(()) - } - - fn store_backtrace_unwind_row_from_ptr( - &self, - row_ptr: PointerValue<'ctx>, - row_index: IntValue<'ctx>, - scratch: &RuntimeBtRowScratch<'ctx>, - ) -> Result<()> { - self.builder - .build_store(scratch.found_ptr, row_index) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let cfa_register = self.load_row_i16( - row_ptr, - crate::BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, - "bt_tree_row_cfa_reg", - )?; - let cfa_offset = self.load_row_i64( - row_ptr, - crate::BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, - "bt_tree_row_cfa_off", - )?; - let ra_kind = self.load_row_i8( - row_ptr, - crate::BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, - "bt_tree_row_ra_kind", - )?; - let ra_register = self.load_row_i16( - row_ptr, - crate::BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, - "bt_tree_row_ra_reg", - )?; - let ra_offset = self.load_row_i64( - row_ptr, - crate::BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, - "bt_tree_row_ra_off", - )?; - let rbp_kind = self.load_row_i8( - row_ptr, - crate::BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, - "bt_tree_row_rbp_kind", - )?; - let rbp_register = self.load_row_i16( - row_ptr, - crate::BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, - "bt_tree_row_rbp_reg", - )?; - let rbp_offset = self.load_row_i64( - row_ptr, - crate::BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, - "bt_tree_row_rbp_off", - )?; - self.builder - .build_store(scratch.cfa_register_ptr, cfa_register) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.cfa_offset_ptr, cfa_offset) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.ra_kind_ptr, ra_kind) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.ra_register_ptr, ra_register) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.ra_offset_ptr, ra_offset) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.rbp_kind_ptr, rbp_kind) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.rbp_register_ptr, rbp_register) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(scratch.rbp_offset_ptr, rbp_offset) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(()) - } - - fn recover_next_frame_from_runtime_row( - &mut self, - row: &RuntimeBtUnwindRow<'ctx>, - state: BtRegisterState<'ctx>, - scratch: &BtScratch<'ctx>, - ) -> Result> { - let cfa_base = self.select_register_state(row.cfa_register, state, "bt_cfa_base")?; - let cfa = self - .builder - .build_int_add(cfa_base, row.cfa_offset, "bt_runtime_cfa") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - let ra_addr = self - .builder - .build_int_add(cfa, row.ra_offset, "bt_runtime_ra_addr") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store( - scratch.next_error_code_ptr, - self.context - .i16_type() - .const_int(BACKTRACE_ERROR_NONE as u64, false), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let (ra_from_memory, ra_read_failed) = self.generate_memory_read_with_fail_flag( - RuntimeAddress::available(ra_addr, self.context), - MemoryAccessSize::U64, - "bt_ra_read", - )?; - let ra_from_memory = ra_from_memory.into_int_value(); - let ra_uses_memory = self.is_recovery_kind( - row.ra_kind, - crate::BACKTRACE_RECOVERY_AT_CFA_OFFSET, - "bt_ra_at_kind", - )?; - let ra_read_failed = self - .builder - .build_and(ra_read_failed, ra_uses_memory, "bt_ra_read_failed") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.store_backtrace_error_code_if( - scratch.next_error_code_ptr, - ra_read_failed, - BACKTRACE_ERROR_RETURN_ADDRESS_READ, - "bt_ra_error_code", - )?; - let ra_from_val = self - .builder - .build_int_add(cfa, row.ra_offset, "bt_runtime_ra_val") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let ra_from_register = self.select_register_state(row.ra_register, state, "bt_ra_reg")?; - let ra_is_val = self.is_recovery_kind( - row.ra_kind, - crate::BACKTRACE_RECOVERY_VAL_CFA_OFFSET, - "bt_ra_val_kind", - )?; - let ra_is_register = self.is_recovery_kind( - row.ra_kind, - crate::BACKTRACE_RECOVERY_REGISTER, - "bt_ra_reg_kind", - )?; - let ra_is_same = self.is_recovery_kind( - row.ra_kind, - crate::BACKTRACE_RECOVERY_SAME_VALUE, - "bt_ra_same_kind", - )?; - let ra_is_register_like = self - .builder - .build_or(ra_is_register, ra_is_same, "bt_ra_register_like") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let ra_value_or_memory = self - .builder - .build_select::, _>( - ra_is_val, - ra_from_val.into(), - ra_from_memory.into(), - "bt_ra_val_or_memory", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let next_ip = self - .builder - .build_select::, _>( - ra_is_register_like, - ra_from_register.into(), - ra_value_or_memory.into(), - "bt_next_ip", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let next_rbp = self.recover_rbp_from_runtime_row( - row, - cfa, - state, - scratch.next_rbp_ptr, - scratch.next_error_code_ptr, - )?; - let error_code = self.load_i16(scratch.next_error_code_ptr, "bt_next_error_code_value")?; - - Ok(BtNextFrame { - ip: next_ip, - rsp: cfa, - rbp: next_rbp, - error_code, - }) - } - - fn validate_backtrace_next_frame( - &self, - state: BtRegisterState<'ctx>, - next: BtNextFrame<'ctx>, - ) -> Result> { - let i64_type = self.context.i64_type(); - let i16_type = self.context.i16_type(); - let zero = i64_type.const_zero(); - let zero_i16 = i16_type.const_zero(); - let min_user_ip = i64_type.const_int(0x1000, false); - let high_byte_mask = i64_type.const_int(0xff00_0000_0000_0000, false); - - let no_read_error = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - next.error_code, - zero_i16, - "bt_no_read_error", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let ip_high_enough = self - .builder - .build_int_compare( - inkwell::IntPredicate::UGE, - next.ip, - min_user_ip, - "bt_next_ip_user_min", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let ip_high_byte = self - .builder - .build_and(next.ip, high_byte_mask, "bt_next_ip_high_byte") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let ip_is_zero = self - .builder - .build_int_compare(inkwell::IntPredicate::EQ, next.ip, zero, "bt_next_ip_zero") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let ip_not_kernel_like = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - ip_high_byte, - zero, - "bt_next_ip_not_kernel_like", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let cfa_nonzero = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - next.rsp, - zero, - "bt_next_cfa_nonzero", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let cfa_changed = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - next.rsp, - state.rsp, - "bt_next_cfa_changed", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let ip_changed = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - next.ip, - state.ip, - "bt_next_ip_changed", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let cfa_progress = self - .builder - .build_or(cfa_changed, ip_changed, "bt_next_frame_progress") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - let ip_valid = self - .builder - .build_and(ip_high_enough, ip_not_kernel_like, "bt_next_ip_valid") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let cfa_valid = self - .builder - .build_and(cfa_nonzero, cfa_progress, "bt_next_cfa_valid") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let frame_shape_valid = self - .builder - .build_and(ip_valid, cfa_valid, "bt_next_frame_valid") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let valid = self - .builder - .build_and( - no_read_error, - frame_shape_valid, - "bt_next_frame_valid_no_read_error", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let complete = self - .builder - .build_and(no_read_error, ip_is_zero, "bt_next_frame_complete") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - let mut error_code = next.error_code; - error_code = self.select_backtrace_error_code_if( - error_code, - ip_high_enough, - BACKTRACE_ERROR_NEXT_IP_BELOW_USER, - "bt_next_ip_below_user_code", - )?; - error_code = self.select_backtrace_error_code_if( - error_code, - ip_not_kernel_like, - BACKTRACE_ERROR_NEXT_IP_KERNEL_LIKE, - "bt_next_ip_kernel_like_code", - )?; - error_code = self.select_backtrace_error_code_if( - error_code, - cfa_nonzero, - BACKTRACE_ERROR_NEXT_CFA_ZERO, - "bt_next_cfa_zero_code", - )?; - error_code = self.select_backtrace_error_code_if( - error_code, - cfa_progress, - BACKTRACE_ERROR_NEXT_CFA_NOT_ADVANCING, - "bt_next_cfa_not_advancing_code", - )?; - error_code = self - .builder - .build_select::, _>( - complete, - self.context - .i16_type() - .const_int(BACKTRACE_ERROR_NONE as u64, false) - .into(), - error_code.into(), - "bt_complete_error_code", - ) - .map(|value| value.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - Ok(BtFrameValidation { - valid, - complete, - error_code, - }) - } - - fn select_backtrace_error_code_if( - &self, - current: IntValue<'ctx>, - condition_ok: IntValue<'ctx>, - error_code: u16, - name: &str, - ) -> Result> { - let i16_type = self.context.i16_type(); - let current_is_none = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - current, - i16_type.const_int(BACKTRACE_ERROR_NONE as u64, false), - &format!("{name}_current_none"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let condition_failed = self - .builder - .build_not(condition_ok, &format!("{name}_condition_failed")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let should_set = self - .builder - .build_and( - current_is_none, - condition_failed, - &format!("{name}_should_set"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_select::, _>( - should_set, - i16_type.const_int(error_code as u64, false).into(), - current.into(), - name, - ) - .map(|value| value.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - } - - fn store_backtrace_error_code_if( - &self, - error_code_ptr: PointerValue<'ctx>, - condition: IntValue<'ctx>, - error_code: u16, - name: &str, - ) -> Result<()> { - let current = self.load_i16(error_code_ptr, &format!("{name}_current"))?; - let i16_type = self.context.i16_type(); - let current_is_none = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - current, - i16_type.const_int(BACKTRACE_ERROR_NONE as u64, false), - &format!("{name}_current_none"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let should_set = self - .builder - .build_and(condition, current_is_none, &format!("{name}_should_set")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let next = self - .builder - .build_select::, _>( - should_set, - i16_type.const_int(error_code as u64, false).into(), - current.into(), - name, - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(error_code_ptr, next) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(()) - } - - fn recover_rbp_from_runtime_row( - &mut self, - row: &RuntimeBtUnwindRow<'ctx>, - cfa: IntValue<'ctx>, - state: BtRegisterState<'ctx>, - next_rbp_ptr: PointerValue<'ctx>, - next_error_code_ptr: PointerValue<'ctx>, - ) -> Result> { - let is_at = self.is_recovery_kind( - row.rbp_kind, - crate::BACKTRACE_RECOVERY_AT_CFA_OFFSET, - "bt_rbp_at_kind", - )?; - let cfa_uses_rbp = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - row.cfa_register, - self.context - .i16_type() - .const_int(X86_64_DWARF_RBP as u64, false), - "bt_rbp_cfa_uses_rbp", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let cfa_offset_is_frame_pointer_call_frame = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - row.cfa_offset, - self.context.i64_type().const_int(16, false), - "bt_rbp_cfa_offset_is_16", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let frame_pointer_call_frame = self - .builder - .build_and( - cfa_uses_rbp, - cfa_offset_is_frame_pointer_call_frame, - "bt_rbp_frame_pointer_call_frame", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let is_at = self - .builder - .build_or( - is_at, - frame_pointer_call_frame, - "bt_rbp_at_or_frame_pointer", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let rbp_offset = self - .builder - .build_select::, _>( - frame_pointer_call_frame, - self.context - .i64_type() - .const_int((-16i64) as u64, true) - .into(), - row.rbp_offset.into(), - "bt_rbp_effective_offset", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let current_fn = self.current_function("recover bt rbp")?; - let at_block = self.context.append_basic_block(current_fn, "bt_rbp_at"); - let non_at_block = self.context.append_basic_block(current_fn, "bt_rbp_non_at"); - let join_block = self.context.append_basic_block(current_fn, "bt_rbp_join"); - self.builder - .build_conditional_branch(is_at, at_block, non_at_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(at_block); - let rbp_addr = self - .builder - .build_int_add(cfa, rbp_offset, "bt_runtime_rbp_addr") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let (rbp_from_memory, rbp_read_failed) = self.generate_memory_read_with_fail_flag( - RuntimeAddress::available(rbp_addr, self.context), - MemoryAccessSize::U64, - "bt_rbp_read", - )?; - self.store_backtrace_error_code_if( - next_error_code_ptr, - rbp_read_failed, - BACKTRACE_ERROR_FRAME_POINTER_READ, - "bt_rbp_error_code", - )?; - self.builder - .build_store(next_rbp_ptr, rbp_from_memory.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_unconditional_branch(join_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(non_at_block); - let rbp_from_val = self - .builder - .build_int_add(cfa, row.rbp_offset, "bt_runtime_rbp_val") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let rbp_from_register = - self.select_register_state(row.rbp_register, state, "bt_rbp_reg")?; - let rbp_is_val = self.is_recovery_kind( - row.rbp_kind, - crate::BACKTRACE_RECOVERY_VAL_CFA_OFFSET, - "bt_rbp_val_kind", - )?; - let rbp_is_register = self.is_recovery_kind( - row.rbp_kind, - crate::BACKTRACE_RECOVERY_REGISTER, - "bt_rbp_reg_kind", - )?; - let rbp_is_same = self.is_recovery_kind( - row.rbp_kind, - crate::BACKTRACE_RECOVERY_SAME_VALUE, - "bt_rbp_same_kind", - )?; - let rbp_is_register_like = self - .builder - .build_or(rbp_is_register, rbp_is_same, "bt_rbp_register_like") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let rbp_value_or_current = self - .builder - .build_select::, _>( - rbp_is_val, - rbp_from_val.into(), - state.rbp.into(), - "bt_rbp_val_or_current", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let rbp_non_at = self - .builder - .build_select::, _>( - rbp_is_register_like, - rbp_from_register.into(), - rbp_value_or_current.into(), - "bt_rbp_non_at_value", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - self.builder - .build_store(next_rbp_ptr, rbp_non_at) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_unconditional_branch(join_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(join_block); - self.load_i64(next_rbp_ptr, "bt_next_rbp_value") - } - - fn select_register_state( - &self, - register: IntValue<'ctx>, - state: BtRegisterState<'ctx>, - name: &str, - ) -> Result> { - let is_rbp = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - register, - self.context - .i16_type() - .const_int(X86_64_DWARF_RBP as u64, false), - &format!("{name}_is_rbp"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let is_rip = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - register, - self.context - .i16_type() - .const_int(X86_64_DWARF_RIP as u64, false), - &format!("{name}_is_rip"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let rbp_or_rsp = self - .builder - .build_select::, _>( - is_rbp, - state.rbp.into(), - state.rsp.into(), - &format!("{name}_rbp_or_rsp"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - self.builder - .build_select::, _>( - is_rip, - state.ip.into(), - rbp_or_rsp.into(), - name, - ) - .map(|value| value.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - } - - fn is_recovery_kind( - &self, - kind: IntValue<'ctx>, - expected: u8, - name: &str, - ) -> Result> { - self.builder - .build_int_compare( - inkwell::IntPredicate::EQ, - kind, - self.context.i8_type().const_int(expected as u64, false), - name, - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - } - - fn lookup_bt_unwind_row_ptr( - &mut self, - row_index: IntValue<'ctx>, - ) -> Result> { - let ptr_type = self.context.ptr_type(AddressSpace::default()); - let i32_type = self.context.i32_type(); - let map_global = self - .module - .get_global("bt_unwind_rows") - .ok_or_else(|| CodeGenError::LLVMError("bt_unwind_rows map not found".to_string()))?; - let map_ptr = self - .builder - .build_bit_cast( - map_global.as_pointer_value(), - ptr_type, - "bt_unwind_rows_map_ptr", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let key_alloca = self.pm_key_alloca.ok_or_else(|| { - CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) - })?; - let key_arr_ty = i32_type.array_type(4); - let zero = i32_type.const_zero(); - // SAFETY: pm_key_alloca is a [4 x i32] entry-block alloca and [0, 0] - // addresses its first element, which is sufficient for an Array u32 key. - let key_ptr = unsafe { - self.builder - .build_gep( - key_arr_ty, - key_alloca, - &[zero, zero], - "bt_unwind_row_key_ptr", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - self.builder - .build_store(key_ptr, row_index) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let key_ptr = self - .builder - .build_bit_cast(key_ptr, ptr_type, "bt_unwind_row_key_void") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let result = self.create_bpf_helper_call( - BPF_FUNC_map_lookup_elem as u64, - &[map_ptr, key_ptr], - ptr_type.into(), - "bt_unwind_row_lookup", - )?; - match result { - BasicValueEnum::PointerValue(ptr) => Ok(ptr), - _ => Err(CodeGenError::LLVMError( - "bt_unwind_rows lookup did not return pointer".to_string(), - )), - } - } - - fn lookup_bt_state_ptr(&mut self, key_const: u32) -> Result> { - self.lookup_percpu_value_ptr("bt_state", key_const) - } - - fn lookup_bt_state_ptr_dynamic( - &mut self, - state_index: IntValue<'ctx>, - ) -> Result> { - let ptr_type = self.context.ptr_type(AddressSpace::default()); - let i32_type = self.context.i32_type(); - let map_global = self - .map_manager - .get_map(&self.module, "bt_state") - .map_err(|e| CodeGenError::LLVMError(format!("Map not found bt_state: {e}")))?; - let map_ptr = self - .builder - .build_bit_cast(map_global, ptr_type, "bt_state_map_ptr") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let key = match state_index - .get_type() - .get_bit_width() - .cmp(&i32_type.get_bit_width()) - { - std::cmp::Ordering::Equal => state_index, - std::cmp::Ordering::Less => self - .builder - .build_int_z_extend(state_index, i32_type, "bt_state_key_i32") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?, - std::cmp::Ordering::Greater => self - .builder - .build_int_truncate(state_index, i32_type, "bt_state_key_i32") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?, - }; - - let key_arr_ty = i32_type.array_type(4); - let key_alloca = self.pm_key_alloca.ok_or_else(|| { - CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) - })?; - let zero = i32_type.const_zero(); - // SAFETY: pm_key_alloca is a [4 x i32] entry-block alloca and [0, 0] - // addresses its first element, which is sufficient for an Array u32 key. - let key_ptr = unsafe { - self.builder - .build_gep(key_arr_ty, key_alloca, &[zero, zero], "bt_state_key_ptr") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - self.builder - .build_store(key_ptr, key) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let key_ptr = self - .builder - .build_bit_cast(key_ptr, ptr_type, "bt_state_key_void") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let result = self.create_bpf_helper_call( - BPF_FUNC_map_lookup_elem as u64, - &[map_ptr, key_ptr], - ptr_type.into(), - "bt_state_lookup", - )?; - match result { - BasicValueEnum::PointerValue(ptr) => Ok(ptr), - _ => Err(CodeGenError::LLVMError( - "bt_state lookup did not return pointer".to_string(), - )), - } - } - - fn lookup_bt_prog_array_ptr(&mut self) -> Result> { - let ptr_type = self.context.ptr_type(AddressSpace::default()); - let map_global = self - .module - .get_global("bt_prog_array") - .ok_or_else(|| CodeGenError::LLVMError("bt_prog_array map not found".to_string()))?; - let map_ptr = self - .builder - .build_bit_cast(map_global.as_pointer_value(), ptr_type, "bt_prog_array_ptr") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - match map_ptr { - BasicValueEnum::PointerValue(ptr) => Ok(ptr), - _ => Err(CodeGenError::LLVMError( - "bt_prog_array cast did not return pointer".to_string(), - )), - } - } - - fn get_or_create_backtrace_tail_enabled_flag(&mut self) -> Result> { - if let Some(ptr) = self.backtrace_tail_enabled_alloca { - return Ok(ptr); - } - - let current_block = self.builder.get_insert_block().ok_or_else(|| { - CodeGenError::LLVMError("no current block for bt tail flag allocation".to_string()) - })?; - let current_fn = self.current_function("allocate bt tail flag")?; - let entry_block = current_fn.get_first_basic_block().ok_or_else(|| { - CodeGenError::LLVMError("no entry block for bt tail flag allocation".to_string()) - })?; - - if let Some(first_instruction) = entry_block.get_first_instruction() { - self.builder.position_before(&first_instruction); - } else { - self.builder.position_at_end(entry_block); - } - let alloca = self - .builder - .build_alloca(self.context.i8_type(), "bt_tail_enabled") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(alloca, self.context.i8_type().const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder.position_at_end(current_block); - self.backtrace_tail_enabled_alloca = Some(alloca); - Ok(alloca) - } - - fn get_or_create_backtrace_tail_last_slot(&mut self) -> Result> { - if let Some(ptr) = self.backtrace_tail_last_slot_alloca { - return Ok(ptr); - } - - let current_block = self.builder.get_insert_block().ok_or_else(|| { - CodeGenError::LLVMError("no current block for bt tail slot allocation".to_string()) - })?; - let current_fn = self.current_function("allocate bt tail slot")?; - let entry_block = current_fn.get_first_basic_block().ok_or_else(|| { - CodeGenError::LLVMError("no entry block for bt tail slot allocation".to_string()) - })?; - - if let Some(first_instruction) = entry_block.get_first_instruction() { - self.builder.position_before(&first_instruction); - } else { - self.builder.position_at_end(entry_block); - } - let alloca = self - .builder - .build_alloca(self.context.i8_type(), "bt_tail_last_slot") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store( - alloca, - self.context - .i8_type() - .const_int(crate::BACKTRACE_TAIL_NO_NEXT_SLOT as u64, false), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder.position_at_end(current_block); - self.backtrace_tail_last_slot_alloca = Some(alloca); - Ok(alloca) - } - - fn link_backtrace_tail_slot( - &mut self, - tail_slot: u8, - offsets_found_u8: IntValue<'ctx>, - done_block: BasicBlock<'ctx>, - ) -> Result<()> { - let current_fn = self.current_function("link bt tail slot")?; - let offsets_found = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - offsets_found_u8, - self.context.i8_type().const_zero(), - "bt_tail_link_offsets_found", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let link_block = self - .context - .append_basic_block(current_fn, "bt_tail_link_slot"); - self.builder - .build_conditional_branch(offsets_found, link_block, done_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(link_block); - let state0_ptr = self.lookup_bt_state_ptr(0)?; - let state0_is_null = self - .builder - .build_is_null(state0_ptr, "bt_tail_link_state0_null") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let state0_ok_block = self - .context - .append_basic_block(current_fn, "bt_tail_link_state0_ok"); - self.builder - .build_conditional_branch(state0_is_null, done_block, state0_ok_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(state0_ok_block); - let tail_enabled_ptr = self.get_or_create_backtrace_tail_enabled_flag()?; - let last_slot_ptr = self.get_or_create_backtrace_tail_last_slot()?; - let enabled_value = self - .builder - .build_load( - self.context.i8_type(), - tail_enabled_ptr, - "bt_tail_link_enabled_value", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let has_prev_slot = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - enabled_value, - self.context.i8_type().const_zero(), - "bt_tail_link_has_prev", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let first_slot_block = self - .context - .append_basic_block(current_fn, "bt_tail_link_first"); - let append_slot_block = self - .context - .append_basic_block(current_fn, "bt_tail_link_append"); - let linked_block = self - .context - .append_basic_block(current_fn, "bt_tail_linked"); - self.builder - .build_conditional_branch(has_prev_slot, append_slot_block, first_slot_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(first_slot_block); - self.store_u8_const( - state0_ptr, - crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, - tail_slot, - "bt_tail_link_active_slot", - )?; - self.builder - .build_unconditional_branch(linked_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(append_slot_block); - let prev_slot = self.load_i8(last_slot_ptr, "bt_tail_link_prev_slot")?; - let prev_state_ptr = self.lookup_bt_state_ptr_dynamic(prev_slot)?; - let prev_state_is_null = self - .builder - .build_is_null(prev_state_ptr, "bt_tail_link_prev_null") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let prev_state_ok_block = self - .context - .append_basic_block(current_fn, "bt_tail_link_prev_ok"); - self.builder - .build_conditional_branch(prev_state_is_null, first_slot_block, prev_state_ok_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(prev_state_ok_block); - self.store_u8_const( - prev_state_ptr, - crate::BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, - tail_slot, - "bt_tail_link_prev_next_slot", - )?; - self.builder - .build_unconditional_branch(linked_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(linked_block); - self.builder - .build_store( - last_slot_ptr, - self.context.i8_type().const_int(tail_slot as u64, false), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(tail_enabled_ptr, self.context.i8_type().const_int(1, false)) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_unconditional_branch(done_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(()) - } - - fn store_state_i64( - &self, - base: PointerValue<'ctx>, - offset: usize, - value: IntValue<'ctx>, - name: &str, - ) -> Result<()> { - self.store_u64_value(base, offset, value, name) - } - - fn store_state_i32( - &self, - base: PointerValue<'ctx>, - offset: usize, - value: IntValue<'ctx>, - name: &str, - ) -> Result<()> { - let ptr = self.byte_gep(base, offset, name)?; - let ptr = self - .builder - .build_pointer_cast( - ptr, - self.context.ptr_type(AddressSpace::default()), - &format!("{name}_u32_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(ptr, value) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(()) - } - - fn load_state_i32( - &self, - base: PointerValue<'ctx>, - offset: usize, - name: &str, - ) -> Result> { - let ptr = self.byte_gep(base, offset, name)?; - let ptr = self - .builder - .build_pointer_cast( - ptr, - self.context.ptr_type(AddressSpace::default()), - &format!("{name}_u32_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(self - .builder - .build_load(self.context.i32_type(), ptr, name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value()) - } - - fn load_row_i64( - &self, - row_ptr: PointerValue<'ctx>, - offset: usize, - name: &str, - ) -> Result> { - let ptr = self.byte_gep(row_ptr, offset, name)?; - let ptr = self - .builder - .build_pointer_cast( - ptr, - self.context.ptr_type(AddressSpace::default()), - &format!("{name}_i64_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(self - .builder - .build_load(self.context.i64_type(), ptr, name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value()) - } - - fn load_row_i16( - &self, - row_ptr: PointerValue<'ctx>, - offset: usize, - name: &str, - ) -> Result> { - let ptr = self.byte_gep(row_ptr, offset, name)?; - Ok(self - .builder - .build_load(self.context.i16_type(), ptr, name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value()) - } - - fn load_row_i8( - &self, - row_ptr: PointerValue<'ctx>, - offset: usize, - name: &str, - ) -> Result> { - let ptr = self.byte_gep(row_ptr, offset, name)?; - Ok(self - .builder - .build_load(self.context.i8_type(), ptr, name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value()) - } - - fn load_i8(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { - Ok(self - .builder - .build_load(self.context.i8_type(), ptr, name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value()) - } - - fn load_bool(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { - Ok(self - .builder - .build_load(self.context.bool_type(), ptr, name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value()) - } - - fn load_i32(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { - Ok(self - .builder - .build_load(self.context.i32_type(), ptr, name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value()) - } - - fn load_i16(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { - Ok(self - .builder - .build_load(self.context.i16_type(), ptr, name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value()) - } - - fn load_i64(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { - Ok(self - .builder - .build_load(self.context.i64_type(), ptr, name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value()) - } - - fn load_dwarf_register_i64( - &mut self, - reg: u16, - pt_regs: PointerValue<'ctx>, - ) -> Result> { - let value = self.load_register_value(reg, pt_regs)?; - match value { - BasicValueEnum::IntValue(value) => Ok(value), - _ => Err(CodeGenError::RegisterMappingError(format!( - "DWARF register {reg} did not load as integer" - ))), - } - } - - fn add_signed_offset( - &self, - base: IntValue<'ctx>, - offset: i64, - name: &str, - ) -> Result> { - if offset == 0 { - return Ok(base); - } - self.builder - .build_int_add( - base, - self.context.i64_type().const_int(offset as u64, true), - name, - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - } - - fn normalized_pc_from_raw( - &self, - raw_ip: IntValue<'ctx>, - module_bias: IntValue<'ctx>, - offsets_found: IntValue<'ctx>, - ) -> Result> { - let rebased = self - .builder - .build_int_sub(raw_ip, module_bias, "bt_normalized_pc") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_select::, _>( - offsets_found, - rebased.into(), - raw_ip.into(), - "bt_pc_or_raw", - ) - .map(|value| value.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - } - - fn lookup_proc_module_range_meta( - &mut self, - pid: IntValue<'ctx>, - name_prefix: &str, - ) -> Result> { - let i32_type = self.context.i32_type(); - let i64_type = self.context.i64_type(); - let ptr_type = self.context.ptr_type(AddressSpace::default()); - let map_global = self - .module - .get_global("proc_module_range_meta") - .ok_or_else(|| { - CodeGenError::LLVMError("proc_module_range_meta map not found".to_string()) - })?; - let map_ptr = self - .builder - .build_bit_cast( - map_global.as_pointer_value(), - ptr_type, - &format!("{name_prefix}_meta_map_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let key_alloca = self.pm_key_alloca.ok_or_else(|| { - CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) - })?; - let key_arr_ty = i32_type.array_type(4); - let zero = i32_type.const_zero(); - // SAFETY: key_alloca is the [4 x i32] pm_key stack slot and [0, 0] - // addresses the pid key element for proc_module_range_meta. - let key_ptr = unsafe { - self.builder - .build_gep( - key_arr_ty, - key_alloca, - &[zero, zero], - &format!("{name_prefix}_meta_key_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - self.builder - .build_store(key_ptr, pid) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let key_arg = self - .builder - .build_bit_cast(key_ptr, ptr_type, &format!("{name_prefix}_meta_key_arg")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let lookup_id = i64_type.const_int(BPF_FUNC_map_lookup_elem as u64, false); - let lookup_fn_type = ptr_type.fn_type(&[ptr_type.into(), ptr_type.into()], false); - let lookup_fn_ptr = self - .builder - .build_int_to_ptr( - lookup_id, - ptr_type, - &format!("{name_prefix}_meta_lookup_fn"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let args: Vec = vec![map_ptr.into(), key_arg.into()]; - let value_ptr_any = self - .builder - .build_indirect_call( - lookup_fn_type, - lookup_fn_ptr, - &args, - &format!("{name_prefix}_meta_lookup"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .try_as_basic_value() - .left() - .ok_or_else(|| { - CodeGenError::LLVMError("proc_module_range_meta lookup returned void".to_string()) - })?; - let value_ptr = match value_ptr_any { - BasicValueEnum::PointerValue(p) => p, - _ => { - return Err(CodeGenError::LLVMError( - "proc_module_range_meta lookup did not return pointer".to_string(), - )); - } - }; - let current_fn = self.current_function("lookup proc module range meta")?; - let hit_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_meta_hit")); - let miss_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_meta_miss")); - let cont_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_meta_cont")); - let value_ptr_int = self - .builder - .build_ptr_to_int( - value_ptr, - i64_type, - &format!("{name_prefix}_meta_value_ptr_int"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let is_hit = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - value_ptr_int, - i64_type.const_zero(), - &format!("{name_prefix}_meta_found"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_conditional_branch(is_hit, hit_block, miss_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(hit_block); - let load_i32_field = |offset: usize, field_name: &str, ctx: &mut EbpfContext<'ctx, 'dw>| { - let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); - // SAFETY: value_ptr points at ProcModuleRangeMeta returned by - // bpf_map_lookup_elem and offset is an i32 field offset. - let field_ptr = unsafe { - ctx.builder - .build_gep(ctx.context.i8_type(), value_ptr, &[offset_i32], field_name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - ctx.builder - .build_load(ctx.context.i32_type(), field_ptr, field_name) - .map(|value| value.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - }; - let active_slot = load_i32_field( - ghostscope_protocol::PROC_MODULE_RANGE_META_ACTIVE_SLOT_OFFSET, - &format!("{name_prefix}_meta_active_slot"), - self, - )?; - let count = load_i32_field( - ghostscope_protocol::PROC_MODULE_RANGE_META_COUNT_OFFSET, - &format!("{name_prefix}_meta_count"), - self, - )?; - self.builder - .build_unconditional_branch(cont_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let hit_end = self.current_insert_block("finish module range meta hit block")?; - - self.builder.position_at_end(miss_block); - self.builder - .build_unconditional_branch(cont_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let miss_end = self.current_insert_block("finish module range meta miss block")?; - - self.builder.position_at_end(cont_block); - let found_type = self.context.bool_type(); - let found_phi = self - .builder - .build_phi(found_type, &format!("{name_prefix}_meta_found_phi")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - found_phi.add_incoming(&[ - (&found_type.const_int(1, false), hit_end), - (&found_type.const_zero(), miss_end), - ]); - let phi_i32 = |ctx: &mut EbpfContext<'ctx, 'dw>, name: &str, hit_value: IntValue<'ctx>| { - let phi = ctx - .builder - .build_phi(ctx.context.i32_type(), name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - phi.add_incoming(&[ - (&hit_value, hit_end), - (&ctx.context.i32_type().const_zero(), miss_end), - ]); - Ok::<_, CodeGenError>(phi.as_basic_value().into_int_value()) - }; - Ok(BtModuleRangeMeta { - found: found_phi.as_basic_value().into_int_value(), - active_slot: phi_i32(self, &format!("{name_prefix}_meta_slot_phi"), active_slot)?, - count: phi_i32(self, &format!("{name_prefix}_meta_count_phi"), count)?, - }) - } - - fn lookup_proc_module_range_value( - &mut self, - pid: IntValue<'ctx>, - slot: IntValue<'ctx>, - index: IntValue<'ctx>, - name_prefix: &str, - ) -> Result> { - let i32_type = self.context.i32_type(); - let i64_type = self.context.i64_type(); - let ptr_type = self.context.ptr_type(AddressSpace::default()); - let map_global = self - .module - .get_global("proc_module_ranges") - .ok_or_else(|| { - CodeGenError::LLVMError("proc_module_ranges map not found".to_string()) - })?; - let map_ptr = self - .builder - .build_bit_cast( - map_global.as_pointer_value(), - ptr_type, - &format!("{name_prefix}_range_map_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let key_alloca = self.pm_key_alloca.ok_or_else(|| { - CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) - })?; - self.builder - .build_store(key_alloca, i32_type.array_type(4).const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let store_key_u32 = |offset: usize, - value: IntValue<'ctx>, - field_name: &str, - ctx: &mut EbpfContext<'ctx, 'dw>| - -> Result<()> { - let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); - // SAFETY: key_alloca is the ProcModuleRangeKey stack slot and - // offset is one of its u32 field offsets. - let field_ptr = unsafe { - ctx.builder - .build_gep(ctx.context.i8_type(), key_alloca, &[offset_i32], field_name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - ctx.builder - .build_store(field_ptr, value) - .map(|_| ()) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - }; - store_key_u32( - ghostscope_protocol::PROC_MODULE_RANGE_KEY_PID_OFFSET, - pid, - &format!("{name_prefix}_range_key_pid"), - self, - )?; - store_key_u32( - ghostscope_protocol::PROC_MODULE_RANGE_KEY_SLOT_OFFSET, - slot, - &format!("{name_prefix}_range_key_slot"), - self, - )?; - store_key_u32( - ghostscope_protocol::PROC_MODULE_RANGE_KEY_INDEX_OFFSET, - index, - &format!("{name_prefix}_range_key_index"), - self, - )?; - store_key_u32( - ghostscope_protocol::PROC_MODULE_RANGE_KEY_PAD_OFFSET, - i32_type.const_zero(), - &format!("{name_prefix}_range_key_pad"), - self, - )?; - let key_arg = self - .builder - .build_bit_cast( - key_alloca, - ptr_type, - &format!("{name_prefix}_range_key_arg"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let lookup_id = i64_type.const_int(BPF_FUNC_map_lookup_elem as u64, false); - let lookup_fn_type = ptr_type.fn_type(&[ptr_type.into(), ptr_type.into()], false); - let lookup_fn_ptr = self - .builder - .build_int_to_ptr( - lookup_id, - ptr_type, - &format!("{name_prefix}_range_lookup_fn"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let args: Vec = vec![map_ptr.into(), key_arg.into()]; - let value_ptr_any = self - .builder - .build_indirect_call( - lookup_fn_type, - lookup_fn_ptr, - &args, - &format!("{name_prefix}_range_lookup"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .try_as_basic_value() - .left() - .ok_or_else(|| { - CodeGenError::LLVMError("proc_module_ranges lookup returned void".to_string()) - })?; - let value_ptr = match value_ptr_any { - BasicValueEnum::PointerValue(p) => p, - _ => { - return Err(CodeGenError::LLVMError( - "proc_module_ranges lookup did not return pointer".to_string(), - )); - } - }; - let current_fn = self.current_function("lookup proc module range value")?; - let hit_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_range_hit")); - let miss_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_range_miss")); - let cont_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_range_cont")); - let value_ptr_int = self - .builder - .build_ptr_to_int( - value_ptr, - i64_type, - &format!("{name_prefix}_range_value_ptr_int"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let is_hit = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - value_ptr_int, - i64_type.const_zero(), - &format!("{name_prefix}_range_found"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_conditional_branch(is_hit, hit_block, miss_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(hit_block); - let load_i64_field = |offset: usize, field_name: &str, ctx: &mut EbpfContext<'ctx, 'dw>| { - let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); - // SAFETY: value_ptr points at ProcModuleRangeValue returned by - // bpf_map_lookup_elem and offset is a u64 field offset. - let field_ptr = unsafe { - ctx.builder - .build_gep(ctx.context.i8_type(), value_ptr, &[offset_i32], field_name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - ctx.builder - .build_load(ctx.context.i64_type(), field_ptr, field_name) - .map(|value| value.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - }; - let load_i32_field = |offset: usize, field_name: &str, ctx: &mut EbpfContext<'ctx, 'dw>| { - let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); - // SAFETY: value_ptr points at ProcModuleRangeValue returned by - // bpf_map_lookup_elem and offset is a u32 field offset. - let field_ptr = unsafe { - ctx.builder - .build_gep(ctx.context.i8_type(), value_ptr, &[offset_i32], field_name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - ctx.builder - .build_load(ctx.context.i32_type(), field_ptr, field_name) - .map(|value| value.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - }; - let base = load_i64_field( - ghostscope_protocol::PROC_MODULE_RANGE_VALUE_BASE_OFFSET, - &format!("{name_prefix}_range_base"), - self, - )?; - let end = load_i64_field( - ghostscope_protocol::PROC_MODULE_RANGE_VALUE_END_OFFSET, - &format!("{name_prefix}_range_end"), - self, - )?; - let text = load_i64_field( - ghostscope_protocol::PROC_MODULE_RANGE_VALUE_TEXT_OFFSET, - &format!("{name_prefix}_range_text"), - self, - )?; - let cookie_lo = load_i32_field( - ghostscope_protocol::PROC_MODULE_RANGE_VALUE_COOKIE_LO_OFFSET, - &format!("{name_prefix}_range_cookie_lo"), - self, - )?; - let cookie_hi = load_i32_field( - ghostscope_protocol::PROC_MODULE_RANGE_VALUE_COOKIE_HI_OFFSET, - &format!("{name_prefix}_range_cookie_hi"), - self, - )?; - let cookie_lo64 = self - .builder - .build_int_z_extend(cookie_lo, i64_type, &format!("{name_prefix}_cookie_lo64")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let cookie_hi64 = self - .builder - .build_int_z_extend(cookie_hi, i64_type, &format!("{name_prefix}_cookie_hi64")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let cookie_hi_shifted = self - .builder - .build_left_shift( - cookie_hi64, - i64_type.const_int(32, false), - &format!("{name_prefix}_cookie_hi_shift"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let cookie = self - .builder - .build_or( - cookie_lo64, - cookie_hi_shifted, - &format!("{name_prefix}_cookie"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_unconditional_branch(cont_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let hit_end = self.current_insert_block("finish module range value hit block")?; - - self.builder.position_at_end(miss_block); - self.builder - .build_unconditional_branch(cont_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let miss_end = self.current_insert_block("finish module range value miss block")?; - - self.builder.position_at_end(cont_block); - let zero_i64 = i64_type.const_zero(); - let found_type = self.context.bool_type(); - let found_phi = self - .builder - .build_phi(found_type, &format!("{name_prefix}_range_found_phi")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - found_phi.add_incoming(&[ - (&found_type.const_int(1, false), hit_end), - (&found_type.const_zero(), miss_end), - ]); - let phi_i64 = |ctx: &mut EbpfContext<'ctx, 'dw>, name: &str, hit_value: IntValue<'ctx>| { - let phi = ctx - .builder - .build_phi(ctx.context.i64_type(), name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - phi.add_incoming(&[(&hit_value, hit_end), (&zero_i64, miss_end)]); - Ok::<_, CodeGenError>(phi.as_basic_value().into_int_value()) - }; - Ok(BtModuleRangeValue { - found: found_phi.as_basic_value().into_int_value(), - base: phi_i64(self, &format!("{name_prefix}_range_base_phi"), base)?, - end: phi_i64(self, &format!("{name_prefix}_range_end_phi"), end)?, - text: phi_i64(self, &format!("{name_prefix}_range_text_phi"), text)?, - cookie: phi_i64(self, &format!("{name_prefix}_range_cookie_phi"), cookie)?, - }) - } - - fn lookup_backtrace_frame_module_in_ranges( - &mut self, - raw_ip: IntValue<'ctx>, - pid: IntValue<'ctx>, - meta: BtModuleRangeMeta<'ctx>, - fallback_cookie: IntValue<'ctx>, - fallback_bias: IntValue<'ctx>, - name_prefix: &str, - ) -> Result> { - let i32_type = self.context.i32_type(); - let i64_type = self.context.i64_type(); - let bool_type = self.context.bool_type(); - let max_steps = backtrace_row_binary_search_steps( - (self.compile_options.proc_module_offsets_max_entries as usize).saturating_mul(2), - ); - let current_fn = self.current_function("lookup backtrace frame module")?; - - let mut found = bool_type.const_zero(); - let mut cookie = fallback_cookie; - let mut bias = fallback_bias; - let mut lo = i32_type.const_zero(); - let mut hi = meta.count; - - for step in 0..max_steps { - let not_found = self - .builder - .build_not(found, &format!("{name_prefix}_{step}_not_found")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let range_active = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULT, - lo, - hi, - &format!("{name_prefix}_{step}_active"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let pending = self - .builder - .build_and( - not_found, - range_active, - &format!("{name_prefix}_{step}_pending"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let should_search = self - .builder - .build_and( - pending, - meta.found, - &format!("{name_prefix}_{step}_should_search"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let search_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_{step}_search")); - let skip_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_{step}_skip")); - let after_block = self - .context - .append_basic_block(current_fn, &format!("{name_prefix}_{step}_after")); - self.builder - .build_conditional_branch(should_search, search_block, skip_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - self.builder.position_at_end(skip_block); - self.builder - .build_unconditional_branch(after_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let skip_end = self.current_insert_block("finish module range skip block")?; - - self.builder.position_at_end(search_block); - let lo_plus_hi = self - .builder - .build_int_add(lo, hi, &format!("{name_prefix}_{step}_lo_plus_hi")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let mid = self - .builder - .build_right_shift( - lo_plus_hi, - i32_type.const_int(1, false), - false, - &format!("{name_prefix}_{step}_mid"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let range = self.lookup_proc_module_range_value( - pid, - meta.active_slot, - mid, - &format!("{name_prefix}_{step}"), - )?; - let at_or_after_base = self - .builder - .build_int_compare( - inkwell::IntPredicate::UGE, - raw_ip, - range.base, - &format!("{name_prefix}_{step}_after_base"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let before_end = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULT, - raw_ip, - range.end, - &format!("{name_prefix}_{step}_before_end"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let in_bounds = self - .builder - .build_and( - at_or_after_base, - before_end, - &format!("{name_prefix}_{step}_in_bounds"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let in_range = self - .builder - .build_and( - range.found, - in_bounds, - &format!("{name_prefix}_{step}_in_range"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let before_range = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULT, - raw_ip, - range.base, - &format!("{name_prefix}_{step}_before_range"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let after_range = self - .builder - .build_int_compare( - inkwell::IntPredicate::UGE, - raw_ip, - range.end, - &format!("{name_prefix}_{step}_after_range"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let found_next = self - .builder - .build_or(found, in_range, &format!("{name_prefix}_{step}_found_next")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let selected_cookie = self - .builder - .build_select::, _>( - in_range, - range.cookie.into(), - cookie.into(), - &format!("{name_prefix}_{step}_selected_cookie"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let selected_bias = self - .builder - .build_select::, _>( - in_range, - range.text.into(), - bias.into(), - &format!("{name_prefix}_{step}_selected_bias"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let mid_plus_one = self - .builder - .build_int_add( - mid, - i32_type.const_int(1, false), - &format!("{name_prefix}_{step}_mid_plus_one"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let range_missing = self - .builder - .build_not(range.found, &format!("{name_prefix}_{step}_range_missing")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let hi_next = self - .builder - .build_select::, _>( - before_range, - mid.into(), - hi.into(), - &format!("{name_prefix}_{step}_hi_next"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let lo_after = self - .builder - .build_select::, _>( - after_range, - mid_plus_one.into(), - lo.into(), - &format!("{name_prefix}_{step}_lo_after"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let lo_next = self - .builder - .build_select::, _>( - range_missing, - hi.into(), - lo_after.into(), - &format!("{name_prefix}_{step}_lo_next"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - self.builder - .build_unconditional_branch(after_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let search_end = self.current_insert_block("finish module range search block")?; - - self.builder.position_at_end(after_block); - - let found_phi = self - .builder - .build_phi(bool_type, &format!("{name_prefix}_{step}_found_phi")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - found_phi.add_incoming(&[(&found_next, search_end), (&found, skip_end)]); - found = found_phi.as_basic_value().into_int_value(); - - let cookie_phi = self - .builder - .build_phi(i64_type, &format!("{name_prefix}_{step}_cookie_phi")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - cookie_phi.add_incoming(&[(&selected_cookie, search_end), (&cookie, skip_end)]); - cookie = cookie_phi.as_basic_value().into_int_value(); - - let bias_phi = self - .builder - .build_phi(i64_type, &format!("{name_prefix}_{step}_bias_phi")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - bias_phi.add_incoming(&[(&selected_bias, search_end), (&bias, skip_end)]); - bias = bias_phi.as_basic_value().into_int_value(); - - let lo_phi = self - .builder - .build_phi(i32_type, &format!("{name_prefix}_{step}_lo_phi")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - lo_phi.add_incoming(&[(&lo_next, search_end), (&lo, skip_end)]); - lo = lo_phi.as_basic_value().into_int_value(); - - let hi_phi = self - .builder - .build_phi(i32_type, &format!("{name_prefix}_{step}_hi_phi")) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - hi_phi.add_incoming(&[(&hi_next, search_end), (&hi, skip_end)]); - hi = hi_phi.as_basic_value().into_int_value(); - } - - Ok(BtFrameModule { - cookie, - bias, - found, - }) - } - - fn resolve_backtrace_frame_module( - &mut self, - raw_ip: IntValue<'ctx>, - fallback_cookie: IntValue<'ctx>, - fallback_bias: IntValue<'ctx>, - fallback_found: IntValue<'ctx>, - name_prefix: &str, - ) -> Result> { - if self.backtrace_module_row_ranges.is_empty() { - return Ok(BtFrameModule { - cookie: fallback_cookie, - bias: fallback_bias, - found: fallback_found, - }); - } - - let pid = self.proc_module_pid_key(name_prefix)?; - let meta = self.lookup_proc_module_range_meta(pid, name_prefix)?; - self.lookup_backtrace_frame_module_in_ranges( - raw_ip, - pid, - meta, - fallback_cookie, - fallback_bias, - name_prefix, - ) - } - - fn backtrace_module_fallback_found(&self, found: IntValue<'ctx>) -> IntValue<'ctx> { - if self.backtrace_module_row_ranges.is_empty() { - found - } else { - self.context.bool_type().const_zero() - } - } - - fn backtrace_lookup_pc_from_raw( - &self, - raw_ip: IntValue<'ctx>, - module_bias: IntValue<'ctx>, - offsets_found: IntValue<'ctx>, - ) -> Result> { - self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found) - } - - fn bool_to_u8(&self, value: IntValue<'ctx>, name: &str) -> Result> { - self.builder - .build_select::, _>( - value, - self.context.i8_type().const_int(1, false).into(), - self.context.i8_type().const_zero().into(), - name, - ) - .map(|value| value.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - } - - fn status_or_offsets_unavailable( - &self, - status: BacktraceStatus, - offsets_found: IntValue<'ctx>, - ) -> Result> { - self.builder - .build_select::, _>( - offsets_found, - self.context - .i8_type() - .const_int(status as u64, false) - .into(), - self.context - .i8_type() - .const_int(BacktraceStatus::OffsetsUnavailable as u64, false) - .into(), - "bt_status_or_offsets", - ) - .map(|value| value.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - } - - fn status_for_backtrace_stop( - &self, - complete: IntValue<'ctx>, - error_code: IntValue<'ctx>, - offsets_found: IntValue<'ctx>, - ) -> Result> { - let i8_type = self.context.i8_type(); - let i16_type = self.context.i16_type(); - let ra_read_error = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - error_code, - i16_type.const_int(BACKTRACE_ERROR_RETURN_ADDRESS_READ as u64, false), - "bt_status_ra_read_error", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let rbp_read_error = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - error_code, - i16_type.const_int(BACKTRACE_ERROR_FRAME_POINTER_READ as u64, false), - "bt_status_rbp_read_error", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let read_error = self - .builder - .build_or(ra_read_error, rbp_read_error, "bt_status_read_error_flag") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let error_status = self - .builder - .build_select::, _>( - read_error, - i8_type - .const_int(BacktraceStatus::ReadError as u64, false) - .into(), - i8_type - .const_int(BacktraceStatus::InvalidFrame as u64, false) - .into(), - "bt_status_for_error_code", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let backtrace_status = self - .builder - .build_select::, _>( - complete, - i8_type - .const_int(BacktraceStatus::Complete as u64, false) - .into(), - error_status, - "bt_status_for_stop", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_select::, _>( - offsets_found, - backtrace_status, - i8_type - .const_int(BacktraceStatus::OffsetsUnavailable as u64, false) - .into(), - "bt_status_or_offsets_for_error_code", - ) - .map(|value| value.into_int_value()) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - } - - fn store_backtrace_frame( - &self, - inst_buffer: PointerValue<'ctx>, - frame_index: usize, - module_cookie: IntValue<'ctx>, - pc: IntValue<'ctx>, - raw_ip: IntValue<'ctx>, - flags: u16, - ) -> Result<()> { - let frame_base = - INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_SIZE + frame_index * BACKTRACE_FRAME_DATA_SIZE; - self.store_u64_value( - inst_buffer, - frame_base + BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, - module_cookie, - "bt_frame_cookie", - )?; - self.store_u64_value( - inst_buffer, - frame_base + BACKTRACE_FRAME_PC_OFFSET, - pc, - "bt_frame_pc", - )?; - self.store_u64_value( - inst_buffer, - frame_base + BACKTRACE_FRAME_RAW_IP_OFFSET, - raw_ip, - "bt_frame_raw_ip", - )?; - self.store_u16_const( - inst_buffer, - frame_base + BACKTRACE_FRAME_FLAGS_OFFSET, - flags, - "bt_frame_flags", - ) - } - - fn store_backtrace_frame_dynamic( - &self, - inst_buffer: PointerValue<'ctx>, - frame_index: IntValue<'ctx>, - max_frame_index: u8, - module_cookie: IntValue<'ctx>, - pc: IntValue<'ctx>, - raw_ip: IntValue<'ctx>, - ) -> Result<()> { - let i64_type = self.context.i64_type(); - let frame_index_i64 = if frame_index.get_type().get_bit_width() == i64_type.get_bit_width() - { - frame_index - } else { - self.builder - .build_int_z_extend(frame_index, i64_type, "bt_frame_index_i64") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - }; - let max_frame_index = i64_type.const_int(max_frame_index as u64, false); - let frame_index_in_bounds = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULE, - frame_index_i64, - max_frame_index, - "bt_frame_index_in_bounds", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let frame_index_i64 = self - .builder - .build_select::, _>( - frame_index_in_bounds, - frame_index_i64.into(), - max_frame_index.into(), - "bt_frame_index_bounded", - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let frame_stride = i64_type.const_int(BACKTRACE_FRAME_DATA_SIZE as u64, false); - let frame_offset = self - .builder - .build_int_mul(frame_index_i64, frame_stride, "bt_dynamic_frame_offset") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let base_offset = i64_type.const_int( - (INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_SIZE) as u64, - false, - ); - let frame_base_offset = self - .builder - .build_int_add(base_offset, frame_offset, "bt_dynamic_frame_base_offset") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let frame_base = - self.dynamic_byte_gep(inst_buffer, frame_base_offset, "bt_dynamic_frame")?; - - self.store_u64_value( - frame_base, - BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, - module_cookie, - "bt_frame_cookie", - )?; - self.store_u64_value(frame_base, BACKTRACE_FRAME_PC_OFFSET, pc, "bt_frame_pc")?; - self.store_u64_value( - frame_base, - BACKTRACE_FRAME_RAW_IP_OFFSET, - raw_ip, - "bt_frame_raw_ip", - )?; - self.store_u16_const( - frame_base, - BACKTRACE_FRAME_FLAGS_OFFSET, - 0, - "bt_frame_flags", - ) - } - - fn store_u8_const( - &self, - base: PointerValue<'ctx>, - offset: usize, - value: u8, - name: &str, - ) -> Result<()> { - self.store_u8_value( - base, - offset, - self.context.i8_type().const_int(value as u64, false), - name, - ) - } - - fn store_u8_value( - &self, - base: PointerValue<'ctx>, - offset: usize, - value: IntValue<'ctx>, - name: &str, - ) -> Result<()> { - let ptr = self.byte_gep(base, offset, name)?; - self.builder - .build_store(ptr, value) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(()) - } - - fn store_u16_const( - &self, - base: PointerValue<'ctx>, - offset: usize, - value: u16, - name: &str, - ) -> Result<()> { - self.store_u16_value( - base, - offset, - self.context.i16_type().const_int(value as u64, false), - name, - ) - } - - fn store_u16_value( - &self, - base: PointerValue<'ctx>, - offset: usize, - value: IntValue<'ctx>, - name: &str, - ) -> Result<()> { - let ptr = self.byte_gep(base, offset, name)?; - let ptr = self - .builder - .build_pointer_cast( - ptr, - self.context.ptr_type(AddressSpace::default()), - &format!("{name}_u16_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(ptr, value) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(()) - } - - fn store_u64_value( - &self, - base: PointerValue<'ctx>, - offset: usize, - value: IntValue<'ctx>, - name: &str, - ) -> Result<()> { - let ptr = self.byte_gep(base, offset, name)?; - let ptr = self - .builder - .build_pointer_cast( - ptr, - self.context.ptr_type(AddressSpace::default()), - &format!("{name}_u64_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder - .build_store(ptr, value) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - Ok(()) - } - - fn byte_gep( - &self, - base: PointerValue<'ctx>, - offset: usize, - name: &str, - ) -> Result> { - // SAFETY: callers pass offsets within the instruction region reserved for - // this Backtrace instruction. - unsafe { - self.builder - .build_gep( - self.context.i8_type(), - base, - &[self.context.i32_type().const_int(offset as u64, false)], - &format!("{name}_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - } - } - - fn dynamic_byte_gep( - &self, - base: PointerValue<'ctx>, - offset: IntValue<'ctx>, - name: &str, - ) -> Result> { - // SAFETY: callers guard the dynamic offset against the per-CPU buffer - // size before using the returned pointer. - unsafe { - self.builder - .build_gep( - self.context.i8_type(), - base, - &[offset], - &format!("{name}_ptr"), - ) - .map_err(|e| CodeGenError::LLVMError(e.to_string())) - } - } - - fn build_entry_alloca(&self, ty: T, name: &str) -> Result> - where - T: inkwell::types::BasicType<'ctx>, - { - let current_block = self.builder.get_insert_block().ok_or_else(|| { - CodeGenError::LLVMError("no current block for bt stack allocation".to_string()) - })?; - let current_fn = self.current_function("allocate bt scratch")?; - let entry_block = current_fn.get_first_basic_block().ok_or_else(|| { - CodeGenError::LLVMError("no entry block for bt stack allocation".to_string()) - })?; - - if let Some(first_instruction) = entry_block.get_first_instruction() { - self.builder.position_before(&first_instruction); - } else { - self.builder.position_at_end(entry_block); - } - let alloca = self - .builder - .build_alloca(ty, name) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - self.builder.position_at_end(current_block); - Ok(alloca) - } -} - -fn backtrace_row_binary_search_steps(row_count: usize) -> usize { - if row_count <= 1 { - 1 - } else { - (usize::BITS - (row_count - 1).leading_zeros()) as usize + 1 - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ebpf::context::BacktraceModuleRowRangeEntry; - use crate::CompileOptions; - use inkwell::AddressSpace; - - #[test] - fn binary_search_steps_cover_power_of_two_row_counts() { - assert_eq!(backtrace_row_binary_search_steps(0), 1); - assert_eq!(backtrace_row_binary_search_steps(1), 1); - assert_eq!(backtrace_row_binary_search_steps(2), 2); - assert_eq!(backtrace_row_binary_search_steps(4), 3); - assert_eq!(backtrace_row_binary_search_steps(8), 4); - } - - #[test] - fn binary_search_steps_cover_non_power_of_two_row_counts() { - assert_eq!(backtrace_row_binary_search_steps(3), 3); - assert_eq!(backtrace_row_binary_search_steps(5), 4); - assert_eq!(backtrace_row_binary_search_steps(9), 5); - } - - #[test] - fn runtime_backtrace_frame_module_resolution_generates_range_lookups() { - let context = inkwell::context::Context::create(); - let opts = CompileOptions::default(); - let mut ctx = - EbpfContext::new(&context, "bt_frame_module_test", Some(0), &opts).expect("ctx"); - let i64_type = context.i64_type(); - for map_name in ["proc_module_range_meta", "proc_module_ranges"] { - let map_global = - ctx.module - .add_global(i64_type, Some(AddressSpace::default()), map_name); - map_global.set_initializer(&i64_type.const_zero()); - } - - let fn_type = context.i32_type().fn_type(&[], false); - let function = ctx.module.add_function("bt_frame_module", fn_type, None); - let entry = context.append_basic_block(function, "entry"); - ctx.builder.position_at_end(entry); - let key_type = context.i32_type().array_type(4); - ctx.pm_key_alloca = Some( - ctx.builder - .build_alloca(key_type, "pm_key") - .expect("pm_key alloca"), - ); - ctx.backtrace_module_row_ranges = vec![ - BacktraceModuleRowRangeEntry { - cookie: 0x1111, - range: ghostscope_protocol::BacktraceModuleRowRange { - row_start: 0, - row_end: 1, - }, - }, - BacktraceModuleRowRangeEntry { - cookie: 0x2222, - range: ghostscope_protocol::BacktraceModuleRowRange { - row_start: 1, - row_end: 2, - }, - }, - ]; - - let frame_module = ctx - .resolve_backtrace_frame_module( - i64_type.const_int(0x7f00_1234, false), - i64_type.const_int(0x1111, false), - i64_type.const_zero(), - context.bool_type().const_zero(), - "test_bt_frame_module", - ) - .expect("resolve frame module"); - ctx.store_u64_value( - ctx.pm_key_alloca.expect("pm key"), - 0, - frame_module.cookie, - "selected_cookie", - ) - .expect("store selected cookie"); - - let ir = ctx.module.print_to_string().to_string(); - assert!( - ir.contains("proc_module_range_meta") - && ir.contains("proc_module_ranges") - && ir.contains("test_bt_frame_module_0_range_base") - && ir.contains("test_bt_frame_module_0_range_end"), - "resolver should use the per-PID module range index\nIR:\n{ir}" - ); - assert!( - ir.contains("test_bt_frame_module_0_range_cookie") - && !ir.contains("proc_module_offsets"), - "resolver should select a module cookie without scanning offsets\nIR:\n{ir}" - ); - } - - #[test] - fn runtime_backtrace_row_bounds_cover_all_prepared_modules() { - let context = inkwell::context::Context::create(); - let opts = CompileOptions::default(); - let mut ctx = - EbpfContext::new(&context, "bt_row_bounds_test", Some(0), &opts).expect("ctx"); - - let fn_type = context - .i32_type() - .fn_type(&[context.i64_type().into()], false); - let function = ctx.module.add_function("bt_row_bounds", fn_type, None); - let entry = context.append_basic_block(function, "entry"); - ctx.builder.position_at_end(entry); - let map_global = ctx.module.add_global( - context.i64_type(), - Some(AddressSpace::default()), - "bt_module_row_ranges", - ); - map_global.set_initializer(&context.i64_type().const_zero()); - let key_type = context.i32_type().array_type(4); - ctx.pm_key_alloca = Some( - ctx.builder - .build_alloca(key_type, "pm_key") - .expect("pm_key alloca"), - ); - - ctx.backtrace_module_row_ranges = (0..=32) - .map(|idx| BacktraceModuleRowRangeEntry { - cookie: 0x1000 + idx as u64, - range: ghostscope_protocol::BacktraceModuleRowRange { - row_start: (idx * 2) as u32, - row_end: (idx * 2 + 2) as u32, - }, - }) - .collect(); - - let bounds = ctx - .backtrace_unwind_row_bounds_for_module( - function - .get_first_param() - .expect("module cookie param") - .into_int_value(), - "test_bt_row_bounds", - ) - .expect("row bounds"); - ctx.builder - .build_return(Some(&bounds.end)) - .expect("return bounds end"); - - let ir = ctx.module.print_to_string().to_string(); - assert!( - ir.contains("bt_module_row_ranges") - && ir.contains("test_bt_row_bounds_row_range_lookup"), - "row bounds lookup should use the module range map\nIR:\n{ir}" - ); - assert!( - !ir.contains("module_matches"), - "row bounds lookup should not use static candidate comparisons\nIR:\n{ir}" - ); - assert!( - !ir.contains("row_range_cookie_lo") && !ir.contains("row_range_cookie_hi"), - "row bounds lookup should store the native u64 module cookie\nIR:\n{ir}" - ); - } -} diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/frame_recovery.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/frame_recovery.rs new file mode 100644 index 00000000..958824d6 --- /dev/null +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/frame_recovery.rs @@ -0,0 +1,1078 @@ +use super::*; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + pub(super) fn recover_next_frame_from_runtime_row( + &mut self, + row: &RuntimeBtUnwindRow<'ctx>, + state: BtRegisterState<'ctx>, + scratch: &BtScratch<'ctx>, + ) -> Result> { + let cfa_base = self.select_register_state(row.cfa_register, state, "bt_cfa_base")?; + let cfa = self + .builder + .build_int_add(cfa_base, row.cfa_offset, "bt_runtime_cfa") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let ra_addr = self + .builder + .build_int_add(cfa, row.ra_offset, "bt_runtime_ra_addr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store( + scratch.next_error_code_ptr, + self.context + .i16_type() + .const_int(BACKTRACE_ERROR_NONE as u64, false), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let (ra_from_memory, ra_read_failed) = self.generate_memory_read_with_fail_flag( + RuntimeAddress::available(ra_addr, self.context), + MemoryAccessSize::U64, + "bt_ra_read", + )?; + let ra_from_memory = ra_from_memory.into_int_value(); + let ra_uses_memory = self.is_recovery_kind( + row.ra_kind, + crate::BACKTRACE_RECOVERY_AT_CFA_OFFSET, + "bt_ra_at_kind", + )?; + let ra_read_failed = self + .builder + .build_and(ra_read_failed, ra_uses_memory, "bt_ra_read_failed") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.store_backtrace_error_code_if( + scratch.next_error_code_ptr, + ra_read_failed, + BACKTRACE_ERROR_RETURN_ADDRESS_READ, + "bt_ra_error_code", + )?; + let ra_from_val = self + .builder + .build_int_add(cfa, row.ra_offset, "bt_runtime_ra_val") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ra_from_register = self.select_register_state(row.ra_register, state, "bt_ra_reg")?; + let ra_is_val = self.is_recovery_kind( + row.ra_kind, + crate::BACKTRACE_RECOVERY_VAL_CFA_OFFSET, + "bt_ra_val_kind", + )?; + let ra_is_register = self.is_recovery_kind( + row.ra_kind, + crate::BACKTRACE_RECOVERY_REGISTER, + "bt_ra_reg_kind", + )?; + let ra_is_same = self.is_recovery_kind( + row.ra_kind, + crate::BACKTRACE_RECOVERY_SAME_VALUE, + "bt_ra_same_kind", + )?; + let ra_is_register_like = self + .builder + .build_or(ra_is_register, ra_is_same, "bt_ra_register_like") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ra_value_or_memory = self + .builder + .build_select::, _>( + ra_is_val, + ra_from_val.into(), + ra_from_memory.into(), + "bt_ra_val_or_memory", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let next_ip = self + .builder + .build_select::, _>( + ra_is_register_like, + ra_from_register.into(), + ra_value_or_memory.into(), + "bt_next_ip", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let next_rbp = self.recover_rbp_from_runtime_row( + row, + cfa, + state, + scratch.next_rbp_ptr, + scratch.next_error_code_ptr, + )?; + let error_code = self.load_i16(scratch.next_error_code_ptr, "bt_next_error_code_value")?; + + Ok(BtNextFrame { + ip: next_ip, + rsp: cfa, + rbp: next_rbp, + error_code, + }) + } + + pub(super) fn validate_backtrace_next_frame( + &self, + state: BtRegisterState<'ctx>, + next: BtNextFrame<'ctx>, + ) -> Result> { + let i64_type = self.context.i64_type(); + let i16_type = self.context.i16_type(); + let zero = i64_type.const_zero(); + let zero_i16 = i16_type.const_zero(); + let min_user_ip = i64_type.const_int(0x1000, false); + let high_byte_mask = i64_type.const_int(0xff00_0000_0000_0000, false); + + let no_read_error = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + next.error_code, + zero_i16, + "bt_no_read_error", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ip_high_enough = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + next.ip, + min_user_ip, + "bt_next_ip_user_min", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ip_high_byte = self + .builder + .build_and(next.ip, high_byte_mask, "bt_next_ip_high_byte") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ip_is_zero = self + .builder + .build_int_compare(inkwell::IntPredicate::EQ, next.ip, zero, "bt_next_ip_zero") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ip_not_kernel_like = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + ip_high_byte, + zero, + "bt_next_ip_not_kernel_like", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_nonzero = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + next.rsp, + zero, + "bt_next_cfa_nonzero", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_changed = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + next.rsp, + state.rsp, + "bt_next_cfa_changed", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let ip_changed = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + next.ip, + state.ip, + "bt_next_ip_changed", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_progress = self + .builder + .build_or(cfa_changed, ip_changed, "bt_next_frame_progress") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let ip_valid = self + .builder + .build_and(ip_high_enough, ip_not_kernel_like, "bt_next_ip_valid") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_valid = self + .builder + .build_and(cfa_nonzero, cfa_progress, "bt_next_cfa_valid") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let frame_shape_valid = self + .builder + .build_and(ip_valid, cfa_valid, "bt_next_frame_valid") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let valid = self + .builder + .build_and( + no_read_error, + frame_shape_valid, + "bt_next_frame_valid_no_read_error", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let complete = self + .builder + .build_and(no_read_error, ip_is_zero, "bt_next_frame_complete") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let mut error_code = next.error_code; + error_code = self.select_backtrace_error_code_if( + error_code, + ip_high_enough, + BACKTRACE_ERROR_NEXT_IP_BELOW_USER, + "bt_next_ip_below_user_code", + )?; + error_code = self.select_backtrace_error_code_if( + error_code, + ip_not_kernel_like, + BACKTRACE_ERROR_NEXT_IP_KERNEL_LIKE, + "bt_next_ip_kernel_like_code", + )?; + error_code = self.select_backtrace_error_code_if( + error_code, + cfa_nonzero, + BACKTRACE_ERROR_NEXT_CFA_ZERO, + "bt_next_cfa_zero_code", + )?; + error_code = self.select_backtrace_error_code_if( + error_code, + cfa_progress, + BACKTRACE_ERROR_NEXT_CFA_NOT_ADVANCING, + "bt_next_cfa_not_advancing_code", + )?; + error_code = self + .builder + .build_select::, _>( + complete, + self.context + .i16_type() + .const_int(BACKTRACE_ERROR_NONE as u64, false) + .into(), + error_code.into(), + "bt_complete_error_code", + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + Ok(BtFrameValidation { + valid, + complete, + error_code, + }) + } + + pub(super) fn select_backtrace_error_code_if( + &self, + current: IntValue<'ctx>, + condition_ok: IntValue<'ctx>, + error_code: u16, + name: &str, + ) -> Result> { + let i16_type = self.context.i16_type(); + let current_is_none = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + current, + i16_type.const_int(BACKTRACE_ERROR_NONE as u64, false), + &format!("{name}_current_none"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let condition_failed = self + .builder + .build_not(condition_ok, &format!("{name}_condition_failed")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let should_set = self + .builder + .build_and( + current_is_none, + condition_failed, + &format!("{name}_should_set"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_select::, _>( + should_set, + i16_type.const_int(error_code as u64, false).into(), + current.into(), + name, + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + pub(super) fn store_backtrace_error_code_if( + &self, + error_code_ptr: PointerValue<'ctx>, + condition: IntValue<'ctx>, + error_code: u16, + name: &str, + ) -> Result<()> { + let current = self.load_i16(error_code_ptr, &format!("{name}_current"))?; + let i16_type = self.context.i16_type(); + let current_is_none = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + current, + i16_type.const_int(BACKTRACE_ERROR_NONE as u64, false), + &format!("{name}_current_none"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let should_set = self + .builder + .build_and(condition, current_is_none, &format!("{name}_should_set")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let next = self + .builder + .build_select::, _>( + should_set, + i16_type.const_int(error_code as u64, false).into(), + current.into(), + name, + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(error_code_ptr, next) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + pub(super) fn recover_rbp_from_runtime_row( + &mut self, + row: &RuntimeBtUnwindRow<'ctx>, + cfa: IntValue<'ctx>, + state: BtRegisterState<'ctx>, + next_rbp_ptr: PointerValue<'ctx>, + next_error_code_ptr: PointerValue<'ctx>, + ) -> Result> { + let is_at = self.is_recovery_kind( + row.rbp_kind, + crate::BACKTRACE_RECOVERY_AT_CFA_OFFSET, + "bt_rbp_at_kind", + )?; + let cfa_uses_rbp = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + row.cfa_register, + self.context + .i16_type() + .const_int(X86_64_DWARF_RBP as u64, false), + "bt_rbp_cfa_uses_rbp", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_offset_is_frame_pointer_call_frame = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + row.cfa_offset, + self.context.i64_type().const_int(16, false), + "bt_rbp_cfa_offset_is_16", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let frame_pointer_call_frame = self + .builder + .build_and( + cfa_uses_rbp, + cfa_offset_is_frame_pointer_call_frame, + "bt_rbp_frame_pointer_call_frame", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let is_at = self + .builder + .build_or( + is_at, + frame_pointer_call_frame, + "bt_rbp_at_or_frame_pointer", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let rbp_offset = self + .builder + .build_select::, _>( + frame_pointer_call_frame, + self.context + .i64_type() + .const_int((-16i64) as u64, true) + .into(), + row.rbp_offset.into(), + "bt_rbp_effective_offset", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let current_fn = self.current_function("recover bt rbp")?; + let at_block = self.context.append_basic_block(current_fn, "bt_rbp_at"); + let non_at_block = self.context.append_basic_block(current_fn, "bt_rbp_non_at"); + let join_block = self.context.append_basic_block(current_fn, "bt_rbp_join"); + self.builder + .build_conditional_branch(is_at, at_block, non_at_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(at_block); + let rbp_addr = self + .builder + .build_int_add(cfa, rbp_offset, "bt_runtime_rbp_addr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let (rbp_from_memory, rbp_read_failed) = self.generate_memory_read_with_fail_flag( + RuntimeAddress::available(rbp_addr, self.context), + MemoryAccessSize::U64, + "bt_rbp_read", + )?; + self.store_backtrace_error_code_if( + next_error_code_ptr, + rbp_read_failed, + BACKTRACE_ERROR_FRAME_POINTER_READ, + "bt_rbp_error_code", + )?; + self.builder + .build_store(next_rbp_ptr, rbp_from_memory.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(join_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(non_at_block); + let rbp_from_val = self + .builder + .build_int_add(cfa, row.rbp_offset, "bt_runtime_rbp_val") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let rbp_from_register = + self.select_register_state(row.rbp_register, state, "bt_rbp_reg")?; + let rbp_is_val = self.is_recovery_kind( + row.rbp_kind, + crate::BACKTRACE_RECOVERY_VAL_CFA_OFFSET, + "bt_rbp_val_kind", + )?; + let rbp_is_register = self.is_recovery_kind( + row.rbp_kind, + crate::BACKTRACE_RECOVERY_REGISTER, + "bt_rbp_reg_kind", + )?; + let rbp_is_same = self.is_recovery_kind( + row.rbp_kind, + crate::BACKTRACE_RECOVERY_SAME_VALUE, + "bt_rbp_same_kind", + )?; + let rbp_is_register_like = self + .builder + .build_or(rbp_is_register, rbp_is_same, "bt_rbp_register_like") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let rbp_value_or_current = self + .builder + .build_select::, _>( + rbp_is_val, + rbp_from_val.into(), + state.rbp.into(), + "bt_rbp_val_or_current", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let rbp_non_at = self + .builder + .build_select::, _>( + rbp_is_register_like, + rbp_from_register.into(), + rbp_value_or_current.into(), + "bt_rbp_non_at_value", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + self.builder + .build_store(next_rbp_ptr, rbp_non_at) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(join_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(join_block); + self.load_i64(next_rbp_ptr, "bt_next_rbp_value") + } + + pub(super) fn select_register_state( + &self, + register: IntValue<'ctx>, + state: BtRegisterState<'ctx>, + name: &str, + ) -> Result> { + let is_rbp = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + register, + self.context + .i16_type() + .const_int(X86_64_DWARF_RBP as u64, false), + &format!("{name}_is_rbp"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let is_rip = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + register, + self.context + .i16_type() + .const_int(X86_64_DWARF_RIP as u64, false), + &format!("{name}_is_rip"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let rbp_or_rsp = self + .builder + .build_select::, _>( + is_rbp, + state.rbp.into(), + state.rsp.into(), + &format!("{name}_rbp_or_rsp"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + self.builder + .build_select::, _>( + is_rip, + state.ip.into(), + rbp_or_rsp.into(), + name, + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + pub(super) fn is_recovery_kind( + &self, + kind: IntValue<'ctx>, + expected: u8, + name: &str, + ) -> Result> { + self.builder + .build_int_compare( + inkwell::IntPredicate::EQ, + kind, + self.context.i8_type().const_int(expected as u64, false), + name, + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + pub(super) fn lookup_bt_unwind_row_ptr( + &mut self, + row_index: IntValue<'ctx>, + ) -> Result> { + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let i32_type = self.context.i32_type(); + let map_global = self + .module + .get_global("bt_unwind_rows") + .ok_or_else(|| CodeGenError::LLVMError("bt_unwind_rows map not found".to_string()))?; + let map_ptr = self + .builder + .build_bit_cast( + map_global.as_pointer_value(), + ptr_type, + "bt_unwind_rows_map_ptr", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_alloca = self.pm_key_alloca.ok_or_else(|| { + CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) + })?; + let key_arr_ty = i32_type.array_type(4); + let zero = i32_type.const_zero(); + // SAFETY: pm_key_alloca is a [4 x i32] entry-block alloca and [0, 0] + // addresses its first element, which is sufficient for an Array u32 key. + let key_ptr = unsafe { + self.builder + .build_gep( + key_arr_ty, + key_alloca, + &[zero, zero], + "bt_unwind_row_key_ptr", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + self.builder + .build_store(key_ptr, row_index) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_ptr = self + .builder + .build_bit_cast(key_ptr, ptr_type, "bt_unwind_row_key_void") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let result = self.create_bpf_helper_call( + BPF_FUNC_map_lookup_elem as u64, + &[map_ptr, key_ptr], + ptr_type.into(), + "bt_unwind_row_lookup", + )?; + match result { + BasicValueEnum::PointerValue(ptr) => Ok(ptr), + _ => Err(CodeGenError::LLVMError( + "bt_unwind_rows lookup did not return pointer".to_string(), + )), + } + } + + pub(super) fn lookup_bt_state_ptr(&mut self, key_const: u32) -> Result> { + self.lookup_percpu_value_ptr("bt_state", key_const) + } + + pub(super) fn lookup_bt_state_ptr_dynamic( + &mut self, + state_index: IntValue<'ctx>, + ) -> Result> { + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let i32_type = self.context.i32_type(); + let map_global = self + .map_manager + .get_map(&self.module, "bt_state") + .map_err(|e| CodeGenError::LLVMError(format!("Map not found bt_state: {e}")))?; + let map_ptr = self + .builder + .build_bit_cast(map_global, ptr_type, "bt_state_map_ptr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key = match state_index + .get_type() + .get_bit_width() + .cmp(&i32_type.get_bit_width()) + { + std::cmp::Ordering::Equal => state_index, + std::cmp::Ordering::Less => self + .builder + .build_int_z_extend(state_index, i32_type, "bt_state_key_i32") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?, + std::cmp::Ordering::Greater => self + .builder + .build_int_truncate(state_index, i32_type, "bt_state_key_i32") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?, + }; + + let key_arr_ty = i32_type.array_type(4); + let key_alloca = self.pm_key_alloca.ok_or_else(|| { + CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) + })?; + let zero = i32_type.const_zero(); + // SAFETY: pm_key_alloca is a [4 x i32] entry-block alloca and [0, 0] + // addresses its first element, which is sufficient for an Array u32 key. + let key_ptr = unsafe { + self.builder + .build_gep(key_arr_ty, key_alloca, &[zero, zero], "bt_state_key_ptr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + self.builder + .build_store(key_ptr, key) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_ptr = self + .builder + .build_bit_cast(key_ptr, ptr_type, "bt_state_key_void") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let result = self.create_bpf_helper_call( + BPF_FUNC_map_lookup_elem as u64, + &[map_ptr, key_ptr], + ptr_type.into(), + "bt_state_lookup", + )?; + match result { + BasicValueEnum::PointerValue(ptr) => Ok(ptr), + _ => Err(CodeGenError::LLVMError( + "bt_state lookup did not return pointer".to_string(), + )), + } + } + + pub(super) fn lookup_bt_prog_array_ptr(&mut self) -> Result> { + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let map_global = self + .module + .get_global("bt_prog_array") + .ok_or_else(|| CodeGenError::LLVMError("bt_prog_array map not found".to_string()))?; + let map_ptr = self + .builder + .build_bit_cast(map_global.as_pointer_value(), ptr_type, "bt_prog_array_ptr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + match map_ptr { + BasicValueEnum::PointerValue(ptr) => Ok(ptr), + _ => Err(CodeGenError::LLVMError( + "bt_prog_array cast did not return pointer".to_string(), + )), + } + } + + pub(super) fn get_or_create_backtrace_tail_enabled_flag( + &mut self, + ) -> Result> { + if let Some(ptr) = self.backtrace_tail_enabled_alloca { + return Ok(ptr); + } + + let current_block = self.builder.get_insert_block().ok_or_else(|| { + CodeGenError::LLVMError("no current block for bt tail flag allocation".to_string()) + })?; + let current_fn = self.current_function("allocate bt tail flag")?; + let entry_block = current_fn.get_first_basic_block().ok_or_else(|| { + CodeGenError::LLVMError("no entry block for bt tail flag allocation".to_string()) + })?; + + if let Some(first_instruction) = entry_block.get_first_instruction() { + self.builder.position_before(&first_instruction); + } else { + self.builder.position_at_end(entry_block); + } + let alloca = self + .builder + .build_alloca(self.context.i8_type(), "bt_tail_enabled") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(alloca, self.context.i8_type().const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder.position_at_end(current_block); + self.backtrace_tail_enabled_alloca = Some(alloca); + Ok(alloca) + } + + pub(super) fn get_or_create_backtrace_tail_last_slot(&mut self) -> Result> { + if let Some(ptr) = self.backtrace_tail_last_slot_alloca { + return Ok(ptr); + } + + let current_block = self.builder.get_insert_block().ok_or_else(|| { + CodeGenError::LLVMError("no current block for bt tail slot allocation".to_string()) + })?; + let current_fn = self.current_function("allocate bt tail slot")?; + let entry_block = current_fn.get_first_basic_block().ok_or_else(|| { + CodeGenError::LLVMError("no entry block for bt tail slot allocation".to_string()) + })?; + + if let Some(first_instruction) = entry_block.get_first_instruction() { + self.builder.position_before(&first_instruction); + } else { + self.builder.position_at_end(entry_block); + } + let alloca = self + .builder + .build_alloca(self.context.i8_type(), "bt_tail_last_slot") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store( + alloca, + self.context + .i8_type() + .const_int(crate::BACKTRACE_TAIL_NO_NEXT_SLOT as u64, false), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder.position_at_end(current_block); + self.backtrace_tail_last_slot_alloca = Some(alloca); + Ok(alloca) + } + + pub(super) fn link_backtrace_tail_slot( + &mut self, + tail_slot: u8, + offsets_found_u8: IntValue<'ctx>, + done_block: BasicBlock<'ctx>, + ) -> Result<()> { + let current_fn = self.current_function("link bt tail slot")?; + let offsets_found = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + offsets_found_u8, + self.context.i8_type().const_zero(), + "bt_tail_link_offsets_found", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let link_block = self + .context + .append_basic_block(current_fn, "bt_tail_link_slot"); + self.builder + .build_conditional_branch(offsets_found, link_block, done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(link_block); + let state0_ptr = self.lookup_bt_state_ptr(0)?; + let state0_is_null = self + .builder + .build_is_null(state0_ptr, "bt_tail_link_state0_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let state0_ok_block = self + .context + .append_basic_block(current_fn, "bt_tail_link_state0_ok"); + self.builder + .build_conditional_branch(state0_is_null, done_block, state0_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(state0_ok_block); + let tail_enabled_ptr = self.get_or_create_backtrace_tail_enabled_flag()?; + let last_slot_ptr = self.get_or_create_backtrace_tail_last_slot()?; + let enabled_value = self + .builder + .build_load( + self.context.i8_type(), + tail_enabled_ptr, + "bt_tail_link_enabled_value", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let has_prev_slot = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + enabled_value, + self.context.i8_type().const_zero(), + "bt_tail_link_has_prev", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let first_slot_block = self + .context + .append_basic_block(current_fn, "bt_tail_link_first"); + let append_slot_block = self + .context + .append_basic_block(current_fn, "bt_tail_link_append"); + let linked_block = self + .context + .append_basic_block(current_fn, "bt_tail_linked"); + self.builder + .build_conditional_branch(has_prev_slot, append_slot_block, first_slot_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(first_slot_block); + self.store_u8_const( + state0_ptr, + crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, + tail_slot, + "bt_tail_link_active_slot", + )?; + self.builder + .build_unconditional_branch(linked_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(append_slot_block); + let prev_slot = self.load_i8(last_slot_ptr, "bt_tail_link_prev_slot")?; + let prev_state_ptr = self.lookup_bt_state_ptr_dynamic(prev_slot)?; + let prev_state_is_null = self + .builder + .build_is_null(prev_state_ptr, "bt_tail_link_prev_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let prev_state_ok_block = self + .context + .append_basic_block(current_fn, "bt_tail_link_prev_ok"); + self.builder + .build_conditional_branch(prev_state_is_null, first_slot_block, prev_state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(prev_state_ok_block); + self.store_u8_const( + prev_state_ptr, + crate::BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, + tail_slot, + "bt_tail_link_prev_next_slot", + )?; + self.builder + .build_unconditional_branch(linked_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(linked_block); + self.builder + .build_store( + last_slot_ptr, + self.context.i8_type().const_int(tail_slot as u64, false), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(tail_enabled_ptr, self.context.i8_type().const_int(1, false)) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + pub(super) fn store_state_i64( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: IntValue<'ctx>, + name: &str, + ) -> Result<()> { + self.store_u64_value(base, offset, value, name) + } + + pub(super) fn store_state_i32( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: IntValue<'ctx>, + name: &str, + ) -> Result<()> { + let ptr = self.byte_gep(base, offset, name)?; + let ptr = self + .builder + .build_pointer_cast( + ptr, + self.context.ptr_type(AddressSpace::default()), + &format!("{name}_u32_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(ptr, value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + pub(super) fn load_state_i32( + &self, + base: PointerValue<'ctx>, + offset: usize, + name: &str, + ) -> Result> { + let ptr = self.byte_gep(base, offset, name)?; + let ptr = self + .builder + .build_pointer_cast( + ptr, + self.context.ptr_type(AddressSpace::default()), + &format!("{name}_u32_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(self + .builder + .build_load(self.context.i32_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + pub(super) fn load_row_i64( + &self, + row_ptr: PointerValue<'ctx>, + offset: usize, + name: &str, + ) -> Result> { + let ptr = self.byte_gep(row_ptr, offset, name)?; + let ptr = self + .builder + .build_pointer_cast( + ptr, + self.context.ptr_type(AddressSpace::default()), + &format!("{name}_i64_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(self + .builder + .build_load(self.context.i64_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + pub(super) fn load_row_i16( + &self, + row_ptr: PointerValue<'ctx>, + offset: usize, + name: &str, + ) -> Result> { + let ptr = self.byte_gep(row_ptr, offset, name)?; + Ok(self + .builder + .build_load(self.context.i16_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + pub(super) fn load_row_i8( + &self, + row_ptr: PointerValue<'ctx>, + offset: usize, + name: &str, + ) -> Result> { + let ptr = self.byte_gep(row_ptr, offset, name)?; + Ok(self + .builder + .build_load(self.context.i8_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + pub(super) fn load_i8(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { + Ok(self + .builder + .build_load(self.context.i8_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + pub(super) fn load_bool(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { + Ok(self + .builder + .build_load(self.context.bool_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + pub(super) fn load_i32(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { + Ok(self + .builder + .build_load(self.context.i32_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + pub(super) fn load_i16(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { + Ok(self + .builder + .build_load(self.context.i16_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + pub(super) fn load_i64(&self, ptr: PointerValue<'ctx>, name: &str) -> Result> { + Ok(self + .builder + .build_load(self.context.i64_type(), ptr, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + } + + pub(super) fn load_dwarf_register_i64( + &mut self, + reg: u16, + pt_regs: PointerValue<'ctx>, + ) -> Result> { + let value = self.load_register_value(reg, pt_regs)?; + match value { + BasicValueEnum::IntValue(value) => Ok(value), + _ => Err(CodeGenError::RegisterMappingError(format!( + "DWARF register {reg} did not load as integer" + ))), + } + } + + pub(super) fn add_signed_offset( + &self, + base: IntValue<'ctx>, + offset: i64, + name: &str, + ) -> Result> { + if offset == 0 { + return Ok(base); + } + self.builder + .build_int_add( + base, + self.context.i64_type().const_int(offset as u64, true), + name, + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + pub(super) fn normalized_pc_from_raw( + &self, + raw_ip: IntValue<'ctx>, + module_bias: IntValue<'ctx>, + offsets_found: IntValue<'ctx>, + ) -> Result> { + let rebased = self + .builder + .build_int_sub(raw_ip, module_bias, "bt_normalized_pc") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_select::, _>( + offsets_found, + rebased.into(), + raw_ip.into(), + "bt_pc_or_raw", + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } +} diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/inline.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/inline.rs new file mode 100644 index 00000000..096d490a --- /dev/null +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/inline.rs @@ -0,0 +1,414 @@ +use super::*; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + /// Generate a DWARF-backed Backtrace instruction. + /// + /// eBPF records `(module_cookie, normalized_pc, raw_ip)` frames and advances + /// the unwind state from compact DWARF CFI rows. Userspace owns final source + /// line and inline-chain symbolization. + pub(super) fn generate_inline_backtrace_instruction( + &mut self, + plan: &BacktraceInstructionPlan, + ) -> Result<()> { + let depth = plan.depth; + let flags = plan.flags; + info!("Generating Backtrace instruction: depth={}", depth); + + let payload_size = plan.payload_size; + let instruction_size = plan.instruction_size; + let inst_buffer = self + .reserve_instruction_region_or_return_zero(instruction_size as u64)? + .into_value_after_runtime_returns(); + + self.store_u8_const( + inst_buffer, + std::mem::offset_of!(InstructionHeader, inst_type), + InstructionType::Backtrace as u8, + "bt_inst_type", + )?; + self.store_u16_const( + inst_buffer, + std::mem::offset_of!(InstructionHeader, data_length), + payload_size as u16, + "bt_data_length", + )?; + + let data_base = INSTRUCTION_HEADER_SIZE; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_REQUESTED_DEPTH_OFFSET, + depth, + "bt_requested_depth", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 1, + "bt_frame_count", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FLAGS_OFFSET, + flags, + "bt_flags", + )?; + self.store_u16_const( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + 0, + "bt_error_code", + )?; + + let Some(compile_ctx) = self.current_compile_time_context.clone() else { + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 0, + "bt_frame_count_no_context", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + BacktraceStatus::DwarfUnavailable as u8, + "bt_status_no_context", + )?; + return Ok(()); + }; + + let module_cookie = self.cookie_for_module_or_fallback(&compile_ctx.module_path); + let module_cookie_value = self.context.i64_type().const_int(module_cookie, false); + let pt_regs = self.get_pt_regs_parameter()?; + let raw_ip = self.load_dwarf_register_i64(X86_64_DWARF_RIP, pt_regs)?; + let (module_bias, offsets_found) = self.generate_runtime_address_from_offsets( + self.context.i64_type().const_zero(), + 0, + module_cookie, + )?; + let normalized_pc = self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found)?; + let caller_fallback_found = self.backtrace_module_fallback_found(offsets_found); + + self.store_backtrace_frame( + inst_buffer, + 0, + module_cookie_value, + normalized_pc, + raw_ip, + 0, + )?; + + if depth == 1 { + let status = + self.status_or_offsets_unavailable(BacktraceStatus::Truncated, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_depth_one", + )?; + return Ok(()); + } + + let row = self + .usable_backtrace_unwind_row_for_pc(&compile_ctx.module_path, compile_ctx.pc_address); + let initial_status = self.status_for_backtrace_unwind_row_for_pc(&row); + let status = self.status_or_offsets_unavailable(initial_status, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_initial_status", + )?; + + let BacktraceUnwindRowForPc::Usable(row) = row else { + return Ok(()); + }; + + let i64_type = self.context.i64_type(); + let ip_ptr = self.build_entry_alloca(i64_type, "bt_state_ip")?; + let rsp_ptr = self.build_entry_alloca(i64_type, "bt_state_rsp")?; + let rbp_ptr = self.build_entry_alloca(i64_type, "bt_state_rbp")?; + let module_bias_ptr = self.build_entry_alloca(i64_type, "bt_state_module_bias")?; + let module_cookie_ptr = self.build_entry_alloca(i64_type, "bt_state_module_cookie")?; + let module_found_ptr = + self.build_entry_alloca(self.context.bool_type(), "bt_state_module_found")?; + let scratch = self.allocate_backtrace_scratch()?; + self.builder + .build_store(ip_ptr, raw_ip) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let initial_rsp = self.load_dwarf_register_i64(X86_64_DWARF_RSP, pt_regs)?; + let initial_rbp = self.load_dwarf_register_i64(X86_64_DWARF_RBP, pt_regs)?; + self.builder + .build_store(rsp_ptr, initial_rsp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_ptr, initial_rbp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_bias_ptr, module_bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, module_cookie_value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_found_ptr, offsets_found) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let current_fn = self.current_function("generate DWARF backtrace")?; + let done = self.context.append_basic_block(current_fn, "bt_done"); + + let runtime_row = self.runtime_row_from_static(row); + let state = BtRegisterState { + ip: self.load_i64(ip_ptr, "bt_initial_current_ip")?, + rsp: self.load_i64(rsp_ptr, "bt_initial_current_rsp")?, + rbp: self.load_i64(rbp_ptr, "bt_initial_current_rbp")?, + }; + let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; + let validation = self.validate_backtrace_next_frame(state, next)?; + let initial_store_block = self + .context + .append_basic_block(current_fn, "bt_initial_store_frame"); + let initial_stop_block = self + .context + .append_basic_block(current_fn, "bt_initial_stop"); + self.builder + .build_conditional_branch(validation.valid, initial_store_block, initial_stop_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(initial_stop_block); + let stop_status = self.status_for_backtrace_stop( + validation.complete, + validation.error_code, + offsets_found, + )?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + stop_status, + "bt_initial_stop_status", + )?; + self.store_u16_value( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + validation.error_code, + "bt_initial_stop_error_code", + )?; + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(initial_store_block); + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie_value, + module_bias, + caller_fallback_found, + "bt_initial_frame_module", + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; + self.store_backtrace_frame(inst_buffer, 1, frame_module.cookie, next_pc, next.ip, 0)?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 2, + "bt_initial_frame_count", + )?; + let status = if depth == 2 { + BacktraceStatus::Truncated + } else { + BacktraceStatus::ReadError + }; + let status = self.status_or_offsets_unavailable(status, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_initial_status_after_frame", + )?; + self.builder + .build_store(ip_ptr, next.ip) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rsp_ptr, next.rsp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_ptr, next.rbp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_bias_ptr, frame_module.bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, frame_module.cookie) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_found_ptr, frame_module.found) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + if depth == 2 { + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } else if self.backtrace_unwind_rows.is_empty() { + let status = self + .status_or_offsets_unavailable(BacktraceStatus::NoUnwindRowsForPc, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_no_rows", + )?; + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } else { + let inline_depth = depth.min(BPF_INLINE_BACKTRACE_FRAME_LIMIT); + for frame_index in 2..inline_depth { + let current_ip = self.load_i64(ip_ptr, "bt_lookup_ip")?; + let current_module_bias = + self.load_i64(module_bias_ptr, "bt_lookup_module_bias")?; + let current_module_cookie = + self.load_i64(module_cookie_ptr, "bt_lookup_module_cookie")?; + let current_module_found = + self.load_bool(module_found_ptr, "bt_lookup_module_found")?; + let lookup_raw = self.add_signed_offset(current_ip, -1, "bt_lookup_raw")?; + let lookup_pc = self.backtrace_lookup_pc_from_raw( + lookup_raw, + current_module_bias, + current_module_found, + )?; + let runtime_row = self.lookup_backtrace_unwind_row( + lookup_pc, + current_module_cookie, + &scratch.row, + &format!("bt_frame_{frame_index}_row"), + )?; + let found_block = self.context.append_basic_block(current_fn, "bt_row_found"); + let missing_block = self + .context + .append_basic_block(current_fn, "bt_row_missing"); + self.builder + .build_conditional_branch(runtime_row.found, found_block, missing_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(missing_block); + let status = self.status_or_offsets_unavailable( + BacktraceStatus::NoUnwindRowsForPc, + current_module_found, + )?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_missing_row", + )?; + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(found_block); + let state = BtRegisterState { + ip: self.load_i64(ip_ptr, "bt_current_ip")?, + rsp: self.load_i64(rsp_ptr, "bt_current_rsp")?, + rbp: self.load_i64(rbp_ptr, "bt_current_rbp")?, + }; + let next = + self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; + let validation = self.validate_backtrace_next_frame(state, next)?; + let store_block = self + .context + .append_basic_block(current_fn, "bt_store_frame"); + let stop_block = self.context.append_basic_block(current_fn, "bt_stop"); + self.builder + .build_conditional_branch(validation.valid, store_block, stop_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(stop_block); + let stop_status = self.status_for_backtrace_stop( + validation.complete, + validation.error_code, + current_module_found, + )?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + stop_status, + "bt_status_stop", + )?; + self.store_u16_value( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + validation.error_code, + "bt_error_code_stop", + )?; + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(store_block); + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + current_module_cookie, + current_module_bias, + self.backtrace_module_fallback_found(current_module_found), + &format!("bt_frame_{frame_index}_module"), + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; + self.store_backtrace_frame( + inst_buffer, + frame_index as usize, + frame_module.cookie, + next_pc, + next.ip, + 0, + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + frame_index + 1, + "bt_frame_count", + )?; + let next_status = if frame_index + 1 == inline_depth { + BacktraceStatus::Truncated + } else { + BacktraceStatus::ReadError + }; + let next_status = + self.status_or_offsets_unavailable(next_status, current_module_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + next_status, + "bt_status_after_frame", + )?; + self.builder + .build_store(ip_ptr, next.ip) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rsp_ptr, next.rsp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_ptr, next.rbp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_bias_ptr, frame_module.bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, frame_module.cookie) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_found_ptr, frame_module.found) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + if frame_index + 1 == inline_depth { + self.builder + .build_unconditional_branch(done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } + } + } + + self.builder.position_at_end(done); + Ok(()) + } +} diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/mod.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/mod.rs new file mode 100644 index 00000000..e50bab83 --- /dev/null +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/mod.rs @@ -0,0 +1,287 @@ +use super::*; +use crate::script::BacktraceStatement; +use aya_ebpf_bindings::bindings::bpf_func_id::{BPF_FUNC_map_lookup_elem, BPF_FUNC_tail_call}; +use ghostscope_dwarf::{CompactUnwindRow, MemoryAccessSize, ModuleAddress}; +use inkwell::basic_block::BasicBlock; +use inkwell::values::BasicMetadataValueEnum; +use std::path::PathBuf; + +use plan::{BacktraceEmitMode, BacktraceInstructionPlan, BPF_INLINE_BACKTRACE_FRAME_LIMIT}; + +const X86_64_DWARF_RIP: u16 = 16; +const X86_64_DWARF_RBP: u16 = 6; +const X86_64_DWARF_RSP: u16 = 7; +const BPF_BACKTRACE_FRAMES_PER_TAIL_CALL: u8 = 4; +const BPF_BACKTRACE_MAX_STEP_INVOCATIONS: u8 = 32; +const BPF_BACKTRACE_STEP_PROG_INDEX: u32 = 0; + +struct RuntimeBtUnwindRow<'ctx> { + found: IntValue<'ctx>, + cfa_register: IntValue<'ctx>, + cfa_offset: IntValue<'ctx>, + ra_kind: IntValue<'ctx>, + ra_register: IntValue<'ctx>, + ra_offset: IntValue<'ctx>, + rbp_kind: IntValue<'ctx>, + rbp_register: IntValue<'ctx>, + rbp_offset: IntValue<'ctx>, +} + +struct BtFrameModule<'ctx> { + cookie: IntValue<'ctx>, + bias: IntValue<'ctx>, + found: IntValue<'ctx>, +} + +struct BtRowBounds<'ctx> { + start: IntValue<'ctx>, + end: IntValue<'ctx>, +} + +struct BtModuleRangeMeta<'ctx> { + found: IntValue<'ctx>, + active_slot: IntValue<'ctx>, + count: IntValue<'ctx>, +} + +struct BtModuleRangeValue<'ctx> { + found: IntValue<'ctx>, + base: IntValue<'ctx>, + end: IntValue<'ctx>, + text: IntValue<'ctx>, + cookie: IntValue<'ctx>, +} + +struct RuntimeBtRowScratch<'ctx> { + found_ptr: PointerValue<'ctx>, + cfa_register_ptr: PointerValue<'ctx>, + cfa_offset_ptr: PointerValue<'ctx>, + ra_kind_ptr: PointerValue<'ctx>, + ra_register_ptr: PointerValue<'ctx>, + ra_offset_ptr: PointerValue<'ctx>, + rbp_kind_ptr: PointerValue<'ctx>, + rbp_register_ptr: PointerValue<'ctx>, + rbp_offset_ptr: PointerValue<'ctx>, +} + +struct BtScratch<'ctx> { + row: RuntimeBtRowScratch<'ctx>, + next_rbp_ptr: PointerValue<'ctx>, + next_error_code_ptr: PointerValue<'ctx>, +} + +#[derive(Clone, Copy)] +struct BtRegisterState<'ctx> { + ip: IntValue<'ctx>, + rsp: IntValue<'ctx>, + rbp: IntValue<'ctx>, +} + +#[derive(Clone, Copy)] +struct BtNextFrame<'ctx> { + ip: IntValue<'ctx>, + rsp: IntValue<'ctx>, + rbp: IntValue<'ctx>, + error_code: IntValue<'ctx>, +} + +struct BtFrameValidation<'ctx> { + valid: IntValue<'ctx>, + complete: IntValue<'ctx>, + error_code: IntValue<'ctx>, +} + +enum BacktraceUnwindRowForPc { + Usable(ghostscope_protocol::BacktraceUnwindRow), + Missing, + Unsupported, +} + +mod frame_recovery; +mod inline; +mod module_ranges; +mod payload; +mod plan; +mod tail_call; +mod unwind_rows; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + pub fn generate_backtrace_instruction(&mut self, stmt: &BacktraceStatement) -> Result<()> { + let plan = self.plan_backtrace_instruction(stmt); + if matches!(plan.mode, BacktraceEmitMode::TailCall) { + return self.generate_tail_call_backtrace_instruction(&plan); + } + + self.generate_inline_backtrace_instruction(&plan) + } +} + +pub(super) fn backtrace_row_binary_search_steps(row_count: usize) -> usize { + if row_count <= 1 { + 1 + } else { + (usize::BITS - (row_count - 1).leading_zeros()) as usize + 1 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ebpf::context::BacktraceModuleRowRangeEntry; + use crate::CompileOptions; + use inkwell::AddressSpace; + + #[test] + fn binary_search_steps_cover_power_of_two_row_counts() { + assert_eq!(backtrace_row_binary_search_steps(0), 1); + assert_eq!(backtrace_row_binary_search_steps(1), 1); + assert_eq!(backtrace_row_binary_search_steps(2), 2); + assert_eq!(backtrace_row_binary_search_steps(4), 3); + assert_eq!(backtrace_row_binary_search_steps(8), 4); + } + + #[test] + fn binary_search_steps_cover_non_power_of_two_row_counts() { + assert_eq!(backtrace_row_binary_search_steps(3), 3); + assert_eq!(backtrace_row_binary_search_steps(5), 4); + assert_eq!(backtrace_row_binary_search_steps(9), 5); + } + + #[test] + fn runtime_backtrace_frame_module_resolution_generates_range_lookups() { + let context = inkwell::context::Context::create(); + let opts = CompileOptions::default(); + let mut ctx = + EbpfContext::new(&context, "bt_frame_module_test", Some(0), &opts).expect("ctx"); + let i64_type = context.i64_type(); + for map_name in ["proc_module_range_meta", "proc_module_ranges"] { + let map_global = + ctx.module + .add_global(i64_type, Some(AddressSpace::default()), map_name); + map_global.set_initializer(&i64_type.const_zero()); + } + + let fn_type = context.i32_type().fn_type(&[], false); + let function = ctx.module.add_function("bt_frame_module", fn_type, None); + let entry = context.append_basic_block(function, "entry"); + ctx.builder.position_at_end(entry); + let key_type = context.i32_type().array_type(4); + ctx.pm_key_alloca = Some( + ctx.builder + .build_alloca(key_type, "pm_key") + .expect("pm_key alloca"), + ); + ctx.backtrace_module_row_ranges = vec![ + BacktraceModuleRowRangeEntry { + cookie: 0x1111, + range: ghostscope_protocol::BacktraceModuleRowRange { + row_start: 0, + row_end: 1, + }, + }, + BacktraceModuleRowRangeEntry { + cookie: 0x2222, + range: ghostscope_protocol::BacktraceModuleRowRange { + row_start: 1, + row_end: 2, + }, + }, + ]; + + let frame_module = ctx + .resolve_backtrace_frame_module( + i64_type.const_int(0x7f00_1234, false), + i64_type.const_int(0x1111, false), + i64_type.const_zero(), + context.bool_type().const_zero(), + "test_bt_frame_module", + ) + .expect("resolve frame module"); + ctx.store_u64_value( + ctx.pm_key_alloca.expect("pm key"), + 0, + frame_module.cookie, + "selected_cookie", + ) + .expect("store selected cookie"); + + let ir = ctx.module.print_to_string().to_string(); + assert!( + ir.contains("proc_module_range_meta") + && ir.contains("proc_module_ranges") + && ir.contains("test_bt_frame_module_0_range_base") + && ir.contains("test_bt_frame_module_0_range_end"), + "resolver should use the per-PID module range index\nIR:\n{ir}" + ); + assert!( + ir.contains("test_bt_frame_module_0_range_cookie") + && !ir.contains("proc_module_offsets"), + "resolver should select a module cookie without scanning offsets\nIR:\n{ir}" + ); + } + + #[test] + fn runtime_backtrace_row_bounds_cover_all_prepared_modules() { + let context = inkwell::context::Context::create(); + let opts = CompileOptions::default(); + let mut ctx = + EbpfContext::new(&context, "bt_row_bounds_test", Some(0), &opts).expect("ctx"); + + let fn_type = context + .i32_type() + .fn_type(&[context.i64_type().into()], false); + let function = ctx.module.add_function("bt_row_bounds", fn_type, None); + let entry = context.append_basic_block(function, "entry"); + ctx.builder.position_at_end(entry); + let map_global = ctx.module.add_global( + context.i64_type(), + Some(AddressSpace::default()), + "bt_module_row_ranges", + ); + map_global.set_initializer(&context.i64_type().const_zero()); + let key_type = context.i32_type().array_type(4); + ctx.pm_key_alloca = Some( + ctx.builder + .build_alloca(key_type, "pm_key") + .expect("pm_key alloca"), + ); + + ctx.backtrace_module_row_ranges = (0..=32) + .map(|idx| BacktraceModuleRowRangeEntry { + cookie: 0x1000 + idx as u64, + range: ghostscope_protocol::BacktraceModuleRowRange { + row_start: (idx * 2) as u32, + row_end: (idx * 2 + 2) as u32, + }, + }) + .collect(); + + let bounds = ctx + .backtrace_unwind_row_bounds_for_module( + function + .get_first_param() + .expect("module cookie param") + .into_int_value(), + "test_bt_row_bounds", + ) + .expect("row bounds"); + ctx.builder + .build_return(Some(&bounds.end)) + .expect("return bounds end"); + + let ir = ctx.module.print_to_string().to_string(); + assert!( + ir.contains("bt_module_row_ranges") + && ir.contains("test_bt_row_bounds_row_range_lookup"), + "row bounds lookup should use the module range map\nIR:\n{ir}" + ); + assert!( + !ir.contains("module_matches"), + "row bounds lookup should not use static candidate comparisons\nIR:\n{ir}" + ); + assert!( + !ir.contains("row_range_cookie_lo") && !ir.contains("row_range_cookie_hi"), + "row bounds lookup should store the native u64 module cookie\nIR:\n{ir}" + ); + } +} diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/module_ranges.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/module_ranges.rs new file mode 100644 index 00000000..0d7593db --- /dev/null +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/module_ranges.rs @@ -0,0 +1,743 @@ +use super::*; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + pub(super) fn lookup_proc_module_range_meta( + &mut self, + pid: IntValue<'ctx>, + name_prefix: &str, + ) -> Result> { + let i32_type = self.context.i32_type(); + let i64_type = self.context.i64_type(); + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let map_global = self + .module + .get_global("proc_module_range_meta") + .ok_or_else(|| { + CodeGenError::LLVMError("proc_module_range_meta map not found".to_string()) + })?; + let map_ptr = self + .builder + .build_bit_cast( + map_global.as_pointer_value(), + ptr_type, + &format!("{name_prefix}_meta_map_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_alloca = self.pm_key_alloca.ok_or_else(|| { + CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) + })?; + let key_arr_ty = i32_type.array_type(4); + let zero = i32_type.const_zero(); + // SAFETY: key_alloca is the [4 x i32] pm_key stack slot and [0, 0] + // addresses the pid key element for proc_module_range_meta. + let key_ptr = unsafe { + self.builder + .build_gep( + key_arr_ty, + key_alloca, + &[zero, zero], + &format!("{name_prefix}_meta_key_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + self.builder + .build_store(key_ptr, pid) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_arg = self + .builder + .build_bit_cast(key_ptr, ptr_type, &format!("{name_prefix}_meta_key_arg")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let lookup_id = i64_type.const_int(BPF_FUNC_map_lookup_elem as u64, false); + let lookup_fn_type = ptr_type.fn_type(&[ptr_type.into(), ptr_type.into()], false); + let lookup_fn_ptr = self + .builder + .build_int_to_ptr( + lookup_id, + ptr_type, + &format!("{name_prefix}_meta_lookup_fn"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let args: Vec = vec![map_ptr.into(), key_arg.into()]; + let value_ptr_any = self + .builder + .build_indirect_call( + lookup_fn_type, + lookup_fn_ptr, + &args, + &format!("{name_prefix}_meta_lookup"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .try_as_basic_value() + .left() + .ok_or_else(|| { + CodeGenError::LLVMError("proc_module_range_meta lookup returned void".to_string()) + })?; + let value_ptr = match value_ptr_any { + BasicValueEnum::PointerValue(p) => p, + _ => { + return Err(CodeGenError::LLVMError( + "proc_module_range_meta lookup did not return pointer".to_string(), + )); + } + }; + let current_fn = self.current_function("lookup proc module range meta")?; + let hit_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_meta_hit")); + let miss_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_meta_miss")); + let cont_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_meta_cont")); + let value_ptr_int = self + .builder + .build_ptr_to_int( + value_ptr, + i64_type, + &format!("{name_prefix}_meta_value_ptr_int"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let is_hit = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + value_ptr_int, + i64_type.const_zero(), + &format!("{name_prefix}_meta_found"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(is_hit, hit_block, miss_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(hit_block); + let load_i32_field = |offset: usize, field_name: &str, ctx: &mut EbpfContext<'ctx, 'dw>| { + let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); + // SAFETY: value_ptr points at ProcModuleRangeMeta returned by + // bpf_map_lookup_elem and offset is an i32 field offset. + let field_ptr = unsafe { + ctx.builder + .build_gep(ctx.context.i8_type(), value_ptr, &[offset_i32], field_name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + ctx.builder + .build_load(ctx.context.i32_type(), field_ptr, field_name) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + }; + let active_slot = load_i32_field( + ghostscope_protocol::PROC_MODULE_RANGE_META_ACTIVE_SLOT_OFFSET, + &format!("{name_prefix}_meta_active_slot"), + self, + )?; + let count = load_i32_field( + ghostscope_protocol::PROC_MODULE_RANGE_META_COUNT_OFFSET, + &format!("{name_prefix}_meta_count"), + self, + )?; + self.builder + .build_unconditional_branch(cont_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let hit_end = self.current_insert_block("finish module range meta hit block")?; + + self.builder.position_at_end(miss_block); + self.builder + .build_unconditional_branch(cont_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let miss_end = self.current_insert_block("finish module range meta miss block")?; + + self.builder.position_at_end(cont_block); + let found_type = self.context.bool_type(); + let found_phi = self + .builder + .build_phi(found_type, &format!("{name_prefix}_meta_found_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + found_phi.add_incoming(&[ + (&found_type.const_int(1, false), hit_end), + (&found_type.const_zero(), miss_end), + ]); + let phi_i32 = |ctx: &mut EbpfContext<'ctx, 'dw>, name: &str, hit_value: IntValue<'ctx>| { + let phi = ctx + .builder + .build_phi(ctx.context.i32_type(), name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + phi.add_incoming(&[ + (&hit_value, hit_end), + (&ctx.context.i32_type().const_zero(), miss_end), + ]); + Ok::<_, CodeGenError>(phi.as_basic_value().into_int_value()) + }; + Ok(BtModuleRangeMeta { + found: found_phi.as_basic_value().into_int_value(), + active_slot: phi_i32(self, &format!("{name_prefix}_meta_slot_phi"), active_slot)?, + count: phi_i32(self, &format!("{name_prefix}_meta_count_phi"), count)?, + }) + } + + pub(super) fn lookup_proc_module_range_value( + &mut self, + pid: IntValue<'ctx>, + slot: IntValue<'ctx>, + index: IntValue<'ctx>, + name_prefix: &str, + ) -> Result> { + let i32_type = self.context.i32_type(); + let i64_type = self.context.i64_type(); + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let map_global = self + .module + .get_global("proc_module_ranges") + .ok_or_else(|| { + CodeGenError::LLVMError("proc_module_ranges map not found".to_string()) + })?; + let map_ptr = self + .builder + .build_bit_cast( + map_global.as_pointer_value(), + ptr_type, + &format!("{name_prefix}_range_map_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_alloca = self.pm_key_alloca.ok_or_else(|| { + CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) + })?; + self.builder + .build_store(key_alloca, i32_type.array_type(4).const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let store_key_u32 = |offset: usize, + value: IntValue<'ctx>, + field_name: &str, + ctx: &mut EbpfContext<'ctx, 'dw>| + -> Result<()> { + let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); + // SAFETY: key_alloca is the ProcModuleRangeKey stack slot and + // offset is one of its u32 field offsets. + let field_ptr = unsafe { + ctx.builder + .build_gep(ctx.context.i8_type(), key_alloca, &[offset_i32], field_name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + ctx.builder + .build_store(field_ptr, value) + .map(|_| ()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + }; + store_key_u32( + ghostscope_protocol::PROC_MODULE_RANGE_KEY_PID_OFFSET, + pid, + &format!("{name_prefix}_range_key_pid"), + self, + )?; + store_key_u32( + ghostscope_protocol::PROC_MODULE_RANGE_KEY_SLOT_OFFSET, + slot, + &format!("{name_prefix}_range_key_slot"), + self, + )?; + store_key_u32( + ghostscope_protocol::PROC_MODULE_RANGE_KEY_INDEX_OFFSET, + index, + &format!("{name_prefix}_range_key_index"), + self, + )?; + store_key_u32( + ghostscope_protocol::PROC_MODULE_RANGE_KEY_PAD_OFFSET, + i32_type.const_zero(), + &format!("{name_prefix}_range_key_pad"), + self, + )?; + let key_arg = self + .builder + .build_bit_cast( + key_alloca, + ptr_type, + &format!("{name_prefix}_range_key_arg"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let lookup_id = i64_type.const_int(BPF_FUNC_map_lookup_elem as u64, false); + let lookup_fn_type = ptr_type.fn_type(&[ptr_type.into(), ptr_type.into()], false); + let lookup_fn_ptr = self + .builder + .build_int_to_ptr( + lookup_id, + ptr_type, + &format!("{name_prefix}_range_lookup_fn"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let args: Vec = vec![map_ptr.into(), key_arg.into()]; + let value_ptr_any = self + .builder + .build_indirect_call( + lookup_fn_type, + lookup_fn_ptr, + &args, + &format!("{name_prefix}_range_lookup"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .try_as_basic_value() + .left() + .ok_or_else(|| { + CodeGenError::LLVMError("proc_module_ranges lookup returned void".to_string()) + })?; + let value_ptr = match value_ptr_any { + BasicValueEnum::PointerValue(p) => p, + _ => { + return Err(CodeGenError::LLVMError( + "proc_module_ranges lookup did not return pointer".to_string(), + )); + } + }; + let current_fn = self.current_function("lookup proc module range value")?; + let hit_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_range_hit")); + let miss_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_range_miss")); + let cont_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_range_cont")); + let value_ptr_int = self + .builder + .build_ptr_to_int( + value_ptr, + i64_type, + &format!("{name_prefix}_range_value_ptr_int"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let is_hit = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + value_ptr_int, + i64_type.const_zero(), + &format!("{name_prefix}_range_found"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(is_hit, hit_block, miss_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(hit_block); + let load_i64_field = |offset: usize, field_name: &str, ctx: &mut EbpfContext<'ctx, 'dw>| { + let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); + // SAFETY: value_ptr points at ProcModuleRangeValue returned by + // bpf_map_lookup_elem and offset is a u64 field offset. + let field_ptr = unsafe { + ctx.builder + .build_gep(ctx.context.i8_type(), value_ptr, &[offset_i32], field_name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + ctx.builder + .build_load(ctx.context.i64_type(), field_ptr, field_name) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + }; + let load_i32_field = |offset: usize, field_name: &str, ctx: &mut EbpfContext<'ctx, 'dw>| { + let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); + // SAFETY: value_ptr points at ProcModuleRangeValue returned by + // bpf_map_lookup_elem and offset is a u32 field offset. + let field_ptr = unsafe { + ctx.builder + .build_gep(ctx.context.i8_type(), value_ptr, &[offset_i32], field_name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + ctx.builder + .build_load(ctx.context.i32_type(), field_ptr, field_name) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + }; + let base = load_i64_field( + ghostscope_protocol::PROC_MODULE_RANGE_VALUE_BASE_OFFSET, + &format!("{name_prefix}_range_base"), + self, + )?; + let end = load_i64_field( + ghostscope_protocol::PROC_MODULE_RANGE_VALUE_END_OFFSET, + &format!("{name_prefix}_range_end"), + self, + )?; + let text = load_i64_field( + ghostscope_protocol::PROC_MODULE_RANGE_VALUE_TEXT_OFFSET, + &format!("{name_prefix}_range_text"), + self, + )?; + let cookie_lo = load_i32_field( + ghostscope_protocol::PROC_MODULE_RANGE_VALUE_COOKIE_LO_OFFSET, + &format!("{name_prefix}_range_cookie_lo"), + self, + )?; + let cookie_hi = load_i32_field( + ghostscope_protocol::PROC_MODULE_RANGE_VALUE_COOKIE_HI_OFFSET, + &format!("{name_prefix}_range_cookie_hi"), + self, + )?; + let cookie_lo64 = self + .builder + .build_int_z_extend(cookie_lo, i64_type, &format!("{name_prefix}_cookie_lo64")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cookie_hi64 = self + .builder + .build_int_z_extend(cookie_hi, i64_type, &format!("{name_prefix}_cookie_hi64")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cookie_hi_shifted = self + .builder + .build_left_shift( + cookie_hi64, + i64_type.const_int(32, false), + &format!("{name_prefix}_cookie_hi_shift"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cookie = self + .builder + .build_or( + cookie_lo64, + cookie_hi_shifted, + &format!("{name_prefix}_cookie"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(cont_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let hit_end = self.current_insert_block("finish module range value hit block")?; + + self.builder.position_at_end(miss_block); + self.builder + .build_unconditional_branch(cont_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let miss_end = self.current_insert_block("finish module range value miss block")?; + + self.builder.position_at_end(cont_block); + let zero_i64 = i64_type.const_zero(); + let found_type = self.context.bool_type(); + let found_phi = self + .builder + .build_phi(found_type, &format!("{name_prefix}_range_found_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + found_phi.add_incoming(&[ + (&found_type.const_int(1, false), hit_end), + (&found_type.const_zero(), miss_end), + ]); + let phi_i64 = |ctx: &mut EbpfContext<'ctx, 'dw>, name: &str, hit_value: IntValue<'ctx>| { + let phi = ctx + .builder + .build_phi(ctx.context.i64_type(), name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + phi.add_incoming(&[(&hit_value, hit_end), (&zero_i64, miss_end)]); + Ok::<_, CodeGenError>(phi.as_basic_value().into_int_value()) + }; + Ok(BtModuleRangeValue { + found: found_phi.as_basic_value().into_int_value(), + base: phi_i64(self, &format!("{name_prefix}_range_base_phi"), base)?, + end: phi_i64(self, &format!("{name_prefix}_range_end_phi"), end)?, + text: phi_i64(self, &format!("{name_prefix}_range_text_phi"), text)?, + cookie: phi_i64(self, &format!("{name_prefix}_range_cookie_phi"), cookie)?, + }) + } + + pub(super) fn lookup_backtrace_frame_module_in_ranges( + &mut self, + raw_ip: IntValue<'ctx>, + pid: IntValue<'ctx>, + meta: BtModuleRangeMeta<'ctx>, + fallback_cookie: IntValue<'ctx>, + fallback_bias: IntValue<'ctx>, + name_prefix: &str, + ) -> Result> { + let i32_type = self.context.i32_type(); + let i64_type = self.context.i64_type(); + let bool_type = self.context.bool_type(); + let max_steps = backtrace_row_binary_search_steps( + (self.compile_options.proc_module_offsets_max_entries as usize).saturating_mul(2), + ); + let current_fn = self.current_function("lookup backtrace frame module")?; + + let mut found = bool_type.const_zero(); + let mut cookie = fallback_cookie; + let mut bias = fallback_bias; + let mut lo = i32_type.const_zero(); + let mut hi = meta.count; + + for step in 0..max_steps { + let not_found = self + .builder + .build_not(found, &format!("{name_prefix}_{step}_not_found")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let range_active = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + lo, + hi, + &format!("{name_prefix}_{step}_active"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let pending = self + .builder + .build_and( + not_found, + range_active, + &format!("{name_prefix}_{step}_pending"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let should_search = self + .builder + .build_and( + pending, + meta.found, + &format!("{name_prefix}_{step}_should_search"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let search_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_{step}_search")); + let skip_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_{step}_skip")); + let after_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_{step}_after")); + self.builder + .build_conditional_branch(should_search, search_block, skip_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(skip_block); + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let skip_end = self.current_insert_block("finish module range skip block")?; + + self.builder.position_at_end(search_block); + let lo_plus_hi = self + .builder + .build_int_add(lo, hi, &format!("{name_prefix}_{step}_lo_plus_hi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let mid = self + .builder + .build_right_shift( + lo_plus_hi, + i32_type.const_int(1, false), + false, + &format!("{name_prefix}_{step}_mid"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let range = self.lookup_proc_module_range_value( + pid, + meta.active_slot, + mid, + &format!("{name_prefix}_{step}"), + )?; + let at_or_after_base = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + raw_ip, + range.base, + &format!("{name_prefix}_{step}_after_base"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let before_end = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + raw_ip, + range.end, + &format!("{name_prefix}_{step}_before_end"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let in_bounds = self + .builder + .build_and( + at_or_after_base, + before_end, + &format!("{name_prefix}_{step}_in_bounds"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let in_range = self + .builder + .build_and( + range.found, + in_bounds, + &format!("{name_prefix}_{step}_in_range"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let before_range = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + raw_ip, + range.base, + &format!("{name_prefix}_{step}_before_range"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let after_range = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + raw_ip, + range.end, + &format!("{name_prefix}_{step}_after_range"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let found_next = self + .builder + .build_or(found, in_range, &format!("{name_prefix}_{step}_found_next")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let selected_cookie = self + .builder + .build_select::, _>( + in_range, + range.cookie.into(), + cookie.into(), + &format!("{name_prefix}_{step}_selected_cookie"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let selected_bias = self + .builder + .build_select::, _>( + in_range, + range.text.into(), + bias.into(), + &format!("{name_prefix}_{step}_selected_bias"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let mid_plus_one = self + .builder + .build_int_add( + mid, + i32_type.const_int(1, false), + &format!("{name_prefix}_{step}_mid_plus_one"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let range_missing = self + .builder + .build_not(range.found, &format!("{name_prefix}_{step}_range_missing")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let hi_next = self + .builder + .build_select::, _>( + before_range, + mid.into(), + hi.into(), + &format!("{name_prefix}_{step}_hi_next"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let lo_after = self + .builder + .build_select::, _>( + after_range, + mid_plus_one.into(), + lo.into(), + &format!("{name_prefix}_{step}_lo_after"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let lo_next = self + .builder + .build_select::, _>( + range_missing, + hi.into(), + lo_after.into(), + &format!("{name_prefix}_{step}_lo_next"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let search_end = self.current_insert_block("finish module range search block")?; + + self.builder.position_at_end(after_block); + + let found_phi = self + .builder + .build_phi(bool_type, &format!("{name_prefix}_{step}_found_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + found_phi.add_incoming(&[(&found_next, search_end), (&found, skip_end)]); + found = found_phi.as_basic_value().into_int_value(); + + let cookie_phi = self + .builder + .build_phi(i64_type, &format!("{name_prefix}_{step}_cookie_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + cookie_phi.add_incoming(&[(&selected_cookie, search_end), (&cookie, skip_end)]); + cookie = cookie_phi.as_basic_value().into_int_value(); + + let bias_phi = self + .builder + .build_phi(i64_type, &format!("{name_prefix}_{step}_bias_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + bias_phi.add_incoming(&[(&selected_bias, search_end), (&bias, skip_end)]); + bias = bias_phi.as_basic_value().into_int_value(); + + let lo_phi = self + .builder + .build_phi(i32_type, &format!("{name_prefix}_{step}_lo_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + lo_phi.add_incoming(&[(&lo_next, search_end), (&lo, skip_end)]); + lo = lo_phi.as_basic_value().into_int_value(); + + let hi_phi = self + .builder + .build_phi(i32_type, &format!("{name_prefix}_{step}_hi_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + hi_phi.add_incoming(&[(&hi_next, search_end), (&hi, skip_end)]); + hi = hi_phi.as_basic_value().into_int_value(); + } + + Ok(BtFrameModule { + cookie, + bias, + found, + }) + } + + pub(super) fn resolve_backtrace_frame_module( + &mut self, + raw_ip: IntValue<'ctx>, + fallback_cookie: IntValue<'ctx>, + fallback_bias: IntValue<'ctx>, + fallback_found: IntValue<'ctx>, + name_prefix: &str, + ) -> Result> { + if self.backtrace_module_row_ranges.is_empty() { + return Ok(BtFrameModule { + cookie: fallback_cookie, + bias: fallback_bias, + found: fallback_found, + }); + } + + let pid = self.proc_module_pid_key(name_prefix)?; + let meta = self.lookup_proc_module_range_meta(pid, name_prefix)?; + self.lookup_backtrace_frame_module_in_ranges( + raw_ip, + pid, + meta, + fallback_cookie, + fallback_bias, + name_prefix, + ) + } + + pub(super) fn backtrace_module_fallback_found(&self, found: IntValue<'ctx>) -> IntValue<'ctx> { + if self.backtrace_module_row_ranges.is_empty() { + found + } else { + self.context.bool_type().const_zero() + } + } + + pub(super) fn backtrace_lookup_pc_from_raw( + &self, + raw_ip: IntValue<'ctx>, + module_bias: IntValue<'ctx>, + offsets_found: IntValue<'ctx>, + ) -> Result> { + self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found) + } +} diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/payload.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/payload.rs new file mode 100644 index 00000000..0b060f43 --- /dev/null +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/payload.rs @@ -0,0 +1,369 @@ +use super::*; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + pub(super) fn bool_to_u8(&self, value: IntValue<'ctx>, name: &str) -> Result> { + self.builder + .build_select::, _>( + value, + self.context.i8_type().const_int(1, false).into(), + self.context.i8_type().const_zero().into(), + name, + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + pub(super) fn status_or_offsets_unavailable( + &self, + status: BacktraceStatus, + offsets_found: IntValue<'ctx>, + ) -> Result> { + self.builder + .build_select::, _>( + offsets_found, + self.context + .i8_type() + .const_int(status as u64, false) + .into(), + self.context + .i8_type() + .const_int(BacktraceStatus::OffsetsUnavailable as u64, false) + .into(), + "bt_status_or_offsets", + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + pub(super) fn status_for_backtrace_stop( + &self, + complete: IntValue<'ctx>, + error_code: IntValue<'ctx>, + offsets_found: IntValue<'ctx>, + ) -> Result> { + let i8_type = self.context.i8_type(); + let i16_type = self.context.i16_type(); + let ra_read_error = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + error_code, + i16_type.const_int(BACKTRACE_ERROR_RETURN_ADDRESS_READ as u64, false), + "bt_status_ra_read_error", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let rbp_read_error = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + error_code, + i16_type.const_int(BACKTRACE_ERROR_FRAME_POINTER_READ as u64, false), + "bt_status_rbp_read_error", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let read_error = self + .builder + .build_or(ra_read_error, rbp_read_error, "bt_status_read_error_flag") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let error_status = self + .builder + .build_select::, _>( + read_error, + i8_type + .const_int(BacktraceStatus::ReadError as u64, false) + .into(), + i8_type + .const_int(BacktraceStatus::InvalidFrame as u64, false) + .into(), + "bt_status_for_error_code", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let backtrace_status = self + .builder + .build_select::, _>( + complete, + i8_type + .const_int(BacktraceStatus::Complete as u64, false) + .into(), + error_status, + "bt_status_for_stop", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_select::, _>( + offsets_found, + backtrace_status, + i8_type + .const_int(BacktraceStatus::OffsetsUnavailable as u64, false) + .into(), + "bt_status_or_offsets_for_error_code", + ) + .map(|value| value.into_int_value()) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + + pub(super) fn store_backtrace_frame( + &self, + inst_buffer: PointerValue<'ctx>, + frame_index: usize, + module_cookie: IntValue<'ctx>, + pc: IntValue<'ctx>, + raw_ip: IntValue<'ctx>, + flags: u16, + ) -> Result<()> { + let frame_base = + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_SIZE + frame_index * BACKTRACE_FRAME_DATA_SIZE; + self.store_u64_value( + inst_buffer, + frame_base + BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, + module_cookie, + "bt_frame_cookie", + )?; + self.store_u64_value( + inst_buffer, + frame_base + BACKTRACE_FRAME_PC_OFFSET, + pc, + "bt_frame_pc", + )?; + self.store_u64_value( + inst_buffer, + frame_base + BACKTRACE_FRAME_RAW_IP_OFFSET, + raw_ip, + "bt_frame_raw_ip", + )?; + self.store_u16_const( + inst_buffer, + frame_base + BACKTRACE_FRAME_FLAGS_OFFSET, + flags, + "bt_frame_flags", + ) + } + + pub(super) fn store_backtrace_frame_dynamic( + &self, + inst_buffer: PointerValue<'ctx>, + frame_index: IntValue<'ctx>, + max_frame_index: u8, + module_cookie: IntValue<'ctx>, + pc: IntValue<'ctx>, + raw_ip: IntValue<'ctx>, + ) -> Result<()> { + let i64_type = self.context.i64_type(); + let frame_index_i64 = if frame_index.get_type().get_bit_width() == i64_type.get_bit_width() + { + frame_index + } else { + self.builder + .build_int_z_extend(frame_index, i64_type, "bt_frame_index_i64") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + let max_frame_index = i64_type.const_int(max_frame_index as u64, false); + let frame_index_in_bounds = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULE, + frame_index_i64, + max_frame_index, + "bt_frame_index_in_bounds", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let frame_index_i64 = self + .builder + .build_select::, _>( + frame_index_in_bounds, + frame_index_i64.into(), + max_frame_index.into(), + "bt_frame_index_bounded", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let frame_stride = i64_type.const_int(BACKTRACE_FRAME_DATA_SIZE as u64, false); + let frame_offset = self + .builder + .build_int_mul(frame_index_i64, frame_stride, "bt_dynamic_frame_offset") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let base_offset = i64_type.const_int( + (INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_SIZE) as u64, + false, + ); + let frame_base_offset = self + .builder + .build_int_add(base_offset, frame_offset, "bt_dynamic_frame_base_offset") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let frame_base = + self.dynamic_byte_gep(inst_buffer, frame_base_offset, "bt_dynamic_frame")?; + + self.store_u64_value( + frame_base, + BACKTRACE_FRAME_MODULE_COOKIE_OFFSET, + module_cookie, + "bt_frame_cookie", + )?; + self.store_u64_value(frame_base, BACKTRACE_FRAME_PC_OFFSET, pc, "bt_frame_pc")?; + self.store_u64_value( + frame_base, + BACKTRACE_FRAME_RAW_IP_OFFSET, + raw_ip, + "bt_frame_raw_ip", + )?; + self.store_u16_const( + frame_base, + BACKTRACE_FRAME_FLAGS_OFFSET, + 0, + "bt_frame_flags", + ) + } + + pub(super) fn store_u8_const( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: u8, + name: &str, + ) -> Result<()> { + self.store_u8_value( + base, + offset, + self.context.i8_type().const_int(value as u64, false), + name, + ) + } + + pub(super) fn store_u8_value( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: IntValue<'ctx>, + name: &str, + ) -> Result<()> { + let ptr = self.byte_gep(base, offset, name)?; + self.builder + .build_store(ptr, value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + pub(super) fn store_u16_const( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: u16, + name: &str, + ) -> Result<()> { + self.store_u16_value( + base, + offset, + self.context.i16_type().const_int(value as u64, false), + name, + ) + } + + pub(super) fn store_u16_value( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: IntValue<'ctx>, + name: &str, + ) -> Result<()> { + let ptr = self.byte_gep(base, offset, name)?; + let ptr = self + .builder + .build_pointer_cast( + ptr, + self.context.ptr_type(AddressSpace::default()), + &format!("{name}_u16_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(ptr, value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + pub(super) fn store_u64_value( + &self, + base: PointerValue<'ctx>, + offset: usize, + value: IntValue<'ctx>, + name: &str, + ) -> Result<()> { + let ptr = self.byte_gep(base, offset, name)?; + let ptr = self + .builder + .build_pointer_cast( + ptr, + self.context.ptr_type(AddressSpace::default()), + &format!("{name}_u64_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(ptr, value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + pub(super) fn byte_gep( + &self, + base: PointerValue<'ctx>, + offset: usize, + name: &str, + ) -> Result> { + // SAFETY: callers pass offsets within the instruction region reserved for + // this Backtrace instruction. + unsafe { + self.builder + .build_gep( + self.context.i8_type(), + base, + &[self.context.i32_type().const_int(offset as u64, false)], + &format!("{name}_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + } + + pub(super) fn dynamic_byte_gep( + &self, + base: PointerValue<'ctx>, + offset: IntValue<'ctx>, + name: &str, + ) -> Result> { + // SAFETY: callers guard the dynamic offset against the per-CPU buffer + // size before using the returned pointer. + unsafe { + self.builder + .build_gep( + self.context.i8_type(), + base, + &[offset], + &format!("{name}_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string())) + } + } + + pub(super) fn build_entry_alloca(&self, ty: T, name: &str) -> Result> + where + T: inkwell::types::BasicType<'ctx>, + { + let current_block = self.builder.get_insert_block().ok_or_else(|| { + CodeGenError::LLVMError("no current block for bt stack allocation".to_string()) + })?; + let current_fn = self.current_function("allocate bt scratch")?; + let entry_block = current_fn.get_first_basic_block().ok_or_else(|| { + CodeGenError::LLVMError("no entry block for bt stack allocation".to_string()) + })?; + + if let Some(first_instruction) = entry_block.get_first_instruction() { + self.builder.position_before(&first_instruction); + } else { + self.builder.position_at_end(entry_block); + } + let alloca = self + .builder + .build_alloca(ty, name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder.position_at_end(current_block); + Ok(alloca) + } +} diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace_plan.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/plan.rs similarity index 99% rename from ghostscope-compiler/src/ebpf/codegen/backtrace_plan.rs rename to ghostscope-compiler/src/ebpf/codegen/backtrace/plan.rs index 6f4fd09c..9d72c277 100644 --- a/ghostscope-compiler/src/ebpf/codegen/backtrace_plan.rs +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/plan.rs @@ -1,7 +1,7 @@ //! Backtrace semantic planning for eBPF codegen. //! //! This module decides which backtrace strategy and data dependencies are -//! required. The LLVM emitter in `backtrace.rs` owns the actual IR blocks, +//! required. The LLVM emitters in sibling modules own the actual IR blocks, //! helper calls, and ABI stores. use super::*; diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/tail_call.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/tail_call.rs new file mode 100644 index 00000000..6fe6cb21 --- /dev/null +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/tail_call.rs @@ -0,0 +1,1319 @@ +use super::*; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + pub(super) fn generate_tail_call_backtrace_instruction( + &mut self, + plan: &BacktraceInstructionPlan, + ) -> Result<()> { + let depth = plan.depth; + let flags = plan.flags; + info!( + "Generating tail-call Backtrace instruction: depth={}", + depth + ); + + let payload_size = plan.payload_size; + let instruction_size = plan.instruction_size; + let offset_ptr = self.event_offset_alloca.ok_or_else(|| { + CodeGenError::LLVMError("event_offset not allocated in entry block".to_string()) + })?; + let inst_offset = self + .builder + .build_load(self.context.i32_type(), offset_ptr, "bt_tail_inst_offset") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let inst_buffer = self + .reserve_instruction_region_or_return_zero(instruction_size as u64)? + .into_value_after_runtime_returns(); + + self.store_u8_const( + inst_buffer, + std::mem::offset_of!(InstructionHeader, inst_type), + InstructionType::Backtrace as u8, + "bt_inst_type", + )?; + self.store_u16_const( + inst_buffer, + std::mem::offset_of!(InstructionHeader, data_length), + payload_size as u16, + "bt_data_length", + )?; + + let data_base = INSTRUCTION_HEADER_SIZE; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_REQUESTED_DEPTH_OFFSET, + depth, + "bt_requested_depth", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 1, + "bt_frame_count", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FLAGS_OFFSET, + flags, + "bt_flags", + )?; + self.store_u16_const( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + 0, + "bt_error_code", + )?; + + let Some(compile_ctx) = self.current_compile_time_context.clone() else { + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 0, + "bt_frame_count_no_context", + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + BacktraceStatus::DwarfUnavailable as u8, + "bt_status_no_context", + )?; + return Ok(()); + }; + + let module_cookie = self.cookie_for_module_or_fallback(&compile_ctx.module_path); + let module_cookie_value = self.context.i64_type().const_int(module_cookie, false); + let pt_regs = self.get_pt_regs_parameter()?; + let raw_ip = self.load_dwarf_register_i64(X86_64_DWARF_RIP, pt_regs)?; + let initial_rsp = self.load_dwarf_register_i64(X86_64_DWARF_RSP, pt_regs)?; + let initial_rbp = self.load_dwarf_register_i64(X86_64_DWARF_RBP, pt_regs)?; + let (module_bias, offsets_found) = self.generate_runtime_address_from_offsets( + self.context.i64_type().const_zero(), + 0, + module_cookie, + )?; + let normalized_pc = self.normalized_pc_from_raw(raw_ip, module_bias, offsets_found)?; + let caller_fallback_found = self.backtrace_module_fallback_found(offsets_found); + + self.store_backtrace_frame( + inst_buffer, + 0, + module_cookie_value, + normalized_pc, + raw_ip, + 0, + )?; + + if depth == 1 { + let status = + self.status_or_offsets_unavailable(BacktraceStatus::Truncated, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_depth_one", + )?; + return Ok(()); + } + + if self.backtrace_unwind_rows.is_empty() { + let status = self + .status_or_offsets_unavailable(BacktraceStatus::NoUnwindRowsForPc, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_no_rows", + )?; + return Ok(()); + } + + let row = match self + .usable_backtrace_unwind_row_for_pc(&compile_ctx.module_path, compile_ctx.pc_address) + { + BacktraceUnwindRowForPc::Usable(row) => row, + row_status => { + let initial_status = self.status_for_backtrace_unwind_row_for_pc(&row_status); + let status = self.status_or_offsets_unavailable(initial_status, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_status_no_initial_row", + )?; + return Ok(()); + } + }; + + let status = + self.status_or_offsets_unavailable(BacktraceStatus::ReadError, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_tail_initial_status", + )?; + + let scratch = self.allocate_backtrace_scratch()?; + let current_fn = self.current_function("initialize bt tail-call state")?; + let done_block = self + .context + .append_basic_block(current_fn, "bt_tail_state_done"); + let i64_type = self.context.i64_type(); + let ip_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_ip")?; + let rsp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rsp")?; + let rbp_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_rbp")?; + let module_bias_ptr = self.build_entry_alloca(i64_type, "bt_tail_prefix_module_bias")?; + let module_cookie_ptr = + self.build_entry_alloca(i64_type, "bt_tail_prefix_module_cookie")?; + let module_found_ptr = + self.build_entry_alloca(self.context.bool_type(), "bt_tail_prefix_module_found")?; + self.builder + .build_store(module_bias_ptr, module_bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, module_cookie_value) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_found_ptr, offsets_found) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let runtime_row = self.runtime_row_from_static(row); + let state = BtRegisterState { + ip: raw_ip, + rsp: initial_rsp, + rbp: initial_rbp, + }; + let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; + let validation = self.validate_backtrace_next_frame(state, next)?; + let initial_store_block = self + .context + .append_basic_block(current_fn, "bt_tail_initial_store_frame"); + let initial_stop_block = self + .context + .append_basic_block(current_fn, "bt_tail_initial_stop"); + self.builder + .build_conditional_branch(validation.valid, initial_store_block, initial_stop_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(initial_stop_block); + let stop_status = self.status_for_backtrace_stop( + validation.complete, + validation.error_code, + offsets_found, + )?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + stop_status, + "bt_tail_initial_stop_status", + )?; + self.store_u16_value( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + validation.error_code, + "bt_tail_initial_stop_error_code", + )?; + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(initial_store_block); + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie_value, + module_bias, + caller_fallback_found, + "bt_tail_initial_frame_module", + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; + self.store_backtrace_frame(inst_buffer, 1, frame_module.cookie, next_pc, next.ip, 0)?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + 2, + "bt_tail_initial_frame_count", + )?; + let status = if depth == 2 { + BacktraceStatus::Truncated + } else { + BacktraceStatus::ReadError + }; + let status = self.status_or_offsets_unavailable(status, offsets_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_tail_initial_status_after_frame", + )?; + self.builder + .build_store(ip_ptr, next.ip) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rsp_ptr, next.rsp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_ptr, next.rbp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_bias_ptr, frame_module.bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, frame_module.cookie) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_found_ptr, frame_module.found) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + if depth == 2 { + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder.position_at_end(done_block); + return Ok(()); + } + + let prefix_depth = depth.min(BPF_INLINE_BACKTRACE_FRAME_LIMIT); + for frame_index in 2..prefix_depth { + let current_ip = self.load_i64(ip_ptr, "bt_tail_prefix_lookup_ip")?; + let current_module_bias = + self.load_i64(module_bias_ptr, "bt_tail_prefix_lookup_module_bias")?; + let current_module_cookie = + self.load_i64(module_cookie_ptr, "bt_tail_prefix_lookup_module_cookie")?; + let current_module_found = + self.load_bool(module_found_ptr, "bt_tail_prefix_lookup_module_found")?; + let lookup_raw = self.add_signed_offset(current_ip, -1, "bt_tail_prefix_lookup_raw")?; + let lookup_pc = self.backtrace_lookup_pc_from_raw( + lookup_raw, + current_module_bias, + current_module_found, + )?; + let runtime_row = self.lookup_backtrace_unwind_row( + lookup_pc, + current_module_cookie, + &scratch.row, + &format!("bt_tail_prefix_frame_{frame_index}_row"), + )?; + let found_block = self + .context + .append_basic_block(current_fn, "bt_tail_prefix_row_found"); + let missing_block = self + .context + .append_basic_block(current_fn, "bt_tail_prefix_row_missing"); + self.builder + .build_conditional_branch(runtime_row.found, found_block, missing_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(missing_block); + let status = self.status_or_offsets_unavailable( + BacktraceStatus::NoUnwindRowsForPc, + current_module_found, + )?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_tail_prefix_status_missing_row", + )?; + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(found_block); + let state = BtRegisterState { + ip: self.load_i64(ip_ptr, "bt_tail_prefix_current_ip")?, + rsp: self.load_i64(rsp_ptr, "bt_tail_prefix_current_rsp")?, + rbp: self.load_i64(rbp_ptr, "bt_tail_prefix_current_rbp")?, + }; + let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, &scratch)?; + let validation = self.validate_backtrace_next_frame(state, next)?; + let store_block = self + .context + .append_basic_block(current_fn, "bt_tail_prefix_store_frame"); + let stop_block = self + .context + .append_basic_block(current_fn, "bt_tail_prefix_stop"); + self.builder + .build_conditional_branch(validation.valid, store_block, stop_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(stop_block); + let stop_status = self.status_for_backtrace_stop( + validation.complete, + validation.error_code, + current_module_found, + )?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + stop_status, + "bt_tail_prefix_stop_status", + )?; + self.store_u16_value( + inst_buffer, + data_base + BACKTRACE_DATA_ERROR_CODE_OFFSET, + validation.error_code, + "bt_tail_prefix_stop_error_code", + )?; + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(store_block); + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + current_module_cookie, + current_module_bias, + self.backtrace_module_fallback_found(current_module_found), + &format!("bt_tail_prefix_frame_{frame_index}_module"), + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; + self.store_backtrace_frame( + inst_buffer, + frame_index as usize, + frame_module.cookie, + next_pc, + next.ip, + 0, + )?; + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + frame_index + 1, + "bt_tail_prefix_frame_count", + )?; + let status = if frame_index + 1 == depth { + BacktraceStatus::Truncated + } else { + BacktraceStatus::ReadError + }; + let status = self.status_or_offsets_unavailable(status, current_module_found)?; + self.store_u8_value( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + status, + "bt_tail_prefix_status_after_frame", + )?; + self.builder + .build_store(ip_ptr, next.ip) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rsp_ptr, next.rsp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(rbp_ptr, next.rbp) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_bias_ptr, frame_module.bias) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_cookie_ptr, frame_module.cookie) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(module_found_ptr, frame_module.found) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + if frame_index + 1 == depth { + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } + } + + if prefix_depth == depth { + self.builder.position_at_end(done_block); + return Ok(()); + } + + let tail_slot = self.next_backtrace_tail_call_slot; + self.next_backtrace_tail_call_slot = self.next_backtrace_tail_call_slot.saturating_add(1); + if self.pending_backtrace_tail_call.is_none() { + let step_program_name = format!( + "{}_bt_step", + self.current_function("name bt tail-call step")? + .get_name() + .to_string_lossy() + ); + self.pending_backtrace_tail_call = + Some(crate::ebpf::context::PendingBacktraceTailCall { + step_program_name, + depth, + instruction_size, + }); + } + + let tail_enabled_ptr = self.get_or_create_backtrace_tail_enabled_flag()?; + + let state_ptr = self.lookup_bt_state_ptr(tail_slot as u32)?; + let state_is_null = self + .builder + .build_is_null(state_ptr, "bt_state_is_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let init_block = self + .context + .append_basic_block(current_fn, "bt_tail_state_init"); + let null_block = self + .context + .append_basic_block(current_fn, "bt_tail_state_null"); + self.builder + .build_conditional_branch(state_is_null, null_block, init_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(null_block); + self.store_u8_const( + inst_buffer, + data_base + BACKTRACE_DATA_STATUS_OFFSET, + BacktraceStatus::InternalError as u8, + "bt_status_state_null", + )?; + self.builder + .build_store(tail_enabled_ptr, self.context.i8_type().const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(done_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(init_block); + let tail_ip = self.load_i64(ip_ptr, "bt_tail_state_prefix_ip")?; + let tail_rsp = self.load_i64(rsp_ptr, "bt_tail_state_prefix_rsp")?; + let tail_rbp = self.load_i64(rbp_ptr, "bt_tail_state_prefix_rbp")?; + let tail_module_bias = self.load_i64(module_bias_ptr, "bt_tail_state_module_bias")?; + let tail_module_cookie = self.load_i64(module_cookie_ptr, "bt_tail_state_module_cookie")?; + let tail_module_found = self.load_bool(module_found_ptr, "bt_tail_state_module_found")?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, + tail_ip, + "bt_state_ip", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, + tail_rsp, + "bt_state_rsp", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, + tail_rbp, + "bt_state_rbp", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, + tail_module_bias, + "bt_state_module_bias", + )?; + self.store_u64_value( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, + tail_module_cookie, + "bt_state_module_cookie", + )?; + self.store_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, + inst_offset, + "bt_state_inst_offset", + )?; + self.store_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + self.context.i32_type().const_zero(), + "bt_state_event_size", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, + prefix_depth, + "bt_state_frame_count", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_REQUESTED_DEPTH_OFFSET, + depth, + "bt_state_requested_depth", + )?; + let offsets_found_u8 = self.bool_to_u8(tail_module_found, "bt_offsets_found_u8")?; + self.store_u8_value( + state_ptr, + crate::BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, + offsets_found_u8, + "bt_state_offsets_found", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + 1, + "bt_state_tail_calls", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_FLAGS_OFFSET, + flags, + "bt_state_flags", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, + tail_slot, + "bt_state_active_slot", + )?; + self.store_u16_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_ERROR_CODE_OFFSET, + BACKTRACE_ERROR_NONE, + "bt_state_error_code", + )?; + self.store_u8_const( + state_ptr, + crate::BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, + crate::BACKTRACE_TAIL_NO_NEXT_SLOT, + "bt_state_next_slot", + )?; + self.link_backtrace_tail_slot(tail_slot, offsets_found_u8, done_block)?; + + self.builder.position_at_end(done_block); + Ok(()) + } + + pub(crate) fn finish_event_after_instructions(&mut self) -> Result<()> { + let Some(plan) = self.pending_backtrace_tail_call.clone() else { + return self.emit_accumulated_event_output_from_stack_offset(); + }; + + let main_block = self.current_insert_block("finish bt tail-call event")?; + let main_pm_key_alloca = self.pm_key_alloca; + self.generate_backtrace_tail_call_step_program(&plan)?; + self.pm_key_alloca = main_pm_key_alloca; + self.builder.position_at_end(main_block); + + let tail_enabled_ptr = self.get_or_create_backtrace_tail_enabled_flag()?; + let enabled_value = self + .builder + .build_load( + self.context.i8_type(), + tail_enabled_ptr, + "bt_tail_enabled_value", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let enabled = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + enabled_value, + self.context.i8_type().const_zero(), + "bt_tail_enabled", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let current_fn = self.current_function("finish bt tail-call event")?; + let tail_block = self + .context + .append_basic_block(current_fn, "bt_tail_dispatch"); + let output_block = self + .context + .append_basic_block(current_fn, "bt_tail_fallback_output"); + self.builder + .build_conditional_branch(enabled, tail_block, output_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(tail_block); + let state0_ptr = self.lookup_bt_state_ptr(0)?; + let state0_is_null = self + .builder + .build_is_null(state0_ptr, "bt_tail_dispatch_state_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let state_ok_block = self + .context + .append_basic_block(current_fn, "bt_tail_dispatch_state_ok"); + self.builder + .build_conditional_branch(state0_is_null, output_block, state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(state_ok_block); + let active_slot = self.load_row_i8( + state0_ptr, + crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, + "bt_tail_dispatch_active_slot", + )?; + let state_ptr = self.lookup_bt_state_ptr_dynamic(active_slot)?; + let state_is_null = self + .builder + .build_is_null(state_ptr, "bt_tail_dispatch_active_state_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let active_state_ok_block = self + .context + .append_basic_block(current_fn, "bt_tail_dispatch_active_state_ok"); + self.builder + .build_conditional_branch(state_is_null, output_block, active_state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(active_state_ok_block); + let event_size = self + .builder + .build_load( + self.context.i32_type(), + self.event_offset_alloca.ok_or_else(|| { + CodeGenError::LLVMError("event_offset not allocated in entry block".to_string()) + })?, + "bt_tail_event_size", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + self.store_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + event_size, + "bt_tail_state_event_size", + )?; + self.emit_bpf_tail_call(BPF_BACKTRACE_STEP_PROG_INDEX)?; + self.builder + .build_unconditional_branch(output_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(output_block); + self.emit_accumulated_event_output_from_stack_offset() + } + + pub(super) fn generate_backtrace_tail_call_step_program( + &mut self, + plan: &crate::ebpf::context::PendingBacktraceTailCall, + ) -> Result<()> { + self.create_tail_call_function(&plan.step_program_name)?; + let current_fn = self.current_function("generate bt tail-call step")?; + let return_block = self + .context + .append_basic_block(current_fn, "bt_step_return"); + let state_ok_block = self + .context + .append_basic_block(current_fn, "bt_step_state_ok"); + let accum_ok_block = self + .context + .append_basic_block(current_fn, "bt_step_accum_ok"); + let bounds_ok_block = self + .context + .append_basic_block(current_fn, "bt_step_bounds_ok"); + let inst_bounds_ok_block = self + .context + .append_basic_block(current_fn, "bt_step_inst_bounds_ok"); + let finalize_block = self + .context + .append_basic_block(current_fn, "bt_step_finalize"); + + let state0_ptr = self.lookup_bt_state_ptr(0)?; + let state_is_null = self + .builder + .build_is_null(state0_ptr, "bt_step_state_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(state_is_null, return_block, state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(state_ok_block); + let active_slot = self.load_row_i8( + state0_ptr, + crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, + "bt_step_active_slot", + )?; + let state_ptr = self.lookup_bt_state_ptr_dynamic(active_slot)?; + let active_state_is_null = self + .builder + .build_is_null(state_ptr, "bt_step_active_state_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let active_state_ok_block = self + .context + .append_basic_block(current_fn, "bt_step_active_state_ok"); + self.builder + .build_conditional_branch(active_state_is_null, return_block, active_state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(active_state_ok_block); + let accum_buffer = self.lookup_percpu_value_ptr("event_accum_buffer", 0)?; + let accum_is_null = self + .builder + .build_is_null(accum_buffer, "bt_step_accum_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(accum_is_null, return_block, accum_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(accum_ok_block); + let inst_offset = self.load_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_INST_OFFSET_OFFSET, + "bt_step_inst_offset", + )?; + let event_size = self.load_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + "bt_step_event_size", + )?; + let max_event_size = self + .context + .i32_type() + .const_int(self.compile_options.max_trace_event_size as u64, false); + let max_inst_offset = self.context.i32_type().const_int( + self.compile_options + .max_trace_event_size + .saturating_sub(plan.instruction_size as u32) as u64, + false, + ); + let inst_in_bounds = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULE, + inst_offset, + max_inst_offset, + "bt_step_inst_offset_in_bounds", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(inst_in_bounds, inst_bounds_ok_block, return_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(inst_bounds_ok_block); + let event_in_bounds = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULE, + event_size, + max_event_size, + "bt_step_event_size_in_bounds", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(event_in_bounds, bounds_ok_block, return_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(bounds_ok_block); + let inst_offset_i64 = self + .builder + .build_int_z_extend( + inst_offset, + self.context.i64_type(), + "bt_step_inst_offset_i64", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let inst_buffer = + self.dynamic_byte_gep(accum_buffer, inst_offset_i64, "bt_step_inst_buffer")?; + let scratch = self.allocate_backtrace_scratch()?; + for _ in 0..BPF_BACKTRACE_FRAMES_PER_TAIL_CALL { + self.generate_backtrace_tail_call_step_iteration( + plan.depth, + state_ptr, + inst_buffer, + &scratch, + finalize_block, + )?; + } + + let tail_calls = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + "bt_step_tail_calls", + )?; + let can_tail_call = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + tail_calls, + self.context + .i8_type() + .const_int(BPF_BACKTRACE_MAX_STEP_INVOCATIONS as u64, false), + "bt_step_can_tail_call", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let self_tail_block = self + .context + .append_basic_block(current_fn, "bt_step_self_tail"); + let tail_budget_done = self + .context + .append_basic_block(current_fn, "bt_step_tail_budget_done"); + self.builder + .build_conditional_branch(can_tail_call, self_tail_block, tail_budget_done) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(tail_budget_done); + self.store_tail_backtrace_status( + inst_buffer, + BacktraceStatus::Truncated, + BACKTRACE_ERROR_NONE, + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(self_tail_block); + let next_tail_calls = self + .builder + .build_int_add( + tail_calls, + self.context.i8_type().const_int(1, false), + "bt_step_next_tail_calls", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.store_u8_value( + state_ptr, + crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + next_tail_calls, + "bt_step_store_tail_calls", + )?; + self.emit_bpf_tail_call(BPF_BACKTRACE_STEP_PROG_INDEX)?; + self.store_tail_backtrace_status( + inst_buffer, + BacktraceStatus::InternalError, + BACKTRACE_ERROR_NONE, + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(finalize_block); + let next_slot = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_NEXT_SLOT_OFFSET, + "bt_final_next_slot", + )?; + let has_next_slot = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + next_slot, + self.context + .i8_type() + .const_int(crate::BACKTRACE_TAIL_NO_NEXT_SLOT as u64, false), + "bt_final_has_next_slot", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let tail_calls = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + "bt_final_tail_calls", + )?; + let can_tail_call_next = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + tail_calls, + self.context + .i8_type() + .const_int(BPF_BACKTRACE_MAX_STEP_INVOCATIONS as u64, false), + "bt_final_can_tail_call_next", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let should_continue_next = self + .builder + .build_and( + has_next_slot, + can_tail_call_next, + "bt_final_should_continue_next_slot", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let next_slot_block = self + .context + .append_basic_block(current_fn, "bt_final_next_slot"); + let emit_block = self.context.append_basic_block(current_fn, "bt_final_emit"); + self.builder + .build_conditional_branch(should_continue_next, next_slot_block, emit_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(next_slot_block); + self.store_u8_value( + state0_ptr, + crate::BACKTRACE_TAIL_STATE_ACTIVE_SLOT_OFFSET, + next_slot, + "bt_store_active_slot", + )?; + let next_state_ptr = self.lookup_bt_state_ptr_dynamic(next_slot)?; + let next_state_is_null = self + .builder + .build_is_null(next_state_ptr, "bt_next_state_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let next_state_ok_block = self + .context + .append_basic_block(current_fn, "bt_next_state_ok"); + self.builder + .build_conditional_branch(next_state_is_null, emit_block, next_state_ok_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(next_state_ok_block); + self.store_state_i32( + next_state_ptr, + crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + event_size, + "bt_next_slot_event_size", + )?; + let next_tail_calls = self + .builder + .build_int_add( + tail_calls, + self.context.i8_type().const_int(1, false), + "bt_next_slot_tail_calls", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.store_u8_value( + next_state_ptr, + crate::BACKTRACE_TAIL_STATE_TAIL_CALLS_OFFSET, + next_tail_calls, + "bt_next_slot_store_tail_calls", + )?; + self.emit_bpf_tail_call(BPF_BACKTRACE_STEP_PROG_INDEX)?; + self.builder + .build_unconditional_branch(emit_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(emit_block); + self.emit_tail_final_event(state_ptr, accum_buffer)?; + self.builder + .build_unconditional_branch(return_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(return_block); + self.build_return_zero() + } + + pub(super) fn generate_backtrace_tail_call_step_iteration( + &mut self, + depth: u8, + state_ptr: PointerValue<'ctx>, + inst_buffer: PointerValue<'ctx>, + scratch: &BtScratch<'ctx>, + finalize_block: BasicBlock<'ctx>, + ) -> Result<()> { + let current_fn = self.current_function("generate bt tail-call step iteration")?; + let depth_block = self + .context + .append_basic_block(current_fn, "bt_step_depth_done"); + let unwind_block = self + .context + .append_basic_block(current_fn, "bt_step_unwind"); + let frame_count = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, + "bt_step_frame_count", + )?; + let depth_value = self.context.i8_type().const_int(depth as u64, false); + let at_depth = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + frame_count, + depth_value, + "bt_step_at_depth", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_conditional_branch(at_depth, depth_block, unwind_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(depth_block); + self.store_tail_backtrace_status( + inst_buffer, + BacktraceStatus::Truncated, + BACKTRACE_ERROR_NONE, + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(unwind_block); + let current_ip = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, + "bt_step_current_ip", + )?; + let current_rsp = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, + "bt_step_current_rsp", + )?; + let current_rbp = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, + "bt_step_current_rbp", + )?; + let module_bias = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, + "bt_step_module_bias", + )?; + let module_cookie = self.load_row_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, + "bt_step_module_cookie", + )?; + let offsets_found_u8 = self.load_row_i8( + state_ptr, + crate::BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, + "bt_step_offsets_found_u8", + )?; + let offsets_found = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + offsets_found_u8, + self.context.i8_type().const_zero(), + "bt_step_offsets_found", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let caller_fallback_found = self.backtrace_module_fallback_found(offsets_found); + let is_first_unwind = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + frame_count, + self.context.i8_type().const_int(1, false), + "bt_step_first_unwind", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let caller_lookup_ip = + self.add_signed_offset(current_ip, -1, "bt_step_caller_lookup_ip")?; + let lookup_raw = self + .builder + .build_select::, _>( + is_first_unwind, + current_ip.into(), + caller_lookup_ip.into(), + "bt_step_lookup_raw", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let lookup_pc = + self.backtrace_lookup_pc_from_raw(lookup_raw, module_bias, offsets_found)?; + let runtime_row = self.lookup_backtrace_unwind_row( + lookup_pc, + module_cookie, + &scratch.row, + "bt_step_row", + )?; + let row_found_block = self + .context + .append_basic_block(current_fn, "bt_step_row_found"); + let row_missing_block = self + .context + .append_basic_block(current_fn, "bt_step_row_missing"); + self.builder + .build_conditional_branch(runtime_row.found, row_found_block, row_missing_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(row_missing_block); + self.store_tail_backtrace_status( + inst_buffer, + BacktraceStatus::NoUnwindRowsForPc, + BACKTRACE_ERROR_NONE, + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(row_found_block); + let state = BtRegisterState { + ip: current_ip, + rsp: current_rsp, + rbp: current_rbp, + }; + let next = self.recover_next_frame_from_runtime_row(&runtime_row, state, scratch)?; + let validation = self.validate_backtrace_next_frame(state, next)?; + let store_block = self + .context + .append_basic_block(current_fn, "bt_step_store_frame"); + let stop_block = self.context.append_basic_block(current_fn, "bt_step_stop"); + self.builder + .build_conditional_branch(validation.valid, store_block, stop_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(stop_block); + let stop_status = self.status_for_backtrace_stop( + validation.complete, + validation.error_code, + offsets_found, + )?; + self.store_u8_value( + inst_buffer, + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_STATUS_OFFSET, + stop_status, + "bt_step_stop_status", + )?; + self.store_u16_value( + inst_buffer, + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_ERROR_CODE_OFFSET, + validation.error_code, + "bt_step_stop_error", + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(store_block); + let frame_module = self.resolve_backtrace_frame_module( + next.ip, + module_cookie, + module_bias, + caller_fallback_found, + "bt_step_frame_module", + )?; + let next_pc = + self.normalized_pc_from_raw(next.ip, frame_module.bias, frame_module.found)?; + self.store_backtrace_frame_dynamic( + inst_buffer, + frame_count, + depth.saturating_sub(1), + frame_module.cookie, + next_pc, + next.ip, + )?; + let next_count = self + .builder + .build_int_add( + frame_count, + self.context.i8_type().const_int(1, false), + "bt_step_next_frame_count", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.store_u8_value( + state_ptr, + crate::BACKTRACE_TAIL_STATE_FRAME_COUNT_OFFSET, + next_count, + "bt_step_state_frame_count", + )?; + self.store_u8_value( + inst_buffer, + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_FRAME_COUNT_OFFSET, + next_count, + "bt_step_inst_frame_count", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_IP_OFFSET, + next.ip, + "bt_step_state_next_ip", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RSP_OFFSET, + next.rsp, + "bt_step_state_next_rsp", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_CURRENT_RBP_OFFSET, + next.rbp, + "bt_step_state_next_rbp", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_BIAS_OFFSET, + frame_module.bias, + "bt_step_state_next_module_bias", + )?; + self.store_state_i64( + state_ptr, + crate::BACKTRACE_TAIL_STATE_MODULE_COOKIE_OFFSET, + frame_module.cookie, + "bt_step_state_next_module_cookie", + )?; + let next_offsets_found_u8 = + self.bool_to_u8(frame_module.found, "bt_step_next_offsets_found_u8")?; + self.store_u8_value( + state_ptr, + crate::BACKTRACE_TAIL_STATE_OFFSETS_FOUND_OFFSET, + next_offsets_found_u8, + "bt_step_state_next_offsets_found", + )?; + + let reached_depth = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + next_count, + depth_value, + "bt_step_reached_depth", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let reached_depth_block = self + .context + .append_basic_block(current_fn, "bt_step_reached_depth"); + let continue_block = self + .context + .append_basic_block(current_fn, "bt_step_continue"); + self.builder + .build_conditional_branch(reached_depth, reached_depth_block, continue_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(reached_depth_block); + self.store_tail_backtrace_status( + inst_buffer, + BacktraceStatus::Truncated, + BACKTRACE_ERROR_NONE, + )?; + self.builder + .build_unconditional_branch(finalize_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(continue_block); + Ok(()) + } + + pub(super) fn emit_bpf_tail_call(&mut self, index: u32) -> Result<()> { + let ctx = self.get_pt_regs_parameter()?; + let prog_array = self.lookup_bt_prog_array_ptr()?; + let args = [ + ctx.into(), + prog_array.into(), + self.context + .i32_type() + .const_int(index as u64, false) + .into(), + ]; + let _ = self.create_bpf_helper_call( + BPF_FUNC_tail_call as u64, + &args, + self.context.i64_type().into(), + "bt_bpf_tail_call", + )?; + Ok(()) + } + + pub(super) fn store_tail_backtrace_status( + &self, + inst_buffer: PointerValue<'ctx>, + status: BacktraceStatus, + error_code: u16, + ) -> Result<()> { + self.store_u8_const( + inst_buffer, + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_STATUS_OFFSET, + status as u8, + "bt_tail_status", + )?; + self.store_u16_const( + inst_buffer, + INSTRUCTION_HEADER_SIZE + BACKTRACE_DATA_ERROR_CODE_OFFSET, + error_code, + "bt_tail_error_code", + ) + } + + pub(super) fn emit_tail_final_event( + &mut self, + state_ptr: PointerValue<'ctx>, + accum_buffer: PointerValue<'ctx>, + ) -> Result<()> { + let event_size = self.load_state_i32( + state_ptr, + crate::BACKTRACE_TAIL_STATE_EVENT_SIZE_OFFSET, + "bt_final_event_size", + )?; + self.emit_accumulated_event_output(accum_buffer, event_size) + } + + pub(super) fn build_return_zero(&mut self) -> Result<()> { + self.builder + .build_return(Some(&self.context.i32_type().const_zero())) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } +} diff --git a/ghostscope-compiler/src/ebpf/codegen/backtrace/unwind_rows.rs b/ghostscope-compiler/src/ebpf/codegen/backtrace/unwind_rows.rs new file mode 100644 index 00000000..98281e6d --- /dev/null +++ b/ghostscope-compiler/src/ebpf/codegen/backtrace/unwind_rows.rs @@ -0,0 +1,593 @@ +use super::*; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + pub(super) fn compact_unwind_row_for_backtrace( + &self, + module_path: &str, + pc: u64, + ) -> Option { + let analyzer = self.process_analyzer?; + let module_address = ModuleAddress::new(PathBuf::from(module_path), pc); + let ctx = analyzer.resolve_pc(&module_address).ok()?; + analyzer.compact_unwind_row_for_context(&ctx).ok().flatten() + } + + pub(super) fn usable_backtrace_unwind_row_for_pc( + &self, + module_path: &str, + pc: u64, + ) -> BacktraceUnwindRowForPc { + let Some(row) = self.compact_unwind_row_for_backtrace(module_path, pc) else { + return BacktraceUnwindRowForPc::Missing; + }; + match crate::backtrace_unwind_row_from_compact(&row) { + Some(row) => BacktraceUnwindRowForPc::Usable(row), + None => BacktraceUnwindRowForPc::Unsupported, + } + } + + pub(super) fn status_for_backtrace_unwind_row_for_pc( + &self, + row: &BacktraceUnwindRowForPc, + ) -> BacktraceStatus { + match row { + BacktraceUnwindRowForPc::Usable(_) => BacktraceStatus::ReadError, + BacktraceUnwindRowForPc::Missing if self.process_analyzer.is_some() => { + BacktraceStatus::NoUnwindRowsForPc + } + BacktraceUnwindRowForPc::Unsupported if self.process_analyzer.is_some() => { + BacktraceStatus::UnsupportedCfi + } + BacktraceUnwindRowForPc::Missing | BacktraceUnwindRowForPc::Unsupported => { + BacktraceStatus::DwarfUnavailable + } + } + } + + pub(super) fn runtime_row_from_static( + &self, + row: ghostscope_protocol::BacktraceUnwindRow, + ) -> RuntimeBtUnwindRow<'ctx> { + let i8_type = self.context.i8_type(); + let i16_type = self.context.i16_type(); + let i64_type = self.context.i64_type(); + RuntimeBtUnwindRow { + found: self.context.bool_type().const_int(1, false), + cfa_register: i16_type.const_int(row.cfa_register as u64, false), + cfa_offset: i64_type.const_int(row.cfa_offset as u64, true), + ra_kind: i8_type.const_int(row.ra_kind as u64, false), + ra_register: i16_type.const_int(row.ra_register as u64, false), + ra_offset: i64_type.const_int(row.ra_offset as u64, true), + rbp_kind: i8_type.const_int(row.rbp_kind as u64, false), + rbp_register: i16_type.const_int(row.rbp_register as u64, false), + rbp_offset: i64_type.const_int(row.rbp_offset as u64, true), + } + } + + pub(super) fn allocate_backtrace_scratch(&self) -> Result> { + let i16_type = self.context.i16_type(); + let i32_type = self.context.i32_type(); + let i64_type = self.context.i64_type(); + + Ok(BtScratch { + row: RuntimeBtRowScratch { + found_ptr: self.build_entry_alloca(i32_type, "bt_row_found")?, + cfa_register_ptr: self.build_entry_alloca(i16_type, "bt_row_cfa_register")?, + cfa_offset_ptr: self.build_entry_alloca(i64_type, "bt_row_cfa_offset")?, + ra_kind_ptr: self.build_entry_alloca(self.context.i8_type(), "bt_row_ra_kind")?, + ra_register_ptr: self.build_entry_alloca(i16_type, "bt_row_ra_register")?, + ra_offset_ptr: self.build_entry_alloca(i64_type, "bt_row_ra_offset")?, + rbp_kind_ptr: self.build_entry_alloca(self.context.i8_type(), "bt_row_rbp_kind")?, + rbp_register_ptr: self.build_entry_alloca(i16_type, "bt_row_rbp_register")?, + rbp_offset_ptr: self.build_entry_alloca(i64_type, "bt_row_rbp_offset")?, + }, + next_rbp_ptr: self.build_entry_alloca(i64_type, "bt_next_rbp")?, + next_error_code_ptr: self.build_entry_alloca(i16_type, "bt_next_error_code")?, + }) + } + + pub(super) fn lookup_backtrace_unwind_row( + &mut self, + normalized_pc: IntValue<'ctx>, + module_cookie: IntValue<'ctx>, + scratch: &RuntimeBtRowScratch<'ctx>, + name_prefix: &str, + ) -> Result> { + let bounds = self.backtrace_unwind_row_bounds_for_module(module_cookie, name_prefix)?; + self.lookup_backtrace_unwind_row_in_range(normalized_pc, bounds.start, bounds.end, scratch) + } + + pub(super) fn backtrace_unwind_row_bounds_for_module( + &mut self, + module_cookie: IntValue<'ctx>, + name_prefix: &str, + ) -> Result> { + let i32_type = self.context.i32_type(); + if self.backtrace_module_row_ranges.is_empty() { + return Ok(BtRowBounds { + start: i32_type.const_zero(), + end: i32_type.const_int(self.backtrace_unwind_rows.len() as u64, false), + }); + } + + let i64_type = self.context.i64_type(); + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let map_global = self + .module + .get_global("bt_module_row_ranges") + .ok_or_else(|| { + CodeGenError::LLVMError("bt_module_row_ranges map not found".to_string()) + })?; + let map_ptr = self + .builder + .build_bit_cast( + map_global.as_pointer_value(), + ptr_type, + &format!("{name_prefix}_row_ranges_map_ptr"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_alloca = + self.build_entry_alloca(i64_type, &format!("{name_prefix}_row_range_key"))?; + self.builder + .build_store(key_alloca, module_cookie) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let key_ptr = self + .builder + .build_bit_cast( + key_alloca, + ptr_type, + &format!("{name_prefix}_row_range_key_void"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let result = self.create_bpf_helper_call( + BPF_FUNC_map_lookup_elem as u64, + &[map_ptr, key_ptr], + ptr_type.into(), + &format!("{name_prefix}_row_range_lookup"), + )?; + let range_ptr = match result { + BasicValueEnum::PointerValue(ptr) => ptr, + _ => { + return Err(CodeGenError::LLVMError( + "bt_module_row_ranges lookup did not return pointer".to_string(), + )) + } + }; + let is_null = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + self.builder + .build_ptr_to_int( + range_ptr, + i64_type, + &format!("{name_prefix}_row_range_ptr_i64"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?, + i64_type.const_zero(), + &format!("{name_prefix}_row_range_is_null"), + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let current_fn = self.current_function("lookup bt row range")?; + let found_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_found_row_range")); + let miss_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_miss_row_range")); + let cont_block = self + .context + .append_basic_block(current_fn, &format!("{name_prefix}_cont_row_range")); + self.builder + .build_conditional_branch(is_null, miss_block, found_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(found_block); + let load_range_field = |offset: usize, + field_name: &str, + ctx: &mut EbpfContext<'ctx, 'dw>| + -> Result> { + let offset_i32 = ctx.context.i32_type().const_int(offset as u64, false); + // SAFETY: range_ptr is a non-null BacktraceModuleRowRange pointer + // returned by bpf_map_lookup_elem, and offsets are shared ABI + // constants from ghostscope-protocol. + let field_ptr = unsafe { + ctx.builder + .build_gep(ctx.context.i8_type(), range_ptr, &[offset_i32], field_name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + }; + Ok(ctx + .builder + .build_load(ctx.context.i32_type(), field_ptr, field_name) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value()) + }; + let found_start = load_range_field( + ghostscope_protocol::BACKTRACE_MODULE_ROW_RANGE_ROW_START_OFFSET, + &format!("{name_prefix}_row_range_start"), + self, + )?; + let found_end = load_range_field( + ghostscope_protocol::BACKTRACE_MODULE_ROW_RANGE_ROW_END_OFFSET, + &format!("{name_prefix}_row_range_end"), + self, + )?; + self.builder + .build_unconditional_branch(cont_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let found_end_block = self.current_insert_block("finish bt row range found block")?; + + self.builder.position_at_end(miss_block); + self.builder + .build_unconditional_branch(cont_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let miss_end_block = self.current_insert_block("finish bt row range miss block")?; + + self.builder.position_at_end(cont_block); + let start_phi = self + .builder + .build_phi(i32_type, &format!("{name_prefix}_row_start_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + start_phi.add_incoming(&[ + (&found_start, found_end_block), + (&i32_type.const_zero(), miss_end_block), + ]); + let end_phi = self + .builder + .build_phi(i32_type, &format!("{name_prefix}_row_end_phi")) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + end_phi.add_incoming(&[ + (&found_end, found_end_block), + (&i32_type.const_zero(), miss_end_block), + ]); + + Ok(BtRowBounds { + start: start_phi.as_basic_value().into_int_value(), + end: end_phi.as_basic_value().into_int_value(), + }) + } + + pub(super) fn lookup_backtrace_unwind_row_in_range( + &mut self, + normalized_pc: IntValue<'ctx>, + row_start: IntValue<'ctx>, + row_end: IntValue<'ctx>, + scratch: &RuntimeBtRowScratch<'ctx>, + ) -> Result> { + let row_count = self.backtrace_unwind_row_map_entries() as usize; + let i16_type = self.context.i16_type(); + let i32_type = self.context.i32_type(); + let i64_type = self.context.i64_type(); + let i8_type = self.context.i8_type(); + let sentinel = i32_type.const_int(row_count as u64, false); + + self.builder + .build_store(scratch.found_ptr, sentinel) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.cfa_register_ptr, i16_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.cfa_offset_ptr, i64_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_kind_ptr, i8_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_register_ptr, i16_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_offset_ptr, i64_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_kind_ptr, i8_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_register_ptr, i16_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_offset_ptr, i64_type.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let current_fn = self.current_function("lookup bt unwind row")?; + let return_block = self + .context + .append_basic_block(current_fn, "bt_row_lookup_return"); + if row_count == 0 { + self.builder + .build_unconditional_branch(return_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + } else { + let lo_ptr = self.build_entry_alloca(i32_type, "bt_row_lo")?; + let hi_ptr = self.build_entry_alloca(i32_type, "bt_row_hi")?; + self.builder + .build_store(lo_ptr, row_start) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(hi_ptr, row_end) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.emit_backtrace_row_runtime_binary_search( + normalized_pc, + scratch, + lo_ptr, + hi_ptr, + row_count, + return_block, + )?; + } + self.builder.position_at_end(return_block); + let final_found_idx = self.load_i32(scratch.found_ptr, "bt_final_found_idx")?; + let found = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + final_found_idx, + sentinel, + "bt_final_row_found", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(RuntimeBtUnwindRow { + found, + cfa_register: self.load_i16(scratch.cfa_register_ptr, "bt_final_cfa_reg")?, + cfa_offset: self.load_i64(scratch.cfa_offset_ptr, "bt_final_cfa_off")?, + ra_kind: self.load_i8(scratch.ra_kind_ptr, "bt_final_ra_kind")?, + ra_register: self.load_i16(scratch.ra_register_ptr, "bt_final_ra_reg")?, + ra_offset: self.load_i64(scratch.ra_offset_ptr, "bt_final_ra_off")?, + rbp_kind: self.load_i8(scratch.rbp_kind_ptr, "bt_final_rbp_kind")?, + rbp_register: self.load_i16(scratch.rbp_register_ptr, "bt_final_rbp_reg")?, + rbp_offset: self.load_i64(scratch.rbp_offset_ptr, "bt_final_rbp_off")?, + }) + } + + pub(super) fn emit_backtrace_row_runtime_binary_search( + &mut self, + normalized_pc: IntValue<'ctx>, + scratch: &RuntimeBtRowScratch<'ctx>, + lo_ptr: PointerValue<'ctx>, + hi_ptr: PointerValue<'ctx>, + row_count: usize, + return_block: BasicBlock<'ctx>, + ) -> Result<()> { + let current_fn = self.current_function("emit bt row lookup tree")?; + let i32_type = self.context.i32_type(); + let sentinel = i32_type.const_int(row_count as u64, false); + let max_steps = backtrace_row_binary_search_steps(row_count); + + for _ in 0..max_steps { + let found_idx = self.load_i32(scratch.found_ptr, "bt_lookup_found_idx")?; + let not_found = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + found_idx, + sentinel, + "bt_lookup_not_found", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let lo = self.load_i32(lo_ptr, "bt_lookup_lo")?; + let hi = self.load_i32(hi_ptr, "bt_lookup_hi")?; + let range_active = self + .builder + .build_int_compare(inkwell::IntPredicate::ULT, lo, hi, "bt_lookup_range_active") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let should_search = self + .builder + .build_and(not_found, range_active, "bt_lookup_should_search") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let search_block = self + .context + .append_basic_block(current_fn, "bt_lookup_search"); + let skip_block = self + .context + .append_basic_block(current_fn, "bt_lookup_skip"); + let after_block = self + .context + .append_basic_block(current_fn, "bt_lookup_after"); + self.builder + .build_conditional_branch(should_search, search_block, skip_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(skip_block); + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(search_block); + let lo_plus_hi = self + .builder + .build_int_add(lo, hi, "bt_lookup_lo_plus_hi") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let mid = self + .builder + .build_right_shift( + lo_plus_hi, + i32_type.const_int(1, false), + false, + "bt_lookup_mid", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let row_ptr = self.lookup_bt_unwind_row_ptr(mid)?; + let row_is_null = self + .builder + .build_is_null(row_ptr, "bt_lookup_row_is_null") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let row_null_block = self + .context + .append_basic_block(current_fn, "bt_lookup_row_null"); + let row_load_block = self + .context + .append_basic_block(current_fn, "bt_lookup_row_load"); + self.builder + .build_conditional_branch(row_is_null, row_null_block, row_load_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(row_null_block); + self.builder + .build_store(lo_ptr, hi) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(row_load_block); + let pc_start = self.load_row_i64( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_PC_START_OFFSET, + "bt_lookup_row_pc_start", + )?; + let pc_end = self.load_row_i64( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_PC_END_OFFSET, + "bt_lookup_row_pc_end", + )?; + let before = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + normalized_pc, + pc_start, + "bt_lookup_pc_before", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let before_block = self + .context + .append_basic_block(current_fn, "bt_lookup_before"); + let not_before_block = self + .context + .append_basic_block(current_fn, "bt_lookup_not_before"); + self.builder + .build_conditional_branch(before, before_block, not_before_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(before_block); + self.builder + .build_store(hi_ptr, mid) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(not_before_block); + let after = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGE, + normalized_pc, + pc_end, + "bt_lookup_pc_after", + ) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let after_range_block = self + .context + .append_basic_block(current_fn, "bt_lookup_after_range"); + let match_block = self + .context + .append_basic_block(current_fn, "bt_lookup_match"); + self.builder + .build_conditional_branch(after, after_range_block, match_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(after_range_block); + let mid_plus_one = self + .builder + .build_int_add(mid, i32_type.const_int(1, false), "bt_lookup_mid_plus_one") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(lo_ptr, mid_plus_one) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(match_block); + self.store_backtrace_unwind_row_from_ptr(row_ptr, mid, scratch)?; + self.builder + .build_unconditional_branch(after_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + self.builder.position_at_end(after_block); + } + + self.builder + .build_unconditional_branch(return_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } + + pub(super) fn store_backtrace_unwind_row_from_ptr( + &self, + row_ptr: PointerValue<'ctx>, + row_index: IntValue<'ctx>, + scratch: &RuntimeBtRowScratch<'ctx>, + ) -> Result<()> { + self.builder + .build_store(scratch.found_ptr, row_index) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let cfa_register = self.load_row_i16( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_CFA_REGISTER_OFFSET, + "bt_tree_row_cfa_reg", + )?; + let cfa_offset = self.load_row_i64( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_CFA_OFFSET_OFFSET, + "bt_tree_row_cfa_off", + )?; + let ra_kind = self.load_row_i8( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RA_KIND_OFFSET, + "bt_tree_row_ra_kind", + )?; + let ra_register = self.load_row_i16( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RA_REGISTER_OFFSET, + "bt_tree_row_ra_reg", + )?; + let ra_offset = self.load_row_i64( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RA_OFFSET_OFFSET, + "bt_tree_row_ra_off", + )?; + let rbp_kind = self.load_row_i8( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RBP_KIND_OFFSET, + "bt_tree_row_rbp_kind", + )?; + let rbp_register = self.load_row_i16( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RBP_REGISTER_OFFSET, + "bt_tree_row_rbp_reg", + )?; + let rbp_offset = self.load_row_i64( + row_ptr, + crate::BACKTRACE_UNWIND_ROW_RBP_OFFSET_OFFSET, + "bt_tree_row_rbp_off", + )?; + self.builder + .build_store(scratch.cfa_register_ptr, cfa_register) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.cfa_offset_ptr, cfa_offset) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_kind_ptr, ra_kind) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_register_ptr, ra_register) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.ra_offset_ptr, ra_offset) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_kind_ptr, rbp_kind) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_register_ptr, rbp_register) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + self.builder + .build_store(scratch.rbp_offset_ptr, rbp_offset) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + Ok(()) + } +} diff --git a/ghostscope-compiler/src/ebpf/codegen/mod.rs b/ghostscope-compiler/src/ebpf/codegen/mod.rs index 0d08b264..a6e34fce 100644 --- a/ghostscope-compiler/src/ebpf/codegen/mod.rs +++ b/ghostscope-compiler/src/ebpf/codegen/mod.rs @@ -173,7 +173,6 @@ fn allocate_dynamic_payload_reservations(max_lens: &[usize], available: usize) - mod args; mod backtrace; -mod backtrace_plan; mod expr_error; mod format; mod instruction_common; diff --git a/ghostscope-compiler/src/ebpf/expression.rs b/ghostscope-compiler/src/ebpf/expression.rs deleted file mode 100644 index e4e28fb8..00000000 --- a/ghostscope-compiler/src/ebpf/expression.rs +++ /dev/null @@ -1,4932 +0,0 @@ -//! Expression compilation for eBPF code generation -//! -//! This module handles compilation of various expression types to LLVM IR. - -use super::context::{CodeGenError, EbpfContext, Result, RuntimeAddress}; -use super::expression_plan::{ - BinaryEmitKind, BinaryIntegerSemantics, BuiltinCallPlan, SpecialVarPlan, -}; -use crate::script::{BinaryOp, Expr}; -use aya_ebpf_bindings::bindings::bpf_func_id::BPF_FUNC_probe_read_user; -use ghostscope_dwarf::{ - AmbiguityReason, Availability, CIntegerComparisonPlan, CIntegerComparisonType, - RuntimeRequirement, TypeInfo as DwarfType, TypeLayoutError, UnsupportedReason, - VariableReadPlan, -}; -use inkwell::values::{BasicValueEnum, IntValue, PointerValue}; -use inkwell::AddressSpace; -use std::path::{Path, PathBuf}; -use tracing::debug; - -// compare cap is provided via compile_options.compare_cap (config: ebpf.compare_cap) - -#[derive(Clone)] -pub(super) struct DynamicTypeInfo { - pub(super) dwarf_type: DwarfType, - pub(super) module_path: Option, -} - -pub(super) struct DynamicLvalue<'ctx> { - pub(super) address: RuntimeAddress<'ctx>, - pub(super) type_info: DynamicTypeInfo, -} - -struct IndexableElementInfo { - element_type: DwarfType, - stride: u64, - module_path: Option, -} - -impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { - pub(crate) fn get_host_pid_tid_values(&mut self) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { - let i32_type = self.context.i32_type(); - let i64_type = self.context.i64_type(); - - // bpf_get_current_pid_tgid() returns: - // - high 32 bits: TGID (process ID / getpid() view) - // - low 32 bits: PID (thread ID / gettid() view) - let host_pid_tgid = self.get_current_pid_tgid()?; - let host_tid = self - .builder - .build_and( - host_pid_tgid, - i64_type.const_int(0xFFFF_FFFF, false), - "host_tid", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let host_pid = self - .builder - .build_right_shift( - host_pid_tgid, - i64_type.const_int(32, false), - false, - "host_pid", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - let host_pid_i32 = self - .builder - .build_int_truncate(host_pid, i32_type, "host_pid_i32") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let host_tid_i32 = self - .builder - .build_int_truncate(host_tid, i32_type, "host_tid_i32") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - Ok((host_pid_i32, host_tid_i32)) - } - - pub(crate) fn get_special_pid_tid_values( - &mut self, - ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { - const BPF_FUNC_GET_NS_CURRENT_PID_TGID: u64 = 120; - const BPF_PIDNS_INFO_SIZE: u64 = 8; // struct { u32 pid; u32 tgid; } - - let i32_type = self.context.i32_type(); - let i64_type = self.context.i64_type(); - let (host_pid_i32, host_tid_i32) = self.get_host_pid_tid_values()?; - - let ns_spec = if let Some(crate::PidFilterSpec::NamespaceTgid { pid_ns, .. }) = - self.compile_options.pid_filter_spec - { - pid_ns.helper_dev_inode() - } else { - self.compile_options - .special_pid_ns - .and_then(|pid_ns| pid_ns.helper_dev_inode()) - }; - let Some((pid_ns_dev, pid_ns_inode)) = ns_spec else { - let host_pid = self - .builder - .build_int_z_extend(host_pid_i32, i64_type, "selected_host_pid") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let host_tid = self - .builder - .build_int_z_extend(host_tid_i32, i64_type, "selected_host_tid") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok((host_pid, host_tid)); - }; - - let ptr_type = self.context.ptr_type(AddressSpace::default()); - let key_arr_ty = i32_type.array_type(4); - let key_alloca = self.pm_key_alloca.ok_or_else(|| { - CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) - })?; - // Reuse entry-allocated stack key storage: helper only needs first 8 bytes. - self.builder - .build_store(key_alloca, key_arr_ty.const_zero()) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - let pidns_info_ptr = self - .builder - .build_bit_cast(key_alloca, ptr_type, "special_pidns_info_ptr") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - let helper_args = [ - i64_type.const_int(pid_ns_dev, false).into(), - i64_type.const_int(pid_ns_inode, false).into(), - pidns_info_ptr, - i64_type.const_int(BPF_PIDNS_INFO_SIZE, false).into(), - ]; - let helper_ret = self.create_bpf_helper_call( - BPF_FUNC_GET_NS_CURRENT_PID_TGID, - &helper_args, - i64_type.into(), - "special_ns_pid_tgid_ret", - )?; - let helper_ret = match helper_ret { - BasicValueEnum::IntValue(v) => v, - _ => { - return Err(CodeGenError::LLVMError( - "bpf_get_ns_current_pid_tgid did not return integer".to_string(), - )) - } - }; - - let helper_ok = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - helper_ret, - i64_type.const_zero(), - "special_ns_helper_ok", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - // SAFETY: key_alloca temporarily holds the two-field pid namespace helper - // result, so [0, 0] addresses the pid field. - let ns_pid_ptr = unsafe { - self.builder.build_gep( - key_arr_ty, - key_alloca, - &[i32_type.const_zero(), i32_type.const_zero()], - "special_ns_pid_ptr", - ) - } - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - // SAFETY: key_alloca temporarily holds the two-field pid namespace helper - // result, so [0, 1] addresses the tgid field. - let ns_tgid_ptr = unsafe { - self.builder.build_gep( - key_arr_ty, - key_alloca, - &[i32_type.const_zero(), i32_type.const_int(1, false)], - "special_ns_tgid_ptr", - ) - } - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - let ns_pid = self - .builder - .build_load(i32_type, ns_pid_ptr, "special_ns_pid") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - let ns_tgid = self - .builder - .build_load(i32_type, ns_tgid_ptr, "special_ns_tgid") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))? - .into_int_value(); - - let selected_pid_i32 = self - .builder - .build_select(helper_ok, ns_tgid, host_pid_i32, "selected_pid_i32") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let selected_tid_i32 = self - .builder - .build_select(helper_ok, ns_pid, host_tid_i32, "selected_tid_i32") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - - let selected_pid = self - .builder - .build_int_z_extend(selected_pid_i32, i64_type, "selected_pid") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let selected_tid = self - .builder - .build_int_z_extend(selected_tid_i32, i64_type, "selected_tid") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - Ok((selected_pid, selected_tid)) - } - - pub(super) fn is_dwarf_aggregate_expr(&mut self, expr: &Expr) -> bool { - if let Expr::Cast { target_type, .. } = expr { - return self - .resolve_cast_target_type(target_type) - .ok() - .is_some_and(|ty| ghostscope_dwarf::is_c_aggregate_type(&ty)); - } - - if let Ok(Some(var)) = self.query_dwarf_for_complex_expr(expr) { - if let Some(ref ty) = var.dwarf_type { - return ghostscope_dwarf::is_c_aggregate_type(ty); - } - } - false - } - - /// Heuristic check: whether an expression should be treated as a pointer/address - /// Returns true for: - /// - Explicit address-of forms (&expr) - /// - Script string literals (compile to pointer data) - /// - Alias variables bound to addresses - /// - DWARF-backed expressions whose type is pointer or array - pub(super) fn is_pointer_like_expr(&mut self, expr: &Expr) -> bool { - use crate::script::Expr as E; - match expr { - E::AddressOf(_) => return true, - E::String(_) => return true, - E::Cast { target_type, .. } => { - if self - .resolve_cast_target_type(target_type) - .ok() - .is_some_and(|ty| { - matches!( - ghostscope_dwarf::strip_type_aliases(&ty), - DwarfType::PointerType { .. } | DwarfType::ArrayType { .. } - ) - }) - { - return true; - } - } - E::Variable(name) => { - if self.alias_variable_exists(name) { - return true; - } - } - _ => {} - } - - if let Ok(Some(var)) = self.query_dwarf_for_complex_expr(expr) { - if let Some(ref ty) = var.dwarf_type { - if ghostscope_dwarf::is_c_pointer_or_array_type(ty) { - return true; - } - } - } - false - } - - pub(super) fn resolve_cast_target_type(&self, target_type: &str) -> Result { - let analyzer = self.process_analyzer; - let resolved = if let Some(context) = self.current_compile_time_context.as_ref() { - let module_path = Path::new(&context.module_path); - analyzer - .map(|analyzer| analyzer.try_resolve_type_spec_in_module(module_path, target_type)) - .transpose() - .map_err(|err| CodeGenError::DwarfError(err.to_string()))? - .flatten() - .or_else(|| ghostscope_dwarf::DwarfAnalyzer::resolve_builtin_type_spec(target_type)) - } else { - analyzer - .map(|analyzer| analyzer.try_resolve_type_spec(target_type)) - .transpose() - .map_err(|err| CodeGenError::DwarfError(err.to_string()))? - .flatten() - .or_else(|| ghostscope_dwarf::DwarfAnalyzer::resolve_builtin_type_spec(target_type)) - }; - - resolved.ok_or_else(|| { - CodeGenError::DwarfError(format!("cast target type '{target_type}' was not found")) - }) - } - - pub(super) fn cast_pointer_target_type(target_type: &DwarfType) -> Option { - match ghostscope_dwarf::strip_type_aliases(target_type) { - DwarfType::PointerType { target_type, .. } => Some(target_type.as_ref().clone()), - _ => None, - } - } - - fn is_float_dwarf_type(target_type: &DwarfType) -> bool { - match ghostscope_dwarf::strip_type_aliases(target_type) { - DwarfType::BaseType { encoding, .. } => { - *encoding == ghostscope_dwarf::constants::DW_ATE_float.0 as u16 - } - _ => false, - } - } - - fn is_bool_dwarf_type(target_type: &DwarfType) -> bool { - match ghostscope_dwarf::strip_type_aliases(target_type) { - DwarfType::BaseType { encoding, .. } => { - *encoding == ghostscope_dwarf::constants::DW_ATE_boolean.0 as u16 - } - _ => false, - } - } - - pub(super) fn cast_value_byte_len(target_type: &DwarfType) -> Option { - if matches!( - ghostscope_dwarf::strip_type_aliases(target_type), - DwarfType::PointerType { .. } - ) { - return Some(8); - } - - if let Some(integer_type) = ghostscope_dwarf::c_integer_comparison_type(target_type) { - return Some(integer_type.size.clamp(1, 8) as usize); - } - - None - } - - pub(super) fn cast_source_pointer_value( - &mut self, - expr: &Expr, - ) -> Result> { - if let Ok(address) = self.resolve_runtime_address_from_expr(expr) { - return Ok(address); - } - - match self.compile_expr(expr)? { - BasicValueEnum::IntValue(value) => Ok(RuntimeAddress::available( - self.normalize_int_to_i64(value, "cast_ptr_i64")?, - self.context, - )), - BasicValueEnum::PointerValue(value) => self - .builder - .build_ptr_to_int(value, self.context.i64_type(), "cast_ptr_value") - .map(|value| RuntimeAddress::available(value, self.context)) - .map_err(|err| CodeGenError::Builder(err.to_string())), - _ => Err(CodeGenError::TypeError( - "cast source expression did not produce an address-sized value".to_string(), - )), - } - } - - pub(super) fn cast_source_memory_address( - &mut self, - expr: &Expr, - ) -> Result> { - if let Ok(address) = self.resolve_runtime_address_from_expr(expr) { - return Ok(address); - } - - if let Some(plan) = self.query_dwarf_for_complex_expr(expr)? { - let status_ptr = if self.condition_context_active { - Some(self.get_or_create_cond_error_global()) - } else { - None - }; - let pc_address = self.get_compile_time_context()?.pc_address; - if let Ok(address) = - self.variable_read_plan_to_runtime_address(&plan, pc_address, status_ptr) - { - return Ok(address); - } - } - - match self.compile_expr(expr)? { - BasicValueEnum::IntValue(value) => Ok(RuntimeAddress::available( - self.normalize_int_to_i64(value, "cast_mem_i64")?, - self.context, - )), - BasicValueEnum::PointerValue(value) => self - .builder - .build_ptr_to_int(value, self.context.i64_type(), "cast_mem_ptr") - .map(|value| RuntimeAddress::available(value, self.context)) - .map_err(|err| CodeGenError::Builder(err.to_string())), - _ => Err(CodeGenError::TypeError( - "cast source expression is not addressable".to_string(), - )), - } - } - - fn cast_lvalue_address_and_type( - &mut self, - expr: &Expr, - target_type: &str, - ) -> Result> { - let target_type = self.resolve_cast_target_type(target_type)?; - let module_path = self - .current_compile_time_context - .as_ref() - .map(|context| PathBuf::from(&context.module_path)); - - if let Some(pointee_type) = Self::cast_pointer_target_type(&target_type) { - let address = self.cast_source_pointer_value(expr)?; - return Ok(DynamicLvalue { - address, - type_info: DynamicTypeInfo { - dwarf_type: pointee_type, - module_path, - }, - }); - } - - let address = self.cast_source_memory_address(expr)?; - Ok(DynamicLvalue { - address, - type_info: DynamicTypeInfo { - dwarf_type: target_type, - module_path, - }, - }) - } - - fn cast_index_base( - &mut self, - expr: &Expr, - ) -> Result)>> { - let Expr::Cast { - expr: source_expr, - target_type, - } = expr - else { - return Ok(None); - }; - - let target_type = self.resolve_cast_target_type(target_type)?; - let module_path = self - .current_compile_time_context - .as_ref() - .map(|context| PathBuf::from(&context.module_path)); - - match ghostscope_dwarf::strip_type_aliases(&target_type) { - DwarfType::PointerType { .. } => { - let Some(element_info) = Self::indexable_info_from_type(&target_type, module_path) - else { - return Ok(None); - }; - let base_address = self.cast_source_pointer_value(source_expr)?; - Ok(Some((element_info, base_address))) - } - DwarfType::ArrayType { .. } => { - let Some(element_info) = Self::indexable_info_from_type(&target_type, module_path) - else { - return Ok(None); - }; - let base_address = self.cast_source_memory_address(source_expr)?; - Ok(Some((element_info, base_address))) - } - _ => Ok(None), - } - } - - fn indexable_info_from_type( - dwarf_type: &DwarfType, - module_path: Option, - ) -> Option { - ghostscope_dwarf::indexable_element_layout(dwarf_type).map(|layout| IndexableElementInfo { - element_type: layout.element_type, - stride: layout.stride, - module_path, - }) - } - - fn compiled_pointer_value_to_runtime_address( - &mut self, - value: BasicValueEnum<'ctx>, - int_name: &str, - ptr_name: &str, - error_message: &'static str, - ) -> Result> { - match value { - BasicValueEnum::IntValue(value) => Ok(RuntimeAddress::available( - self.normalize_int_to_i64(value, int_name)?, - self.context, - )), - BasicValueEnum::PointerValue(value) => self - .builder - .build_ptr_to_int(value, self.context.i64_type(), ptr_name) - .map(|value| RuntimeAddress::available(value, self.context)) - .map_err(|err| CodeGenError::Builder(err.to_string())), - _ => Err(CodeGenError::TypeError(error_message.to_string())), - } - } - - fn dynamic_lvalue_from_indexable_base( - &mut self, - element_info: IndexableElementInfo, - base_address: RuntimeAddress<'ctx>, - index_value: IntValue<'ctx>, - name: &str, - ) -> Result> { - let stride_value = self - .context - .i64_type() - .const_int(element_info.stride, false); - let byte_offset = self - .builder - .build_int_mul(index_value, stride_value, &format!("{name}_byte_offset")) - .map_err(|err| CodeGenError::Builder(err.to_string()))?; - let element_address = self - .builder - .build_int_add( - base_address.value, - byte_offset, - &format!("{name}_element_address"), - ) - .map_err(|err| CodeGenError::Builder(err.to_string()))?; - - Ok(DynamicLvalue { - address: base_address.with_value(element_address), - type_info: DynamicTypeInfo { - dwarf_type: element_info.element_type, - module_path: element_info.module_path, - }, - }) - } - - fn dynamic_lvalue_from_const_pointer_arithmetic( - &mut self, - expr: &Expr, - ) -> Result>> { - let Some((base_expr, index)) = self.pointer_arithmetic_parts_expanding_aliases(expr)? - else { - return Ok(None); - }; - let Some((element_info, base_address)) = self.cast_index_base(&base_expr)? else { - return Ok(None); - }; - let index_value = self.context.i64_type().const_int(index as u64, true); - self.dynamic_lvalue_from_indexable_base( - element_info, - base_address, - index_value, - "dynamic_cast_ptr_arith", - ) - .map(Some) - } - - fn compile_cast_integer_value( - &mut self, - expr: &Expr, - target_type: &DwarfType, - ) -> Result> { - let value = match self.compile_expr(expr)? { - BasicValueEnum::IntValue(value) => value, - BasicValueEnum::PointerValue(value) => self - .builder - .build_ptr_to_int(value, self.context.i64_type(), "cast_int_ptr") - .map_err(|err| CodeGenError::Builder(err.to_string()))?, - _ => { - return Err(CodeGenError::TypeError( - "integer cast source must be an integer or pointer".to_string(), - )) - } - }; - - if Self::is_bool_dwarf_type(target_type) { - let value = self.normalize_int_to_i64(value, "cast_bool_i64")?; - return self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - value, - self.context.i64_type().const_zero(), - "cast_bool", - ) - .map_err(|err| CodeGenError::Builder(err.to_string())); - } - - let Some(integer_type) = ghostscope_dwarf::c_integer_comparison_type(target_type) else { - return Err(CodeGenError::TypeError(format!( - "cast target '{}' is not an integer type", - target_type.type_name() - ))); - }; - - let bit_width = integer_type.size.saturating_mul(8).clamp(1, 64) as u32; - let target_int_type = self.context.custom_width_int_type(bit_width); - let current_width = value.get_type().get_bit_width(); - let narrowed = if current_width > bit_width { - self.builder - .build_int_truncate(value, target_int_type, "cast_int_trunc") - .map_err(|err| CodeGenError::Builder(err.to_string()))? - } else if current_width < bit_width { - if integer_type.is_unsigned || current_width == 1 { - self.builder - .build_int_z_extend(value, target_int_type, "cast_int_zext") - .map_err(|err| CodeGenError::Builder(err.to_string()))? - } else { - self.builder - .build_int_s_extend(value, target_int_type, "cast_int_sext") - .map_err(|err| CodeGenError::Builder(err.to_string()))? - } - } else { - value - }; - - if bit_width == 64 { - return Ok(narrowed); - } - - if integer_type.is_unsigned { - self.builder - .build_int_z_extend(narrowed, self.context.i64_type(), "cast_int_zext_i64") - .map_err(|err| CodeGenError::Builder(err.to_string())) - } else { - self.builder - .build_int_s_extend(narrowed, self.context.i64_type(), "cast_int_sext_i64") - .map_err(|err| CodeGenError::Builder(err.to_string())) - } - } - - fn compile_cast_expr_value( - &mut self, - expr: &Expr, - target_type: &str, - ) -> Result> { - let target_type = self.resolve_cast_target_type(target_type)?; - - if Self::cast_pointer_target_type(&target_type).is_some() { - let address = self.cast_source_pointer_value(expr)?; - let ptr_ty = self.context.ptr_type(AddressSpace::default()); - return self - .builder - .build_int_to_ptr(address.value, ptr_ty, "cast_as_ptr") - .map(|value| value.into()) - .map_err(|err| CodeGenError::Builder(err.to_string())); - } - - if ghostscope_dwarf::is_c_aggregate_type(&target_type) { - let address = self.cast_source_memory_address(expr)?; - let ptr_ty = self.context.ptr_type(AddressSpace::default()); - return self - .builder - .build_int_to_ptr(address.value, ptr_ty, "cast_aggregate_ptr") - .map(|value| value.into()) - .map_err(|err| CodeGenError::Builder(err.to_string())); - } - - if ghostscope_dwarf::c_integer_comparison_type(&target_type).is_some() { - return self - .compile_cast_integer_value(expr, &target_type) - .map(|value| value.into()); - } - - if Self::is_float_dwarf_type(&target_type) { - return Err(CodeGenError::TypeError( - "floating-point casts are only supported for memory reads/printing".to_string(), - )); - } - - Err(CodeGenError::TypeError(format!( - "cast target '{}' is not supported as a value expression", - target_type.type_name() - ))) - } - - pub(crate) fn integer_literal_value(expr: &Expr) -> Option { - use crate::script::ast::BinaryOp as BO; - use crate::script::ast::Expr as E; - - match expr { - E::Int(value) => Some(*value), - E::BinaryOp { - left, - op: BO::Add, - right, - } => { - Self::integer_literal_value(left)?.checked_add(Self::integer_literal_value(right)?) - } - E::BinaryOp { - left, - op: BO::Subtract, - right, - } => { - Self::integer_literal_value(left)?.checked_sub(Self::integer_literal_value(right)?) - } - E::BinaryOp { - left, - op: BO::Multiply, - right, - } => { - Self::integer_literal_value(left)?.checked_mul(Self::integer_literal_value(right)?) - } - E::BinaryOp { - left, - op: BO::Divide, - right, - } => { - Self::integer_literal_value(left)?.checked_div(Self::integer_literal_value(right)?) - } - E::BinaryOp { - left, - op: BO::Modulo, - right, - } => { - Self::integer_literal_value(left)?.checked_rem(Self::integer_literal_value(right)?) - } - E::BinaryOp { - left, - op: BO::BitAnd, - right, - } => Some(Self::integer_literal_value(left)? & Self::integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::BitXor, - right, - } => Some(Self::integer_literal_value(left)? ^ Self::integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::BitOr, - right, - } => Some(Self::integer_literal_value(left)? | Self::integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::ShiftLeft, - right, - } => { - let shift = u32::try_from(Self::integer_literal_value(right)?).ok()?; - Self::integer_literal_value(left)?.checked_shl(shift) - } - E::BinaryOp { - left, - op: BO::ShiftRight, - right, - } => { - let shift = u32::try_from(Self::integer_literal_value(right)?).ok()?; - Self::integer_literal_value(left)?.checked_shr(shift) - } - E::UnaryBitNot(inner) => Some(!Self::integer_literal_value(inner)?), - _ => None, - } - } - - pub(crate) fn pointer_arithmetic_parts(expr: &Expr) -> Option<(&Expr, i64)> { - use crate::script::ast::Expr as E; - - fn collect_offset(expr: &Expr, acc: i64) -> Option<(&Expr, i64)> { - use crate::script::ast::BinaryOp as BO; - use crate::script::ast::Expr as E; - - match expr { - E::BinaryOp { - left, - op: BO::Add, - right, - } => match (&**left, &**right) { - (ptr_side, int_expr) - if EbpfContext::<'static, 'static>::integer_literal_value(int_expr) - .is_some() => - { - let index = - EbpfContext::<'static, 'static>::integer_literal_value(int_expr)?; - collect_offset(ptr_side, acc.checked_add(index)?) - } - (int_expr, ptr_side) - if EbpfContext::<'static, 'static>::integer_literal_value(int_expr) - .is_some() => - { - let index = - EbpfContext::<'static, 'static>::integer_literal_value(int_expr)?; - collect_offset(ptr_side, acc.checked_add(index)?) - } - _ => Some((expr, acc)), - }, - E::BinaryOp { - left, - op: BO::Subtract, - right, - } => match &**right { - int_expr - if EbpfContext::<'static, 'static>::integer_literal_value(int_expr) - .is_some() => - { - let index = - EbpfContext::<'static, 'static>::integer_literal_value(int_expr)?; - collect_offset(left, acc.checked_sub(index)?) - } - _ => Some((expr, acc)), - }, - _ => Some((expr, acc)), - } - } - - let E::BinaryOp { .. } = expr else { - return None; - }; - - let (base, index) = collect_offset(expr, 0)?; - match base { - E::BinaryOp { .. } => None, - _ => Some((base, index)), - } - } - - pub(crate) fn pointer_arithmetic_parts_expanding_aliases( - &self, - expr: &Expr, - ) -> Result> { - let Some((base, index)) = Self::pointer_arithmetic_parts(expr) else { - return Ok(None); - }; - - let mut base = base.clone(); - let mut index = index; - let mut visited = std::collections::HashSet::new(); - - loop { - let Expr::Variable(name) = &base else { - break; - }; - if !self.alias_variable_exists(name) { - break; - } - if !visited.insert(name.clone()) { - return Err(CodeGenError::TypeError(format!( - "alias cycle detected for '{name}'" - ))); - } - let Some(target) = self.get_alias_variable(name) else { - break; - }; - if let Some((alias_base, alias_index)) = Self::pointer_arithmetic_parts(&target) { - index = alias_index.checked_add(index).ok_or_else(|| { - CodeGenError::TypeError("pointer arithmetic offset overflow".to_string()) - })?; - base = alias_base.clone(); - } else { - base = target; - } - } - - Ok(Some((base, index))) - } - - fn is_dwarf_pointer_or_array_arg(&mut self, expr: &Expr) -> Result { - let Some(var) = self.query_dwarf_for_complex_expr(expr)? else { - return Ok(false); - }; - let Some(ty) = var.dwarf_type.as_ref() else { - return Ok(false); - }; - let ty = ghostscope_dwarf::strip_type_aliases(ty); - Ok(matches!( - ty, - DwarfType::PointerType { .. } | DwarfType::ArrayType { .. } - )) - } - - fn dwarf_integer_comparison_expr(&mut self, expr: &Expr) -> Option { - if let Expr::Cast { target_type, .. } = expr { - return self - .resolve_cast_target_type(target_type) - .ok() - .and_then(|ty| ghostscope_dwarf::c_integer_comparison_type(&ty)); - } - - if let Ok(Some(var)) = self.query_dwarf_for_complex_expr(expr) { - if let Some(ref ty) = var.dwarf_type { - return ghostscope_dwarf::c_integer_comparison_type(ty); - } - } - None - } - - fn integer_comparison_plan_for_exprs( - &mut self, - left: &Expr, - right: &Expr, - ) -> Option { - let left_ty = self.dwarf_integer_comparison_expr(left); - let right_ty = self.dwarf_integer_comparison_expr(right); - if left_ty.is_none() && right_ty.is_none() { - return None; - } - - Some(ghostscope_dwarf::usual_c_arithmetic_comparison_plan( - left_ty.unwrap_or_else(CIntegerComparisonType::signed_i64), - right_ty.unwrap_or_else(CIntegerComparisonType::signed_i64), - )) - } - - pub(super) fn unsigned_ordering_width_for_exprs( - &mut self, - left: &Expr, - right: &Expr, - ) -> Option { - let plan = self.integer_comparison_plan_for_exprs(left, right)?; - if plan.is_unsigned { - Some((plan.size * 8) as u32) - } else { - None - } - } - - pub(super) fn unsigned_shift_width_for_expr(&mut self, expr: &Expr) -> Option { - let c_type = self.dwarf_integer_comparison_expr(expr)?.promoted(); - if c_type.is_unsigned { - Some((c_type.size * 8) as u32) - } else { - None - } - } - - /// Ensure that when an expression refers to a DWARF-backed variable (not via address-of), - /// the variable's DWARF type is a pointer or array (decays to pointer for memcmp/strncmp). - fn ensure_dwarf_pointer_arg(&mut self, e: &Expr, where_ctx: &str) -> Result<()> { - // Allow explicit address-of forms (&expr), which purposefully produce a pointer - if matches!(e, Expr::AddressOf(_)) { - return Ok(()); - } - if let Some((ptr_side, _)) = self.pointer_arithmetic_parts_expanding_aliases(e)? { - if matches!(&ptr_side, Expr::AddressOf(_)) - || self - .is_dwarf_pointer_or_array_arg(&ptr_side) - .unwrap_or(false) - { - return Ok(()); - } - } - if self.is_dynamic_pointer_arithmetic_expr(e)? - || self.expands_to_nonliteral_pointer_arithmetic(e)? - { - return Ok(()); - } - match self.query_dwarf_for_complex_expr(e) { - Ok(Some(var)) => { - let Some(ty) = var.dwarf_type.as_ref() else { - return Err(CodeGenError::TypeError(format!( - "{where_ctx}: DWARF variable has no type information" - ))); - }; - let ty = ghostscope_dwarf::strip_type_aliases(ty); - if !matches!( - ty, - DwarfType::PointerType { .. } | DwarfType::ArrayType { .. } - ) { - return Err(CodeGenError::TypeError(format!( - "{where_ctx}: only pointer or array DWARF variables are supported" - ))); - } - Ok(()) - } - // No DWARF info or analyzer missing: allow script-level pointer values - Ok(None) | Err(_) => match self.compile_expr(e) { - Ok(BasicValueEnum::PointerValue(_)) => Ok(()), - _ => Err(CodeGenError::TypeError(format!( - "{where_ctx}: expression is not a pointer" - ))), - }, - } - } - - fn is_dynamic_pointer_arithmetic_expr(&mut self, expr: &Expr) -> Result { - use crate::script::ast::BinaryOp as BO; - use crate::script::ast::Expr as E; - - let E::BinaryOp { left, op, right } = expr else { - return Ok(false); - }; - - match op { - BO::Add => Ok(self.is_dynamic_indexable_pointer_base(left)? - || self.is_dynamic_indexable_pointer_base(right)?), - BO::Subtract => self.is_dynamic_indexable_pointer_base(left), - _ => Ok(false), - } - } - - fn is_dynamic_indexable_pointer_base(&mut self, expr: &Expr) -> Result { - if matches!(expr, Expr::AddressOf(_)) { - return Ok(true); - } - - if self.cast_index_base(expr)?.is_some() { - return Ok(true); - } - - if self - .query_dwarf_for_complex_expr(expr) - .ok() - .flatten() - .and_then(|var| var.dwarf_type) - .is_some_and(|ty| ghostscope_dwarf::is_c_pointer_or_array_type(&ty)) - { - return Ok(true); - } - - let expanded = self.expand_alias_variable_expr(expr)?; - if matches!(expanded, Expr::AddressOf(_)) { - return Ok(true); - } - let Some((base_expr, _static_index)) = - self.pointer_arithmetic_parts_expanding_aliases(&expanded)? - else { - return Ok(false); - }; - - Ok(self - .query_dwarf_for_complex_expr(&base_expr) - .ok() - .flatten() - .and_then(|var| var.dwarf_type) - .is_some_and(|ty| ghostscope_dwarf::is_c_pointer_or_array_type(&ty))) - } - - fn expands_to_nonliteral_pointer_arithmetic(&mut self, expr: &Expr) -> Result { - let expanded = self.expand_alias_variable_expr(expr)?; - self.is_nonliteral_pointer_arithmetic_expr(&expanded) - } - - fn is_nonliteral_pointer_arithmetic_expr(&mut self, expr: &Expr) -> Result { - use crate::script::ast::BinaryOp as BO; - use crate::script::ast::Expr as E; - - let E::BinaryOp { left, op, right } = expr else { - return Ok(false); - }; - - match op { - BO::Add => { - let left_is_ptr = self.is_dynamic_indexable_pointer_base(left)?; - let right_is_ptr = self.is_dynamic_indexable_pointer_base(right)?; - let left_is_literal = Self::integer_literal_value(left).is_some(); - let right_is_literal = Self::integer_literal_value(right).is_some(); - - if (left_is_ptr && !right_is_ptr && !right_is_literal) - || (right_is_ptr && !left_is_ptr && !left_is_literal) - { - return Ok(true); - } - - Ok(self.expands_to_nonliteral_pointer_arithmetic(left)? - || self.expands_to_nonliteral_pointer_arithmetic(right)?) - } - BO::Subtract => { - let left_is_ptr = self.is_dynamic_indexable_pointer_base(left)?; - let right_is_literal = Self::integer_literal_value(right).is_some(); - - if left_is_ptr && !right_is_literal { - return Ok(true); - } - - self.expands_to_nonliteral_pointer_arithmetic(left) - } - _ => Ok(false), - } - } - - /// Resolve an expression to an i64 pointer value. Accepts integer (address) and pointer values; - /// falls back to DWARF evaluation for complex expressions. - pub(crate) fn resolve_ptr_i64_from_expr( - &mut self, - e: &Expr, - ) -> Result> { - self.resolve_runtime_address_from_expr(e) - .map(|address| address.value) - } - - pub(crate) fn resolve_runtime_address_from_expr( - &mut self, - e: &Expr, - ) -> Result> { - let mut visited = std::collections::HashSet::new(); - self.resolve_runtime_address_from_expr_internal(e, &mut visited, 0) - } - - fn resolve_runtime_address_from_expr_internal( - &mut self, - e: &Expr, - visited: &mut std::collections::HashSet, - depth: usize, - ) -> Result> { - use crate::script::ast::BinaryOp as BO; - use crate::script::ast::Expr as E; - use inkwell::values::BasicValueEnum::*; - const MAX_DEPTH: usize = 64; - if depth > MAX_DEPTH { - return Err(CodeGenError::TypeError( - "alias expansion depth exceeded (cycle?)".into(), - )); - } - if let E::Cast { expr, target_type } = e { - let target_type_info = self.resolve_cast_target_type(target_type)?; - if Self::cast_pointer_target_type(&target_type_info).is_some() { - return self.cast_source_pointer_value(expr); - } - return self.cast_source_memory_address(expr); - } - // Alias variable indirection: resolve its target expression first - if let E::Variable(name) = e { - if self.alias_variable_exists(name) { - if !visited.insert(name.clone()) { - return Err(CodeGenError::TypeError(format!( - "alias cycle detected for '{name}'" - ))); - } - if let Some(target) = self.get_alias_variable(name) { - let r = self.resolve_runtime_address_from_expr_internal( - &target, - visited, - depth + 1, - ); - visited.remove(name); - return r; - } - } - } - // Special-case: explicit address-of must yield a pointer-sized address - if let E::AddressOf(inner) = e { - // Support alias variables transparently: &alias -> address of aliased DWARF expr - let resolved_inner: &E = if let E::Variable(name) = inner.as_ref() { - if self.alias_variable_exists(name) { - // Owned target for query - if let Some(target) = self.get_alias_variable(name) { - if let Some(var) = self.query_dwarf_for_complex_expr(&target)? { - let status_ptr = if self.condition_context_active { - Some(self.get_or_create_cond_error_global()) - } else { - None - }; - let pc_address = self.get_compile_time_context()?.pc_address; - return self.variable_read_plan_to_runtime_address( - &var, pc_address, status_ptr, - ); - } else { - return Err(CodeGenError::TypeError( - "cannot take address of unresolved expression".into(), - )); - } - } else { - return Err(CodeGenError::TypeError( - "cannot take address of unresolved expression".into(), - )); - } - } else { - inner.as_ref() - } - } else { - inner.as_ref() - }; - - if let E::ArrayAccess(array_expr, index_expr) = resolved_inner { - if let Some(element_lvalue) = - self.compile_dynamic_array_element_address(array_expr, index_expr)? - { - return Ok(element_lvalue.address); - } - } - - if let Some(lvalue) = self.dynamic_lvalue_address_and_type(resolved_inner)? { - return Ok(lvalue.address); - } - - if let Some(var) = self.query_dwarf_for_complex_expr(resolved_inner)? { - let status_ptr = if self.condition_context_active { - Some(self.get_or_create_cond_error_global()) - } else { - None - }; - let pc_address = self.get_compile_time_context()?.pc_address; - return self.variable_read_plan_to_runtime_address(&var, pc_address, status_ptr); - } else { - return Err(CodeGenError::TypeError( - "cannot take address of unresolved expression".into(), - )); - } - } - - if let Some(address) = self.dynamic_pointer_arithmetic_address(e)? { - return Ok(address); - } - - if let Some((ptr_side, index)) = self.pointer_arithmetic_parts_expanding_aliases(e)? { - if matches!(&ptr_side, E::AddressOf(_)) { - let base = - self.resolve_runtime_address_from_expr_internal(&ptr_side, visited, depth + 1)?; - let off = self.context.i64_type().const_int(index as u64, false); - let value = self - .builder - .build_int_add(base.value, off, "ptr_add") - .map_err(|err| CodeGenError::Builder(err.to_string()))?; - return Ok(base.with_value(value)); - } else if let Some((element_info, base_address)) = self.cast_index_base(&ptr_side)? { - let index_value = self.context.i64_type().const_int(index as u64, true); - return self - .dynamic_lvalue_from_indexable_base( - element_info, - base_address, - index_value, - "cast_ptr_add", - ) - .map(|lvalue| lvalue.address); - } else if let Some(var) = self.query_dwarf_for_complex_expr(&ptr_side)? { - if var.dwarf_type.is_some() { - let pointed_plan = var - .plan_pointer_element_index(index) - .map_err(|err| CodeGenError::DwarfError(err.to_string()))?; - let status_ptr = if self.condition_context_active { - Some(self.get_or_create_cond_error_global()) - } else { - None - }; - let pc_address = self.get_compile_time_context()?.pc_address; - return self.variable_read_plan_to_runtime_address( - &pointed_plan, - pc_address, - status_ptr, - ); - } - } - } - - // Support constant-offset addressing: (alias_expr + K) or (K + alias_expr) - if let E::BinaryOp { left, op, right } = e { - if matches!(op, BO::Add) { - // alias + K - if let Some(k) = Self::integer_literal_value(right) { - if let Ok(base) = - self.resolve_runtime_address_from_expr_internal(left, visited, depth + 1) - { - let off = self.context.i64_type().const_int(k as u64, false); - let value = self - .builder - .build_int_add(base.value, off, "ptr_add") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(base.with_value(value)); - } - } - // K + alias - if let Some(k) = Self::integer_literal_value(left) { - if let Ok(base) = - self.resolve_runtime_address_from_expr_internal(right, visited, depth + 1) - { - let off = self.context.i64_type().const_int(k as u64, false); - let value = self - .builder - .build_int_add(base.value, off, "ptr_add") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(base.with_value(value)); - } - } - } else if matches!(op, BO::Subtract) { - if let Some(k) = Self::integer_literal_value(right) { - if let Ok(base) = - self.resolve_runtime_address_from_expr_internal(left, visited, depth + 1) - { - let off = self - .context - .i64_type() - .const_int(k.wrapping_neg() as u64, false); - let value = self - .builder - .build_int_add(base.value, off, "ptr_sub") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(base.with_value(value)); - } - } - } - } - // Prefer DWARF-based address resolution first so that array/aggregate - // expressions decay to their base address rather than loading values. - if let Ok(Some(var)) = self.query_dwarf_for_complex_expr(e) { - if let Some(dty) = var.dwarf_type.as_ref() { - let dty = ghostscope_dwarf::strip_type_aliases(dty); - match dty { - DwarfType::PointerType { .. } => { - let pc_address = self.get_compile_time_context()?.pc_address; - let val_any = - self.variable_read_plan_to_llvm_value(&var, pc_address, None)?; - match val_any { - IntValue(iv) => Ok(RuntimeAddress::available(iv, self.context)), - PointerValue(pv) => self - .builder - .build_ptr_to_int(pv, self.context.i64_type(), "ptr_as_i64") - .map(|value| RuntimeAddress::available(value, self.context)) - .map_err(|e| CodeGenError::Builder(e.to_string())), - _ => Err(CodeGenError::TypeError( - "DWARF value is not pointer/integer".into(), - )), - } - } - DwarfType::ArrayType { .. } => { - // Use the base address of the array as pointer - let status_ptr = if self.condition_context_active { - Some(self.get_or_create_cond_error_global()) - } else { - None - }; - let pc_address = self.get_compile_time_context()?.pc_address; - self.variable_read_plan_to_runtime_address(&var, pc_address, status_ptr) - } - _ => Err(CodeGenError::TypeError( - "DWARF value is not pointer/array".into(), - )), - } - } else { - let status_ptr = if self.condition_context_active { - Some(self.get_or_create_cond_error_global()) - } else { - None - }; - let pc_address = self.get_compile_time_context()?.pc_address; - self.variable_read_plan_to_runtime_address(&var, pc_address, status_ptr) - } - } else { - // No DWARF-backed address and not an address-of/alias+const: reject script-level pointers. - Err(CodeGenError::TypeError( - "expression is not a pointer/address".into(), - )) - } - } - - fn dynamic_pointer_arithmetic_address( - &mut self, - expr: &Expr, - ) -> Result>> { - use crate::script::ast::BinaryOp as BO; - use crate::script::ast::Expr as E; - - let E::BinaryOp { left, op, right } = expr else { - return Ok(None); - }; - - match op { - BO::Add => { - if let Some(address) = self.dynamic_raw_address_candidate(left, right, false)? { - return Ok(Some(address)); - } - if let Some(address) = self.dynamic_raw_address_candidate(right, left, false)? { - return Ok(Some(address)); - } - if let Some(address) = self.dynamic_index_address_candidate(left, right)? { - return Ok(Some(address)); - } - self.dynamic_index_address_candidate(right, left) - } - BO::Subtract => { - if let Some(address) = self.dynamic_raw_address_candidate(left, right, true)? { - return Ok(Some(address)); - } - let negative_right = E::BinaryOp { - left: Box::new(E::Int(0)), - op: BO::Subtract, - right: right.clone(), - }; - self.dynamic_index_address_candidate(left, &negative_right) - } - _ => Ok(None), - } - } - - fn dynamic_raw_address_candidate( - &mut self, - base_expr: &Expr, - offset_expr: &Expr, - subtract: bool, - ) -> Result>> { - if Self::integer_literal_value(offset_expr).is_some() { - return Ok(None); - } - - let expanded_base = self.expand_alias_variable_expr(base_expr)?; - if !matches!(expanded_base, Expr::AddressOf(_)) { - return Ok(None); - } - - let base_address = self.resolve_runtime_address_from_expr(&expanded_base)?; - let offset = match self.compile_expr(offset_expr)? { - BasicValueEnum::IntValue(value) => { - self.normalize_int_to_i64(value, "dynamic_raw_offset_i64")? - } - _ => { - return Err(CodeGenError::TypeError( - "raw address offset expression must compile to an integer".to_string(), - )) - } - }; - let offset = if subtract { - self.builder - .build_int_neg(offset, "dynamic_raw_offset_neg") - .map_err(|err| CodeGenError::Builder(err.to_string()))? - } else { - offset - }; - let address = self - .builder - .build_int_add(base_address.value, offset, "dynamic_raw_address") - .map_err(|err| CodeGenError::Builder(err.to_string()))?; - Ok(Some(base_address.with_value(address))) - } - - fn dynamic_index_address_candidate( - &mut self, - base_expr: &Expr, - index_expr: &Expr, - ) -> Result>> { - match self.compile_dynamic_array_element_address(base_expr, index_expr) { - Ok(Some(element_lvalue)) => Ok(Some(element_lvalue.address)), - Ok(None) => Ok(None), - Err(CodeGenError::VariableNotFound(_)) - | Err(CodeGenError::VariableNotInScope(_)) - | Err(CodeGenError::TypeError(_)) => Ok(None), - Err(err) => Err(err), - } - } - - /// Builtin memcmp (boolean variant): returns true iff first `len` bytes equal. - /// Supports dynamic `len` (expr), clamped to [0, compare_cap]. - fn compile_memcmp_builtin( - &mut self, - a_expr: &Expr, - b_expr: &Expr, - len_expr: &Expr, - ) -> Result> { - // Note: constant hex/len validation happens at parse-time; dynamic cases are handled at runtime by masking bytes. - - // Note: do not resolve pointers yet; if either side is hex("...") we will synthesize bytes - - // Compile length expr to i32 and clamp to [0, CAP] - let len_val = self.compile_expr(len_expr)?; - let len_iv = match len_val { - BasicValueEnum::IntValue(iv) => iv, - _ => { - return Err(CodeGenError::TypeError( - "memcmp length must be an integer expression".into(), - )) - } - }; - let i32_ty = self.context.i32_type(); - let len_i32 = if len_iv.get_type().get_bit_width() > 32 { - self.builder - .build_int_truncate(len_iv, i32_ty, "memcmp_len_trunc") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } else if len_iv.get_type().get_bit_width() < 32 { - self.builder - .build_int_z_extend(len_iv, i32_ty, "memcmp_len_zext") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } else { - len_iv - }; - let zero_i32 = i32_ty.const_zero(); - let is_neg = self - .builder - .build_int_compare( - inkwell::IntPredicate::SLT, - len_i32, - zero_i32, - "memcmp_len_neg", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let len_nn = self - .builder - .build_select(is_neg, zero_i32, len_i32, "memcmp_len_nn") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let cap = self.compile_options.compare_cap; - let cap_const = i32_ty.const_int(cap as u64, false); - let gt = self - .builder - .build_int_compare( - inkwell::IntPredicate::UGT, - len_nn, - cap_const, - "memcmp_len_gt", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let sel_len = self - .builder - .build_select(gt, cap_const, len_nn, "memcmp_len_sel") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - - // Fast-path: if effective length is zero, return true without any reads - let len_is_zero = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - sel_len, - i32_ty.const_zero(), - "memcmp_len_is_zero", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let func = self.current_function("compile memcmp length branch")?; - let zero_b = self.context.append_basic_block(func, "memcmp_len_zero"); - let nz_b = self.context.append_basic_block(func, "memcmp_len_nz"); - let cont_b = self.context.append_basic_block(func, "memcmp_len_cont"); - self.builder - .build_conditional_branch(len_is_zero, zero_b, nz_b) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - // Zero-length branch: true - self.builder.position_at_end(zero_b); - let bool_true = self.context.bool_type().const_int(1, false); - self.builder - .build_unconditional_branch(cont_b) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let zero_block = self.current_insert_block("finish memcmp zero-length block")?; - - // Non-zero branch: perform reads and compare - self.builder.position_at_end(nz_b); - - // Prepare static buffers of size CAP for both sides - let (arr_a_ty, buf_a) = self.get_or_create_i8_buffer(cap, "_gs_bi_memcmp_a"); - let (arr_b_ty, buf_b) = self.get_or_create_i8_buffer(cap, "_gs_bi_memcmp_b"); - let ptr_ty = self.context.ptr_type(AddressSpace::default()); - - // Helper: parse hex builtin into bytes - let parse_hex_bytes = |e: &Expr| -> Option> { - if let Expr::BuiltinCall { name, args } = e { - if name == "hex" && args.len() == 1 { - if let Expr::String(s) = &args[0] { - // Parser guarantees only hex digits and even length - if s.is_empty() { - return Some(Vec::new()); - } - let mut out = Vec::with_capacity(s.len() / 2); - let mut i = 0usize; - while i + 1 < s.len() { - let v = u8::from_str_radix(&s[i..i + 2], 16).ok()?; - out.push(v); - i += 2; - } - return Some(out); - } - } - } - None - }; - - // Side A - // If side A is DWARF-backed (and not an explicit address-of), enforce pointer DWARF type - if parse_hex_bytes(a_expr).is_none() { - self.ensure_dwarf_pointer_arg(a_expr, "memcmp arg0")?; - } - let ok_a = if let Some(bytes) = parse_hex_bytes(a_expr) { - let i32_ty = self.context.i32_type(); - let idx0 = i32_ty.const_zero(); - for i in 0..(cap as usize) { - let idx_i = i32_ty.const_int(i as u64, false); - // SAFETY: buf_a is a cap-sized array and i is bounded by cap. - let pa = unsafe { - self.builder - .build_gep(arr_a_ty, buf_a, &[idx0, idx_i], &format!("hex_a_i{i}")) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let byte = if i < bytes.len() { bytes[i] } else { 0 } as u64; - let bv = self.context.i8_type().const_int(byte, false); - self.builder - .build_store(pa, bv) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - } - self.context.bool_type().const_int(1, false) - } else { - // Resolve pointer for A and read from user memory - let ptr_a = self.resolve_runtime_address_from_expr(a_expr)?; - let offsets_found_a = ptr_a.offsets_found; - let dst_a = self - .builder - .build_bit_cast(buf_a, ptr_ty, "memcmp_dst_a") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let base_src_a = self - .builder - .build_int_to_ptr(ptr_a.value, ptr_ty, "memcmp_src_a") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let null_ptr = ptr_ty.const_null(); - let src_a = self - .builder - .build_select::, _>( - offsets_found_a, - base_src_a.into(), - null_ptr.into(), - "memcmp_src_a_or_null", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_pointer_value(); - let zero_i32 = self.context.i32_type().const_zero(); - let effective_len_a = self - .builder - .build_select::, _>( - offsets_found_a, - sel_len.into(), - zero_i32.into(), - "memcmp_len_a_or_zero", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let ret_a = self - .create_bpf_helper_call( - BPF_FUNC_probe_read_user as u64, - &[dst_a, effective_len_a.into(), src_a.into()], - self.context.i64_type().into(), - "probe_read_user_memcmp_a", - )? - .into_int_value(); - let i64_ty = self.context.i64_type(); - let eq_a = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - ret_a, - i64_ty.const_zero(), - "memcmp_ok_a", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder - .build_and(eq_a, offsets_found_a, "memcmp_ok_a") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - - // Side B - if parse_hex_bytes(b_expr).is_none() { - self.ensure_dwarf_pointer_arg(b_expr, "memcmp arg1")?; - } - let ok_b = if let Some(bytes) = parse_hex_bytes(b_expr) { - let i32_ty = self.context.i32_type(); - let idx0 = i32_ty.const_zero(); - for i in 0..(cap as usize) { - let idx_i = i32_ty.const_int(i as u64, false); - // SAFETY: buf_b is a cap-sized array and i is bounded by cap. - let pb = unsafe { - self.builder - .build_gep(arr_b_ty, buf_b, &[idx0, idx_i], &format!("hex_b_i{i}")) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let byte = if i < bytes.len() { bytes[i] } else { 0 } as u64; - let bv = self.context.i8_type().const_int(byte, false); - self.builder - .build_store(pb, bv) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - } - self.context.bool_type().const_int(1, false) - } else { - // Resolve pointer for B and read from user memory - let ptr_b = self.resolve_runtime_address_from_expr(b_expr)?; - let offsets_found_b = ptr_b.offsets_found; - let dst_b = self - .builder - .build_bit_cast(buf_b, ptr_ty, "memcmp_dst_b") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let base_src_b = self - .builder - .build_int_to_ptr(ptr_b.value, ptr_ty, "memcmp_src_b") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let null_ptr = ptr_ty.const_null(); - let src_b = self - .builder - .build_select::, _>( - offsets_found_b, - base_src_b.into(), - null_ptr.into(), - "memcmp_src_b_or_null", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_pointer_value(); - let zero_i32 = self.context.i32_type().const_zero(); - let effective_len_b = self - .builder - .build_select::, _>( - offsets_found_b, - sel_len.into(), - zero_i32.into(), - "memcmp_len_b_or_zero", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let ret_b = self - .create_bpf_helper_call( - BPF_FUNC_probe_read_user as u64, - &[dst_b, effective_len_b.into(), src_b.into()], - self.context.i64_type().into(), - "probe_read_user_memcmp_b", - )? - .into_int_value(); - let i64_ty = self.context.i64_type(); - let eq_b = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - ret_b, - i64_ty.const_zero(), - "memcmp_ok_b", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder - .build_and(eq_b, offsets_found_b, "memcmp_ok_b") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - - let status_ok = self - .builder - .build_and(ok_a, ok_b, "memcmp_status_ok") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - // If in condition context and either side failed, set condition error code = 1 (ProbeReadFailed) - if self.condition_context_active { - let not_a = self - .builder - .build_not(ok_a, "memcmp_fail_a") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let not_b = self - .builder - .build_not(ok_b, "memcmp_fail_b") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let any_fail = self - .builder - .build_or(not_a, not_b, "memcmp_any_fail") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let func = self.current_function("compile memcmp condition error branch")?; - let set_b = self.context.append_basic_block(func, "memcmp_set_err"); - let cont_b = self.context.append_basic_block(func, "memcmp_cont"); - self.builder - .build_conditional_branch(any_fail, set_b, cont_b) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder.position_at_end(set_b); - // Align error_code with VariableStatus::ReadError = 2 - let _ = self.set_condition_error_if_unset(2u8); - // Decide which side failed (prefer recording the actual failing side) - let not_a_val = self - .builder - .build_not(ok_a, "memcmp_fail_a_val") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let not_b_val = self - .builder - .build_not(ok_b, "memcmp_fail_b_val") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let cur_fn = self.current_function("compile memcmp failure address branch")?; - let set_a_bb = self.context.append_basic_block(cur_fn, "set_addr_a"); - let check_b_bb = self.context.append_basic_block(cur_fn, "check_fail_b"); - let set_b_bb = self.context.append_basic_block(cur_fn, "set_addr_b"); - let after_set_bb = self.context.append_basic_block(cur_fn, "after_set_addr"); - - // Branch on A failure first - self.builder - .build_conditional_branch(not_a_val, set_a_bb, check_b_bb) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - // set A address - self.builder.position_at_end(set_a_bb); - if let Some(pa) = match parse_hex_bytes(a_expr) { - Some(_) => None, - None => Some(self.resolve_ptr_i64_from_expr(a_expr)?), - } { - let _ = self.set_condition_error_addr_if_unset(pa); - } - self.builder - .build_unconditional_branch(after_set_bb) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - // check B failure and set B - self.builder.position_at_end(check_b_bb); - self.builder - .build_conditional_branch(not_b_val, set_b_bb, after_set_bb) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder.position_at_end(set_b_bb); - if let Some(pb) = match parse_hex_bytes(b_expr) { - Some(_) => None, - None => Some(self.resolve_ptr_i64_from_expr(b_expr)?), - } { - let _ = self.set_condition_error_addr_if_unset(pb); - } - self.builder - .build_unconditional_branch(after_set_bb) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder.position_at_end(after_set_bb); - // Build flags: bit0=A fail, bit1=B fail, bit2=len clamped, bit3=len<=0 - let i8t = self.context.i8_type(); - let b_a = self - .builder - .build_int_z_extend( - self.builder - .build_not(ok_a, "fa") - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - i8t, - "fa8", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let b_b1 = self - .builder - .build_int_z_extend( - self.builder - .build_not(ok_b, "fb") - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - i8t, - "fb8", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let sh1 = self - .builder - .build_left_shift(b_b1, i8t.const_int(1, false), "b_b_shift") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - // gt: len_nn > cap (len clamped) - let b_c = self - .builder - .build_int_z_extend(gt, i8t, "clamped8") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let sh2 = self - .builder - .build_left_shift(b_c, i8t.const_int(2, false), "b_c_shift") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - // len<=0: reuse len_is_zero - let b_z = self - .builder - .build_int_z_extend(len_is_zero, i8t, "len0_8") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let sh3 = self - .builder - .build_left_shift(b_z, i8t.const_int(3, false), "b_z_shift") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let f01 = self - .builder - .build_or(b_a, sh1, "f01") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let f012 = self - .builder - .build_or(f01, sh2, "f012") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let flags = self - .builder - .build_or(f012, sh3, "flags") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let _ = self.or_condition_error_flags(flags); - self.builder - .build_unconditional_branch(cont_b) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder.position_at_end(cont_b); - } - - // Aggregate XOR/OR across 0..CAP, masked by (i < sel_len) - let i32_ty = self.context.i32_type(); - let idx0 = i32_ty.const_zero(); - let mut acc = self.context.i8_type().const_zero(); - for i in 0..cap as usize { - let idx_i = i32_ty.const_int(i as u64, false); - // active = (i < sel_len) - let active = self - .builder - .build_int_compare( - inkwell::IntPredicate::ULT, - idx_i, - sel_len, - &format!("memcmp_i{i}_active"), - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - // a[i] - // SAFETY: i is bounded by cmp_bound, which is clamped to the buffer cap. - let pa = unsafe { - self.builder - .build_gep(arr_a_ty, buf_a, &[idx0, idx_i], &format!("memcmp_a_i{i}")) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let va = self - .builder - .build_load(self.context.i8_type(), pa, &format!("ld_a_{i}")) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let va = match va { - BasicValueEnum::IntValue(iv) => iv, - _ => return Err(CodeGenError::LLVMError("memcmp load a != i8".into())), - }; - // b[i] - // SAFETY: i is bounded by cmp_bound, which is clamped to the buffer cap. - let pb = unsafe { - self.builder - .build_gep(arr_b_ty, buf_b, &[idx0, idx_i], &format!("memcmp_b_i{i}")) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let vb = self - .builder - .build_load(self.context.i8_type(), pb, &format!("ld_b_{i}")) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let vb = match vb { - BasicValueEnum::IntValue(iv) => iv, - _ => return Err(CodeGenError::LLVMError("memcmp load b != i8".into())), - }; - let diff = self - .builder - .build_xor(va, vb, &format!("memcmp_diff_{i}")) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let zero8 = self.context.i8_type().const_zero(); - let masked = self - .builder - .build_select(active, diff, zero8, &format!("memcmp_masked_{i}")) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - acc = self - .builder - .build_or(acc, masked, &format!("memcmp_acc_{i}")) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - } - let eq_bytes = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - acc, - self.context.i8_type().const_zero(), - "memcmp_acc_zero", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let nz_result = self - .builder - .build_and(status_ok, eq_bytes, "memcmp_and") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - self.builder - .build_unconditional_branch(cont_b) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let nz_block = self.current_insert_block("finish memcmp non-zero block")?; - - // Merge - self.builder.position_at_end(cont_b); - let phi = self - .builder - .build_phi(self.context.bool_type(), "memcmp_phi") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - phi.add_incoming(&[(&bool_true, zero_block), (&nz_result, nz_block)]); - Ok(phi.as_basic_value()) - } - /// Builtin strncmp/starts_with implementation: bounded byte-compare without NUL requirement. - fn compile_bounded_compare_len_i32( - &mut self, - len_expr: &Expr, - max_len: u32, - name_prefix: &str, - ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { - let len_val = self.compile_expr(len_expr)?; - let len_iv = match len_val { - BasicValueEnum::IntValue(iv) => iv, - _ => { - return Err(CodeGenError::TypeError(format!( - "{name_prefix} length must be an integer expression" - ))) - } - }; - let i32_ty = self.context.i32_type(); - let len_i32 = if len_iv.get_type().get_bit_width() > 32 { - self.builder - .build_int_truncate(len_iv, i32_ty, &format!("{name_prefix}_len_trunc")) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } else if len_iv.get_type().get_bit_width() < 32 { - self.builder - .build_int_z_extend(len_iv, i32_ty, &format!("{name_prefix}_len_zext")) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } else { - len_iv - }; - let zero_i32 = i32_ty.const_zero(); - let is_neg = self - .builder - .build_int_compare( - inkwell::IntPredicate::SLT, - len_i32, - zero_i32, - &format!("{name_prefix}_len_neg"), - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let len_nn = self - .builder - .build_select(is_neg, zero_i32, len_i32, &format!("{name_prefix}_len_nn")) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let max_const = i32_ty.const_int(max_len as u64, false); - let gt = self - .builder - .build_int_compare( - inkwell::IntPredicate::UGT, - len_nn, - max_const, - &format!("{name_prefix}_len_gt"), - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let bounded_len = self - .builder - .build_select(gt, max_const, len_nn, &format!("{name_prefix}_len_sel")) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let is_zero = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - bounded_len, - zero_i32, - &format!("{name_prefix}_len_is_zero"), - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok((bounded_len, is_zero)) - } - - fn compile_strncmp_builtin( - &mut self, - dwarf_expr: &Expr, - lit: &str, - n_expr: &Expr, - ) -> Result> { - // Fast path: if the first argument is a script string variable or a string literal, - // perform a compile-time bounded comparison and return a constant boolean. - let immediate_bytes_opt = match dwarf_expr { - Expr::Variable(name) => { - if self - .get_variable_type(name) - .is_some_and(|t| matches!(t, crate::script::VarType::String)) - { - self.get_string_variable_bytes(name).cloned() - } else { - None - } - } - Expr::String(s) => { - let mut b = s.as_bytes().to_vec(); - b.push(0); - Some(b) - } - _ => None, - }; - - if let Some(bytes) = immediate_bytes_opt { - if let Expr::Int(n) = n_expr { - let n_usize = std::cmp::min( - (*n).max(0) as usize, - self.compile_options.compare_cap as usize, - ); - let content_len = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len()); - let cmp_len = std::cmp::min(n_usize, std::cmp::min(content_len, lit.len())); - let equal = bytes.get(0..cmp_len).unwrap_or(&[]) - == lit.as_bytes().get(0..cmp_len).unwrap_or(&[]); - let bool_val = self - .context - .bool_type() - .const_int(if equal { 1 } else { 0 }, false); - return Ok(bool_val.into()); - } - - // Treat as bounded byte compare between two immediate strings - let cap = self.compile_options.compare_cap as usize; - let content_len = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len()); - let cmp_bound = std::cmp::min(cap, std::cmp::min(content_len, lit.len())) as u32; - if cmp_bound == 0 { - return Ok(self.context.bool_type().const_int(1, false).into()); - } - let (bounded_len, _len_is_zero) = - self.compile_bounded_compare_len_i32(n_expr, cmp_bound, "strncmp")?; - let i32_ty = self.context.i32_type(); - let i8_ty = self.context.i8_type(); - let mut acc = i8_ty.const_zero(); - for (i, (byte, lit_byte)) in bytes - .iter() - .copied() - .zip(lit.as_bytes().iter().copied()) - .take(cmp_bound as usize) - .enumerate() - { - let active = self - .builder - .build_int_compare( - inkwell::IntPredicate::UGT, - bounded_len, - i32_ty.const_int(i as u64, false), - "strncmp_imm_active", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let diff = i8_ty.const_int((byte ^ lit_byte) as u64, false); - let active_diff = self - .builder - .build_select(active, diff, i8_ty.const_zero(), "strncmp_imm_diff") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - acc = self - .builder - .build_or(acc, active_diff, "strncmp_imm_acc") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - } - let equal = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - acc, - i8_ty.const_zero(), - "strncmp_imm_eq", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(equal.into()); - } - - // Determine pointer value (i64) of the target memory (DWARF or alias) - // Prefer DWARF resolution for richer status/hints; fallback to generic pointer resolver. - let ptr_i64 = match self.query_dwarf_for_complex_expr(dwarf_expr)? { - Some(var) => { - if let Some(ty) = var.dwarf_type.as_ref() { - let ty = ghostscope_dwarf::strip_type_aliases(ty); - match ty { - DwarfType::PointerType { .. } => { - let pc_address = self.get_compile_time_context()?.pc_address; - let val_any = - self.variable_read_plan_to_llvm_value(&var, pc_address, None)?; - match val_any { - BasicValueEnum::IntValue(iv) => { - RuntimeAddress::available(iv, self.context) - } - BasicValueEnum::PointerValue(pv) => self - .builder - .build_ptr_to_int(pv, self.context.i64_type(), "ptr_as_i64") - .map(|value| RuntimeAddress::available(value, self.context)) - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - _ => { - return Err(CodeGenError::TypeError( - "strncmp requires pointer/integer value for pointer; got unsupported DWARF value".into(), - )) - } - } - } - DwarfType::ArrayType { .. } => { - let status_ptr = if self.condition_context_active { - Some(self.get_or_create_cond_error_global()) - } else { - None - }; - let pc_address = self.get_compile_time_context()?.pc_address; - self.variable_read_plan_to_runtime_address( - &var, pc_address, status_ptr, - )? - } - _ => { - // Not a pointer/array -> treat as error - return Err(CodeGenError::TypeError( - "strncmp requires the non-string side to be an address expression (pointer/array)".into(), - )); - } - } - } else { - return Err(CodeGenError::TypeError( - "strncmp non-string side lacks DWARF type info".into(), - )); - } - } - None => { - // Generic pointer expr (e.g., alias); resolve to i64 - self.resolve_runtime_address_from_expr(dwarf_expr).map_err(|_| { - CodeGenError::TypeError( - "strncmp requires at least one string argument, and the other side must be an address expression (DWARF pointer/array or alias)".to_string(), - ) - })? - } - }; - - let cap = self.compile_options.compare_cap; - let cmp_bound = std::cmp::min(lit.len() as u32, cap); - if cmp_bound == 0 { - return Ok(self.context.bool_type().const_int(1, false).into()); - } - let (bounded_len, len_is_zero) = - self.compile_bounded_compare_len_i32(n_expr, cmp_bound, "strncmp")?; - - let func = self.current_function("compile strncmp length branch")?; - let zero_b = self.context.append_basic_block(func, "strncmp_len_zero"); - let nz_b = self.context.append_basic_block(func, "strncmp_len_nz"); - let final_b = self.context.append_basic_block(func, "strncmp_len_cont"); - self.builder - .build_conditional_branch(len_is_zero, zero_b, nz_b) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - self.builder.position_at_end(zero_b); - let bool_true = self.context.bool_type().const_int(1, false); - self.builder - .build_unconditional_branch(final_b) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let zero_block = self.current_insert_block("finish strncmp zero-length block")?; - - self.builder.position_at_end(nz_b); - - let (arr_ty, buf_global) = self.get_or_create_i8_buffer(cmp_bound, "_gs_bi_strncmp"); - let ptr_ty = self.context.ptr_type(AddressSpace::default()); - let dst_ptr = self - .builder - .build_bit_cast(buf_global, ptr_ty, "strncmp_dst_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let base_src_ptr = self - .builder - .build_int_to_ptr(ptr_i64.value, ptr_ty, "strncmp_src_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let src_ptr = self - .builder - .build_select::, _>( - ptr_i64.offsets_found, - base_src_ptr.into(), - ptr_ty.const_null().into(), - "strncmp_src_or_null", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_pointer_value(); - let effective_len = self - .builder - .build_select::, _>( - ptr_i64.offsets_found, - bounded_len.into(), - self.context.i32_type().const_zero().into(), - "strncmp_len_or_zero", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let ret = self - .create_bpf_helper_call( - BPF_FUNC_probe_read_user as u64, - &[dst_ptr, effective_len.into(), src_ptr.into()], - self.context.i64_type().into(), - "probe_read_user_strncmp", - )? - .into_int_value(); - let read_ok = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - ret, - self.context.i64_type().const_zero(), - "rd_ok", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let status_ok = self - .builder - .build_and(read_ok, ptr_i64.offsets_found, "strncmp_ok_with_offsets") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - // If in condition context and read failed, set condition error code = 1 (ProbeReadFailed) - if self.condition_context_active { - let func = self.current_function("compile strncmp condition error branch")?; - let set_b = self.context.append_basic_block(func, "strncmp_set_err"); - let cont_b = self.context.append_basic_block(func, "strncmp_cont"); - let not_ok = self - .builder - .build_not(status_ok, "rd_fail") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder - .build_conditional_branch(not_ok, set_b, cont_b) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder.position_at_end(set_b); - // VariableStatus::ReadError = 2 - let _ = self.set_condition_error_if_unset(2u8); - let _ = self.set_condition_error_addr_if_unset(ptr_i64.value); - // flags: bit0 = read failure for strncmp - let one = self.context.i8_type().const_int(1, false); - let _ = self.or_condition_error_flags(one); - self.builder - .build_unconditional_branch(cont_b) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder.position_at_end(cont_b); - } - - // XOR/OR accumulation over the bounded maximum; inactive bytes do not contribute. - let i32_ty = self.context.i32_type(); - let idx0 = i32_ty.const_zero(); - let mut acc = self.context.i8_type().const_zero(); - for (i, b) in lit.as_bytes().iter().take(cmp_bound as usize).enumerate() { - let idx_i = i32_ty.const_int(i as u64, false); - // SAFETY: i is bounded by cmp_bound, which is clamped to the buffer cap. - let ptr_i = unsafe { - self.builder - .build_gep(arr_ty, buf_global, &[idx0, idx_i], "ch_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let ch = self - .builder - .build_load(self.context.i8_type(), ptr_i, "ch") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let ch = match ch { - BasicValueEnum::IntValue(iv) => iv, - _ => return Err(CodeGenError::LLVMError("load did not return i8".into())), - }; - let expect = self.context.i8_type().const_int(*b as u64, false); - let diff = self - .builder - .build_xor(ch, expect, "diff") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let active = self - .builder - .build_int_compare( - inkwell::IntPredicate::UGT, - bounded_len, - idx_i, - "strncmp_byte_active", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let diff = self - .builder - .build_select( - active, - diff, - self.context.i8_type().const_zero(), - "strncmp_active_diff", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - acc = self - .builder - .build_or(acc, diff, "acc_or") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - } - let eq_bytes = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - acc, - self.context.i8_type().const_zero(), - "acc_zero", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - let result = self - .builder - .build_and(status_ok, eq_bytes, "strncmp_and") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder - .build_unconditional_branch(final_b) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let nz_block = self.current_insert_block("finish strncmp non-zero block")?; - - self.builder.position_at_end(final_b); - let result_phi = self - .builder - .build_phi(self.context.bool_type(), "strncmp_result") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - result_phi.add_incoming(&[(&bool_true, zero_block), (&result, nz_block)]); - Ok(result_phi.as_basic_value()) - } - /// Compile an expression - pub fn compile_expr(&mut self, expr: &Expr) -> Result> { - match expr { - Expr::Int(value) => { - // Treat script integer literals as signed i64 constants - let int_value = self.context.i64_type().const_int(*value as u64, true); - debug!( - "compile_expr: Int literal {} compiled to IntValue with bit width {}", - value, - int_value.get_type().get_bit_width() - ); - Ok(int_value.into()) - } - Expr::Float(_value) => Err(CodeGenError::TypeError( - "Floating point expressions are not supported".to_string(), - )), - Expr::String(value) => { - // Create string constant using a simpler approach - let string_value = self.context.const_string(value.as_bytes(), true); - let global = self - .module - .add_global(string_value.get_type(), None, "str_const"); - global.set_initializer(&string_value); - - let ptr_type = self.context.ptr_type(AddressSpace::default()); - let cast_ptr = self - .builder - .build_bit_cast(global.as_pointer_value(), ptr_type, "str_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(cast_ptr) - } - Expr::Bool(value) => { - // Represent booleans as i1 for logical/compare consistency - let b = self - .context - .bool_type() - .const_int(if *value { 1 } else { 0 }, false); - Ok(b.into()) - } - Expr::UnaryNot(inner) => { - // Compile operand to integer and compare EQ to zero to produce boolean not - let v = self.compile_expr(inner)?; - let iv = match v { - BasicValueEnum::IntValue(iv) => iv, - _ => { - return Err(CodeGenError::TypeError( - "Logical NOT requires integer/boolean operand".to_string(), - )) - } - }; - let zero = iv.get_type().const_zero(); - let res = self - .builder - .build_int_compare(inkwell::IntPredicate::EQ, iv, zero, "not_eq0") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(res.into()) - } - Expr::UnaryBitNot(inner) => { - let unsigned_width = self - .dwarf_integer_comparison_expr(inner) - .map(CIntegerComparisonType::promoted) - .and_then(|integer_type| { - integer_type - .is_unsigned - .then_some((integer_type.size * 8) as u32) - }); - let v = self.compile_expr(inner)?; - let iv = match v { - BasicValueEnum::IntValue(iv) => iv, - _ => { - return Err(CodeGenError::TypeError( - "Bitwise NOT requires integer/boolean operand".to_string(), - )) - } - }; - if let Some(bit_width) = unsigned_width { - let iv = - self.normalize_int_for_unsigned_compare(iv, bit_width, "bitnot_unsigned")?; - let result = self - .builder - .build_xor(iv, iv.get_type().const_all_ones(), "bitnot") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return self - .zero_extend_int_to_i64_if_needed(result, "bitnot_zext_i64") - .map(|value| value.into()); - } - let iv = if iv.get_type().get_bit_width() == 1 { - self.builder - .build_int_z_extend(iv, self.context.i64_type(), "bitnot_bool_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } else { - iv - }; - let all_ones = iv.get_type().const_all_ones(); - let result = self - .builder - .build_xor(iv, all_ones, "bitnot") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - Expr::Variable(var_name) => { - debug!("compile_expr: Compiling variable expression: {}", var_name); - - // First: DWARF alias variable takes precedence - if self.alias_variable_exists(var_name) { - debug!( - "compile_expr: '{}' is an alias variable; resolving to runtime address", - var_name - ); - let aliased = self - .get_alias_variable(var_name) - .expect("alias existence just checked"); - // Resolve to i64 address then cast to ptr - let addr_i64 = self.resolve_ptr_i64_from_expr(&aliased)?; - let ptr_ty = self.context.ptr_type(AddressSpace::default()); - let as_ptr = self - .builder - .build_int_to_ptr(addr_i64, ptr_ty, "alias_as_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(as_ptr.into()); - } - - // Then check if it's a concrete script-defined variable - if self.variable_exists(var_name) { - debug!("compile_expr: Found script variable: {}", var_name); - let loaded_value = self.load_variable(var_name)?; - debug!( - "compile_expr: Loaded variable '{}' with type: {:?}", - var_name, - loaded_value.get_type() - ); - match &loaded_value { - BasicValueEnum::IntValue(iv) => debug!( - "compile_expr: Variable '{}' is IntValue with bit width {}", - var_name, - iv.get_type().get_bit_width() - ), - BasicValueEnum::FloatValue(_) => { - debug!("compile_expr: Variable '{}' is FloatValue", var_name) - } - BasicValueEnum::PointerValue(_) => { - debug!("compile_expr: Variable '{}' is PointerValue", var_name) - } - _ => debug!("compile_expr: Variable '{}' is other type", var_name), - } - return Ok(loaded_value); - } - - // If not found in script variables nor alias map, try DWARF variables - debug!( - "Variable '{}' not found in script variables, checking DWARF", - var_name - ); - // If not a DWARF variable either, treat as out-of-scope script name for friendliness - match self.query_dwarf_for_variable(var_name) { - Ok(Some(_)) => self.compile_dwarf_expression(expr), - Ok(None) => Err(CodeGenError::VariableNotInScope(var_name.clone())), - Err(e) => Err(CodeGenError::DwarfError(e.to_string())), - } - } - Expr::SpecialVar(name) => { - // Accept both "$pid" and "pid" forms from the parser - let sanitized = name.trim_start_matches('$'); - self.handle_special_variable(sanitized) - } - Expr::BuiltinCall { name, args } => match self.plan_builtin_call(name, args)? { - BuiltinCallPlan::Memcmp => { - self.compile_memcmp_builtin(&args[0], &args[1], &args[2]) - } - BuiltinCallPlan::Strncmp => { - // Accept string on either side: string literal or script string variable - fn extract_script_string( - this: &mut EbpfContext<'_, '_>, - e: &Expr, - ) -> Option { - match e { - Expr::String(s) => Some(s.clone()), - Expr::Variable(name) => this - .get_variable_type(name) - .is_some_and(|t| matches!(t, crate::script::VarType::String)) - .then(|| { - this.get_string_variable_bytes(name).map(|b| { - let cut = b.iter().position(|&x| x == 0).unwrap_or(b.len()); - String::from_utf8_lossy(&b[..cut]).to_string() - }) - }) - .flatten(), - _ => None, - } - } - let left_str = extract_script_string(self, &args[0]); - let right_str = extract_script_string(self, &args[1]); - match (left_str, right_str) { - (Some(ls), Some(rs)) => { - let left_expr = Expr::String(ls); - self.compile_strncmp_builtin(&left_expr, &rs, &args[2]) - } - (Some(ls), None) => self.compile_strncmp_builtin(&args[1], &ls, &args[2]), - (None, Some(rs)) => self.compile_strncmp_builtin(&args[0], &rs, &args[2]), - (None, None) => Err(CodeGenError::TypeError( - "strncmp requires at least one string argument (string literal or script string variable) as the first or second parameter".into(), - )), - } - } - BuiltinCallPlan::StartsWith => { - // Accept string on either side (literal or script string var) - fn extract_script_string( - this: &mut EbpfContext<'_, '_>, - e: &Expr, - ) -> Option { - match e { - Expr::String(s) => Some(s.clone()), - Expr::Variable(name) => this - .get_variable_type(name) - .is_some_and(|t| matches!(t, crate::script::VarType::String)) - .then(|| { - this.get_string_variable_bytes(name).map(|b| { - let cut = b.iter().position(|&x| x == 0).unwrap_or(b.len()); - String::from_utf8_lossy(&b[..cut]).to_string() - }) - }) - .flatten(), - _ => None, - } - } - let s0 = extract_script_string(self, &args[0]); - let s1 = extract_script_string(self, &args[1]); - match (s0, s1) { - (Some(a), Some(b)) => { - // both strings -> compile-time fold - let ok = a.as_bytes().starts_with(b.as_bytes()); - let bv = self.context.bool_type().const_int(ok as u64, false); - Ok(bv.into()) - } - (Some(a), None) => { - let n_expr = Expr::Int(a.len() as i64); - self.compile_strncmp_builtin(&args[1], &a, &n_expr) - } - (None, Some(b)) => { - let n_expr = Expr::Int(b.len() as i64); - self.compile_strncmp_builtin(&args[0], &b, &n_expr) - } - (None, None) => Err(CodeGenError::TypeError( - "starts_with requires at least one string argument (string literal or script string variable) as the first or second parameter".into(), - )), - } - } - }, - Expr::BinaryOp { left, op, right } => { - let binary_plan = self.plan_binary_expr(left, op, right)?; - if let BinaryEmitKind::StringComparison(string_plan) = &binary_plan.emit_kind { - let other = if string_plan.literal_on_left { - right.as_ref() - } else { - left.as_ref() - }; - return self.compile_string_comparison( - other, - &string_plan.literal, - string_plan.equal, - ); - } - // Implement short-circuit for logical OR (||) and logical AND (&&) - if matches!(&binary_plan.emit_kind, BinaryEmitKind::LogicalOr) { - // Evaluate LHS to boolean (non-zero => true). Accept integer or pointer. - let lhs_val = self.compile_expr(left)?; - let lhs_int = match lhs_val { - BasicValueEnum::IntValue(iv) => iv, - BasicValueEnum::PointerValue(pv) => self - .builder - .build_ptr_to_int(pv, self.context.i64_type(), "lor_lhs_ptr_as_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - _ => { - return Err(CodeGenError::TypeError( - "Logical OR requires integer or pointer operands".to_string(), - )) - } - }; - let lhs_zero = lhs_int.get_type().const_zero(); - let lhs_bool = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - lhs_int, - lhs_zero, - "lor_lhs_nz", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - // Prepare control flow blocks - let curr_block = self.builder.get_insert_block().ok_or_else(|| { - CodeGenError::LLVMError("No current basic block".to_string()) - })?; - let func = curr_block - .get_parent() - .ok_or_else(|| CodeGenError::LLVMError("No parent function".to_string()))?; - let rhs_block = self.context.append_basic_block(func, "lor_rhs"); - let merge_block = self.context.append_basic_block(func, "lor_merge"); - - // If lhs is true, jump directly to merge (short-circuit) - self.builder - .build_conditional_branch(lhs_bool, merge_block, rhs_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - // RHS path: compute boolean only if needed - self.builder.position_at_end(rhs_block); - let rhs_val = self.compile_expr(right)?; - let rhs_int = match rhs_val { - BasicValueEnum::IntValue(iv) => iv, - BasicValueEnum::PointerValue(pv) => self - .builder - .build_ptr_to_int(pv, self.context.i64_type(), "lor_rhs_ptr_as_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - _ => { - return Err(CodeGenError::TypeError( - "Logical OR requires integer or pointer operands".to_string(), - )) - } - }; - let rhs_zero = rhs_int.get_type().const_zero(); - let rhs_bool = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - rhs_int, - rhs_zero, - "lor_rhs_nz", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - // Capture the actual block where RHS computation ended - let rhs_end_block = self.builder.get_insert_block().ok_or_else(|| { - CodeGenError::LLVMError("No current basic block after RHS".to_string()) - })?; - self.builder - .build_unconditional_branch(merge_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - // Merge: phi of i1: true from LHS-true, RHS bool from rhs_block - self.builder.position_at_end(merge_block); - let i1 = self.context.bool_type(); - let phi = self - .builder - .build_phi(i1, "lor_phi") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let one = i1.const_int(1, false); - phi.add_incoming(&[(&one, curr_block), (&rhs_bool, rhs_end_block)]); - return Ok(phi.as_basic_value()); - } else if matches!(&binary_plan.emit_kind, BinaryEmitKind::LogicalAnd) { - // Evaluate LHS to boolean (non-zero => true). Accept integer or pointer. - let lhs_val = self.compile_expr(left)?; - let lhs_int = match lhs_val { - BasicValueEnum::IntValue(iv) => iv, - BasicValueEnum::PointerValue(pv) => self - .builder - .build_ptr_to_int(pv, self.context.i64_type(), "land_lhs_ptr_as_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - _ => { - return Err(CodeGenError::TypeError( - "Logical AND requires integer or pointer operands".to_string(), - )) - } - }; - let lhs_zero = lhs_int.get_type().const_zero(); - let lhs_bool = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - lhs_int, - lhs_zero, - "land_lhs_nz", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - // Prepare control flow: if lhs is true, evaluate rhs; else short-circuit to false - let curr_block = self.builder.get_insert_block().ok_or_else(|| { - CodeGenError::LLVMError("No current basic block".to_string()) - })?; - let func = curr_block - .get_parent() - .ok_or_else(|| CodeGenError::LLVMError("No parent function".to_string()))?; - let rhs_block = self.context.append_basic_block(func, "land_rhs"); - let merge_block = self.context.append_basic_block(func, "land_merge"); - - // If lhs is true, go compute rhs; else jump to merge with false - self.builder - .build_conditional_branch(lhs_bool, rhs_block, merge_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - // RHS path - self.builder.position_at_end(rhs_block); - let rhs_val = self.compile_expr(right)?; - let rhs_int = match rhs_val { - BasicValueEnum::IntValue(iv) => iv, - BasicValueEnum::PointerValue(pv) => self - .builder - .build_ptr_to_int(pv, self.context.i64_type(), "land_rhs_ptr_as_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - _ => { - return Err(CodeGenError::TypeError( - "Logical AND requires integer or pointer operands".to_string(), - )) - } - }; - let rhs_zero = rhs_int.get_type().const_zero(); - let rhs_bool = self - .builder - .build_int_compare( - inkwell::IntPredicate::NE, - rhs_int, - rhs_zero, - "land_rhs_nz", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let rhs_end_block = self.builder.get_insert_block().ok_or_else(|| { - CodeGenError::LLVMError("No current basic block after RHS".to_string()) - })?; - self.builder - .build_unconditional_branch(merge_block) - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - - // Merge: phi(i1) with false from LHS=false path, RHS bool from rhs path - self.builder.position_at_end(merge_block); - let i1 = self.context.bool_type(); - let phi = self - .builder - .build_phi(i1, "land_phi") - .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; - let zero = i1.const_zero(); - phi.add_incoming(&[(&rhs_bool, rhs_end_block), (&zero, curr_block)]); - return Ok(phi.as_basic_value()); - } - - // Default eager evaluation for other binary ops - let left_val = self.compile_expr(left)?; - let right_val = self.compile_expr(right)?; - self.compile_binary_op_with_ordering( - left_val, - binary_plan.op, - right_val, - binary_plan.integer_semantics, - ) - } - Expr::MemberAccess(_, _) => { - // Use unified DWARF expression compilation - self.compile_dwarf_expression(expr) - } - Expr::PointerDeref(_) => { - // Use unified DWARF expression compilation - self.compile_dwarf_expression(expr) - } - Expr::AddressOf(inner) => { - // Address-of computes runtime addresses using the module origin carried by the plan. - // Transparently support alias variables: &alias -> address of aliased DWARF expression - let target_inner: &Expr = if let Expr::Variable(var_name) = inner.as_ref() { - if self.alias_variable_exists(var_name) { - // Use the aliased target expression (by-value) and query DWARF on it - let aliased = self - .get_alias_variable(var_name) - .expect("alias existence just checked"); - let var = - self.query_dwarf_for_complex_expr(&aliased)? - .ok_or_else(|| { - super::context::CodeGenError::TypeError( - "cannot take address of unresolved expression".to_string(), - ) - })?; - let pc_address = self.get_compile_time_context()?.pc_address; - match self.variable_read_plan_to_runtime_address(&var, pc_address, None) { - Ok(address) => { - let ptr_ty = self.context.ptr_type(AddressSpace::default()); - let as_ptr = self - .builder - .build_int_to_ptr(address.value, ptr_ty, "addr_as_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(as_ptr.into()); - } - Err(_) => { - return Err(super::context::CodeGenError::TypeError( - "cannot take address of rvalue".to_string(), - )); - } - } - } else { - inner.as_ref() - } - } else { - inner.as_ref() - }; - - if let Some(lvalue) = self.dynamic_lvalue_address_and_type(target_inner)? { - let ptr_ty = self.context.ptr_type(AddressSpace::default()); - let as_ptr = self - .builder - .build_int_to_ptr(lvalue.address.value, ptr_ty, "addr_as_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(as_ptr.into()); - } - - let var = self - .query_dwarf_for_complex_expr(target_inner)? - .ok_or_else(|| { - super::context::CodeGenError::TypeError( - "cannot take address of unresolved expression".to_string(), - ) - })?; - let pc_address = self.get_compile_time_context()?.pc_address; - match self.variable_read_plan_to_runtime_address(&var, pc_address, None) { - Ok(address) => { - let ptr_ty = self.context.ptr_type(AddressSpace::default()); - let as_ptr = self - .builder - .build_int_to_ptr(address.value, ptr_ty, "addr_as_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(as_ptr.into()) - } - Err(_) => Err(super::context::CodeGenError::TypeError( - "cannot take address of rvalue".to_string(), - )), - } - } - Expr::ArrayAccess(_, _) => { - // Use unified DWARF expression compilation - self.compile_dwarf_expression(expr) - } - Expr::Cast { - expr: inner, - target_type, - } => self.compile_cast_expr_value(inner, target_type), - Expr::ChainAccess(_) => { - // Use unified DWARF expression compilation - self.compile_dwarf_expression(expr) - } - } - } - - /// Handle special variables like $pid, $tid, etc. - pub fn handle_special_variable(&mut self, name: &str) -> Result> { - match self.plan_special_variable(name)? { - SpecialVarPlan::Pid => { - let (pid, _tid) = self.get_special_pid_tid_values()?; - Ok(pid.into()) - } - SpecialVarPlan::Tid => { - let (_pid, tid) = self.get_special_pid_tid_values()?; - Ok(tid.into()) - } - SpecialVarPlan::HostPid => { - let (host_pid, _host_tid) = self.get_host_pid_tid_values()?; - let host_pid = self - .builder - .build_int_z_extend(host_pid, self.context.i64_type(), "selected_host_pid") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(host_pid.into()) - } - SpecialVarPlan::InputPid => { - let input_pid = self.compile_options.input_pid.ok_or_else(|| { - CodeGenError::NotImplemented( - "Special variable '$input_pid' is only available in -p mode".to_string(), - ) - })?; - Ok(self - .context - .i64_type() - .const_int(input_pid as u64, false) - .into()) - } - SpecialVarPlan::Timestamp => { - // Use BPF helper to get current timestamp - let ts = self.get_current_timestamp()?; - Ok(ts.into()) - } - SpecialVarPlan::Pc => self.load_special_register_value(16), - SpecialVarPlan::Sp => self.load_special_register_value(7), - } - } - - fn load_special_register_value(&mut self, dwarf_reg: u16) -> Result> { - let pt_regs = self.get_pt_regs_parameter()?; - self.load_register_value(dwarf_reg, pt_regs) - } - - /// Compile binary operations - pub fn compile_binary_op( - &mut self, - left: BasicValueEnum<'ctx>, - op: BinaryOp, - right: BasicValueEnum<'ctx>, - ) -> Result> { - self.compile_binary_op_with_ordering(left, op, right, BinaryIntegerSemantics::default()) - } - - pub(crate) fn build_signed_int_div_via_udiv( - &mut self, - left: IntValue<'ctx>, - right: IntValue<'ctx>, - name: &str, - ) -> Result> { - let int_type = left.get_type(); - let zero = int_type.const_zero(); - let left_is_neg = self - .builder - .build_int_compare(inkwell::IntPredicate::SLT, left, zero, "sdiv_lhs_neg") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let right_is_neg = self - .builder - .build_int_compare(inkwell::IntPredicate::SLT, right, zero, "sdiv_rhs_neg") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let neg_left = self - .builder - .build_int_sub(zero, left, "sdiv_lhs_negated") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let neg_right = self - .builder - .build_int_sub(zero, right, "sdiv_rhs_negated") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let abs_left = self - .builder - .build_select::, _>( - left_is_neg, - neg_left.into(), - left.into(), - "sdiv_lhs_abs", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let abs_right = self - .builder - .build_select::, _>( - right_is_neg, - neg_right.into(), - right.into(), - "sdiv_rhs_abs", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let abs_quotient = self - .builder - .build_int_unsigned_div(abs_left, abs_right, "sdiv_abs_udiv") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let negative_result = self - .builder - .build_xor(left_is_neg, right_is_neg, "sdiv_result_neg") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let neg_quotient = self - .builder - .build_int_sub(zero, abs_quotient, "sdiv_negated") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder - .build_select::, _>( - negative_result, - neg_quotient.into(), - abs_quotient.into(), - name, - ) - .map_err(|e| CodeGenError::Builder(e.to_string())) - .map(|value| value.into_int_value()) - } - - pub(crate) fn build_signed_int_rem_via_urem( - &mut self, - left: IntValue<'ctx>, - right: IntValue<'ctx>, - name: &str, - ) -> Result> { - let int_type = left.get_type(); - let zero = int_type.const_zero(); - let left_is_neg = self - .builder - .build_int_compare(inkwell::IntPredicate::SLT, left, zero, "srem_lhs_neg") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let right_is_neg = self - .builder - .build_int_compare(inkwell::IntPredicate::SLT, right, zero, "srem_rhs_neg") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let neg_left = self - .builder - .build_int_sub(zero, left, "srem_lhs_negated") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let neg_right = self - .builder - .build_int_sub(zero, right, "srem_rhs_negated") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let abs_left = self - .builder - .build_select::, _>( - left_is_neg, - neg_left.into(), - left.into(), - "srem_lhs_abs", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let abs_right = self - .builder - .build_select::, _>( - right_is_neg, - neg_right.into(), - right.into(), - "srem_rhs_abs", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - .into_int_value(); - let abs_remainder = self - .builder - .build_int_unsigned_rem(abs_left, abs_right, "srem_abs_urem") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let neg_remainder = self - .builder - .build_int_sub(zero, abs_remainder, "srem_negated") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder - .build_select::, _>( - left_is_neg, - neg_remainder.into(), - abs_remainder.into(), - name, - ) - .map_err(|e| CodeGenError::Builder(e.to_string())) - .map(|value| value.into_int_value()) - } - - fn normalize_int_for_unsigned_compare( - &mut self, - value: IntValue<'ctx>, - bit_width: u32, - name: &str, - ) -> Result> { - let current_width = value.get_type().get_bit_width(); - if current_width == bit_width { - return Ok(value); - } - - let target_type = self.context.custom_width_int_type(bit_width); - if current_width > bit_width { - self.builder - .build_int_truncate(value, target_type, name) - .map_err(|e| CodeGenError::Builder(e.to_string())) - } else { - self.builder - .build_int_z_extend(value, target_type, name) - .map_err(|e| CodeGenError::Builder(e.to_string())) - } - } - - fn align_int_widths_for_binary_op( - &mut self, - left: IntValue<'ctx>, - right: IntValue<'ctx>, - ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { - let left_width = left.get_type().get_bit_width(); - let right_width = right.get_type().get_bit_width(); - if left_width == right_width { - return Ok((left, right)); - } - - let target_width = left_width.max(right_width); - let target_type = self.context.custom_width_int_type(target_width); - let left = if left_width < target_width { - self.builder - .build_int_z_extend(left, target_type, "lhs_width_align") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } else { - left - }; - let right = if right_width < target_width { - self.builder - .build_int_z_extend(right, target_type, "rhs_width_align") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } else { - right - }; - Ok((left, right)) - } - - fn mask_shift_amount(&mut self, amount: IntValue<'ctx>, name: &str) -> Result> { - let bit_width = amount.get_type().get_bit_width(); - let mask = amount - .get_type() - .const_int(u64::from(bit_width.saturating_sub(1)), false); - self.builder - .build_and(amount, mask, name) - .map_err(|e| CodeGenError::Builder(e.to_string())) - } - - fn normalize_ints_for_unsigned_width( - &mut self, - left: IntValue<'ctx>, - right: IntValue<'ctx>, - bit_width: u32, - name: &str, - ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { - let left = self.normalize_int_for_unsigned_compare( - left, - bit_width, - &format!("{name}_lhs_unsigned"), - )?; - let right = self.normalize_int_for_unsigned_compare( - right, - bit_width, - &format!("{name}_rhs_unsigned"), - )?; - Ok((left, right)) - } - - fn zero_extend_int_to_i64_if_needed( - &mut self, - value: IntValue<'ctx>, - name: &str, - ) -> Result> { - if value.get_type().get_bit_width() >= 64 { - return Ok(value); - } - self.builder - .build_int_z_extend(value, self.context.i64_type(), name) - .map_err(|e| CodeGenError::Builder(e.to_string())) - } - - fn compile_binary_op_with_ordering( - &mut self, - left: BasicValueEnum<'ctx>, - op: BinaryOp, - right: BasicValueEnum<'ctx>, - integer_semantics: BinaryIntegerSemantics, - ) -> Result> { - use inkwell::values::BasicValueEnum::*; - - // Debug logging to understand the actual types - debug!("compile_binary_op: op={:?}", op); - debug!("compile_binary_op: left type = {:?}", left.get_type()); - debug!("compile_binary_op: right type = {:?}", right.get_type()); - match &left { - IntValue(iv) => debug!( - "compile_binary_op: left is IntValue with bit width {}", - iv.get_type().get_bit_width() - ), - FloatValue(_) => debug!("compile_binary_op: left is FloatValue"), - PointerValue(_) => debug!("compile_binary_op: left is PointerValue"), - _ => debug!("compile_binary_op: left is other type"), - } - match &right { - IntValue(iv) => debug!( - "compile_binary_op: right is IntValue with bit width {}", - iv.get_type().get_bit_width() - ), - FloatValue(_) => debug!("compile_binary_op: right is FloatValue"), - PointerValue(_) => debug!("compile_binary_op: right is PointerValue"), - _ => debug!("compile_binary_op: right is other type"), - } - - match (left, right) { - (IntValue(left_int), IntValue(right_int)) => { - let (left_int, right_int) = - self.align_int_widths_for_binary_op(left_int, right_int)?; - let unsigned_cmp_values = if let Some(bit_width) = - integer_semantics.unsigned_ordering_width - { - Some(( - self.normalize_int_for_unsigned_compare( - left_int, - bit_width, - "lhs_unsigned_cmp", - )?, - self.normalize_int_for_unsigned_compare( - right_int, - bit_width, - "rhs_unsigned_cmp", - )?, - )) - } else { - None - }; - let result = match op { - BinaryOp::Add => self - .builder - .build_int_add(left_int, right_int, "add") - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - BinaryOp::Subtract => self - .builder - .build_int_sub(left_int, right_int, "sub") - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - BinaryOp::Multiply => self - .builder - .build_int_mul(left_int, right_int, "mul") - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - BinaryOp::Divide => { - if let Some(bit_width) = integer_semantics.unsigned_division_width { - let (left_int, right_int) = self.normalize_ints_for_unsigned_width( - left_int, - right_int, - bit_width, - "div", - )?; - let result = self - .builder - .build_int_unsigned_div(left_int, right_int, "div") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.zero_extend_int_to_i64_if_needed(result, "div_zext_i64")? - } else { - self.build_signed_int_div_via_udiv(left_int, right_int, "div")? - } - } - BinaryOp::Modulo => { - if let Some(bit_width) = integer_semantics.unsigned_division_width { - let (left_int, right_int) = self.normalize_ints_for_unsigned_width( - left_int, - right_int, - bit_width, - "mod", - )?; - let result = self - .builder - .build_int_unsigned_rem(left_int, right_int, "mod") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.zero_extend_int_to_i64_if_needed(result, "mod_zext_i64")? - } else { - self.build_signed_int_rem_via_urem(left_int, right_int, "mod")? - } - } - BinaryOp::BitAnd => { - let (left_int, right_int) = - if let Some(bit_width) = integer_semantics.unsigned_bitwise_width { - self.normalize_ints_for_unsigned_width( - left_int, - right_int, - bit_width, - "bitand", - )? - } else { - (left_int, right_int) - }; - let result = self - .builder - .build_and(left_int, right_int, "bitand") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - if integer_semantics.unsigned_bitwise_width.is_some() { - self.zero_extend_int_to_i64_if_needed(result, "bitand_zext_i64")? - } else { - result - } - } - BinaryOp::BitXor => { - let (left_int, right_int) = - if let Some(bit_width) = integer_semantics.unsigned_bitwise_width { - self.normalize_ints_for_unsigned_width( - left_int, - right_int, - bit_width, - "bitxor", - )? - } else { - (left_int, right_int) - }; - let result = self - .builder - .build_xor(left_int, right_int, "bitxor") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - if integer_semantics.unsigned_bitwise_width.is_some() { - self.zero_extend_int_to_i64_if_needed(result, "bitxor_zext_i64")? - } else { - result - } - } - BinaryOp::BitOr => { - let (left_int, right_int) = - if let Some(bit_width) = integer_semantics.unsigned_bitwise_width { - self.normalize_ints_for_unsigned_width( - left_int, - right_int, - bit_width, - "bitor", - )? - } else { - (left_int, right_int) - }; - let result = self - .builder - .build_or(left_int, right_int, "bitor") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - if integer_semantics.unsigned_bitwise_width.is_some() { - self.zero_extend_int_to_i64_if_needed(result, "bitor_zext_i64")? - } else { - result - } - } - BinaryOp::ShiftLeft => { - let right_int = self.mask_shift_amount(right_int, "shl_rhs_mask")?; - self.builder - .build_left_shift(left_int, right_int, "shl") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } - BinaryOp::ShiftRight => { - let right_int = self.mask_shift_amount(right_int, "shr_rhs_mask")?; - self.builder - .build_right_shift( - left_int, - right_int, - integer_semantics.unsigned_right_shift_width.is_none(), - "shr", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } - // Comparison operators - BinaryOp::Equal => { - let result = self - .builder - .build_int_compare(inkwell::IntPredicate::EQ, left_int, right_int, "eq") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(result.into()); - } - BinaryOp::NotEqual => { - let result = self - .builder - .build_int_compare(inkwell::IntPredicate::NE, left_int, right_int, "ne") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(result.into()); - } - BinaryOp::LessThan => { - let predicate = if integer_semantics.unsigned_ordering_width.is_some() { - inkwell::IntPredicate::ULT - } else { - inkwell::IntPredicate::SLT - }; - let (left_cmp, right_cmp) = - unsigned_cmp_values.unwrap_or((left_int, right_int)); - let result = self - .builder - .build_int_compare(predicate, left_cmp, right_cmp, "lt") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(result.into()); - } - BinaryOp::LessEqual => { - let predicate = if integer_semantics.unsigned_ordering_width.is_some() { - inkwell::IntPredicate::ULE - } else { - inkwell::IntPredicate::SLE - }; - let (left_cmp, right_cmp) = - unsigned_cmp_values.unwrap_or((left_int, right_int)); - let result = self - .builder - .build_int_compare(predicate, left_cmp, right_cmp, "le") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(result.into()); - } - BinaryOp::GreaterThan => { - let predicate = if integer_semantics.unsigned_ordering_width.is_some() { - inkwell::IntPredicate::UGT - } else { - inkwell::IntPredicate::SGT - }; - let (left_cmp, right_cmp) = - unsigned_cmp_values.unwrap_or((left_int, right_int)); - let result = self - .builder - .build_int_compare(predicate, left_cmp, right_cmp, "gt") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(result.into()); - } - BinaryOp::GreaterEqual => { - let predicate = if integer_semantics.unsigned_ordering_width.is_some() { - inkwell::IntPredicate::UGE - } else { - inkwell::IntPredicate::SGE - }; - let (left_cmp, right_cmp) = - unsigned_cmp_values.unwrap_or((left_int, right_int)); - let result = self - .builder - .build_int_compare(predicate, left_cmp, right_cmp, "ge") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(result.into()); - } - // Logical operators with boolean semantics (non-zero is true) - BinaryOp::LogicalAnd => { - let lz = left_int.get_type().const_zero(); - let rz = right_int.get_type().const_zero(); - let lbool = self - .builder - .build_int_compare(inkwell::IntPredicate::NE, left_int, lz, "lhs_nz") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let rbool = self - .builder - .build_int_compare(inkwell::IntPredicate::NE, right_int, rz, "rhs_nz") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let result = self - .builder - .build_and(lbool, rbool, "and_bool") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(result.into()); - } - BinaryOp::LogicalOr => { - let lz = left_int.get_type().const_zero(); - let rz = right_int.get_type().const_zero(); - let lbool = self - .builder - .build_int_compare(inkwell::IntPredicate::NE, left_int, lz, "lhs_nz") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let rbool = self - .builder - .build_int_compare(inkwell::IntPredicate::NE, right_int, rz, "rhs_nz") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let result = self - .builder - .build_or(lbool, rbool, "or_bool") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - return Ok(result.into()); - } - }; - Ok(result.into()) - } - // Pointer equality/inequality comparisons - (PointerValue(lp), IntValue(ri)) | (IntValue(ri), PointerValue(lp)) => { - match op { - BinaryOp::Equal | BinaryOp::NotEqual => { - let lpi64 = self - .builder - .build_ptr_to_int(lp, self.context.i64_type(), "ptr_as_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - // Normalize RHS to i64 - let rbw = ri.get_type().get_bit_width(); - let ri64 = if rbw < 64 { - self.builder - .build_int_z_extend(ri, self.context.i64_type(), "rhs_zext_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } else if rbw > 64 { - self.builder - .build_int_truncate(ri, self.context.i64_type(), "rhs_trunc_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } else { - ri - }; - let pred = if matches!(op, BinaryOp::Equal) { - inkwell::IntPredicate::EQ - } else { - inkwell::IntPredicate::NE - }; - let cmp = self - .builder - .build_int_compare(pred, lpi64, ri64, "ptr_cmp") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(cmp.into()) - } - _ => Err(CodeGenError::TypeError( - "Unsupported operation between aggregate address/pointer and integer: only '==' and '!=' are allowed. If you meant to offset an address, use '&expr +/- ' in an alias/address context, or access a scalar field.".to_string(), - )), - } - } - (PointerValue(lp), PointerValue(rp)) => match op { - BinaryOp::Equal | BinaryOp::NotEqual => { - let lpi64 = self - .builder - .build_ptr_to_int(lp, self.context.i64_type(), "l_ptr_as_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let rpi64 = self - .builder - .build_ptr_to_int(rp, self.context.i64_type(), "r_ptr_as_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let pred = if matches!(op, BinaryOp::Equal) { - inkwell::IntPredicate::EQ - } else { - inkwell::IntPredicate::NE - }; - let cmp = self - .builder - .build_int_compare(pred, lpi64, rpi64, "ptr_ptr_cmp") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(cmp.into()) - } - _ => Err(CodeGenError::TypeError( - "Pointer ordered comparison ('<', '<=', '>', '>=') is not supported. Use '==' or '!=' to compare addresses. If you need to adjust an address, use '&expr +/- ' in an alias/address context; to compare values, select a scalar field (e.g., 'obj.field')." - .to_string(), - )), - }, - (FloatValue(left_float), FloatValue(right_float)) => match op { - BinaryOp::Add => { - let result = self - .builder - .build_float_add(left_float, right_float, "add") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - BinaryOp::Subtract => { - let result = self - .builder - .build_float_sub(left_float, right_float, "sub") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - BinaryOp::Multiply => { - let result = self - .builder - .build_float_mul(left_float, right_float, "mul") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - BinaryOp::Divide => { - let result = self - .builder - .build_float_div(left_float, right_float, "div") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - // Float comparison operators - BinaryOp::Equal => { - let result = self - .builder - .build_float_compare( - inkwell::FloatPredicate::OEQ, - left_float, - right_float, - "eq", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - BinaryOp::NotEqual => { - let result = self - .builder - .build_float_compare( - inkwell::FloatPredicate::ONE, - left_float, - right_float, - "ne", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - BinaryOp::LessThan => { - let result = self - .builder - .build_float_compare( - inkwell::FloatPredicate::OLT, - left_float, - right_float, - "lt", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - BinaryOp::LessEqual => { - let result = self - .builder - .build_float_compare( - inkwell::FloatPredicate::OLE, - left_float, - right_float, - "le", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - BinaryOp::GreaterThan => { - let result = self - .builder - .build_float_compare( - inkwell::FloatPredicate::OGT, - left_float, - right_float, - "gt", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - BinaryOp::GreaterEqual => { - let result = self - .builder - .build_float_compare( - inkwell::FloatPredicate::OGE, - left_float, - right_float, - "ge", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - Ok(result.into()) - } - _ => Err(CodeGenError::NotImplemented(format!( - "Float binary operation {op:?} not implemented" - ))), - }, - _ => Err(CodeGenError::TypeError(format!( - "Type mismatch in binary operation {op:?}" - ))), - } - } - - /// Compile member access (struct.field) - pub fn compile_member_access( - &mut self, - obj_expr: &Expr, - field: &str, - ) -> Result> { - // Create a MemberAccess expression and use the unified DWARF compilation - let member_access_expr = Expr::MemberAccess(Box::new(obj_expr.clone()), field.to_string()); - self.compile_dwarf_expression(&member_access_expr) - } - - /// Compile pointer dereference (*ptr) - pub fn compile_pointer_deref(&mut self, expr: &Expr) -> Result> { - // Create a PointerDeref expression and use the unified DWARF compilation - let pointer_deref_expr = Expr::PointerDeref(Box::new(expr.clone())); - self.compile_dwarf_expression(&pointer_deref_expr) - } - - /// Compile array access (arr[index]) - pub fn compile_array_access( - &mut self, - array_expr: &Expr, - index_expr: &Expr, - ) -> Result> { - if let Some((value, _element_type)) = - self.compile_dynamic_array_access_value(array_expr, index_expr)? - { - return Ok(value); - } - - // Create an ArrayAccess expression and use the unified DWARF compilation - let array_access_expr = - Expr::ArrayAccess(Box::new(array_expr.clone()), Box::new(index_expr.clone())); - self.compile_dwarf_expression(&array_access_expr) - } - - /// Compile chain access (person.name.first) - pub fn compile_chain_access(&mut self, chain: &[String]) -> Result> { - // Create a ChainAccess expression and use the unified DWARF compilation - let chain_access_expr = Expr::ChainAccess(chain.to_vec()); - self.compile_dwarf_expression(&chain_access_expr) - } - - /// Unified DWARF expression compilation - pub fn compile_dwarf_expression( - &mut self, - expr: &crate::script::Expr, - ) -> Result> { - debug!( - "compile_dwarf_expression: Compiling complex expression: {:?}", - expr - ); - - if let crate::script::Expr::Cast { - expr: inner, - target_type, - } = expr - { - return self.compile_cast_expr_value(inner, target_type); - } - - if let crate::script::Expr::ArrayAccess(array_expr, index_expr) = expr { - if let Some((value, _element_type)) = - self.compile_dynamic_array_access_value(array_expr, index_expr)? - { - return Ok(value); - } - } - if let crate::script::Expr::MemberAccess(obj_expr, field) = expr { - if let Some((value, _member_type)) = - self.compile_dynamic_member_access_value(obj_expr, field)? - { - return Ok(value); - } - } - if matches!(expr, crate::script::Expr::PointerDeref(_)) { - if let Some(lvalue) = self.dynamic_lvalue_address_and_type(expr)? { - return self - .read_dynamic_address_value(lvalue.address, &lvalue.type_info.dwarf_type); - } - } - - // Query DWARF for the complex expression - let compile_context = self.get_compile_time_context()?.clone(); - let variable_plan = match self.query_dwarf_for_complex_expr(expr)? { - Some(var) => var, - None => { - let expr_str = Self::expr_to_debug_string(expr); - return Err(CodeGenError::VariableNotFound(expr_str)); - } - }; - - let materialized = - self.variable_read_plan_to_materialization(variable_plan, compile_context.pc_address)?; - let dwarf_type = materialized.dwarf_type.as_ref().ok_or_else(|| { - CodeGenError::DwarfError("Expression has no DWARF type information".to_string()) - })?; - - debug!( - "compile_dwarf_expression: Found DWARF info for expression '{}' with type: {:?}", - materialized.name, dwarf_type - ); - - self.variable_materialization_to_llvm_value(&materialized, compile_context.pc_address, None) - } - - pub(super) fn compile_dynamic_array_access_value( - &mut self, - array_expr: &Expr, - index_expr: &Expr, - ) -> Result, DwarfType)>> { - let Some(element_lvalue) = - self.compile_dynamic_array_element_address(array_expr, index_expr)? - else { - return Ok(None); - }; - - let value = self.read_dynamic_address_value( - element_lvalue.address, - &element_lvalue.type_info.dwarf_type, - )?; - Ok(Some((value, element_lvalue.type_info.dwarf_type))) - } - - pub(super) fn compile_dynamic_member_access_value( - &mut self, - obj_expr: &Expr, - field: &str, - ) -> Result, DwarfType)>> { - let Some(object_lvalue) = self.dynamic_lvalue_address_and_type(obj_expr)? else { - return Ok(None); - }; - - let Some(element_lvalue) = self.dynamic_member_base_address_and_type(object_lvalue)? else { - return Ok(None); - }; - let (member_offset, member_type) = - self.dynamic_member_offset_and_type(&element_lvalue.type_info, field)?; - - let member_offset = self.context.i64_type().const_int(member_offset, false); - let member_address = self - .builder - .build_int_add( - element_lvalue.address.value, - member_offset, - "dynamic_member_address", - ) - .map_err(|err| CodeGenError::Builder(err.to_string()))?; - let value = self.read_dynamic_address_value( - element_lvalue.address.with_value(member_address), - &member_type, - )?; - Ok(Some((value, member_type))) - } - - pub(super) fn dynamic_lvalue_address_and_type( - &mut self, - expr: &Expr, - ) -> Result>> { - if let Expr::Variable(name) = expr { - if self.alias_variable_exists(name) { - let expanded = self.expand_alias_variable_expr(expr)?; - return self.dynamic_lvalue_address_and_type(&expanded); - } - } - - if let Expr::Cast { - expr: inner, - target_type, - } = expr - { - return self - .cast_lvalue_address_and_type(inner, target_type) - .map(Some); - } - - if let Expr::PointerDeref(inner) = expr { - let expanded_inner = self.expand_alias_variable_expr(inner)?; - if matches!(expanded_inner, Expr::Cast { .. }) { - return self.dynamic_lvalue_address_and_type(&expanded_inner); - } - if let Expr::BinaryOp { .. } = expanded_inner { - if let Some(lvalue) = self.dynamic_lvalue_address_and_type(&expanded_inner)? { - return Ok(Some(lvalue)); - } - } - } - - if let Expr::ArrayAccess(array_expr, index_expr) = expr { - return self.compile_dynamic_array_element_address(array_expr, index_expr); - } - - if let Some(lvalue) = self.dynamic_lvalue_from_const_pointer_arithmetic(expr)? { - return Ok(Some(lvalue)); - } - - if self.expands_to_nonliteral_pointer_arithmetic(expr)? { - let Some(element_info) = self.indexable_element_type_and_stride(expr)? else { - return Ok(None); - }; - let element_address = self.resolve_runtime_address_from_expr(expr)?; - return Ok(Some(DynamicLvalue { - address: element_address, - type_info: DynamicTypeInfo { - dwarf_type: element_info.element_type, - module_path: element_info.module_path, - }, - })); - } - - if let Expr::MemberAccess(obj_expr, field) = expr { - let Some(object_lvalue) = self.dynamic_lvalue_address_and_type(obj_expr)? else { - return Ok(None); - }; - let Some(base_lvalue) = self.dynamic_member_base_address_and_type(object_lvalue)? - else { - return Ok(None); - }; - let (member_offset, member_type) = - self.dynamic_member_offset_and_type(&base_lvalue.type_info, field)?; - let member_offset = self.context.i64_type().const_int(member_offset, false); - let member_address = self - .builder - .build_int_add( - base_lvalue.address.value, - member_offset, - "dynamic_member_lvalue_address", - ) - .map_err(|err| CodeGenError::Builder(err.to_string()))?; - return Ok(Some(DynamicLvalue { - address: base_lvalue.address.with_value(member_address), - type_info: DynamicTypeInfo { - dwarf_type: member_type, - module_path: base_lvalue.type_info.module_path, - }, - })); - } - - Ok(None) - } - - fn dynamic_member_base_address_and_type( - &mut self, - object: DynamicLvalue<'ctx>, - ) -> Result>> { - let module_path = object.type_info.module_path.clone(); - let object_type = self.complete_dynamic_member_element_type( - object.type_info.dwarf_type, - module_path.as_deref(), - ); - match ghostscope_dwarf::strip_type_aliases(&object_type) { - DwarfType::StructType { .. } | DwarfType::UnionType { .. } => Ok(Some(DynamicLvalue { - address: object.address, - type_info: DynamicTypeInfo { - dwarf_type: object_type, - module_path, - }, - })), - DwarfType::PointerType { target_type, .. } => { - let pointer_value = - self.read_dynamic_address_value(object.address, &object_type)?; - let pointer_value = match pointer_value { - BasicValueEnum::IntValue(value) => { - self.normalize_int_to_i64(value, "dynamic_member_pointer_i64")? - } - BasicValueEnum::PointerValue(value) => self - .builder - .build_ptr_to_int( - value, - self.context.i64_type(), - "dynamic_member_pointer_ptr", - ) - .map_err(|err| CodeGenError::Builder(err.to_string()))?, - _ => { - return Err(CodeGenError::TypeError( - "dynamic member pointer base did not compile to an address".to_string(), - )) - } - }; - let target_type = self.complete_dynamic_member_element_type( - target_type.as_ref().clone(), - module_path.as_deref(), - ); - Ok(Some(DynamicLvalue { - address: RuntimeAddress::available(pointer_value, self.context), - type_info: DynamicTypeInfo { - dwarf_type: target_type, - module_path, - }, - })) - } - _ => Ok(None), - } - } - - fn dynamic_member_offset_and_type( - &self, - aggregate: &DynamicTypeInfo, - field: &str, - ) -> Result<(u64, DwarfType)> { - let aggregate_type = self.complete_dynamic_member_element_type( - aggregate.dwarf_type.clone(), - aggregate.module_path.as_deref(), - ); - match ghostscope_dwarf::member_layout(&aggregate_type, field) { - Ok(layout) => Ok((layout.offset, layout.member_type)), - Err(err @ TypeLayoutError::UnknownMember { .. }) => { - Err(CodeGenError::DwarfError(err.to_string())) - } - Err(err @ TypeLayoutError::InvalidMemberBase { .. }) => { - Err(CodeGenError::TypeError(err.to_string())) - } - } - } - - fn dynamic_array_base_from_plan( - &mut self, - array_plan: &VariableReadPlan, - pc_address: u64, - status_ptr: Option>, - static_index: i64, - ) -> Result<(IndexableElementInfo, RuntimeAddress<'ctx>, i64)> { - let module_path = array_plan.module_path.clone(); - let array_type = array_plan.dwarf_type.as_ref().ok_or_else(|| { - CodeGenError::DwarfError("Array expression has no DWARF type information".to_string()) - })?; - let element_info = - Self::indexable_info_from_type(array_type, module_path).ok_or_else(|| { - CodeGenError::TypeError(format!( - "dynamic array index requires array or pointer type, got '{}'", - array_type.type_name() - )) - })?; - - match ghostscope_dwarf::strip_type_aliases(array_type) { - DwarfType::ArrayType { .. } => { - let base_address = - self.variable_read_plan_to_runtime_address(array_plan, pc_address, status_ptr)?; - Ok((element_info, base_address, static_index)) - } - DwarfType::PointerType { .. } => { - let pointer_value = - self.variable_read_plan_to_llvm_value(array_plan, pc_address, status_ptr)?; - let base_address = self.compiled_pointer_value_to_runtime_address( - pointer_value, - "dynamic_array_base_i64", - "dynamic_array_base_ptr", - "array base pointer did not compile to an address", - )?; - Ok((element_info, base_address, static_index)) - } - _ => unreachable!("indexable_info_from_type accepts only array or pointer types"), - } - } - - fn compile_dynamic_array_element_address( - &mut self, - array_expr: &Expr, - index_expr: &Expr, - ) -> Result>> { - let literal_index = Self::integer_literal_value(index_expr); - let expanded_array_expr = self.expand_alias_variable_expr(array_expr)?; - let has_dynamic_base = - self.expands_to_nonliteral_pointer_arithmetic(&expanded_array_expr)?; - let cast_base = self.cast_index_base(&expanded_array_expr)?; - - if literal_index.is_some() && !has_dynamic_base && cast_base.is_none() { - return Ok(None); - } - - let compile_context = self.get_compile_time_context()?.clone(); - let status_ptr = if self.condition_context_active { - Some(self.get_or_create_cond_error_global()) - } else { - None - }; - - let (element_info, base_address, static_index) = if let Some((element_info, base_address)) = - cast_base - { - (element_info, base_address, 0) - } else { - match self.query_dwarf_for_complex_expr(array_expr)? { - Some(array_plan) => self.dynamic_array_base_from_plan( - &array_plan, - compile_context.pc_address, - status_ptr, - 0, - )?, - None => { - if let Some((base_expr, static_index)) = - self.pointer_arithmetic_parts_expanding_aliases(&expanded_array_expr)? - { - let array_plan = self - .query_dwarf_for_complex_expr(&base_expr)? - .ok_or_else(|| { - CodeGenError::VariableNotFound(Self::expr_to_debug_string( - &base_expr, - )) - })?; - self.dynamic_array_base_from_plan( - &array_plan, - compile_context.pc_address, - status_ptr, - static_index, - )? - } else if has_dynamic_base { - let element_info = self - .indexable_element_type_and_stride(&expanded_array_expr)? - .ok_or_else(|| { - CodeGenError::VariableNotFound(Self::expr_to_debug_string( - array_expr, - )) - })?; - let base_address = - self.resolve_runtime_address_from_expr(&expanded_array_expr)?; - (element_info, base_address, 0) - } else if let Some(array_lvalue) = - self.dynamic_lvalue_address_and_type(&expanded_array_expr)? - { - let module_path = array_lvalue.type_info.module_path.clone(); - let element_info = Self::indexable_info_from_type( - &array_lvalue.type_info.dwarf_type, - module_path, - ) - .ok_or_else(|| { - CodeGenError::TypeError(format!( - "dynamic array index requires array or pointer type, got '{}'", - array_lvalue.type_info.dwarf_type.type_name() - )) - })?; - match ghostscope_dwarf::strip_type_aliases( - &array_lvalue.type_info.dwarf_type, - ) { - DwarfType::ArrayType { .. } => (element_info, array_lvalue.address, 0), - DwarfType::PointerType { .. } => { - let pointer_value = self.read_dynamic_address_value( - array_lvalue.address, - &array_lvalue.type_info.dwarf_type, - )?; - let base_address = self.compiled_pointer_value_to_runtime_address( - pointer_value, - "dynamic_array_member_ptr_i64", - "dynamic_array_member_ptr", - "array member pointer did not compile to an address", - )?; - (element_info, base_address, 0) - } - _ => unreachable!( - "indexable_info_from_type accepts only array or pointer types" - ), - } - } else { - return Err(CodeGenError::VariableNotFound(Self::expr_to_debug_string( - array_expr, - ))); - } - } - } - }; - - let index_value = if let Some(index) = literal_index { - self.context.i64_type().const_int(index as u64, true) - } else { - match self.compile_expr(index_expr)? { - BasicValueEnum::IntValue(value) => { - self.normalize_int_to_i64(value, "dynamic_array_index_i64")? - } - _ => { - return Err(CodeGenError::TypeError( - "array index expression must compile to an integer".to_string(), - )) - } - } - }; - let index_value = if static_index == 0 { - index_value - } else { - let static_index_value = self.context.i64_type().const_int(static_index as u64, true); - self.builder - .build_int_add( - index_value, - static_index_value, - "dynamic_array_static_index", - ) - .map_err(|err| CodeGenError::Builder(err.to_string()))? - }; - let stride_value = self - .context - .i64_type() - .const_int(element_info.stride, false); - let byte_offset = self - .builder - .build_int_mul(index_value, stride_value, "dynamic_array_byte_offset") - .map_err(|err| CodeGenError::Builder(err.to_string()))?; - let element_address = self - .builder - .build_int_add( - base_address.value, - byte_offset, - "dynamic_array_element_address", - ) - .map_err(|err| CodeGenError::Builder(err.to_string()))?; - - Ok(Some(DynamicLvalue { - address: base_address.with_value(element_address), - type_info: DynamicTypeInfo { - dwarf_type: element_info.element_type, - module_path: element_info.module_path, - }, - })) - } - - fn indexable_element_type_and_stride( - &mut self, - expr: &Expr, - ) -> Result> { - use crate::script::ast::BinaryOp as BO; - use crate::script::ast::Expr as E; - - let expanded = self.expand_alias_variable_expr(expr)?; - - if let Some((element_info, _base_address)) = self.cast_index_base(&expanded)? { - return Ok(Some(element_info)); - } - - if let Some(plan) = self.query_dwarf_for_complex_expr(&expanded)? { - if let Some(dwarf_type) = plan.dwarf_type.as_ref() { - if let Some(info) = - Self::indexable_info_from_type(dwarf_type, plan.module_path.clone()) - { - return Ok(Some(info)); - } - } - } - - if let Some((base_expr, _static_index)) = - self.pointer_arithmetic_parts_expanding_aliases(&expanded)? - { - if let Some(plan) = self.query_dwarf_for_complex_expr(&base_expr)? { - if let Some(dwarf_type) = plan.dwarf_type.as_ref() { - if let Some(info) = - Self::indexable_info_from_type(dwarf_type, plan.module_path.clone()) - { - return Ok(Some(info)); - } - } - } - } - - match expanded { - E::BinaryOp { - ref left, - op: BO::Add, - ref right, - } => { - if let Some(info) = self.indexable_element_type_and_stride(left)? { - return Ok(Some(info)); - } - self.indexable_element_type_and_stride(right) - } - E::BinaryOp { - ref left, - op: BO::Subtract, - .. - } => self.indexable_element_type_and_stride(left), - _ => Ok(None), - } - } - - fn complete_dynamic_member_element_type( - &self, - element_type: DwarfType, - module_path: Option<&Path>, - ) -> DwarfType { - let Some(analyzer) = self.process_analyzer else { - return element_type; - }; - let fallback_module_path = self - .current_compile_time_context - .as_ref() - .map(|ctx| PathBuf::from(&ctx.module_path)); - let lookup_module_path = module_path.or(fallback_module_path.as_deref()); - - if let Some(module_path) = lookup_module_path { - analyzer.complete_shallow_unknown_aggregate_type_in_module(module_path, element_type) - } else { - analyzer.complete_shallow_unknown_aggregate_type(element_type) - } - } - - fn read_dynamic_address_value( - &mut self, - address: RuntimeAddress<'ctx>, - dwarf_type: &DwarfType, - ) -> Result> { - if ghostscope_dwarf::is_c_aggregate_type(dwarf_type) { - let ptr_ty = self.context.ptr_type(AddressSpace::default()); - let as_ptr = self - .builder - .build_int_to_ptr(address.value, ptr_ty, "dynamic_aggregate_ptr") - .map_err(|err| CodeGenError::Builder(err.to_string()))?; - return Ok(as_ptr.into()); - } - - let access_size = self.dwarf_type_to_memory_access_size(dwarf_type); - let value = if self.condition_context_active { - self.generate_memory_read_with_status(address, access_size)? - } else { - self.generate_memory_read(address, access_size, None)? - }; - self.sign_extend_memory_read_if_needed(value, dwarf_type, access_size) - } - - fn expand_alias_variable_expr(&self, expr: &Expr) -> Result { - let mut expanded = expr.clone(); - let mut visited = std::collections::HashSet::new(); - - loop { - let Expr::Variable(name) = &expanded else { - return Ok(expanded); - }; - if !self.alias_variable_exists(name) { - return Ok(expanded); - } - if !visited.insert(name.clone()) { - return Err(CodeGenError::TypeError(format!( - "alias cycle detected for '{name}'" - ))); - } - let Some(target) = self.get_alias_variable(name) else { - return Ok(expanded); - }; - expanded = target; - } - } - - fn normalize_int_to_i64(&self, value: IntValue<'ctx>, name: &str) -> Result> { - let width = value.get_type().get_bit_width(); - if width == 64 { - return Ok(value); - } - - if width < 64 { - return self - .builder - .build_int_s_extend(value, self.context.i64_type(), name) - .map_err(|err| CodeGenError::Builder(err.to_string())); - } - - self.builder - .build_int_truncate(value, self.context.i64_type(), name) - .map_err(|err| CodeGenError::Builder(err.to_string())) - } - - pub(crate) fn dwarf_expression_unavailable_error( - name: &str, - availability: &Availability, - pc_address: u64, - ) -> CodeGenError { - let reason = Self::format_availability_reason(availability); - CodeGenError::VariableUnavailable(format!( - "'{name}' is {reason}; cannot use it as a value expression at PC 0x{pc_address:x}" - )) - } - - pub(crate) fn dwarf_lvalue_address_unavailable_error( - name: &str, - availability: &Availability, - pc_address: u64, - ) -> CodeGenError { - let reason = Self::format_availability_reason(availability); - CodeGenError::VariableUnavailable(format!( - "'{name}' is {reason}; cannot take its address at PC 0x{pc_address:x}" - )) - } - - fn format_availability_reason(availability: &Availability) -> String { - match availability { - Availability::OptimizedOut => "optimized out at the selected probe PC".to_string(), - Availability::NotInScope => "not in scope at the selected probe PC".to_string(), - Availability::Unsupported(reason) => { - format!( - "unsupported DWARF semantic shape: {}", - Self::format_unsupported_reason(reason) - ) - } - Availability::Requires(requirement) => { - format!( - "requires unavailable runtime support: {}", - Self::format_runtime_requirement(requirement) - ) - } - Availability::Ambiguous(reason) => { - format!( - "ambiguous DWARF semantic result: {}", - Self::format_ambiguity_reason(reason) - ) - } - Availability::Available | Availability::PartiallyAvailable => "available".to_string(), - } - } - - fn format_unsupported_reason(reason: &UnsupportedReason) -> String { - match reason { - UnsupportedReason::DwarfOp { op } => format!("unsupported DWARF op {op}"), - UnsupportedReason::ExpressionShape { detail } => { - format!("unsupported DWARF expression shape: {detail}") - } - UnsupportedReason::TypeLayout { detail } => { - format!("unsupported type layout: {detail}") - } - UnsupportedReason::AddressClass { detail } => { - format!("unsupported address class: {detail}") - } - UnsupportedReason::RegisterMapping { dwarf_reg } => { - format!("unsupported DWARF register mapping for register {dwarf_reg}") - } - } - } - - fn format_runtime_requirement(requirement: &RuntimeRequirement) -> &'static str { - match requirement { - RuntimeRequirement::CallerFrame => "caller-frame recovery", - RuntimeRequirement::SleepableUprobe => "sleepable uprobe support", - RuntimeRequirement::UserMemoryRead => "user-memory read support", - RuntimeRequirement::DwarfCfiRecovery => "DWARF CFI recovery", - } - } - - fn format_ambiguity_reason(reason: &AmbiguityReason) -> String { - match reason { - AmbiguityReason::InlineContext { detail } => { - format!("ambiguous inline context: {detail}") - } - AmbiguityReason::VariableDeclaration { detail } => { - format!("ambiguous variable declaration: {detail}") - } - AmbiguityReason::TypeResolution { detail } => { - format!("ambiguous type resolution: {detail}") - } - } - } - - /// Helper: Convert expression to string for debugging - fn expr_to_debug_string(expr: &crate::script::Expr) -> String { - use crate::script::Expr; - - match expr { - Expr::Variable(name) => name.clone(), - Expr::MemberAccess(obj, field) => { - format!("{}.{}", Self::expr_to_debug_string(obj), field) - } - Expr::ArrayAccess(arr, _) => format!("{}[index]", Self::expr_to_debug_string(arr)), - Expr::Cast { expr, target_type } => format!( - "cast({}, \"{}\")", - Self::expr_to_debug_string(expr), - target_type - ), - Expr::ChainAccess(chain) => chain.join("."), - Expr::PointerDeref(expr) => format!("*{}", Self::expr_to_debug_string(expr)), - _ => "expr".to_string(), - } - } -} - -impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { - /// Compile comparison between a DWARF-side expression and a script string literal. - /// Supports char* and char[N] according to design in string_comparison.md. - fn compile_string_comparison( - &mut self, - dwarf_expr: &Expr, - lit: &str, - is_equal: bool, - ) -> Result> { - use ghostscope_dwarf::TypeInfo as TI; - - // Query DWARF for the non-string side to obtain evaluation and type info - let var = self - .query_dwarf_for_complex_expr(dwarf_expr)? - .ok_or_else(|| { - CodeGenError::TypeError( - "string comparison requires DWARF variable/expression".into(), - ) - })?; - // Try DWARF type first; if unavailable, fall back to type_name string parsing - let dwarf_type_opt = var.dwarf_type.as_ref(); - - enum ParsedKind { - PtrChar, - ArrChar(Option), - Other, - } - fn parse_type_name(name: &str) -> ParsedKind { - let lower = name.to_lowercase(); - let has_char = lower.contains("char"); - let is_ptr = lower.contains('*'); - if has_char && is_ptr { - return ParsedKind::PtrChar; - } - if has_char && lower.contains('[') { - // Try to extract N inside brackets - let mut n: Option = None; - if let Some(start) = lower.find('[') { - if let Some(end) = lower[start + 1..].find(']') { - let inside = &lower[start + 1..start + 1 + end]; - let digits: String = - inside.chars().filter(|c| c.is_ascii_digit()).collect(); - if !digits.is_empty() { - if let Ok(v) = digits.parse::() { - n = Some(v); - } - } - } - } - return ParsedKind::ArrChar(n); - } - ParsedKind::Other - } - - let lit_bytes = lit.as_bytes(); - let lit_len = lit_bytes.len() as u32; - let one = self.context.bool_type().const_int(1, false); - let zero = self.context.bool_type().const_zero(); - - // Build final boolean accumulator - let result = match dwarf_type_opt.map(ghostscope_dwarf::strip_type_aliases) { - // char* / const char* - Some(TI::PointerType { target_type, .. }) => { - // Ensure pointee is char-like - let base = ghostscope_dwarf::strip_type_aliases(target_type.as_ref()); - let is_char_like = matches!(base, TI::BaseType { name, size, .. } if name.contains("char") && *size == 1); - if !is_char_like { - return Err(CodeGenError::TypeError( - "automatic string comparison only supports char*".into(), - )); - } - - // Evaluate expression to pointer value and read up to L+1 bytes - let pc_address = self.get_compile_time_context()?.pc_address; - let val_any = self.variable_read_plan_to_llvm_value(&var, pc_address, None)?; - let ptr_i64 = match val_any { - BasicValueEnum::IntValue(iv) => iv, - BasicValueEnum::PointerValue(pv) => self - .builder - .build_ptr_to_int(pv, self.context.i64_type(), "ptr_as_i64") - .map_err(|e| CodeGenError::Builder(e.to_string()))?, - _ => { - return Err(CodeGenError::TypeError( - "pointer value must be integer or pointer".into(), - )) - } - }; - let need = lit_len + 1; - let (buf_global, ret_len, arr_ty) = self.read_user_cstr_into_buffer( - RuntimeAddress::available(ptr_i64, self.context), - need, - "_gs_strbuf", - )?; - - // ret_len must equal L+1 - let i64_ty = self.context.i64_type(); - let expect_len = i64_ty.const_int(need as u64, false); - let len_ok = self - .builder - .build_int_compare(inkwell::IntPredicate::EQ, ret_len, expect_len, "str_len_ok") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - // buf[L] must be '\0' - let i32_ty = self.context.i32_type(); - let idx0 = i32_ty.const_zero(); - let idx_l = i32_ty.const_int(lit_len as u64, false); - // SAFETY: the string read requested lit_len + 1 bytes, so index - // lit_len is within the scratch buffer. - let char_ptr = unsafe { - self.builder - .build_gep(arr_ty, buf_global, &[idx0, idx_l], "nul_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let c = self - .builder - .build_load(self.context.i8_type(), char_ptr, "c_l") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let c = match c { - BasicValueEnum::IntValue(iv) => iv, - _ => return Err(CodeGenError::LLVMError("load did not return i8".into())), - }; - let nul_ok = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - c, - self.context.i8_type().const_zero(), - "nul_ok", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - // Compare first L bytes using XOR/OR accumulation to reduce branchiness - let mut acc = self.context.i8_type().const_zero(); - for (i, b) in lit_bytes.iter().enumerate() { - let idx_i = i32_ty.const_int(i as u64, false); - // SAFETY: lit_bytes length is bounded by the scratch buffer size. - let ptr_i = unsafe { - self.builder - .build_gep(arr_ty, buf_global, &[idx0, idx_i], "ch_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let ch = self - .builder - .build_load(self.context.i8_type(), ptr_i, "ch") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let ch = match ch { - BasicValueEnum::IntValue(iv) => iv, - _ => return Err(CodeGenError::LLVMError("load did not return i8".into())), - }; - let expect = self.context.i8_type().const_int(*b as u64, false); - let diff = self - .builder - .build_xor(ch, expect, "diff") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - acc = self - .builder - .build_or(acc, diff, "acc_or") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - } - let eq_bytes = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - acc, - self.context.i8_type().const_zero(), - "acc_zero", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let ok1 = self - .builder - .build_and(len_ok, nul_ok, "ok_len_nul") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder - .build_and(ok1, eq_bytes, "str_eq") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } - // char[N] - Some(TI::ArrayType { - element_type, - element_count, - total_size, - }) => { - let elem = ghostscope_dwarf::strip_type_aliases(element_type.as_ref()); - let is_char_like = matches!(elem, TI::BaseType { name, size, .. } if name.contains("char") && *size == 1); - if !is_char_like { - return Err(CodeGenError::TypeError( - "automatic string comparison only supports char[N]".into(), - )); - } - // Determine N (element count) - let n_opt = element_count.or_else(|| total_size.map(|ts| ts)); - let n = if let Some(nv) = n_opt { nv as u32 } else { 0 }; - if n == 0 { - return Err(CodeGenError::TypeError( - "array size unknown for char[N] comparison".into(), - )); - } - // If L+1 > N, compile-time false - if lit_len + 1 > n { - // Return const false (or true if '!=' requested) - return Ok((if is_equal { zero } else { one }).into()); - } - let status_ptr = if self.condition_context_active { - Some(self.get_or_create_cond_error_global()) - } else { - None - }; - let pc_address = self.get_compile_time_context()?.pc_address; - let addr = - self.variable_read_plan_to_runtime_address(&var, pc_address, status_ptr)?; - // Read exactly L+1 bytes - let (buf_global, status, arr_ty) = - self.read_user_bytes_into_buffer(addr, lit_len + 1, "_gs_arrbuf")?; - // status == 0 - let status_ok = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - status, - self.context.i64_type().const_zero(), - "rd_ok", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - // buf[L] must be '\0' - let i32_ty = self.context.i32_type(); - let idx0 = i32_ty.const_zero(); - let idx_l = i32_ty.const_int(lit_len as u64, false); - // SAFETY: the string read requested lit_len + 1 bytes, so index - // lit_len is within the scratch buffer. - let char_ptr = unsafe { - self.builder - .build_gep(arr_ty, buf_global, &[idx0, idx_l], "nul_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let c = self - .builder - .build_load(self.context.i8_type(), char_ptr, "c_l") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let c = match c { - BasicValueEnum::IntValue(iv) => iv, - _ => return Err(CodeGenError::LLVMError("load did not return i8".into())), - }; - let nul_ok = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - c, - self.context.i8_type().const_zero(), - "nul_ok", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - // Compare first L bytes using XOR/OR accumulation - let mut acc = self.context.i8_type().const_zero(); - for (i, b) in lit_bytes.iter().enumerate() { - let idx_i = i32_ty.const_int(i as u64, false); - // SAFETY: lit_bytes length is bounded by the scratch buffer size. - let ptr_i = unsafe { - self.builder - .build_gep(arr_ty, buf_global, &[idx0, idx_i], "ch_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let ch = self - .builder - .build_load(self.context.i8_type(), ptr_i, "ch") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let ch = match ch { - BasicValueEnum::IntValue(iv) => iv, - _ => return Err(CodeGenError::LLVMError("load did not return i8".into())), - }; - let expect = self.context.i8_type().const_int(*b as u64, false); - let diff = self - .builder - .build_xor(ch, expect, "diff") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - acc = self - .builder - .build_or(acc, diff, "acc_or") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - } - let eq_bytes = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - acc, - self.context.i8_type().const_zero(), - "acc_zero", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let ok1 = self - .builder - .build_and(status_ok, nul_ok, "ok_len_nul") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder - .build_and(ok1, eq_bytes, "arr_eq") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } - None => { - let status_ptr = if self.condition_context_active { - Some(self.get_or_create_cond_error_global()) - } else { - None - }; - let pc_address = self.get_compile_time_context()?.pc_address; - let addr = - self.variable_read_plan_to_runtime_address(&var, pc_address, status_ptr)?; - // Fallback using type_name string - match parse_type_name(&var.type_name) { - ParsedKind::PtrChar => { - // Load pointer value from variable location (assume 64-bit) - let ptr_any = self.generate_memory_read( - addr, - ghostscope_dwarf::MemoryAccessSize::U64, - None, - )?; - let ptr_i64 = match ptr_any { - BasicValueEnum::IntValue(iv) => iv, - _ => { - return Err(CodeGenError::LLVMError( - "pointer load did not return integer".to_string(), - )) - } - }; - let need = lit_len + 1; - let (buf_global, ret_len, arr_ty) = self.read_user_cstr_into_buffer( - RuntimeAddress::available(ptr_i64, self.context), - need, - "_gs_strbuf", - )?; - - let i64_ty = self.context.i64_type(); - let expect_len = i64_ty.const_int(need as u64, false); - let len_ok = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - ret_len, - expect_len, - "str_len_ok", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - let i32_ty = self.context.i32_type(); - let idx0 = i32_ty.const_zero(); - let idx_l = i32_ty.const_int(lit_len as u64, false); - // SAFETY: the string read requested lit_len + 1 bytes, so - // index lit_len is within the scratch buffer. - let char_ptr = unsafe { - self.builder - .build_gep(arr_ty, buf_global, &[idx0, idx_l], "nul_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let c = self - .builder - .build_load(self.context.i8_type(), char_ptr, "c_l") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let c = match c { - BasicValueEnum::IntValue(iv) => iv, - _ => { - return Err(CodeGenError::LLVMError( - "load did not return i8".into(), - )) - } - }; - let nul_ok = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - c, - self.context.i8_type().const_zero(), - "nul_ok", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - - let mut acc = self.context.i8_type().const_zero(); - for (i, b) in lit_bytes.iter().enumerate() { - let idx_i = i32_ty.const_int(i as u64, false); - // SAFETY: lit_bytes length is bounded by the scratch buffer size. - let ptr_i = unsafe { - self.builder - .build_gep(arr_ty, buf_global, &[idx0, idx_i], "ch_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let ch = self - .builder - .build_load(self.context.i8_type(), ptr_i, "ch") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let ch = match ch { - BasicValueEnum::IntValue(iv) => iv, - _ => { - return Err(CodeGenError::LLVMError( - "load did not return i8".into(), - )) - } - }; - let expect = self.context.i8_type().const_int(*b as u64, false); - let diff = self - .builder - .build_xor(ch, expect, "diff") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - acc = self - .builder - .build_or(acc, diff, "acc_or") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - } - let eq_bytes = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - acc, - self.context.i8_type().const_zero(), - "acc_zero", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let ok1 = self - .builder - .build_and(len_ok, nul_ok, "ok_len_nul") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder - .build_and(ok1, eq_bytes, "str_eq") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } - ParsedKind::ArrChar(n_opt) => { - // If we know N and L+1>N, return false; else read L+1 bytes - if let Some(n) = n_opt { - if lit_len + 1 > n { - return Ok((if is_equal { zero } else { one }).into()); - } - } - let (buf_global, status, arr_ty) = - self.read_user_bytes_into_buffer(addr, lit_len + 1, "_gs_arrbuf")?; - let status_ok = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - status, - self.context.i64_type().const_zero(), - "rd_ok", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let i32_ty = self.context.i32_type(); - let idx0 = i32_ty.const_zero(); - let idx_l = i32_ty.const_int(lit_len as u64, false); - // SAFETY: the string read requested lit_len + 1 bytes, so - // index lit_len is within the scratch buffer. - let char_ptr = unsafe { - self.builder - .build_gep(arr_ty, buf_global, &[idx0, idx_l], "nul_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let c = self - .builder - .build_load(self.context.i8_type(), char_ptr, "c_l") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let c = match c { - BasicValueEnum::IntValue(iv) => iv, - _ => { - return Err(CodeGenError::LLVMError( - "load did not return i8".into(), - )) - } - }; - let nul_ok = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - c, - self.context.i8_type().const_zero(), - "nul_ok", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let mut acc = self.context.i8_type().const_zero(); - for (i, b) in lit_bytes.iter().enumerate() { - let idx_i = i32_ty.const_int(i as u64, false); - // SAFETY: lit_bytes length is bounded by the scratch buffer size. - let ptr_i = unsafe { - self.builder - .build_gep(arr_ty, buf_global, &[idx0, idx_i], "ch_ptr") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - let ch = self - .builder - .build_load(self.context.i8_type(), ptr_i, "ch") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let ch = match ch { - BasicValueEnum::IntValue(iv) => iv, - _ => { - return Err(CodeGenError::LLVMError( - "load did not return i8".into(), - )) - } - }; - let expect = self.context.i8_type().const_int(*b as u64, false); - let diff = self - .builder - .build_xor(ch, expect, "diff") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - acc = self - .builder - .build_or(acc, diff, "acc_or") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - } - let eq_bytes = self - .builder - .build_int_compare( - inkwell::IntPredicate::EQ, - acc, - self.context.i8_type().const_zero(), - "acc_zero", - ) - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - let ok1 = self - .builder - .build_and(status_ok, nul_ok, "ok_len_nul") - .map_err(|e| CodeGenError::Builder(e.to_string()))?; - self.builder - .build_and(ok1, eq_bytes, "arr_eq") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - } - ParsedKind::Other => { - return Err(CodeGenError::TypeError(format!( - "string comparison unsupported for type name '{}' without DWARF type", - var.type_name - ))); - } - } - } - Some(_) => { - return Err(CodeGenError::TypeError( - "string comparison only supports char* or char[N]".into(), - )); - } - }; - - // Apply == / != - let final_bool = if is_equal { - result - } else { - self.builder - .build_not(result, "not_eq") - .map_err(|e| CodeGenError::Builder(e.to_string()))? - }; - Ok(final_bool.into()) - } -} diff --git a/ghostscope-compiler/src/ebpf/expression/binary.rs b/ghostscope-compiler/src/ebpf/expression/binary.rs new file mode 100644 index 00000000..83d136c2 --- /dev/null +++ b/ghostscope-compiler/src/ebpf/expression/binary.rs @@ -0,0 +1,710 @@ +use crate::ebpf::context::{CodeGenError, EbpfContext, Result}; +use crate::ebpf::expression_plan::BinaryIntegerSemantics; +use crate::script::BinaryOp; +use inkwell::values::{BasicValueEnum, IntValue}; +use tracing::debug; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + pub fn compile_binary_op( + &mut self, + left: BasicValueEnum<'ctx>, + op: BinaryOp, + right: BasicValueEnum<'ctx>, + ) -> Result> { + self.compile_binary_op_with_ordering(left, op, right, BinaryIntegerSemantics::default()) + } + + pub(crate) fn build_signed_int_div_via_udiv( + &mut self, + left: IntValue<'ctx>, + right: IntValue<'ctx>, + name: &str, + ) -> Result> { + let int_type = left.get_type(); + let zero = int_type.const_zero(); + let left_is_neg = self + .builder + .build_int_compare(inkwell::IntPredicate::SLT, left, zero, "sdiv_lhs_neg") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let right_is_neg = self + .builder + .build_int_compare(inkwell::IntPredicate::SLT, right, zero, "sdiv_rhs_neg") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let neg_left = self + .builder + .build_int_sub(zero, left, "sdiv_lhs_negated") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let neg_right = self + .builder + .build_int_sub(zero, right, "sdiv_rhs_negated") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let abs_left = self + .builder + .build_select::, _>( + left_is_neg, + neg_left.into(), + left.into(), + "sdiv_lhs_abs", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let abs_right = self + .builder + .build_select::, _>( + right_is_neg, + neg_right.into(), + right.into(), + "sdiv_rhs_abs", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let abs_quotient = self + .builder + .build_int_unsigned_div(abs_left, abs_right, "sdiv_abs_udiv") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let negative_result = self + .builder + .build_xor(left_is_neg, right_is_neg, "sdiv_result_neg") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let neg_quotient = self + .builder + .build_int_sub(zero, abs_quotient, "sdiv_negated") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder + .build_select::, _>( + negative_result, + neg_quotient.into(), + abs_quotient.into(), + name, + ) + .map_err(|e| CodeGenError::Builder(e.to_string())) + .map(|value| value.into_int_value()) + } + + pub(crate) fn build_signed_int_rem_via_urem( + &mut self, + left: IntValue<'ctx>, + right: IntValue<'ctx>, + name: &str, + ) -> Result> { + let int_type = left.get_type(); + let zero = int_type.const_zero(); + let left_is_neg = self + .builder + .build_int_compare(inkwell::IntPredicate::SLT, left, zero, "srem_lhs_neg") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let right_is_neg = self + .builder + .build_int_compare(inkwell::IntPredicate::SLT, right, zero, "srem_rhs_neg") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let neg_left = self + .builder + .build_int_sub(zero, left, "srem_lhs_negated") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let neg_right = self + .builder + .build_int_sub(zero, right, "srem_rhs_negated") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let abs_left = self + .builder + .build_select::, _>( + left_is_neg, + neg_left.into(), + left.into(), + "srem_lhs_abs", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let abs_right = self + .builder + .build_select::, _>( + right_is_neg, + neg_right.into(), + right.into(), + "srem_rhs_abs", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let abs_remainder = self + .builder + .build_int_unsigned_rem(abs_left, abs_right, "srem_abs_urem") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let neg_remainder = self + .builder + .build_int_sub(zero, abs_remainder, "srem_negated") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder + .build_select::, _>( + left_is_neg, + neg_remainder.into(), + abs_remainder.into(), + name, + ) + .map_err(|e| CodeGenError::Builder(e.to_string())) + .map(|value| value.into_int_value()) + } + + pub(super) fn normalize_int_for_unsigned_compare( + &mut self, + value: IntValue<'ctx>, + bit_width: u32, + name: &str, + ) -> Result> { + let current_width = value.get_type().get_bit_width(); + if current_width == bit_width { + return Ok(value); + } + + let target_type = self.context.custom_width_int_type(bit_width); + if current_width > bit_width { + self.builder + .build_int_truncate(value, target_type, name) + .map_err(|e| CodeGenError::Builder(e.to_string())) + } else { + self.builder + .build_int_z_extend(value, target_type, name) + .map_err(|e| CodeGenError::Builder(e.to_string())) + } + } + + fn align_int_widths_for_binary_op( + &mut self, + left: IntValue<'ctx>, + right: IntValue<'ctx>, + ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { + let left_width = left.get_type().get_bit_width(); + let right_width = right.get_type().get_bit_width(); + if left_width == right_width { + return Ok((left, right)); + } + + let target_width = left_width.max(right_width); + let target_type = self.context.custom_width_int_type(target_width); + let left = if left_width < target_width { + self.builder + .build_int_z_extend(left, target_type, "lhs_width_align") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } else { + left + }; + let right = if right_width < target_width { + self.builder + .build_int_z_extend(right, target_type, "rhs_width_align") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } else { + right + }; + Ok((left, right)) + } + + fn mask_shift_amount(&mut self, amount: IntValue<'ctx>, name: &str) -> Result> { + let bit_width = amount.get_type().get_bit_width(); + let mask = amount + .get_type() + .const_int(u64::from(bit_width.saturating_sub(1)), false); + self.builder + .build_and(amount, mask, name) + .map_err(|e| CodeGenError::Builder(e.to_string())) + } + + fn normalize_ints_for_unsigned_width( + &mut self, + left: IntValue<'ctx>, + right: IntValue<'ctx>, + bit_width: u32, + name: &str, + ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { + let left = self.normalize_int_for_unsigned_compare( + left, + bit_width, + &format!("{name}_lhs_unsigned"), + )?; + let right = self.normalize_int_for_unsigned_compare( + right, + bit_width, + &format!("{name}_rhs_unsigned"), + )?; + Ok((left, right)) + } + + pub(super) fn zero_extend_int_to_i64_if_needed( + &mut self, + value: IntValue<'ctx>, + name: &str, + ) -> Result> { + if value.get_type().get_bit_width() >= 64 { + return Ok(value); + } + self.builder + .build_int_z_extend(value, self.context.i64_type(), name) + .map_err(|e| CodeGenError::Builder(e.to_string())) + } + + pub(super) fn compile_binary_op_with_ordering( + &mut self, + left: BasicValueEnum<'ctx>, + op: BinaryOp, + right: BasicValueEnum<'ctx>, + integer_semantics: BinaryIntegerSemantics, + ) -> Result> { + use inkwell::values::BasicValueEnum::*; + + // Debug logging to understand the actual types + debug!("compile_binary_op: op={:?}", op); + debug!("compile_binary_op: left type = {:?}", left.get_type()); + debug!("compile_binary_op: right type = {:?}", right.get_type()); + match &left { + IntValue(iv) => debug!( + "compile_binary_op: left is IntValue with bit width {}", + iv.get_type().get_bit_width() + ), + FloatValue(_) => debug!("compile_binary_op: left is FloatValue"), + PointerValue(_) => debug!("compile_binary_op: left is PointerValue"), + _ => debug!("compile_binary_op: left is other type"), + } + match &right { + IntValue(iv) => debug!( + "compile_binary_op: right is IntValue with bit width {}", + iv.get_type().get_bit_width() + ), + FloatValue(_) => debug!("compile_binary_op: right is FloatValue"), + PointerValue(_) => debug!("compile_binary_op: right is PointerValue"), + _ => debug!("compile_binary_op: right is other type"), + } + + match (left, right) { + (IntValue(left_int), IntValue(right_int)) => { + let (left_int, right_int) = + self.align_int_widths_for_binary_op(left_int, right_int)?; + let unsigned_cmp_values = if let Some(bit_width) = + integer_semantics.unsigned_ordering_width + { + Some(( + self.normalize_int_for_unsigned_compare( + left_int, + bit_width, + "lhs_unsigned_cmp", + )?, + self.normalize_int_for_unsigned_compare( + right_int, + bit_width, + "rhs_unsigned_cmp", + )?, + )) + } else { + None + }; + let result = match op { + BinaryOp::Add => self + .builder + .build_int_add(left_int, right_int, "add") + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + BinaryOp::Subtract => self + .builder + .build_int_sub(left_int, right_int, "sub") + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + BinaryOp::Multiply => self + .builder + .build_int_mul(left_int, right_int, "mul") + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + BinaryOp::Divide => { + if let Some(bit_width) = integer_semantics.unsigned_division_width { + let (left_int, right_int) = self.normalize_ints_for_unsigned_width( + left_int, + right_int, + bit_width, + "div", + )?; + let result = self + .builder + .build_int_unsigned_div(left_int, right_int, "div") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.zero_extend_int_to_i64_if_needed(result, "div_zext_i64")? + } else { + self.build_signed_int_div_via_udiv(left_int, right_int, "div")? + } + } + BinaryOp::Modulo => { + if let Some(bit_width) = integer_semantics.unsigned_division_width { + let (left_int, right_int) = self.normalize_ints_for_unsigned_width( + left_int, + right_int, + bit_width, + "mod", + )?; + let result = self + .builder + .build_int_unsigned_rem(left_int, right_int, "mod") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.zero_extend_int_to_i64_if_needed(result, "mod_zext_i64")? + } else { + self.build_signed_int_rem_via_urem(left_int, right_int, "mod")? + } + } + BinaryOp::BitAnd => { + let (left_int, right_int) = + if let Some(bit_width) = integer_semantics.unsigned_bitwise_width { + self.normalize_ints_for_unsigned_width( + left_int, + right_int, + bit_width, + "bitand", + )? + } else { + (left_int, right_int) + }; + let result = self + .builder + .build_and(left_int, right_int, "bitand") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + if integer_semantics.unsigned_bitwise_width.is_some() { + self.zero_extend_int_to_i64_if_needed(result, "bitand_zext_i64")? + } else { + result + } + } + BinaryOp::BitXor => { + let (left_int, right_int) = + if let Some(bit_width) = integer_semantics.unsigned_bitwise_width { + self.normalize_ints_for_unsigned_width( + left_int, + right_int, + bit_width, + "bitxor", + )? + } else { + (left_int, right_int) + }; + let result = self + .builder + .build_xor(left_int, right_int, "bitxor") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + if integer_semantics.unsigned_bitwise_width.is_some() { + self.zero_extend_int_to_i64_if_needed(result, "bitxor_zext_i64")? + } else { + result + } + } + BinaryOp::BitOr => { + let (left_int, right_int) = + if let Some(bit_width) = integer_semantics.unsigned_bitwise_width { + self.normalize_ints_for_unsigned_width( + left_int, + right_int, + bit_width, + "bitor", + )? + } else { + (left_int, right_int) + }; + let result = self + .builder + .build_or(left_int, right_int, "bitor") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + if integer_semantics.unsigned_bitwise_width.is_some() { + self.zero_extend_int_to_i64_if_needed(result, "bitor_zext_i64")? + } else { + result + } + } + BinaryOp::ShiftLeft => { + let right_int = self.mask_shift_amount(right_int, "shl_rhs_mask")?; + self.builder + .build_left_shift(left_int, right_int, "shl") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } + BinaryOp::ShiftRight => { + let right_int = self.mask_shift_amount(right_int, "shr_rhs_mask")?; + self.builder + .build_right_shift( + left_int, + right_int, + integer_semantics.unsigned_right_shift_width.is_none(), + "shr", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } + // Comparison operators + BinaryOp::Equal => { + let result = self + .builder + .build_int_compare(inkwell::IntPredicate::EQ, left_int, right_int, "eq") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(result.into()); + } + BinaryOp::NotEqual => { + let result = self + .builder + .build_int_compare(inkwell::IntPredicate::NE, left_int, right_int, "ne") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(result.into()); + } + BinaryOp::LessThan => { + let predicate = if integer_semantics.unsigned_ordering_width.is_some() { + inkwell::IntPredicate::ULT + } else { + inkwell::IntPredicate::SLT + }; + let (left_cmp, right_cmp) = + unsigned_cmp_values.unwrap_or((left_int, right_int)); + let result = self + .builder + .build_int_compare(predicate, left_cmp, right_cmp, "lt") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(result.into()); + } + BinaryOp::LessEqual => { + let predicate = if integer_semantics.unsigned_ordering_width.is_some() { + inkwell::IntPredicate::ULE + } else { + inkwell::IntPredicate::SLE + }; + let (left_cmp, right_cmp) = + unsigned_cmp_values.unwrap_or((left_int, right_int)); + let result = self + .builder + .build_int_compare(predicate, left_cmp, right_cmp, "le") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(result.into()); + } + BinaryOp::GreaterThan => { + let predicate = if integer_semantics.unsigned_ordering_width.is_some() { + inkwell::IntPredicate::UGT + } else { + inkwell::IntPredicate::SGT + }; + let (left_cmp, right_cmp) = + unsigned_cmp_values.unwrap_or((left_int, right_int)); + let result = self + .builder + .build_int_compare(predicate, left_cmp, right_cmp, "gt") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(result.into()); + } + BinaryOp::GreaterEqual => { + let predicate = if integer_semantics.unsigned_ordering_width.is_some() { + inkwell::IntPredicate::UGE + } else { + inkwell::IntPredicate::SGE + }; + let (left_cmp, right_cmp) = + unsigned_cmp_values.unwrap_or((left_int, right_int)); + let result = self + .builder + .build_int_compare(predicate, left_cmp, right_cmp, "ge") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(result.into()); + } + // Logical operators with boolean semantics (non-zero is true) + BinaryOp::LogicalAnd => { + let lz = left_int.get_type().const_zero(); + let rz = right_int.get_type().const_zero(); + let lbool = self + .builder + .build_int_compare(inkwell::IntPredicate::NE, left_int, lz, "lhs_nz") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let rbool = self + .builder + .build_int_compare(inkwell::IntPredicate::NE, right_int, rz, "rhs_nz") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let result = self + .builder + .build_and(lbool, rbool, "and_bool") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(result.into()); + } + BinaryOp::LogicalOr => { + let lz = left_int.get_type().const_zero(); + let rz = right_int.get_type().const_zero(); + let lbool = self + .builder + .build_int_compare(inkwell::IntPredicate::NE, left_int, lz, "lhs_nz") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let rbool = self + .builder + .build_int_compare(inkwell::IntPredicate::NE, right_int, rz, "rhs_nz") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let result = self + .builder + .build_or(lbool, rbool, "or_bool") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(result.into()); + } + }; + Ok(result.into()) + } + // Pointer equality/inequality comparisons + (PointerValue(lp), IntValue(ri)) | (IntValue(ri), PointerValue(lp)) => { + match op { + BinaryOp::Equal | BinaryOp::NotEqual => { + let lpi64 = self + .builder + .build_ptr_to_int(lp, self.context.i64_type(), "ptr_as_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + // Normalize RHS to i64 + let rbw = ri.get_type().get_bit_width(); + let ri64 = if rbw < 64 { + self.builder + .build_int_z_extend(ri, self.context.i64_type(), "rhs_zext_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } else if rbw > 64 { + self.builder + .build_int_truncate(ri, self.context.i64_type(), "rhs_trunc_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } else { + ri + }; + let pred = if matches!(op, BinaryOp::Equal) { + inkwell::IntPredicate::EQ + } else { + inkwell::IntPredicate::NE + }; + let cmp = self + .builder + .build_int_compare(pred, lpi64, ri64, "ptr_cmp") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(cmp.into()) + } + _ => Err(CodeGenError::TypeError( + "Unsupported operation between aggregate address/pointer and integer: only '==' and '!=' are allowed. If you meant to offset an address, use '&expr +/- ' in an alias/address context, or access a scalar field.".to_string(), + )), + } + } + (PointerValue(lp), PointerValue(rp)) => match op { + BinaryOp::Equal | BinaryOp::NotEqual => { + let lpi64 = self + .builder + .build_ptr_to_int(lp, self.context.i64_type(), "l_ptr_as_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let rpi64 = self + .builder + .build_ptr_to_int(rp, self.context.i64_type(), "r_ptr_as_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let pred = if matches!(op, BinaryOp::Equal) { + inkwell::IntPredicate::EQ + } else { + inkwell::IntPredicate::NE + }; + let cmp = self + .builder + .build_int_compare(pred, lpi64, rpi64, "ptr_ptr_cmp") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(cmp.into()) + } + _ => Err(CodeGenError::TypeError( + "Pointer ordered comparison ('<', '<=', '>', '>=') is not supported. Use '==' or '!=' to compare addresses. If you need to adjust an address, use '&expr +/- ' in an alias/address context; to compare values, select a scalar field (e.g., 'obj.field')." + .to_string(), + )), + }, + (FloatValue(left_float), FloatValue(right_float)) => match op { + BinaryOp::Add => { + let result = self + .builder + .build_float_add(left_float, right_float, "add") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + BinaryOp::Subtract => { + let result = self + .builder + .build_float_sub(left_float, right_float, "sub") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + BinaryOp::Multiply => { + let result = self + .builder + .build_float_mul(left_float, right_float, "mul") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + BinaryOp::Divide => { + let result = self + .builder + .build_float_div(left_float, right_float, "div") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + // Float comparison operators + BinaryOp::Equal => { + let result = self + .builder + .build_float_compare( + inkwell::FloatPredicate::OEQ, + left_float, + right_float, + "eq", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + BinaryOp::NotEqual => { + let result = self + .builder + .build_float_compare( + inkwell::FloatPredicate::ONE, + left_float, + right_float, + "ne", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + BinaryOp::LessThan => { + let result = self + .builder + .build_float_compare( + inkwell::FloatPredicate::OLT, + left_float, + right_float, + "lt", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + BinaryOp::LessEqual => { + let result = self + .builder + .build_float_compare( + inkwell::FloatPredicate::OLE, + left_float, + right_float, + "le", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + BinaryOp::GreaterThan => { + let result = self + .builder + .build_float_compare( + inkwell::FloatPredicate::OGT, + left_float, + right_float, + "gt", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + BinaryOp::GreaterEqual => { + let result = self + .builder + .build_float_compare( + inkwell::FloatPredicate::OGE, + left_float, + right_float, + "ge", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + _ => Err(CodeGenError::NotImplemented(format!( + "Float binary operation {op:?} not implemented" + ))), + }, + _ => Err(CodeGenError::TypeError(format!( + "Type mismatch in binary operation {op:?}" + ))), + } + } +} diff --git a/ghostscope-compiler/src/ebpf/expression/builtins.rs b/ghostscope-compiler/src/ebpf/expression/builtins.rs new file mode 100644 index 00000000..3a7d63b8 --- /dev/null +++ b/ghostscope-compiler/src/ebpf/expression/builtins.rs @@ -0,0 +1,1466 @@ +use crate::ebpf::context::{CodeGenError, EbpfContext, Result, RuntimeAddress}; +use crate::script::Expr; +use aya_ebpf_bindings::bindings::bpf_func_id::BPF_FUNC_probe_read_user; +use ghostscope_dwarf::TypeInfo as DwarfType; +use inkwell::values::{BasicValueEnum, IntValue}; +use inkwell::AddressSpace; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + /// Builtin memcmp (boolean variant): returns true iff first `len` bytes equal. + /// Supports dynamic `len` (expr), clamped to [0, compare_cap]. + pub(super) fn compile_memcmp_builtin( + &mut self, + a_expr: &Expr, + b_expr: &Expr, + len_expr: &Expr, + ) -> Result> { + // Note: constant hex/len validation happens at parse-time; dynamic cases are handled at runtime by masking bytes. + + // Note: do not resolve pointers yet; if either side is hex("...") we will synthesize bytes + + // Compile length expr to i32 and clamp to [0, CAP] + let len_val = self.compile_expr(len_expr)?; + let len_iv = match len_val { + BasicValueEnum::IntValue(iv) => iv, + _ => { + return Err(CodeGenError::TypeError( + "memcmp length must be an integer expression".into(), + )) + } + }; + let i32_ty = self.context.i32_type(); + let len_i32 = if len_iv.get_type().get_bit_width() > 32 { + self.builder + .build_int_truncate(len_iv, i32_ty, "memcmp_len_trunc") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } else if len_iv.get_type().get_bit_width() < 32 { + self.builder + .build_int_z_extend(len_iv, i32_ty, "memcmp_len_zext") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } else { + len_iv + }; + let zero_i32 = i32_ty.const_zero(); + let is_neg = self + .builder + .build_int_compare( + inkwell::IntPredicate::SLT, + len_i32, + zero_i32, + "memcmp_len_neg", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let len_nn = self + .builder + .build_select(is_neg, zero_i32, len_i32, "memcmp_len_nn") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let cap = self.compile_options.compare_cap; + let cap_const = i32_ty.const_int(cap as u64, false); + let gt = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGT, + len_nn, + cap_const, + "memcmp_len_gt", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let sel_len = self + .builder + .build_select(gt, cap_const, len_nn, "memcmp_len_sel") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + + // Fast-path: if effective length is zero, return true without any reads + let len_is_zero = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + sel_len, + i32_ty.const_zero(), + "memcmp_len_is_zero", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let func = self.current_function("compile memcmp length branch")?; + let zero_b = self.context.append_basic_block(func, "memcmp_len_zero"); + let nz_b = self.context.append_basic_block(func, "memcmp_len_nz"); + let cont_b = self.context.append_basic_block(func, "memcmp_len_cont"); + self.builder + .build_conditional_branch(len_is_zero, zero_b, nz_b) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + // Zero-length branch: true + self.builder.position_at_end(zero_b); + let bool_true = self.context.bool_type().const_int(1, false); + self.builder + .build_unconditional_branch(cont_b) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let zero_block = self.current_insert_block("finish memcmp zero-length block")?; + + // Non-zero branch: perform reads and compare + self.builder.position_at_end(nz_b); + + // Prepare static buffers of size CAP for both sides + let (arr_a_ty, buf_a) = self.get_or_create_i8_buffer(cap, "_gs_bi_memcmp_a"); + let (arr_b_ty, buf_b) = self.get_or_create_i8_buffer(cap, "_gs_bi_memcmp_b"); + let ptr_ty = self.context.ptr_type(AddressSpace::default()); + + // Helper: parse hex builtin into bytes + let parse_hex_bytes = |e: &Expr| -> Option> { + if let Expr::BuiltinCall { name, args } = e { + if name == "hex" && args.len() == 1 { + if let Expr::String(s) = &args[0] { + // Parser guarantees only hex digits and even length + if s.is_empty() { + return Some(Vec::new()); + } + let mut out = Vec::with_capacity(s.len() / 2); + let mut i = 0usize; + while i + 1 < s.len() { + let v = u8::from_str_radix(&s[i..i + 2], 16).ok()?; + out.push(v); + i += 2; + } + return Some(out); + } + } + } + None + }; + + // Side A + // If side A is DWARF-backed (and not an explicit address-of), enforce pointer DWARF type + if parse_hex_bytes(a_expr).is_none() { + self.ensure_dwarf_pointer_arg(a_expr, "memcmp arg0")?; + } + let ok_a = if let Some(bytes) = parse_hex_bytes(a_expr) { + let i32_ty = self.context.i32_type(); + let idx0 = i32_ty.const_zero(); + for i in 0..(cap as usize) { + let idx_i = i32_ty.const_int(i as u64, false); + // SAFETY: buf_a is a cap-sized array and i is bounded by cap. + let pa = unsafe { + self.builder + .build_gep(arr_a_ty, buf_a, &[idx0, idx_i], &format!("hex_a_i{i}")) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let byte = if i < bytes.len() { bytes[i] } else { 0 } as u64; + let bv = self.context.i8_type().const_int(byte, false); + self.builder + .build_store(pa, bv) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + } + self.context.bool_type().const_int(1, false) + } else { + // Resolve pointer for A and read from user memory + let ptr_a = self.resolve_runtime_address_from_expr(a_expr)?; + let offsets_found_a = ptr_a.offsets_found; + let dst_a = self + .builder + .build_bit_cast(buf_a, ptr_ty, "memcmp_dst_a") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let base_src_a = self + .builder + .build_int_to_ptr(ptr_a.value, ptr_ty, "memcmp_src_a") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let null_ptr = ptr_ty.const_null(); + let src_a = self + .builder + .build_select::, _>( + offsets_found_a, + base_src_a.into(), + null_ptr.into(), + "memcmp_src_a_or_null", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_pointer_value(); + let zero_i32 = self.context.i32_type().const_zero(); + let effective_len_a = self + .builder + .build_select::, _>( + offsets_found_a, + sel_len.into(), + zero_i32.into(), + "memcmp_len_a_or_zero", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let ret_a = self + .create_bpf_helper_call( + BPF_FUNC_probe_read_user as u64, + &[dst_a, effective_len_a.into(), src_a.into()], + self.context.i64_type().into(), + "probe_read_user_memcmp_a", + )? + .into_int_value(); + let i64_ty = self.context.i64_type(); + let eq_a = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + ret_a, + i64_ty.const_zero(), + "memcmp_ok_a", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder + .build_and(eq_a, offsets_found_a, "memcmp_ok_a") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + + // Side B + if parse_hex_bytes(b_expr).is_none() { + self.ensure_dwarf_pointer_arg(b_expr, "memcmp arg1")?; + } + let ok_b = if let Some(bytes) = parse_hex_bytes(b_expr) { + let i32_ty = self.context.i32_type(); + let idx0 = i32_ty.const_zero(); + for i in 0..(cap as usize) { + let idx_i = i32_ty.const_int(i as u64, false); + // SAFETY: buf_b is a cap-sized array and i is bounded by cap. + let pb = unsafe { + self.builder + .build_gep(arr_b_ty, buf_b, &[idx0, idx_i], &format!("hex_b_i{i}")) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let byte = if i < bytes.len() { bytes[i] } else { 0 } as u64; + let bv = self.context.i8_type().const_int(byte, false); + self.builder + .build_store(pb, bv) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + } + self.context.bool_type().const_int(1, false) + } else { + // Resolve pointer for B and read from user memory + let ptr_b = self.resolve_runtime_address_from_expr(b_expr)?; + let offsets_found_b = ptr_b.offsets_found; + let dst_b = self + .builder + .build_bit_cast(buf_b, ptr_ty, "memcmp_dst_b") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let base_src_b = self + .builder + .build_int_to_ptr(ptr_b.value, ptr_ty, "memcmp_src_b") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let null_ptr = ptr_ty.const_null(); + let src_b = self + .builder + .build_select::, _>( + offsets_found_b, + base_src_b.into(), + null_ptr.into(), + "memcmp_src_b_or_null", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_pointer_value(); + let zero_i32 = self.context.i32_type().const_zero(); + let effective_len_b = self + .builder + .build_select::, _>( + offsets_found_b, + sel_len.into(), + zero_i32.into(), + "memcmp_len_b_or_zero", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let ret_b = self + .create_bpf_helper_call( + BPF_FUNC_probe_read_user as u64, + &[dst_b, effective_len_b.into(), src_b.into()], + self.context.i64_type().into(), + "probe_read_user_memcmp_b", + )? + .into_int_value(); + let i64_ty = self.context.i64_type(); + let eq_b = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + ret_b, + i64_ty.const_zero(), + "memcmp_ok_b", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder + .build_and(eq_b, offsets_found_b, "memcmp_ok_b") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + + let status_ok = self + .builder + .build_and(ok_a, ok_b, "memcmp_status_ok") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + // If in condition context and either side failed, set condition error code = 1 (ProbeReadFailed) + if self.condition_context_active { + let not_a = self + .builder + .build_not(ok_a, "memcmp_fail_a") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let not_b = self + .builder + .build_not(ok_b, "memcmp_fail_b") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let any_fail = self + .builder + .build_or(not_a, not_b, "memcmp_any_fail") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let func = self.current_function("compile memcmp condition error branch")?; + let set_b = self.context.append_basic_block(func, "memcmp_set_err"); + let cont_b = self.context.append_basic_block(func, "memcmp_cont"); + self.builder + .build_conditional_branch(any_fail, set_b, cont_b) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder.position_at_end(set_b); + // Align error_code with VariableStatus::ReadError = 2 + let _ = self.set_condition_error_if_unset(2u8); + // Decide which side failed (prefer recording the actual failing side) + let not_a_val = self + .builder + .build_not(ok_a, "memcmp_fail_a_val") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let not_b_val = self + .builder + .build_not(ok_b, "memcmp_fail_b_val") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let cur_fn = self.current_function("compile memcmp failure address branch")?; + let set_a_bb = self.context.append_basic_block(cur_fn, "set_addr_a"); + let check_b_bb = self.context.append_basic_block(cur_fn, "check_fail_b"); + let set_b_bb = self.context.append_basic_block(cur_fn, "set_addr_b"); + let after_set_bb = self.context.append_basic_block(cur_fn, "after_set_addr"); + + // Branch on A failure first + self.builder + .build_conditional_branch(not_a_val, set_a_bb, check_b_bb) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + // set A address + self.builder.position_at_end(set_a_bb); + if let Some(pa) = match parse_hex_bytes(a_expr) { + Some(_) => None, + None => Some(self.resolve_ptr_i64_from_expr(a_expr)?), + } { + let _ = self.set_condition_error_addr_if_unset(pa); + } + self.builder + .build_unconditional_branch(after_set_bb) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + // check B failure and set B + self.builder.position_at_end(check_b_bb); + self.builder + .build_conditional_branch(not_b_val, set_b_bb, after_set_bb) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder.position_at_end(set_b_bb); + if let Some(pb) = match parse_hex_bytes(b_expr) { + Some(_) => None, + None => Some(self.resolve_ptr_i64_from_expr(b_expr)?), + } { + let _ = self.set_condition_error_addr_if_unset(pb); + } + self.builder + .build_unconditional_branch(after_set_bb) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder.position_at_end(after_set_bb); + // Build flags: bit0=A fail, bit1=B fail, bit2=len clamped, bit3=len<=0 + let i8t = self.context.i8_type(); + let b_a = self + .builder + .build_int_z_extend( + self.builder + .build_not(ok_a, "fa") + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + i8t, + "fa8", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let b_b1 = self + .builder + .build_int_z_extend( + self.builder + .build_not(ok_b, "fb") + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + i8t, + "fb8", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let sh1 = self + .builder + .build_left_shift(b_b1, i8t.const_int(1, false), "b_b_shift") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + // gt: len_nn > cap (len clamped) + let b_c = self + .builder + .build_int_z_extend(gt, i8t, "clamped8") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let sh2 = self + .builder + .build_left_shift(b_c, i8t.const_int(2, false), "b_c_shift") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + // len<=0: reuse len_is_zero + let b_z = self + .builder + .build_int_z_extend(len_is_zero, i8t, "len0_8") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let sh3 = self + .builder + .build_left_shift(b_z, i8t.const_int(3, false), "b_z_shift") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let f01 = self + .builder + .build_or(b_a, sh1, "f01") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let f012 = self + .builder + .build_or(f01, sh2, "f012") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let flags = self + .builder + .build_or(f012, sh3, "flags") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let _ = self.or_condition_error_flags(flags); + self.builder + .build_unconditional_branch(cont_b) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder.position_at_end(cont_b); + } + + // Aggregate XOR/OR across 0..CAP, masked by (i < sel_len) + let i32_ty = self.context.i32_type(); + let idx0 = i32_ty.const_zero(); + let mut acc = self.context.i8_type().const_zero(); + for i in 0..cap as usize { + let idx_i = i32_ty.const_int(i as u64, false); + // active = (i < sel_len) + let active = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + idx_i, + sel_len, + &format!("memcmp_i{i}_active"), + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + // a[i] + // SAFETY: i is bounded by cmp_bound, which is clamped to the buffer cap. + let pa = unsafe { + self.builder + .build_gep(arr_a_ty, buf_a, &[idx0, idx_i], &format!("memcmp_a_i{i}")) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let va = self + .builder + .build_load(self.context.i8_type(), pa, &format!("ld_a_{i}")) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let va = match va { + BasicValueEnum::IntValue(iv) => iv, + _ => return Err(CodeGenError::LLVMError("memcmp load a != i8".into())), + }; + // b[i] + // SAFETY: i is bounded by cmp_bound, which is clamped to the buffer cap. + let pb = unsafe { + self.builder + .build_gep(arr_b_ty, buf_b, &[idx0, idx_i], &format!("memcmp_b_i{i}")) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let vb = self + .builder + .build_load(self.context.i8_type(), pb, &format!("ld_b_{i}")) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let vb = match vb { + BasicValueEnum::IntValue(iv) => iv, + _ => return Err(CodeGenError::LLVMError("memcmp load b != i8".into())), + }; + let diff = self + .builder + .build_xor(va, vb, &format!("memcmp_diff_{i}")) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let zero8 = self.context.i8_type().const_zero(); + let masked = self + .builder + .build_select(active, diff, zero8, &format!("memcmp_masked_{i}")) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + acc = self + .builder + .build_or(acc, masked, &format!("memcmp_acc_{i}")) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + } + let eq_bytes = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + acc, + self.context.i8_type().const_zero(), + "memcmp_acc_zero", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let nz_result = self + .builder + .build_and(status_ok, eq_bytes, "memcmp_and") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + self.builder + .build_unconditional_branch(cont_b) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let nz_block = self.current_insert_block("finish memcmp non-zero block")?; + + // Merge + self.builder.position_at_end(cont_b); + let phi = self + .builder + .build_phi(self.context.bool_type(), "memcmp_phi") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + phi.add_incoming(&[(&bool_true, zero_block), (&nz_result, nz_block)]); + Ok(phi.as_basic_value()) + } + /// Builtin strncmp/starts_with implementation: bounded byte-compare without NUL requirement. + fn compile_bounded_compare_len_i32( + &mut self, + len_expr: &Expr, + max_len: u32, + name_prefix: &str, + ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { + let len_val = self.compile_expr(len_expr)?; + let len_iv = match len_val { + BasicValueEnum::IntValue(iv) => iv, + _ => { + return Err(CodeGenError::TypeError(format!( + "{name_prefix} length must be an integer expression" + ))) + } + }; + let i32_ty = self.context.i32_type(); + let len_i32 = if len_iv.get_type().get_bit_width() > 32 { + self.builder + .build_int_truncate(len_iv, i32_ty, &format!("{name_prefix}_len_trunc")) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } else if len_iv.get_type().get_bit_width() < 32 { + self.builder + .build_int_z_extend(len_iv, i32_ty, &format!("{name_prefix}_len_zext")) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } else { + len_iv + }; + let zero_i32 = i32_ty.const_zero(); + let is_neg = self + .builder + .build_int_compare( + inkwell::IntPredicate::SLT, + len_i32, + zero_i32, + &format!("{name_prefix}_len_neg"), + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let len_nn = self + .builder + .build_select(is_neg, zero_i32, len_i32, &format!("{name_prefix}_len_nn")) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let max_const = i32_ty.const_int(max_len as u64, false); + let gt = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGT, + len_nn, + max_const, + &format!("{name_prefix}_len_gt"), + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let bounded_len = self + .builder + .build_select(gt, max_const, len_nn, &format!("{name_prefix}_len_sel")) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let is_zero = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + bounded_len, + zero_i32, + &format!("{name_prefix}_len_is_zero"), + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok((bounded_len, is_zero)) + } + + pub(super) fn compile_strncmp_builtin( + &mut self, + dwarf_expr: &Expr, + lit: &str, + n_expr: &Expr, + ) -> Result> { + // Fast path: if the first argument is a script string variable or a string literal, + // perform a compile-time bounded comparison and return a constant boolean. + let immediate_bytes_opt = match dwarf_expr { + Expr::Variable(name) => { + if self + .get_variable_type(name) + .is_some_and(|t| matches!(t, crate::script::VarType::String)) + { + self.get_string_variable_bytes(name).cloned() + } else { + None + } + } + Expr::String(s) => { + let mut b = s.as_bytes().to_vec(); + b.push(0); + Some(b) + } + _ => None, + }; + + if let Some(bytes) = immediate_bytes_opt { + if let Expr::Int(n) = n_expr { + let n_usize = std::cmp::min( + (*n).max(0) as usize, + self.compile_options.compare_cap as usize, + ); + let content_len = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len()); + let cmp_len = std::cmp::min(n_usize, std::cmp::min(content_len, lit.len())); + let equal = bytes.get(0..cmp_len).unwrap_or(&[]) + == lit.as_bytes().get(0..cmp_len).unwrap_or(&[]); + let bool_val = self + .context + .bool_type() + .const_int(if equal { 1 } else { 0 }, false); + return Ok(bool_val.into()); + } + + // Treat as bounded byte compare between two immediate strings + let cap = self.compile_options.compare_cap as usize; + let content_len = bytes.iter().position(|&b| b == 0).unwrap_or(bytes.len()); + let cmp_bound = std::cmp::min(cap, std::cmp::min(content_len, lit.len())) as u32; + if cmp_bound == 0 { + return Ok(self.context.bool_type().const_int(1, false).into()); + } + let (bounded_len, _len_is_zero) = + self.compile_bounded_compare_len_i32(n_expr, cmp_bound, "strncmp")?; + let i32_ty = self.context.i32_type(); + let i8_ty = self.context.i8_type(); + let mut acc = i8_ty.const_zero(); + for (i, (byte, lit_byte)) in bytes + .iter() + .copied() + .zip(lit.as_bytes().iter().copied()) + .take(cmp_bound as usize) + .enumerate() + { + let active = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGT, + bounded_len, + i32_ty.const_int(i as u64, false), + "strncmp_imm_active", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let diff = i8_ty.const_int((byte ^ lit_byte) as u64, false); + let active_diff = self + .builder + .build_select(active, diff, i8_ty.const_zero(), "strncmp_imm_diff") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + acc = self + .builder + .build_or(acc, active_diff, "strncmp_imm_acc") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + } + let equal = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + acc, + i8_ty.const_zero(), + "strncmp_imm_eq", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(equal.into()); + } + + // Determine pointer value (i64) of the target memory (DWARF or alias) + // Prefer DWARF resolution for richer status/hints; fallback to generic pointer resolver. + let ptr_i64 = match self.query_dwarf_for_complex_expr(dwarf_expr)? { + Some(var) => { + if let Some(ty) = var.dwarf_type.as_ref() { + let ty = ghostscope_dwarf::strip_type_aliases(ty); + match ty { + DwarfType::PointerType { .. } => { + let pc_address = self.get_compile_time_context()?.pc_address; + let val_any = + self.variable_read_plan_to_llvm_value(&var, pc_address, None)?; + match val_any { + BasicValueEnum::IntValue(iv) => { + RuntimeAddress::available(iv, self.context) + } + BasicValueEnum::PointerValue(pv) => self + .builder + .build_ptr_to_int(pv, self.context.i64_type(), "ptr_as_i64") + .map(|value| RuntimeAddress::available(value, self.context)) + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + _ => { + return Err(CodeGenError::TypeError( + "strncmp requires pointer/integer value for pointer; got unsupported DWARF value".into(), + )) + } + } + } + DwarfType::ArrayType { .. } => { + let status_ptr = if self.condition_context_active { + Some(self.get_or_create_cond_error_global()) + } else { + None + }; + let pc_address = self.get_compile_time_context()?.pc_address; + self.variable_read_plan_to_runtime_address( + &var, pc_address, status_ptr, + )? + } + _ => { + // Not a pointer/array -> treat as error + return Err(CodeGenError::TypeError( + "strncmp requires the non-string side to be an address expression (pointer/array)".into(), + )); + } + } + } else { + return Err(CodeGenError::TypeError( + "strncmp non-string side lacks DWARF type info".into(), + )); + } + } + None => { + // Generic pointer expr (e.g., alias); resolve to i64 + self.resolve_runtime_address_from_expr(dwarf_expr).map_err(|_| { + CodeGenError::TypeError( + "strncmp requires at least one string argument, and the other side must be an address expression (DWARF pointer/array or alias)".to_string(), + ) + })? + } + }; + + let cap = self.compile_options.compare_cap; + let cmp_bound = std::cmp::min(lit.len() as u32, cap); + if cmp_bound == 0 { + return Ok(self.context.bool_type().const_int(1, false).into()); + } + let (bounded_len, len_is_zero) = + self.compile_bounded_compare_len_i32(n_expr, cmp_bound, "strncmp")?; + + let func = self.current_function("compile strncmp length branch")?; + let zero_b = self.context.append_basic_block(func, "strncmp_len_zero"); + let nz_b = self.context.append_basic_block(func, "strncmp_len_nz"); + let final_b = self.context.append_basic_block(func, "strncmp_len_cont"); + self.builder + .build_conditional_branch(len_is_zero, zero_b, nz_b) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + self.builder.position_at_end(zero_b); + let bool_true = self.context.bool_type().const_int(1, false); + self.builder + .build_unconditional_branch(final_b) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let zero_block = self.current_insert_block("finish strncmp zero-length block")?; + + self.builder.position_at_end(nz_b); + + let (arr_ty, buf_global) = self.get_or_create_i8_buffer(cmp_bound, "_gs_bi_strncmp"); + let ptr_ty = self.context.ptr_type(AddressSpace::default()); + let dst_ptr = self + .builder + .build_bit_cast(buf_global, ptr_ty, "strncmp_dst_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let base_src_ptr = self + .builder + .build_int_to_ptr(ptr_i64.value, ptr_ty, "strncmp_src_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let src_ptr = self + .builder + .build_select::, _>( + ptr_i64.offsets_found, + base_src_ptr.into(), + ptr_ty.const_null().into(), + "strncmp_src_or_null", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_pointer_value(); + let effective_len = self + .builder + .build_select::, _>( + ptr_i64.offsets_found, + bounded_len.into(), + self.context.i32_type().const_zero().into(), + "strncmp_len_or_zero", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let ret = self + .create_bpf_helper_call( + BPF_FUNC_probe_read_user as u64, + &[dst_ptr, effective_len.into(), src_ptr.into()], + self.context.i64_type().into(), + "probe_read_user_strncmp", + )? + .into_int_value(); + let read_ok = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + ret, + self.context.i64_type().const_zero(), + "rd_ok", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let status_ok = self + .builder + .build_and(read_ok, ptr_i64.offsets_found, "strncmp_ok_with_offsets") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + // If in condition context and read failed, set condition error code = 1 (ProbeReadFailed) + if self.condition_context_active { + let func = self.current_function("compile strncmp condition error branch")?; + let set_b = self.context.append_basic_block(func, "strncmp_set_err"); + let cont_b = self.context.append_basic_block(func, "strncmp_cont"); + let not_ok = self + .builder + .build_not(status_ok, "rd_fail") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder + .build_conditional_branch(not_ok, set_b, cont_b) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder.position_at_end(set_b); + // VariableStatus::ReadError = 2 + let _ = self.set_condition_error_if_unset(2u8); + let _ = self.set_condition_error_addr_if_unset(ptr_i64.value); + // flags: bit0 = read failure for strncmp + let one = self.context.i8_type().const_int(1, false); + let _ = self.or_condition_error_flags(one); + self.builder + .build_unconditional_branch(cont_b) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder.position_at_end(cont_b); + } + + // XOR/OR accumulation over the bounded maximum; inactive bytes do not contribute. + let i32_ty = self.context.i32_type(); + let idx0 = i32_ty.const_zero(); + let mut acc = self.context.i8_type().const_zero(); + for (i, b) in lit.as_bytes().iter().take(cmp_bound as usize).enumerate() { + let idx_i = i32_ty.const_int(i as u64, false); + // SAFETY: i is bounded by cmp_bound, which is clamped to the buffer cap. + let ptr_i = unsafe { + self.builder + .build_gep(arr_ty, buf_global, &[idx0, idx_i], "ch_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let ch = self + .builder + .build_load(self.context.i8_type(), ptr_i, "ch") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let ch = match ch { + BasicValueEnum::IntValue(iv) => iv, + _ => return Err(CodeGenError::LLVMError("load did not return i8".into())), + }; + let expect = self.context.i8_type().const_int(*b as u64, false); + let diff = self + .builder + .build_xor(ch, expect, "diff") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let active = self + .builder + .build_int_compare( + inkwell::IntPredicate::UGT, + bounded_len, + idx_i, + "strncmp_byte_active", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let diff = self + .builder + .build_select( + active, + diff, + self.context.i8_type().const_zero(), + "strncmp_active_diff", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + acc = self + .builder + .build_or(acc, diff, "acc_or") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + } + let eq_bytes = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + acc, + self.context.i8_type().const_zero(), + "acc_zero", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + let result = self + .builder + .build_and(status_ok, eq_bytes, "strncmp_and") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder + .build_unconditional_branch(final_b) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let nz_block = self.current_insert_block("finish strncmp non-zero block")?; + + self.builder.position_at_end(final_b); + let result_phi = self + .builder + .build_phi(self.context.bool_type(), "strncmp_result") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + result_phi.add_incoming(&[(&bool_true, zero_block), (&result, nz_block)]); + Ok(result_phi.as_basic_value()) + } + + /// Compile comparison between a DWARF-side expression and a script string literal. + /// Supports char* and char[N] according to design in string_comparison.md. + pub(super) fn compile_string_comparison( + &mut self, + dwarf_expr: &Expr, + lit: &str, + is_equal: bool, + ) -> Result> { + use ghostscope_dwarf::TypeInfo as TI; + + // Query DWARF for the non-string side to obtain evaluation and type info + let var = self + .query_dwarf_for_complex_expr(dwarf_expr)? + .ok_or_else(|| { + CodeGenError::TypeError( + "string comparison requires DWARF variable/expression".into(), + ) + })?; + // Try DWARF type first; if unavailable, fall back to type_name string parsing + let dwarf_type_opt = var.dwarf_type.as_ref(); + + enum ParsedKind { + PtrChar, + ArrChar(Option), + Other, + } + fn parse_type_name(name: &str) -> ParsedKind { + let lower = name.to_lowercase(); + let has_char = lower.contains("char"); + let is_ptr = lower.contains('*'); + if has_char && is_ptr { + return ParsedKind::PtrChar; + } + if has_char && lower.contains('[') { + // Try to extract N inside brackets + let mut n: Option = None; + if let Some(start) = lower.find('[') { + if let Some(end) = lower[start + 1..].find(']') { + let inside = &lower[start + 1..start + 1 + end]; + let digits: String = + inside.chars().filter(|c| c.is_ascii_digit()).collect(); + if !digits.is_empty() { + if let Ok(v) = digits.parse::() { + n = Some(v); + } + } + } + } + return ParsedKind::ArrChar(n); + } + ParsedKind::Other + } + + let lit_bytes = lit.as_bytes(); + let lit_len = lit_bytes.len() as u32; + let one = self.context.bool_type().const_int(1, false); + let zero = self.context.bool_type().const_zero(); + + // Build final boolean accumulator + let result = match dwarf_type_opt.map(ghostscope_dwarf::strip_type_aliases) { + // char* / const char* + Some(TI::PointerType { target_type, .. }) => { + // Ensure pointee is char-like + let base = ghostscope_dwarf::strip_type_aliases(target_type.as_ref()); + let is_char_like = matches!(base, TI::BaseType { name, size, .. } if name.contains("char") && *size == 1); + if !is_char_like { + return Err(CodeGenError::TypeError( + "automatic string comparison only supports char*".into(), + )); + } + + // Evaluate expression to pointer value and read up to L+1 bytes + let pc_address = self.get_compile_time_context()?.pc_address; + let val_any = self.variable_read_plan_to_llvm_value(&var, pc_address, None)?; + let ptr_i64 = match val_any { + BasicValueEnum::IntValue(iv) => iv, + BasicValueEnum::PointerValue(pv) => self + .builder + .build_ptr_to_int(pv, self.context.i64_type(), "ptr_as_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + _ => { + return Err(CodeGenError::TypeError( + "pointer value must be integer or pointer".into(), + )) + } + }; + let need = lit_len + 1; + let (buf_global, ret_len, arr_ty) = self.read_user_cstr_into_buffer( + RuntimeAddress::available(ptr_i64, self.context), + need, + "_gs_strbuf", + )?; + + // ret_len must equal L+1 + let i64_ty = self.context.i64_type(); + let expect_len = i64_ty.const_int(need as u64, false); + let len_ok = self + .builder + .build_int_compare(inkwell::IntPredicate::EQ, ret_len, expect_len, "str_len_ok") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + // buf[L] must be '\0' + let i32_ty = self.context.i32_type(); + let idx0 = i32_ty.const_zero(); + let idx_l = i32_ty.const_int(lit_len as u64, false); + // SAFETY: the string read requested lit_len + 1 bytes, so index + // lit_len is within the scratch buffer. + let char_ptr = unsafe { + self.builder + .build_gep(arr_ty, buf_global, &[idx0, idx_l], "nul_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let c = self + .builder + .build_load(self.context.i8_type(), char_ptr, "c_l") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let c = match c { + BasicValueEnum::IntValue(iv) => iv, + _ => return Err(CodeGenError::LLVMError("load did not return i8".into())), + }; + let nul_ok = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + c, + self.context.i8_type().const_zero(), + "nul_ok", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + // Compare first L bytes using XOR/OR accumulation to reduce branchiness + let mut acc = self.context.i8_type().const_zero(); + for (i, b) in lit_bytes.iter().enumerate() { + let idx_i = i32_ty.const_int(i as u64, false); + // SAFETY: lit_bytes length is bounded by the scratch buffer size. + let ptr_i = unsafe { + self.builder + .build_gep(arr_ty, buf_global, &[idx0, idx_i], "ch_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let ch = self + .builder + .build_load(self.context.i8_type(), ptr_i, "ch") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let ch = match ch { + BasicValueEnum::IntValue(iv) => iv, + _ => return Err(CodeGenError::LLVMError("load did not return i8".into())), + }; + let expect = self.context.i8_type().const_int(*b as u64, false); + let diff = self + .builder + .build_xor(ch, expect, "diff") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + acc = self + .builder + .build_or(acc, diff, "acc_or") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + } + let eq_bytes = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + acc, + self.context.i8_type().const_zero(), + "acc_zero", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let ok1 = self + .builder + .build_and(len_ok, nul_ok, "ok_len_nul") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder + .build_and(ok1, eq_bytes, "str_eq") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } + // char[N] + Some(TI::ArrayType { + element_type, + element_count, + total_size, + }) => { + let elem = ghostscope_dwarf::strip_type_aliases(element_type.as_ref()); + let is_char_like = matches!(elem, TI::BaseType { name, size, .. } if name.contains("char") && *size == 1); + if !is_char_like { + return Err(CodeGenError::TypeError( + "automatic string comparison only supports char[N]".into(), + )); + } + // Determine N (element count) + let n_opt = element_count.or_else(|| total_size.map(|ts| ts)); + let n = if let Some(nv) = n_opt { nv as u32 } else { 0 }; + if n == 0 { + return Err(CodeGenError::TypeError( + "array size unknown for char[N] comparison".into(), + )); + } + // If L+1 > N, compile-time false + if lit_len + 1 > n { + // Return const false (or true if '!=' requested) + return Ok((if is_equal { zero } else { one }).into()); + } + let status_ptr = if self.condition_context_active { + Some(self.get_or_create_cond_error_global()) + } else { + None + }; + let pc_address = self.get_compile_time_context()?.pc_address; + let addr = + self.variable_read_plan_to_runtime_address(&var, pc_address, status_ptr)?; + // Read exactly L+1 bytes + let (buf_global, status, arr_ty) = + self.read_user_bytes_into_buffer(addr, lit_len + 1, "_gs_arrbuf")?; + // status == 0 + let status_ok = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + status, + self.context.i64_type().const_zero(), + "rd_ok", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + // buf[L] must be '\0' + let i32_ty = self.context.i32_type(); + let idx0 = i32_ty.const_zero(); + let idx_l = i32_ty.const_int(lit_len as u64, false); + // SAFETY: the string read requested lit_len + 1 bytes, so index + // lit_len is within the scratch buffer. + let char_ptr = unsafe { + self.builder + .build_gep(arr_ty, buf_global, &[idx0, idx_l], "nul_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let c = self + .builder + .build_load(self.context.i8_type(), char_ptr, "c_l") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let c = match c { + BasicValueEnum::IntValue(iv) => iv, + _ => return Err(CodeGenError::LLVMError("load did not return i8".into())), + }; + let nul_ok = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + c, + self.context.i8_type().const_zero(), + "nul_ok", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + // Compare first L bytes using XOR/OR accumulation + let mut acc = self.context.i8_type().const_zero(); + for (i, b) in lit_bytes.iter().enumerate() { + let idx_i = i32_ty.const_int(i as u64, false); + // SAFETY: lit_bytes length is bounded by the scratch buffer size. + let ptr_i = unsafe { + self.builder + .build_gep(arr_ty, buf_global, &[idx0, idx_i], "ch_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let ch = self + .builder + .build_load(self.context.i8_type(), ptr_i, "ch") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let ch = match ch { + BasicValueEnum::IntValue(iv) => iv, + _ => return Err(CodeGenError::LLVMError("load did not return i8".into())), + }; + let expect = self.context.i8_type().const_int(*b as u64, false); + let diff = self + .builder + .build_xor(ch, expect, "diff") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + acc = self + .builder + .build_or(acc, diff, "acc_or") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + } + let eq_bytes = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + acc, + self.context.i8_type().const_zero(), + "acc_zero", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let ok1 = self + .builder + .build_and(status_ok, nul_ok, "ok_len_nul") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder + .build_and(ok1, eq_bytes, "arr_eq") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } + None => { + let status_ptr = if self.condition_context_active { + Some(self.get_or_create_cond_error_global()) + } else { + None + }; + let pc_address = self.get_compile_time_context()?.pc_address; + let addr = + self.variable_read_plan_to_runtime_address(&var, pc_address, status_ptr)?; + // Fallback using type_name string + match parse_type_name(&var.type_name) { + ParsedKind::PtrChar => { + // Load pointer value from variable location (assume 64-bit) + let ptr_any = self.generate_memory_read( + addr, + ghostscope_dwarf::MemoryAccessSize::U64, + None, + )?; + let ptr_i64 = match ptr_any { + BasicValueEnum::IntValue(iv) => iv, + _ => { + return Err(CodeGenError::LLVMError( + "pointer load did not return integer".to_string(), + )) + } + }; + let need = lit_len + 1; + let (buf_global, ret_len, arr_ty) = self.read_user_cstr_into_buffer( + RuntimeAddress::available(ptr_i64, self.context), + need, + "_gs_strbuf", + )?; + + let i64_ty = self.context.i64_type(); + let expect_len = i64_ty.const_int(need as u64, false); + let len_ok = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + ret_len, + expect_len, + "str_len_ok", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + let i32_ty = self.context.i32_type(); + let idx0 = i32_ty.const_zero(); + let idx_l = i32_ty.const_int(lit_len as u64, false); + // SAFETY: the string read requested lit_len + 1 bytes, so + // index lit_len is within the scratch buffer. + let char_ptr = unsafe { + self.builder + .build_gep(arr_ty, buf_global, &[idx0, idx_l], "nul_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let c = self + .builder + .build_load(self.context.i8_type(), char_ptr, "c_l") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let c = match c { + BasicValueEnum::IntValue(iv) => iv, + _ => { + return Err(CodeGenError::LLVMError( + "load did not return i8".into(), + )) + } + }; + let nul_ok = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + c, + self.context.i8_type().const_zero(), + "nul_ok", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + let mut acc = self.context.i8_type().const_zero(); + for (i, b) in lit_bytes.iter().enumerate() { + let idx_i = i32_ty.const_int(i as u64, false); + // SAFETY: lit_bytes length is bounded by the scratch buffer size. + let ptr_i = unsafe { + self.builder + .build_gep(arr_ty, buf_global, &[idx0, idx_i], "ch_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let ch = self + .builder + .build_load(self.context.i8_type(), ptr_i, "ch") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let ch = match ch { + BasicValueEnum::IntValue(iv) => iv, + _ => { + return Err(CodeGenError::LLVMError( + "load did not return i8".into(), + )) + } + }; + let expect = self.context.i8_type().const_int(*b as u64, false); + let diff = self + .builder + .build_xor(ch, expect, "diff") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + acc = self + .builder + .build_or(acc, diff, "acc_or") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + } + let eq_bytes = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + acc, + self.context.i8_type().const_zero(), + "acc_zero", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let ok1 = self + .builder + .build_and(len_ok, nul_ok, "ok_len_nul") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder + .build_and(ok1, eq_bytes, "str_eq") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } + ParsedKind::ArrChar(n_opt) => { + // If we know N and L+1>N, return false; else read L+1 bytes + if let Some(n) = n_opt { + if lit_len + 1 > n { + return Ok((if is_equal { zero } else { one }).into()); + } + } + let (buf_global, status, arr_ty) = + self.read_user_bytes_into_buffer(addr, lit_len + 1, "_gs_arrbuf")?; + let status_ok = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + status, + self.context.i64_type().const_zero(), + "rd_ok", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let i32_ty = self.context.i32_type(); + let idx0 = i32_ty.const_zero(); + let idx_l = i32_ty.const_int(lit_len as u64, false); + // SAFETY: the string read requested lit_len + 1 bytes, so + // index lit_len is within the scratch buffer. + let char_ptr = unsafe { + self.builder + .build_gep(arr_ty, buf_global, &[idx0, idx_l], "nul_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let c = self + .builder + .build_load(self.context.i8_type(), char_ptr, "c_l") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let c = match c { + BasicValueEnum::IntValue(iv) => iv, + _ => { + return Err(CodeGenError::LLVMError( + "load did not return i8".into(), + )) + } + }; + let nul_ok = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + c, + self.context.i8_type().const_zero(), + "nul_ok", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let mut acc = self.context.i8_type().const_zero(); + for (i, b) in lit_bytes.iter().enumerate() { + let idx_i = i32_ty.const_int(i as u64, false); + // SAFETY: lit_bytes length is bounded by the scratch buffer size. + let ptr_i = unsafe { + self.builder + .build_gep(arr_ty, buf_global, &[idx0, idx_i], "ch_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + let ch = self + .builder + .build_load(self.context.i8_type(), ptr_i, "ch") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let ch = match ch { + BasicValueEnum::IntValue(iv) => iv, + _ => { + return Err(CodeGenError::LLVMError( + "load did not return i8".into(), + )) + } + }; + let expect = self.context.i8_type().const_int(*b as u64, false); + let diff = self + .builder + .build_xor(ch, expect, "diff") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + acc = self + .builder + .build_or(acc, diff, "acc_or") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + } + let eq_bytes = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + acc, + self.context.i8_type().const_zero(), + "acc_zero", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let ok1 = self + .builder + .build_and(status_ok, nul_ok, "ok_len_nul") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + self.builder + .build_and(ok1, eq_bytes, "arr_eq") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } + ParsedKind::Other => { + return Err(CodeGenError::TypeError(format!( + "string comparison unsupported for type name '{}' without DWARF type", + var.type_name + ))); + } + } + } + Some(_) => { + return Err(CodeGenError::TypeError( + "string comparison only supports char* or char[N]".into(), + )); + } + }; + + // Apply == / != + let final_bool = if is_equal { + result + } else { + self.builder + .build_not(result, "not_eq") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + }; + Ok(final_bool.into()) + } +} diff --git a/ghostscope-compiler/src/ebpf/expression/casts.rs b/ghostscope-compiler/src/ebpf/expression/casts.rs new file mode 100644 index 00000000..603a2bf5 --- /dev/null +++ b/ghostscope-compiler/src/ebpf/expression/casts.rs @@ -0,0 +1,648 @@ +use super::{DynamicLvalue, DynamicTypeInfo, IndexableElementInfo}; +use crate::ebpf::context::{CodeGenError, EbpfContext, Result, RuntimeAddress}; +use crate::script::Expr; +use ghostscope_dwarf::TypeInfo as DwarfType; +use inkwell::values::{BasicValueEnum, IntValue}; +use inkwell::AddressSpace; +use std::path::{Path, PathBuf}; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + pub(in crate::ebpf) fn is_dwarf_aggregate_expr(&mut self, expr: &Expr) -> bool { + if let Expr::Cast { target_type, .. } = expr { + return self + .resolve_cast_target_type(target_type) + .ok() + .is_some_and(|ty| ghostscope_dwarf::is_c_aggregate_type(&ty)); + } + + if let Ok(Some(var)) = self.query_dwarf_for_complex_expr(expr) { + if let Some(ref ty) = var.dwarf_type { + return ghostscope_dwarf::is_c_aggregate_type(ty); + } + } + false + } + + /// Heuristic check: whether an expression should be treated as a pointer/address + /// Returns true for: + /// - Explicit address-of forms (&expr) + /// - Script string literals (compile to pointer data) + /// - Alias variables bound to addresses + /// - DWARF-backed expressions whose type is pointer or array + pub(in crate::ebpf) fn is_pointer_like_expr(&mut self, expr: &Expr) -> bool { + use crate::script::Expr as E; + match expr { + E::AddressOf(_) => return true, + E::String(_) => return true, + E::Cast { target_type, .. } => { + if self + .resolve_cast_target_type(target_type) + .ok() + .is_some_and(|ty| { + matches!( + ghostscope_dwarf::strip_type_aliases(&ty), + DwarfType::PointerType { .. } | DwarfType::ArrayType { .. } + ) + }) + { + return true; + } + } + E::Variable(name) => { + if self.alias_variable_exists(name) { + return true; + } + } + _ => {} + } + + if let Ok(Some(var)) = self.query_dwarf_for_complex_expr(expr) { + if let Some(ref ty) = var.dwarf_type { + if ghostscope_dwarf::is_c_pointer_or_array_type(ty) { + return true; + } + } + } + false + } + + pub(in crate::ebpf) fn resolve_cast_target_type(&self, target_type: &str) -> Result { + let analyzer = self.process_analyzer; + let resolved = if let Some(context) = self.current_compile_time_context.as_ref() { + let module_path = Path::new(&context.module_path); + analyzer + .map(|analyzer| analyzer.try_resolve_type_spec_in_module(module_path, target_type)) + .transpose() + .map_err(|err| CodeGenError::DwarfError(err.to_string()))? + .flatten() + .or_else(|| ghostscope_dwarf::DwarfAnalyzer::resolve_builtin_type_spec(target_type)) + } else { + analyzer + .map(|analyzer| analyzer.try_resolve_type_spec(target_type)) + .transpose() + .map_err(|err| CodeGenError::DwarfError(err.to_string()))? + .flatten() + .or_else(|| ghostscope_dwarf::DwarfAnalyzer::resolve_builtin_type_spec(target_type)) + }; + + resolved.ok_or_else(|| { + CodeGenError::DwarfError(format!("cast target type '{target_type}' was not found")) + }) + } + + pub(super) fn cast_pointer_target_type(target_type: &DwarfType) -> Option { + match ghostscope_dwarf::strip_type_aliases(target_type) { + DwarfType::PointerType { target_type, .. } => Some(target_type.as_ref().clone()), + _ => None, + } + } + + fn is_float_dwarf_type(target_type: &DwarfType) -> bool { + match ghostscope_dwarf::strip_type_aliases(target_type) { + DwarfType::BaseType { encoding, .. } => { + *encoding == ghostscope_dwarf::constants::DW_ATE_float.0 as u16 + } + _ => false, + } + } + + fn is_bool_dwarf_type(target_type: &DwarfType) -> bool { + match ghostscope_dwarf::strip_type_aliases(target_type) { + DwarfType::BaseType { encoding, .. } => { + *encoding == ghostscope_dwarf::constants::DW_ATE_boolean.0 as u16 + } + _ => false, + } + } + + pub(in crate::ebpf) fn cast_value_byte_len(target_type: &DwarfType) -> Option { + if matches!( + ghostscope_dwarf::strip_type_aliases(target_type), + DwarfType::PointerType { .. } + ) { + return Some(8); + } + + if let Some(integer_type) = ghostscope_dwarf::c_integer_comparison_type(target_type) { + return Some(integer_type.size.clamp(1, 8) as usize); + } + + None + } + + pub(in crate::ebpf) fn cast_source_pointer_value( + &mut self, + expr: &Expr, + ) -> Result> { + if let Ok(address) = self.resolve_runtime_address_from_expr(expr) { + return Ok(address); + } + + match self.compile_expr(expr)? { + BasicValueEnum::IntValue(value) => Ok(RuntimeAddress::available( + self.normalize_int_to_i64(value, "cast_ptr_i64")?, + self.context, + )), + BasicValueEnum::PointerValue(value) => self + .builder + .build_ptr_to_int(value, self.context.i64_type(), "cast_ptr_value") + .map(|value| RuntimeAddress::available(value, self.context)) + .map_err(|err| CodeGenError::Builder(err.to_string())), + _ => Err(CodeGenError::TypeError( + "cast source expression did not produce an address-sized value".to_string(), + )), + } + } + + pub(in crate::ebpf) fn cast_source_memory_address( + &mut self, + expr: &Expr, + ) -> Result> { + if let Ok(address) = self.resolve_runtime_address_from_expr(expr) { + return Ok(address); + } + + if let Some(plan) = self.query_dwarf_for_complex_expr(expr)? { + let status_ptr = if self.condition_context_active { + Some(self.get_or_create_cond_error_global()) + } else { + None + }; + let pc_address = self.get_compile_time_context()?.pc_address; + if let Ok(address) = + self.variable_read_plan_to_runtime_address(&plan, pc_address, status_ptr) + { + return Ok(address); + } + } + + match self.compile_expr(expr)? { + BasicValueEnum::IntValue(value) => Ok(RuntimeAddress::available( + self.normalize_int_to_i64(value, "cast_mem_i64")?, + self.context, + )), + BasicValueEnum::PointerValue(value) => self + .builder + .build_ptr_to_int(value, self.context.i64_type(), "cast_mem_ptr") + .map(|value| RuntimeAddress::available(value, self.context)) + .map_err(|err| CodeGenError::Builder(err.to_string())), + _ => Err(CodeGenError::TypeError( + "cast source expression is not addressable".to_string(), + )), + } + } + + pub(super) fn cast_lvalue_address_and_type( + &mut self, + expr: &Expr, + target_type: &str, + ) -> Result> { + let target_type = self.resolve_cast_target_type(target_type)?; + let module_path = self + .current_compile_time_context + .as_ref() + .map(|context| PathBuf::from(&context.module_path)); + + if let Some(pointee_type) = Self::cast_pointer_target_type(&target_type) { + let address = self.cast_source_pointer_value(expr)?; + return Ok(DynamicLvalue { + address, + type_info: DynamicTypeInfo { + dwarf_type: pointee_type, + module_path, + }, + }); + } + + let address = self.cast_source_memory_address(expr)?; + Ok(DynamicLvalue { + address, + type_info: DynamicTypeInfo { + dwarf_type: target_type, + module_path, + }, + }) + } + + pub(super) fn cast_index_base( + &mut self, + expr: &Expr, + ) -> Result)>> { + let Expr::Cast { + expr: source_expr, + target_type, + } = expr + else { + return Ok(None); + }; + + let target_type = self.resolve_cast_target_type(target_type)?; + let module_path = self + .current_compile_time_context + .as_ref() + .map(|context| PathBuf::from(&context.module_path)); + + match ghostscope_dwarf::strip_type_aliases(&target_type) { + DwarfType::PointerType { .. } => { + let Some(element_info) = Self::indexable_info_from_type(&target_type, module_path) + else { + return Ok(None); + }; + let base_address = self.cast_source_pointer_value(source_expr)?; + Ok(Some((element_info, base_address))) + } + DwarfType::ArrayType { .. } => { + let Some(element_info) = Self::indexable_info_from_type(&target_type, module_path) + else { + return Ok(None); + }; + let base_address = self.cast_source_memory_address(source_expr)?; + Ok(Some((element_info, base_address))) + } + _ => Ok(None), + } + } + + pub(super) fn indexable_info_from_type( + dwarf_type: &DwarfType, + module_path: Option, + ) -> Option { + ghostscope_dwarf::indexable_element_layout(dwarf_type).map(|layout| IndexableElementInfo { + element_type: layout.element_type, + stride: layout.stride, + module_path, + }) + } + + pub(super) fn compiled_pointer_value_to_runtime_address( + &mut self, + value: BasicValueEnum<'ctx>, + int_name: &str, + ptr_name: &str, + error_message: &'static str, + ) -> Result> { + match value { + BasicValueEnum::IntValue(value) => Ok(RuntimeAddress::available( + self.normalize_int_to_i64(value, int_name)?, + self.context, + )), + BasicValueEnum::PointerValue(value) => self + .builder + .build_ptr_to_int(value, self.context.i64_type(), ptr_name) + .map(|value| RuntimeAddress::available(value, self.context)) + .map_err(|err| CodeGenError::Builder(err.to_string())), + _ => Err(CodeGenError::TypeError(error_message.to_string())), + } + } + + pub(super) fn dynamic_lvalue_from_indexable_base( + &mut self, + element_info: IndexableElementInfo, + base_address: RuntimeAddress<'ctx>, + index_value: IntValue<'ctx>, + name: &str, + ) -> Result> { + let stride_value = self + .context + .i64_type() + .const_int(element_info.stride, false); + let byte_offset = self + .builder + .build_int_mul(index_value, stride_value, &format!("{name}_byte_offset")) + .map_err(|err| CodeGenError::Builder(err.to_string()))?; + let element_address = self + .builder + .build_int_add( + base_address.value, + byte_offset, + &format!("{name}_element_address"), + ) + .map_err(|err| CodeGenError::Builder(err.to_string()))?; + + Ok(DynamicLvalue { + address: base_address.with_value(element_address), + type_info: DynamicTypeInfo { + dwarf_type: element_info.element_type, + module_path: element_info.module_path, + }, + }) + } + + pub(super) fn dynamic_lvalue_from_const_pointer_arithmetic( + &mut self, + expr: &Expr, + ) -> Result>> { + let Some((base_expr, index)) = self.pointer_arithmetic_parts_expanding_aliases(expr)? + else { + return Ok(None); + }; + let Some((element_info, base_address)) = self.cast_index_base(&base_expr)? else { + return Ok(None); + }; + let index_value = self.context.i64_type().const_int(index as u64, true); + self.dynamic_lvalue_from_indexable_base( + element_info, + base_address, + index_value, + "dynamic_cast_ptr_arith", + ) + .map(Some) + } + + fn compile_cast_integer_value( + &mut self, + expr: &Expr, + target_type: &DwarfType, + ) -> Result> { + let value = match self.compile_expr(expr)? { + BasicValueEnum::IntValue(value) => value, + BasicValueEnum::PointerValue(value) => self + .builder + .build_ptr_to_int(value, self.context.i64_type(), "cast_int_ptr") + .map_err(|err| CodeGenError::Builder(err.to_string()))?, + _ => { + return Err(CodeGenError::TypeError( + "integer cast source must be an integer or pointer".to_string(), + )) + } + }; + + if Self::is_bool_dwarf_type(target_type) { + let value = self.normalize_int_to_i64(value, "cast_bool_i64")?; + return self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + value, + self.context.i64_type().const_zero(), + "cast_bool", + ) + .map_err(|err| CodeGenError::Builder(err.to_string())); + } + + let Some(integer_type) = ghostscope_dwarf::c_integer_comparison_type(target_type) else { + return Err(CodeGenError::TypeError(format!( + "cast target '{}' is not an integer type", + target_type.type_name() + ))); + }; + + let bit_width = integer_type.size.saturating_mul(8).clamp(1, 64) as u32; + let target_int_type = self.context.custom_width_int_type(bit_width); + let current_width = value.get_type().get_bit_width(); + let narrowed = if current_width > bit_width { + self.builder + .build_int_truncate(value, target_int_type, "cast_int_trunc") + .map_err(|err| CodeGenError::Builder(err.to_string()))? + } else if current_width < bit_width { + if integer_type.is_unsigned || current_width == 1 { + self.builder + .build_int_z_extend(value, target_int_type, "cast_int_zext") + .map_err(|err| CodeGenError::Builder(err.to_string()))? + } else { + self.builder + .build_int_s_extend(value, target_int_type, "cast_int_sext") + .map_err(|err| CodeGenError::Builder(err.to_string()))? + } + } else { + value + }; + + if bit_width == 64 { + return Ok(narrowed); + } + + if integer_type.is_unsigned { + self.builder + .build_int_z_extend(narrowed, self.context.i64_type(), "cast_int_zext_i64") + .map_err(|err| CodeGenError::Builder(err.to_string())) + } else { + self.builder + .build_int_s_extend(narrowed, self.context.i64_type(), "cast_int_sext_i64") + .map_err(|err| CodeGenError::Builder(err.to_string())) + } + } + + pub(super) fn compile_cast_expr_value( + &mut self, + expr: &Expr, + target_type: &str, + ) -> Result> { + let target_type = self.resolve_cast_target_type(target_type)?; + + if Self::cast_pointer_target_type(&target_type).is_some() { + let address = self.cast_source_pointer_value(expr)?; + let ptr_ty = self.context.ptr_type(AddressSpace::default()); + return self + .builder + .build_int_to_ptr(address.value, ptr_ty, "cast_as_ptr") + .map(|value| value.into()) + .map_err(|err| CodeGenError::Builder(err.to_string())); + } + + if ghostscope_dwarf::is_c_aggregate_type(&target_type) { + let address = self.cast_source_memory_address(expr)?; + let ptr_ty = self.context.ptr_type(AddressSpace::default()); + return self + .builder + .build_int_to_ptr(address.value, ptr_ty, "cast_aggregate_ptr") + .map(|value| value.into()) + .map_err(|err| CodeGenError::Builder(err.to_string())); + } + + if ghostscope_dwarf::c_integer_comparison_type(&target_type).is_some() { + return self + .compile_cast_integer_value(expr, &target_type) + .map(|value| value.into()); + } + + if Self::is_float_dwarf_type(&target_type) { + return Err(CodeGenError::TypeError( + "floating-point casts are only supported for memory reads/printing".to_string(), + )); + } + + Err(CodeGenError::TypeError(format!( + "cast target '{}' is not supported as a value expression", + target_type.type_name() + ))) + } + + pub(crate) fn integer_literal_value(expr: &Expr) -> Option { + use crate::script::ast::BinaryOp as BO; + use crate::script::ast::Expr as E; + + match expr { + E::Int(value) => Some(*value), + E::BinaryOp { + left, + op: BO::Add, + right, + } => { + Self::integer_literal_value(left)?.checked_add(Self::integer_literal_value(right)?) + } + E::BinaryOp { + left, + op: BO::Subtract, + right, + } => { + Self::integer_literal_value(left)?.checked_sub(Self::integer_literal_value(right)?) + } + E::BinaryOp { + left, + op: BO::Multiply, + right, + } => { + Self::integer_literal_value(left)?.checked_mul(Self::integer_literal_value(right)?) + } + E::BinaryOp { + left, + op: BO::Divide, + right, + } => { + Self::integer_literal_value(left)?.checked_div(Self::integer_literal_value(right)?) + } + E::BinaryOp { + left, + op: BO::Modulo, + right, + } => { + Self::integer_literal_value(left)?.checked_rem(Self::integer_literal_value(right)?) + } + E::BinaryOp { + left, + op: BO::BitAnd, + right, + } => Some(Self::integer_literal_value(left)? & Self::integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::BitXor, + right, + } => Some(Self::integer_literal_value(left)? ^ Self::integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::BitOr, + right, + } => Some(Self::integer_literal_value(left)? | Self::integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::ShiftLeft, + right, + } => { + let shift = u32::try_from(Self::integer_literal_value(right)?).ok()?; + Self::integer_literal_value(left)?.checked_shl(shift) + } + E::BinaryOp { + left, + op: BO::ShiftRight, + right, + } => { + let shift = u32::try_from(Self::integer_literal_value(right)?).ok()?; + Self::integer_literal_value(left)?.checked_shr(shift) + } + E::UnaryBitNot(inner) => Some(!Self::integer_literal_value(inner)?), + _ => None, + } + } + + pub(in crate::ebpf) fn pointer_arithmetic_parts(expr: &Expr) -> Option<(&Expr, i64)> { + use crate::script::ast::Expr as E; + + fn collect_offset(expr: &Expr, acc: i64) -> Option<(&Expr, i64)> { + use crate::script::ast::BinaryOp as BO; + use crate::script::ast::Expr as E; + + match expr { + E::BinaryOp { + left, + op: BO::Add, + right, + } => match (&**left, &**right) { + (ptr_side, int_expr) + if EbpfContext::<'static, 'static>::integer_literal_value(int_expr) + .is_some() => + { + let index = + EbpfContext::<'static, 'static>::integer_literal_value(int_expr)?; + collect_offset(ptr_side, acc.checked_add(index)?) + } + (int_expr, ptr_side) + if EbpfContext::<'static, 'static>::integer_literal_value(int_expr) + .is_some() => + { + let index = + EbpfContext::<'static, 'static>::integer_literal_value(int_expr)?; + collect_offset(ptr_side, acc.checked_add(index)?) + } + _ => Some((expr, acc)), + }, + E::BinaryOp { + left, + op: BO::Subtract, + right, + } => match &**right { + int_expr + if EbpfContext::<'static, 'static>::integer_literal_value(int_expr) + .is_some() => + { + let index = + EbpfContext::<'static, 'static>::integer_literal_value(int_expr)?; + collect_offset(left, acc.checked_sub(index)?) + } + _ => Some((expr, acc)), + }, + _ => Some((expr, acc)), + } + } + + let E::BinaryOp { .. } = expr else { + return None; + }; + + let (base, index) = collect_offset(expr, 0)?; + match base { + E::BinaryOp { .. } => None, + _ => Some((base, index)), + } + } + + pub(in crate::ebpf) fn pointer_arithmetic_parts_expanding_aliases( + &self, + expr: &Expr, + ) -> Result> { + let Some((base, index)) = Self::pointer_arithmetic_parts(expr) else { + return Ok(None); + }; + + let mut base = base.clone(); + let mut index = index; + let mut visited = std::collections::HashSet::new(); + + loop { + let Expr::Variable(name) = &base else { + break; + }; + if !self.alias_variable_exists(name) { + break; + } + if !visited.insert(name.clone()) { + return Err(CodeGenError::TypeError(format!( + "alias cycle detected for '{name}'" + ))); + } + let Some(target) = self.get_alias_variable(name) else { + break; + }; + if let Some((alias_base, alias_index)) = Self::pointer_arithmetic_parts(&target) { + index = alias_index.checked_add(index).ok_or_else(|| { + CodeGenError::TypeError("pointer arithmetic offset overflow".to_string()) + })?; + base = alias_base.clone(); + } else { + base = target; + } + } + + Ok(Some((base, index))) + } +} diff --git a/ghostscope-compiler/src/ebpf/expression/dwarf_access.rs b/ghostscope-compiler/src/ebpf/expression/dwarf_access.rs new file mode 100644 index 00000000..48f32443 --- /dev/null +++ b/ghostscope-compiler/src/ebpf/expression/dwarf_access.rs @@ -0,0 +1,782 @@ +use super::{DynamicLvalue, DynamicTypeInfo, IndexableElementInfo}; +use crate::ebpf::context::{CodeGenError, EbpfContext, Result, RuntimeAddress}; +use crate::script::Expr; +use ghostscope_dwarf::{ + AmbiguityReason, Availability, RuntimeRequirement, TypeInfo as DwarfType, TypeLayoutError, + UnsupportedReason, VariableReadPlan, +}; +use inkwell::values::{BasicValueEnum, IntValue, PointerValue}; +use inkwell::AddressSpace; +use std::path::{Path, PathBuf}; +use tracing::debug; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + /// Compile member access (struct.field) + pub fn compile_member_access( + &mut self, + obj_expr: &Expr, + field: &str, + ) -> Result> { + // Create a MemberAccess expression and use the unified DWARF compilation + let member_access_expr = Expr::MemberAccess(Box::new(obj_expr.clone()), field.to_string()); + self.compile_dwarf_expression(&member_access_expr) + } + + /// Compile pointer dereference (*ptr) + pub fn compile_pointer_deref(&mut self, expr: &Expr) -> Result> { + // Create a PointerDeref expression and use the unified DWARF compilation + let pointer_deref_expr = Expr::PointerDeref(Box::new(expr.clone())); + self.compile_dwarf_expression(&pointer_deref_expr) + } + + /// Compile array access (arr[index]) + pub fn compile_array_access( + &mut self, + array_expr: &Expr, + index_expr: &Expr, + ) -> Result> { + if let Some((value, _element_type)) = + self.compile_dynamic_array_access_value(array_expr, index_expr)? + { + return Ok(value); + } + + // Create an ArrayAccess expression and use the unified DWARF compilation + let array_access_expr = + Expr::ArrayAccess(Box::new(array_expr.clone()), Box::new(index_expr.clone())); + self.compile_dwarf_expression(&array_access_expr) + } + + /// Compile chain access (person.name.first) + pub fn compile_chain_access(&mut self, chain: &[String]) -> Result> { + // Create a ChainAccess expression and use the unified DWARF compilation + let chain_access_expr = Expr::ChainAccess(chain.to_vec()); + self.compile_dwarf_expression(&chain_access_expr) + } + + /// Unified DWARF expression compilation + pub fn compile_dwarf_expression( + &mut self, + expr: &crate::script::Expr, + ) -> Result> { + debug!( + "compile_dwarf_expression: Compiling complex expression: {:?}", + expr + ); + + if let crate::script::Expr::Cast { + expr: inner, + target_type, + } = expr + { + return self.compile_cast_expr_value(inner, target_type); + } + + if let crate::script::Expr::ArrayAccess(array_expr, index_expr) = expr { + if let Some((value, _element_type)) = + self.compile_dynamic_array_access_value(array_expr, index_expr)? + { + return Ok(value); + } + } + if let crate::script::Expr::MemberAccess(obj_expr, field) = expr { + if let Some((value, _member_type)) = + self.compile_dynamic_member_access_value(obj_expr, field)? + { + return Ok(value); + } + } + if matches!(expr, crate::script::Expr::PointerDeref(_)) { + if let Some(lvalue) = self.dynamic_lvalue_address_and_type(expr)? { + return self + .read_dynamic_address_value(lvalue.address, &lvalue.type_info.dwarf_type); + } + } + + // Query DWARF for the complex expression + let compile_context = self.get_compile_time_context()?.clone(); + let variable_plan = match self.query_dwarf_for_complex_expr(expr)? { + Some(var) => var, + None => { + let expr_str = Self::expr_to_debug_string(expr); + return Err(CodeGenError::VariableNotFound(expr_str)); + } + }; + + let materialized = + self.variable_read_plan_to_materialization(variable_plan, compile_context.pc_address)?; + let dwarf_type = materialized.dwarf_type.as_ref().ok_or_else(|| { + CodeGenError::DwarfError("Expression has no DWARF type information".to_string()) + })?; + + debug!( + "compile_dwarf_expression: Found DWARF info for expression '{}' with type: {:?}", + materialized.name, dwarf_type + ); + + self.variable_materialization_to_llvm_value(&materialized, compile_context.pc_address, None) + } + + pub(in crate::ebpf) fn compile_dynamic_array_access_value( + &mut self, + array_expr: &Expr, + index_expr: &Expr, + ) -> Result, DwarfType)>> { + let Some(element_lvalue) = + self.compile_dynamic_array_element_address(array_expr, index_expr)? + else { + return Ok(None); + }; + + let value = self.read_dynamic_address_value( + element_lvalue.address, + &element_lvalue.type_info.dwarf_type, + )?; + Ok(Some((value, element_lvalue.type_info.dwarf_type))) + } + + pub(in crate::ebpf) fn compile_dynamic_member_access_value( + &mut self, + obj_expr: &Expr, + field: &str, + ) -> Result, DwarfType)>> { + let Some(object_lvalue) = self.dynamic_lvalue_address_and_type(obj_expr)? else { + return Ok(None); + }; + + let Some(element_lvalue) = self.dynamic_member_base_address_and_type(object_lvalue)? else { + return Ok(None); + }; + let (member_offset, member_type) = + self.dynamic_member_offset_and_type(&element_lvalue.type_info, field)?; + + let member_offset = self.context.i64_type().const_int(member_offset, false); + let member_address = self + .builder + .build_int_add( + element_lvalue.address.value, + member_offset, + "dynamic_member_address", + ) + .map_err(|err| CodeGenError::Builder(err.to_string()))?; + let value = self.read_dynamic_address_value( + element_lvalue.address.with_value(member_address), + &member_type, + )?; + Ok(Some((value, member_type))) + } + + pub(in crate::ebpf) fn dynamic_lvalue_address_and_type( + &mut self, + expr: &Expr, + ) -> Result>> { + if let Expr::Variable(name) = expr { + if self.alias_variable_exists(name) { + let expanded = self.expand_alias_variable_expr(expr)?; + return self.dynamic_lvalue_address_and_type(&expanded); + } + } + + if let Expr::Cast { + expr: inner, + target_type, + } = expr + { + return self + .cast_lvalue_address_and_type(inner, target_type) + .map(Some); + } + + if let Expr::PointerDeref(inner) = expr { + let expanded_inner = self.expand_alias_variable_expr(inner)?; + if matches!(expanded_inner, Expr::Cast { .. }) { + return self.dynamic_lvalue_address_and_type(&expanded_inner); + } + if let Expr::BinaryOp { .. } = expanded_inner { + if let Some(lvalue) = self.dynamic_lvalue_address_and_type(&expanded_inner)? { + return Ok(Some(lvalue)); + } + } + } + + if let Expr::ArrayAccess(array_expr, index_expr) = expr { + return self.compile_dynamic_array_element_address(array_expr, index_expr); + } + + if let Some(lvalue) = self.dynamic_lvalue_from_const_pointer_arithmetic(expr)? { + return Ok(Some(lvalue)); + } + + if self.expands_to_nonliteral_pointer_arithmetic(expr)? { + let Some(element_info) = self.indexable_element_type_and_stride(expr)? else { + return Ok(None); + }; + let element_address = self.resolve_runtime_address_from_expr(expr)?; + return Ok(Some(DynamicLvalue { + address: element_address, + type_info: DynamicTypeInfo { + dwarf_type: element_info.element_type, + module_path: element_info.module_path, + }, + })); + } + + if let Expr::MemberAccess(obj_expr, field) = expr { + let Some(object_lvalue) = self.dynamic_lvalue_address_and_type(obj_expr)? else { + return Ok(None); + }; + let Some(base_lvalue) = self.dynamic_member_base_address_and_type(object_lvalue)? + else { + return Ok(None); + }; + let (member_offset, member_type) = + self.dynamic_member_offset_and_type(&base_lvalue.type_info, field)?; + let member_offset = self.context.i64_type().const_int(member_offset, false); + let member_address = self + .builder + .build_int_add( + base_lvalue.address.value, + member_offset, + "dynamic_member_lvalue_address", + ) + .map_err(|err| CodeGenError::Builder(err.to_string()))?; + return Ok(Some(DynamicLvalue { + address: base_lvalue.address.with_value(member_address), + type_info: DynamicTypeInfo { + dwarf_type: member_type, + module_path: base_lvalue.type_info.module_path, + }, + })); + } + + Ok(None) + } + + fn dynamic_member_base_address_and_type( + &mut self, + object: DynamicLvalue<'ctx>, + ) -> Result>> { + let module_path = object.type_info.module_path.clone(); + let object_type = self.complete_dynamic_member_element_type( + object.type_info.dwarf_type, + module_path.as_deref(), + ); + match ghostscope_dwarf::strip_type_aliases(&object_type) { + DwarfType::StructType { .. } | DwarfType::UnionType { .. } => Ok(Some(DynamicLvalue { + address: object.address, + type_info: DynamicTypeInfo { + dwarf_type: object_type, + module_path, + }, + })), + DwarfType::PointerType { target_type, .. } => { + let pointer_value = + self.read_dynamic_address_value(object.address, &object_type)?; + let pointer_value = match pointer_value { + BasicValueEnum::IntValue(value) => { + self.normalize_int_to_i64(value, "dynamic_member_pointer_i64")? + } + BasicValueEnum::PointerValue(value) => self + .builder + .build_ptr_to_int( + value, + self.context.i64_type(), + "dynamic_member_pointer_ptr", + ) + .map_err(|err| CodeGenError::Builder(err.to_string()))?, + _ => { + return Err(CodeGenError::TypeError( + "dynamic member pointer base did not compile to an address".to_string(), + )) + } + }; + let target_type = self.complete_dynamic_member_element_type( + target_type.as_ref().clone(), + module_path.as_deref(), + ); + Ok(Some(DynamicLvalue { + address: RuntimeAddress::available(pointer_value, self.context), + type_info: DynamicTypeInfo { + dwarf_type: target_type, + module_path, + }, + })) + } + _ => Ok(None), + } + } + + fn dynamic_member_offset_and_type( + &self, + aggregate: &DynamicTypeInfo, + field: &str, + ) -> Result<(u64, DwarfType)> { + let aggregate_type = self.complete_dynamic_member_element_type( + aggregate.dwarf_type.clone(), + aggregate.module_path.as_deref(), + ); + match ghostscope_dwarf::member_layout(&aggregate_type, field) { + Ok(layout) => Ok((layout.offset, layout.member_type)), + Err(err @ TypeLayoutError::UnknownMember { .. }) => { + Err(CodeGenError::DwarfError(err.to_string())) + } + Err(err @ TypeLayoutError::InvalidMemberBase { .. }) => { + Err(CodeGenError::TypeError(err.to_string())) + } + } + } + + fn dynamic_array_base_from_plan( + &mut self, + array_plan: &VariableReadPlan, + pc_address: u64, + status_ptr: Option>, + static_index: i64, + ) -> Result<(IndexableElementInfo, RuntimeAddress<'ctx>, i64)> { + let module_path = array_plan.module_path.clone(); + let array_type = array_plan.dwarf_type.as_ref().ok_or_else(|| { + CodeGenError::DwarfError("Array expression has no DWARF type information".to_string()) + })?; + let element_info = + Self::indexable_info_from_type(array_type, module_path).ok_or_else(|| { + CodeGenError::TypeError(format!( + "dynamic array index requires array or pointer type, got '{}'", + array_type.type_name() + )) + })?; + + match ghostscope_dwarf::strip_type_aliases(array_type) { + DwarfType::ArrayType { .. } => { + let base_address = + self.variable_read_plan_to_runtime_address(array_plan, pc_address, status_ptr)?; + Ok((element_info, base_address, static_index)) + } + DwarfType::PointerType { .. } => { + let pointer_value = + self.variable_read_plan_to_llvm_value(array_plan, pc_address, status_ptr)?; + let base_address = self.compiled_pointer_value_to_runtime_address( + pointer_value, + "dynamic_array_base_i64", + "dynamic_array_base_ptr", + "array base pointer did not compile to an address", + )?; + Ok((element_info, base_address, static_index)) + } + _ => unreachable!("indexable_info_from_type accepts only array or pointer types"), + } + } + + pub(super) fn compile_dynamic_array_element_address( + &mut self, + array_expr: &Expr, + index_expr: &Expr, + ) -> Result>> { + let literal_index = Self::integer_literal_value(index_expr); + let expanded_array_expr = self.expand_alias_variable_expr(array_expr)?; + let has_dynamic_base = + self.expands_to_nonliteral_pointer_arithmetic(&expanded_array_expr)?; + let cast_base = self.cast_index_base(&expanded_array_expr)?; + + if literal_index.is_some() && !has_dynamic_base && cast_base.is_none() { + return Ok(None); + } + + let compile_context = self.get_compile_time_context()?.clone(); + let status_ptr = if self.condition_context_active { + Some(self.get_or_create_cond_error_global()) + } else { + None + }; + + let (element_info, base_address, static_index) = if let Some((element_info, base_address)) = + cast_base + { + (element_info, base_address, 0) + } else { + match self.query_dwarf_for_complex_expr(array_expr)? { + Some(array_plan) => self.dynamic_array_base_from_plan( + &array_plan, + compile_context.pc_address, + status_ptr, + 0, + )?, + None => { + if let Some((base_expr, static_index)) = + self.pointer_arithmetic_parts_expanding_aliases(&expanded_array_expr)? + { + let array_plan = self + .query_dwarf_for_complex_expr(&base_expr)? + .ok_or_else(|| { + CodeGenError::VariableNotFound(Self::expr_to_debug_string( + &base_expr, + )) + })?; + self.dynamic_array_base_from_plan( + &array_plan, + compile_context.pc_address, + status_ptr, + static_index, + )? + } else if has_dynamic_base { + let element_info = self + .indexable_element_type_and_stride(&expanded_array_expr)? + .ok_or_else(|| { + CodeGenError::VariableNotFound(Self::expr_to_debug_string( + array_expr, + )) + })?; + let base_address = + self.resolve_runtime_address_from_expr(&expanded_array_expr)?; + (element_info, base_address, 0) + } else if let Some(array_lvalue) = + self.dynamic_lvalue_address_and_type(&expanded_array_expr)? + { + let module_path = array_lvalue.type_info.module_path.clone(); + let element_info = Self::indexable_info_from_type( + &array_lvalue.type_info.dwarf_type, + module_path, + ) + .ok_or_else(|| { + CodeGenError::TypeError(format!( + "dynamic array index requires array or pointer type, got '{}'", + array_lvalue.type_info.dwarf_type.type_name() + )) + })?; + match ghostscope_dwarf::strip_type_aliases( + &array_lvalue.type_info.dwarf_type, + ) { + DwarfType::ArrayType { .. } => (element_info, array_lvalue.address, 0), + DwarfType::PointerType { .. } => { + let pointer_value = self.read_dynamic_address_value( + array_lvalue.address, + &array_lvalue.type_info.dwarf_type, + )?; + let base_address = self.compiled_pointer_value_to_runtime_address( + pointer_value, + "dynamic_array_member_ptr_i64", + "dynamic_array_member_ptr", + "array member pointer did not compile to an address", + )?; + (element_info, base_address, 0) + } + _ => unreachable!( + "indexable_info_from_type accepts only array or pointer types" + ), + } + } else { + return Err(CodeGenError::VariableNotFound(Self::expr_to_debug_string( + array_expr, + ))); + } + } + } + }; + + let index_value = if let Some(index) = literal_index { + self.context.i64_type().const_int(index as u64, true) + } else { + match self.compile_expr(index_expr)? { + BasicValueEnum::IntValue(value) => { + self.normalize_int_to_i64(value, "dynamic_array_index_i64")? + } + _ => { + return Err(CodeGenError::TypeError( + "array index expression must compile to an integer".to_string(), + )) + } + } + }; + let index_value = if static_index == 0 { + index_value + } else { + let static_index_value = self.context.i64_type().const_int(static_index as u64, true); + self.builder + .build_int_add( + index_value, + static_index_value, + "dynamic_array_static_index", + ) + .map_err(|err| CodeGenError::Builder(err.to_string()))? + }; + let stride_value = self + .context + .i64_type() + .const_int(element_info.stride, false); + let byte_offset = self + .builder + .build_int_mul(index_value, stride_value, "dynamic_array_byte_offset") + .map_err(|err| CodeGenError::Builder(err.to_string()))?; + let element_address = self + .builder + .build_int_add( + base_address.value, + byte_offset, + "dynamic_array_element_address", + ) + .map_err(|err| CodeGenError::Builder(err.to_string()))?; + + Ok(Some(DynamicLvalue { + address: base_address.with_value(element_address), + type_info: DynamicTypeInfo { + dwarf_type: element_info.element_type, + module_path: element_info.module_path, + }, + })) + } + + fn indexable_element_type_and_stride( + &mut self, + expr: &Expr, + ) -> Result> { + use crate::script::ast::BinaryOp as BO; + use crate::script::ast::Expr as E; + + let expanded = self.expand_alias_variable_expr(expr)?; + + if let Some((element_info, _base_address)) = self.cast_index_base(&expanded)? { + return Ok(Some(element_info)); + } + + if let Some(plan) = self.query_dwarf_for_complex_expr(&expanded)? { + if let Some(dwarf_type) = plan.dwarf_type.as_ref() { + if let Some(info) = + Self::indexable_info_from_type(dwarf_type, plan.module_path.clone()) + { + return Ok(Some(info)); + } + } + } + + if let Some((base_expr, _static_index)) = + self.pointer_arithmetic_parts_expanding_aliases(&expanded)? + { + if let Some(plan) = self.query_dwarf_for_complex_expr(&base_expr)? { + if let Some(dwarf_type) = plan.dwarf_type.as_ref() { + if let Some(info) = + Self::indexable_info_from_type(dwarf_type, plan.module_path.clone()) + { + return Ok(Some(info)); + } + } + } + } + + match expanded { + E::BinaryOp { + ref left, + op: BO::Add, + ref right, + } => { + if let Some(info) = self.indexable_element_type_and_stride(left)? { + return Ok(Some(info)); + } + self.indexable_element_type_and_stride(right) + } + E::BinaryOp { + ref left, + op: BO::Subtract, + .. + } => self.indexable_element_type_and_stride(left), + _ => Ok(None), + } + } + + fn complete_dynamic_member_element_type( + &self, + element_type: DwarfType, + module_path: Option<&Path>, + ) -> DwarfType { + let Some(analyzer) = self.process_analyzer else { + return element_type; + }; + let fallback_module_path = self + .current_compile_time_context + .as_ref() + .map(|ctx| PathBuf::from(&ctx.module_path)); + let lookup_module_path = module_path.or(fallback_module_path.as_deref()); + + if let Some(module_path) = lookup_module_path { + analyzer.complete_shallow_unknown_aggregate_type_in_module(module_path, element_type) + } else { + analyzer.complete_shallow_unknown_aggregate_type(element_type) + } + } + + fn read_dynamic_address_value( + &mut self, + address: RuntimeAddress<'ctx>, + dwarf_type: &DwarfType, + ) -> Result> { + if ghostscope_dwarf::is_c_aggregate_type(dwarf_type) { + let ptr_ty = self.context.ptr_type(AddressSpace::default()); + let as_ptr = self + .builder + .build_int_to_ptr(address.value, ptr_ty, "dynamic_aggregate_ptr") + .map_err(|err| CodeGenError::Builder(err.to_string()))?; + return Ok(as_ptr.into()); + } + + let access_size = self.dwarf_type_to_memory_access_size(dwarf_type); + let value = if self.condition_context_active { + self.generate_memory_read_with_status(address, access_size)? + } else { + self.generate_memory_read(address, access_size, None)? + }; + self.sign_extend_memory_read_if_needed(value, dwarf_type, access_size) + } + + pub(super) fn expand_alias_variable_expr(&self, expr: &Expr) -> Result { + let mut expanded = expr.clone(); + let mut visited = std::collections::HashSet::new(); + + loop { + let Expr::Variable(name) = &expanded else { + return Ok(expanded); + }; + if !self.alias_variable_exists(name) { + return Ok(expanded); + } + if !visited.insert(name.clone()) { + return Err(CodeGenError::TypeError(format!( + "alias cycle detected for '{name}'" + ))); + } + let Some(target) = self.get_alias_variable(name) else { + return Ok(expanded); + }; + expanded = target; + } + } + + pub(super) fn normalize_int_to_i64( + &self, + value: IntValue<'ctx>, + name: &str, + ) -> Result> { + let width = value.get_type().get_bit_width(); + if width == 64 { + return Ok(value); + } + + if width < 64 { + return self + .builder + .build_int_s_extend(value, self.context.i64_type(), name) + .map_err(|err| CodeGenError::Builder(err.to_string())); + } + + self.builder + .build_int_truncate(value, self.context.i64_type(), name) + .map_err(|err| CodeGenError::Builder(err.to_string())) + } + + pub(crate) fn dwarf_expression_unavailable_error( + name: &str, + availability: &Availability, + pc_address: u64, + ) -> CodeGenError { + let reason = Self::format_availability_reason(availability); + CodeGenError::VariableUnavailable(format!( + "'{name}' is {reason}; cannot use it as a value expression at PC 0x{pc_address:x}" + )) + } + + pub(crate) fn dwarf_lvalue_address_unavailable_error( + name: &str, + availability: &Availability, + pc_address: u64, + ) -> CodeGenError { + let reason = Self::format_availability_reason(availability); + CodeGenError::VariableUnavailable(format!( + "'{name}' is {reason}; cannot take its address at PC 0x{pc_address:x}" + )) + } + + fn format_availability_reason(availability: &Availability) -> String { + match availability { + Availability::OptimizedOut => "optimized out at the selected probe PC".to_string(), + Availability::NotInScope => "not in scope at the selected probe PC".to_string(), + Availability::Unsupported(reason) => { + format!( + "unsupported DWARF semantic shape: {}", + Self::format_unsupported_reason(reason) + ) + } + Availability::Requires(requirement) => { + format!( + "requires unavailable runtime support: {}", + Self::format_runtime_requirement(requirement) + ) + } + Availability::Ambiguous(reason) => { + format!( + "ambiguous DWARF semantic result: {}", + Self::format_ambiguity_reason(reason) + ) + } + Availability::Available | Availability::PartiallyAvailable => "available".to_string(), + } + } + + fn format_unsupported_reason(reason: &UnsupportedReason) -> String { + match reason { + UnsupportedReason::DwarfOp { op } => format!("unsupported DWARF op {op}"), + UnsupportedReason::ExpressionShape { detail } => { + format!("unsupported DWARF expression shape: {detail}") + } + UnsupportedReason::TypeLayout { detail } => { + format!("unsupported type layout: {detail}") + } + UnsupportedReason::AddressClass { detail } => { + format!("unsupported address class: {detail}") + } + UnsupportedReason::RegisterMapping { dwarf_reg } => { + format!("unsupported DWARF register mapping for register {dwarf_reg}") + } + } + } + + fn format_runtime_requirement(requirement: &RuntimeRequirement) -> &'static str { + match requirement { + RuntimeRequirement::CallerFrame => "caller-frame recovery", + RuntimeRequirement::SleepableUprobe => "sleepable uprobe support", + RuntimeRequirement::UserMemoryRead => "user-memory read support", + RuntimeRequirement::DwarfCfiRecovery => "DWARF CFI recovery", + } + } + + fn format_ambiguity_reason(reason: &AmbiguityReason) -> String { + match reason { + AmbiguityReason::InlineContext { detail } => { + format!("ambiguous inline context: {detail}") + } + AmbiguityReason::VariableDeclaration { detail } => { + format!("ambiguous variable declaration: {detail}") + } + AmbiguityReason::TypeResolution { detail } => { + format!("ambiguous type resolution: {detail}") + } + } + } + + /// Helper: Convert expression to string for debugging + fn expr_to_debug_string(expr: &crate::script::Expr) -> String { + use crate::script::Expr; + + match expr { + Expr::Variable(name) => name.clone(), + Expr::MemberAccess(obj, field) => { + format!("{}.{}", Self::expr_to_debug_string(obj), field) + } + Expr::ArrayAccess(arr, _) => format!("{}[index]", Self::expr_to_debug_string(arr)), + Expr::Cast { expr, target_type } => format!( + "cast({}, \"{}\")", + Self::expr_to_debug_string(expr), + target_type + ), + Expr::ChainAccess(chain) => chain.join("."), + Expr::PointerDeref(expr) => format!("*{}", Self::expr_to_debug_string(expr)), + _ => "expr".to_string(), + } + } +} diff --git a/ghostscope-compiler/src/ebpf/expression/mod.rs b/ghostscope-compiler/src/ebpf/expression/mod.rs new file mode 100644 index 00000000..e210bc1f --- /dev/null +++ b/ghostscope-compiler/src/ebpf/expression/mod.rs @@ -0,0 +1,574 @@ +//! Expression compilation for eBPF code generation +//! +//! This module handles compilation of various expression types to LLVM IR. + +use super::context::{CodeGenError, EbpfContext, Result, RuntimeAddress}; +use super::expression_plan::{BinaryEmitKind, BuiltinCallPlan}; +use crate::script::Expr; +use ghostscope_dwarf::{CIntegerComparisonType, TypeInfo as DwarfType}; +use inkwell::values::BasicValueEnum; +use inkwell::AddressSpace; + +mod binary; +mod builtins; +mod casts; +mod dwarf_access; +mod runtime_address; +mod special_vars; +use std::path::PathBuf; +use tracing::debug; + +// compare cap is provided via compile_options.compare_cap (config: ebpf.compare_cap) + +#[derive(Clone)] +pub(super) struct DynamicTypeInfo { + pub(super) dwarf_type: DwarfType, + pub(super) module_path: Option, +} + +pub(super) struct DynamicLvalue<'ctx> { + pub(super) address: RuntimeAddress<'ctx>, + pub(super) type_info: DynamicTypeInfo, +} + +struct IndexableElementInfo { + element_type: DwarfType, + stride: u64, + module_path: Option, +} + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + pub fn compile_expr(&mut self, expr: &Expr) -> Result> { + match expr { + Expr::Int(value) => { + // Treat script integer literals as signed i64 constants + let int_value = self.context.i64_type().const_int(*value as u64, true); + debug!( + "compile_expr: Int literal {} compiled to IntValue with bit width {}", + value, + int_value.get_type().get_bit_width() + ); + Ok(int_value.into()) + } + Expr::Float(_value) => Err(CodeGenError::TypeError( + "Floating point expressions are not supported".to_string(), + )), + Expr::String(value) => { + // Create string constant using a simpler approach + let string_value = self.context.const_string(value.as_bytes(), true); + let global = self + .module + .add_global(string_value.get_type(), None, "str_const"); + global.set_initializer(&string_value); + + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let cast_ptr = self + .builder + .build_bit_cast(global.as_pointer_value(), ptr_type, "str_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(cast_ptr) + } + Expr::Bool(value) => { + // Represent booleans as i1 for logical/compare consistency + let b = self + .context + .bool_type() + .const_int(if *value { 1 } else { 0 }, false); + Ok(b.into()) + } + Expr::UnaryNot(inner) => { + // Compile operand to integer and compare EQ to zero to produce boolean not + let v = self.compile_expr(inner)?; + let iv = match v { + BasicValueEnum::IntValue(iv) => iv, + _ => { + return Err(CodeGenError::TypeError( + "Logical NOT requires integer/boolean operand".to_string(), + )) + } + }; + let zero = iv.get_type().const_zero(); + let res = self + .builder + .build_int_compare(inkwell::IntPredicate::EQ, iv, zero, "not_eq0") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(res.into()) + } + Expr::UnaryBitNot(inner) => { + let unsigned_width = self + .dwarf_integer_comparison_expr(inner) + .map(CIntegerComparisonType::promoted) + .and_then(|integer_type| { + integer_type + .is_unsigned + .then_some((integer_type.size * 8) as u32) + }); + let v = self.compile_expr(inner)?; + let iv = match v { + BasicValueEnum::IntValue(iv) => iv, + _ => { + return Err(CodeGenError::TypeError( + "Bitwise NOT requires integer/boolean operand".to_string(), + )) + } + }; + if let Some(bit_width) = unsigned_width { + let iv = + self.normalize_int_for_unsigned_compare(iv, bit_width, "bitnot_unsigned")?; + let result = self + .builder + .build_xor(iv, iv.get_type().const_all_ones(), "bitnot") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return self + .zero_extend_int_to_i64_if_needed(result, "bitnot_zext_i64") + .map(|value| value.into()); + } + let iv = if iv.get_type().get_bit_width() == 1 { + self.builder + .build_int_z_extend(iv, self.context.i64_type(), "bitnot_bool_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + } else { + iv + }; + let all_ones = iv.get_type().const_all_ones(); + let result = self + .builder + .build_xor(iv, all_ones, "bitnot") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(result.into()) + } + Expr::Variable(var_name) => { + debug!("compile_expr: Compiling variable expression: {}", var_name); + + // First: DWARF alias variable takes precedence + if self.alias_variable_exists(var_name) { + debug!( + "compile_expr: '{}' is an alias variable; resolving to runtime address", + var_name + ); + let aliased = self + .get_alias_variable(var_name) + .expect("alias existence just checked"); + // Resolve to i64 address then cast to ptr + let addr_i64 = self.resolve_ptr_i64_from_expr(&aliased)?; + let ptr_ty = self.context.ptr_type(AddressSpace::default()); + let as_ptr = self + .builder + .build_int_to_ptr(addr_i64, ptr_ty, "alias_as_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(as_ptr.into()); + } + + // Then check if it's a concrete script-defined variable + if self.variable_exists(var_name) { + debug!("compile_expr: Found script variable: {}", var_name); + let loaded_value = self.load_variable(var_name)?; + debug!( + "compile_expr: Loaded variable '{}' with type: {:?}", + var_name, + loaded_value.get_type() + ); + match &loaded_value { + BasicValueEnum::IntValue(iv) => debug!( + "compile_expr: Variable '{}' is IntValue with bit width {}", + var_name, + iv.get_type().get_bit_width() + ), + BasicValueEnum::FloatValue(_) => { + debug!("compile_expr: Variable '{}' is FloatValue", var_name) + } + BasicValueEnum::PointerValue(_) => { + debug!("compile_expr: Variable '{}' is PointerValue", var_name) + } + _ => debug!("compile_expr: Variable '{}' is other type", var_name), + } + return Ok(loaded_value); + } + + // If not found in script variables nor alias map, try DWARF variables + debug!( + "Variable '{}' not found in script variables, checking DWARF", + var_name + ); + // If not a DWARF variable either, treat as out-of-scope script name for friendliness + match self.query_dwarf_for_variable(var_name) { + Ok(Some(_)) => self.compile_dwarf_expression(expr), + Ok(None) => Err(CodeGenError::VariableNotInScope(var_name.clone())), + Err(e) => Err(CodeGenError::DwarfError(e.to_string())), + } + } + Expr::SpecialVar(name) => { + // Accept both "$pid" and "pid" forms from the parser + let sanitized = name.trim_start_matches('$'); + self.handle_special_variable(sanitized) + } + Expr::BuiltinCall { name, args } => match self.plan_builtin_call(name, args)? { + BuiltinCallPlan::Memcmp => { + self.compile_memcmp_builtin(&args[0], &args[1], &args[2]) + } + BuiltinCallPlan::Strncmp => { + // Accept string on either side: string literal or script string variable + fn extract_script_string( + this: &mut EbpfContext<'_, '_>, + e: &Expr, + ) -> Option { + match e { + Expr::String(s) => Some(s.clone()), + Expr::Variable(name) => this + .get_variable_type(name) + .is_some_and(|t| matches!(t, crate::script::VarType::String)) + .then(|| { + this.get_string_variable_bytes(name).map(|b| { + let cut = b.iter().position(|&x| x == 0).unwrap_or(b.len()); + String::from_utf8_lossy(&b[..cut]).to_string() + }) + }) + .flatten(), + _ => None, + } + } + let left_str = extract_script_string(self, &args[0]); + let right_str = extract_script_string(self, &args[1]); + match (left_str, right_str) { + (Some(ls), Some(rs)) => { + let left_expr = Expr::String(ls); + self.compile_strncmp_builtin(&left_expr, &rs, &args[2]) + } + (Some(ls), None) => self.compile_strncmp_builtin(&args[1], &ls, &args[2]), + (None, Some(rs)) => self.compile_strncmp_builtin(&args[0], &rs, &args[2]), + (None, None) => Err(CodeGenError::TypeError( + "strncmp requires at least one string argument (string literal or script string variable) as the first or second parameter".into(), + )), + } + } + BuiltinCallPlan::StartsWith => { + // Accept string on either side (literal or script string var) + fn extract_script_string( + this: &mut EbpfContext<'_, '_>, + e: &Expr, + ) -> Option { + match e { + Expr::String(s) => Some(s.clone()), + Expr::Variable(name) => this + .get_variable_type(name) + .is_some_and(|t| matches!(t, crate::script::VarType::String)) + .then(|| { + this.get_string_variable_bytes(name).map(|b| { + let cut = b.iter().position(|&x| x == 0).unwrap_or(b.len()); + String::from_utf8_lossy(&b[..cut]).to_string() + }) + }) + .flatten(), + _ => None, + } + } + let s0 = extract_script_string(self, &args[0]); + let s1 = extract_script_string(self, &args[1]); + match (s0, s1) { + (Some(a), Some(b)) => { + // both strings -> compile-time fold + let ok = a.as_bytes().starts_with(b.as_bytes()); + let bv = self.context.bool_type().const_int(ok as u64, false); + Ok(bv.into()) + } + (Some(a), None) => { + let n_expr = Expr::Int(a.len() as i64); + self.compile_strncmp_builtin(&args[1], &a, &n_expr) + } + (None, Some(b)) => { + let n_expr = Expr::Int(b.len() as i64); + self.compile_strncmp_builtin(&args[0], &b, &n_expr) + } + (None, None) => Err(CodeGenError::TypeError( + "starts_with requires at least one string argument (string literal or script string variable) as the first or second parameter".into(), + )), + } + } + }, + Expr::BinaryOp { left, op, right } => { + let binary_plan = self.plan_binary_expr(left, op, right)?; + if let BinaryEmitKind::StringComparison(string_plan) = &binary_plan.emit_kind { + let other = if string_plan.literal_on_left { + right.as_ref() + } else { + left.as_ref() + }; + return self.compile_string_comparison( + other, + &string_plan.literal, + string_plan.equal, + ); + } + // Implement short-circuit for logical OR (||) and logical AND (&&) + if matches!(&binary_plan.emit_kind, BinaryEmitKind::LogicalOr) { + // Evaluate LHS to boolean (non-zero => true). Accept integer or pointer. + let lhs_val = self.compile_expr(left)?; + let lhs_int = match lhs_val { + BasicValueEnum::IntValue(iv) => iv, + BasicValueEnum::PointerValue(pv) => self + .builder + .build_ptr_to_int(pv, self.context.i64_type(), "lor_lhs_ptr_as_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + _ => { + return Err(CodeGenError::TypeError( + "Logical OR requires integer or pointer operands".to_string(), + )) + } + }; + let lhs_zero = lhs_int.get_type().const_zero(); + let lhs_bool = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + lhs_int, + lhs_zero, + "lor_lhs_nz", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + // Prepare control flow blocks + let curr_block = self.builder.get_insert_block().ok_or_else(|| { + CodeGenError::LLVMError("No current basic block".to_string()) + })?; + let func = curr_block + .get_parent() + .ok_or_else(|| CodeGenError::LLVMError("No parent function".to_string()))?; + let rhs_block = self.context.append_basic_block(func, "lor_rhs"); + let merge_block = self.context.append_basic_block(func, "lor_merge"); + + // If lhs is true, jump directly to merge (short-circuit) + self.builder + .build_conditional_branch(lhs_bool, merge_block, rhs_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + // RHS path: compute boolean only if needed + self.builder.position_at_end(rhs_block); + let rhs_val = self.compile_expr(right)?; + let rhs_int = match rhs_val { + BasicValueEnum::IntValue(iv) => iv, + BasicValueEnum::PointerValue(pv) => self + .builder + .build_ptr_to_int(pv, self.context.i64_type(), "lor_rhs_ptr_as_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + _ => { + return Err(CodeGenError::TypeError( + "Logical OR requires integer or pointer operands".to_string(), + )) + } + }; + let rhs_zero = rhs_int.get_type().const_zero(); + let rhs_bool = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + rhs_int, + rhs_zero, + "lor_rhs_nz", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + // Capture the actual block where RHS computation ended + let rhs_end_block = self.builder.get_insert_block().ok_or_else(|| { + CodeGenError::LLVMError("No current basic block after RHS".to_string()) + })?; + self.builder + .build_unconditional_branch(merge_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + // Merge: phi of i1: true from LHS-true, RHS bool from rhs_block + self.builder.position_at_end(merge_block); + let i1 = self.context.bool_type(); + let phi = self + .builder + .build_phi(i1, "lor_phi") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let one = i1.const_int(1, false); + phi.add_incoming(&[(&one, curr_block), (&rhs_bool, rhs_end_block)]); + return Ok(phi.as_basic_value()); + } else if matches!(&binary_plan.emit_kind, BinaryEmitKind::LogicalAnd) { + // Evaluate LHS to boolean (non-zero => true). Accept integer or pointer. + let lhs_val = self.compile_expr(left)?; + let lhs_int = match lhs_val { + BasicValueEnum::IntValue(iv) => iv, + BasicValueEnum::PointerValue(pv) => self + .builder + .build_ptr_to_int(pv, self.context.i64_type(), "land_lhs_ptr_as_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + _ => { + return Err(CodeGenError::TypeError( + "Logical AND requires integer or pointer operands".to_string(), + )) + } + }; + let lhs_zero = lhs_int.get_type().const_zero(); + let lhs_bool = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + lhs_int, + lhs_zero, + "land_lhs_nz", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + // Prepare control flow: if lhs is true, evaluate rhs; else short-circuit to false + let curr_block = self.builder.get_insert_block().ok_or_else(|| { + CodeGenError::LLVMError("No current basic block".to_string()) + })?; + let func = curr_block + .get_parent() + .ok_or_else(|| CodeGenError::LLVMError("No parent function".to_string()))?; + let rhs_block = self.context.append_basic_block(func, "land_rhs"); + let merge_block = self.context.append_basic_block(func, "land_merge"); + + // If lhs is true, go compute rhs; else jump to merge with false + self.builder + .build_conditional_branch(lhs_bool, rhs_block, merge_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + // RHS path + self.builder.position_at_end(rhs_block); + let rhs_val = self.compile_expr(right)?; + let rhs_int = match rhs_val { + BasicValueEnum::IntValue(iv) => iv, + BasicValueEnum::PointerValue(pv) => self + .builder + .build_ptr_to_int(pv, self.context.i64_type(), "land_rhs_ptr_as_i64") + .map_err(|e| CodeGenError::Builder(e.to_string()))?, + _ => { + return Err(CodeGenError::TypeError( + "Logical AND requires integer or pointer operands".to_string(), + )) + } + }; + let rhs_zero = rhs_int.get_type().const_zero(); + let rhs_bool = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + rhs_int, + rhs_zero, + "land_rhs_nz", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let rhs_end_block = self.builder.get_insert_block().ok_or_else(|| { + CodeGenError::LLVMError("No current basic block after RHS".to_string()) + })?; + self.builder + .build_unconditional_branch(merge_block) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + // Merge: phi(i1) with false from LHS=false path, RHS bool from rhs path + self.builder.position_at_end(merge_block); + let i1 = self.context.bool_type(); + let phi = self + .builder + .build_phi(i1, "land_phi") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + let zero = i1.const_zero(); + phi.add_incoming(&[(&rhs_bool, rhs_end_block), (&zero, curr_block)]); + return Ok(phi.as_basic_value()); + } + + // Default eager evaluation for other binary ops + let left_val = self.compile_expr(left)?; + let right_val = self.compile_expr(right)?; + self.compile_binary_op_with_ordering( + left_val, + binary_plan.op, + right_val, + binary_plan.integer_semantics, + ) + } + Expr::MemberAccess(_, _) => { + // Use unified DWARF expression compilation + self.compile_dwarf_expression(expr) + } + Expr::PointerDeref(_) => { + // Use unified DWARF expression compilation + self.compile_dwarf_expression(expr) + } + Expr::AddressOf(inner) => { + // Address-of computes runtime addresses using the module origin carried by the plan. + // Transparently support alias variables: &alias -> address of aliased DWARF expression + let target_inner: &Expr = if let Expr::Variable(var_name) = inner.as_ref() { + if self.alias_variable_exists(var_name) { + // Use the aliased target expression (by-value) and query DWARF on it + let aliased = self + .get_alias_variable(var_name) + .expect("alias existence just checked"); + let var = + self.query_dwarf_for_complex_expr(&aliased)? + .ok_or_else(|| { + super::context::CodeGenError::TypeError( + "cannot take address of unresolved expression".to_string(), + ) + })?; + let pc_address = self.get_compile_time_context()?.pc_address; + match self.variable_read_plan_to_runtime_address(&var, pc_address, None) { + Ok(address) => { + let ptr_ty = self.context.ptr_type(AddressSpace::default()); + let as_ptr = self + .builder + .build_int_to_ptr(address.value, ptr_ty, "addr_as_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(as_ptr.into()); + } + Err(_) => { + return Err(super::context::CodeGenError::TypeError( + "cannot take address of rvalue".to_string(), + )); + } + } + } else { + inner.as_ref() + } + } else { + inner.as_ref() + }; + + if let Some(lvalue) = self.dynamic_lvalue_address_and_type(target_inner)? { + let ptr_ty = self.context.ptr_type(AddressSpace::default()); + let as_ptr = self + .builder + .build_int_to_ptr(lvalue.address.value, ptr_ty, "addr_as_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(as_ptr.into()); + } + + let var = self + .query_dwarf_for_complex_expr(target_inner)? + .ok_or_else(|| { + super::context::CodeGenError::TypeError( + "cannot take address of unresolved expression".to_string(), + ) + })?; + let pc_address = self.get_compile_time_context()?.pc_address; + match self.variable_read_plan_to_runtime_address(&var, pc_address, None) { + Ok(address) => { + let ptr_ty = self.context.ptr_type(AddressSpace::default()); + let as_ptr = self + .builder + .build_int_to_ptr(address.value, ptr_ty, "addr_as_ptr") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(as_ptr.into()) + } + Err(_) => Err(super::context::CodeGenError::TypeError( + "cannot take address of rvalue".to_string(), + )), + } + } + Expr::ArrayAccess(_, _) => { + // Use unified DWARF expression compilation + self.compile_dwarf_expression(expr) + } + Expr::Cast { + expr: inner, + target_type, + } => self.compile_cast_expr_value(inner, target_type), + Expr::ChainAccess(_) => { + // Use unified DWARF expression compilation + self.compile_dwarf_expression(expr) + } + } + } +} diff --git a/ghostscope-compiler/src/ebpf/expression/runtime_address.rs b/ghostscope-compiler/src/ebpf/expression/runtime_address.rs new file mode 100644 index 00000000..ebe9b58e --- /dev/null +++ b/ghostscope-compiler/src/ebpf/expression/runtime_address.rs @@ -0,0 +1,581 @@ +use crate::ebpf::context::{CodeGenError, EbpfContext, Result, RuntimeAddress}; +use crate::script::Expr; +use ghostscope_dwarf::{CIntegerComparisonPlan, CIntegerComparisonType, TypeInfo as DwarfType}; +use inkwell::values::BasicValueEnum; + +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + fn is_dwarf_pointer_or_array_arg(&mut self, expr: &Expr) -> Result { + let Some(var) = self.query_dwarf_for_complex_expr(expr)? else { + return Ok(false); + }; + let Some(ty) = var.dwarf_type.as_ref() else { + return Ok(false); + }; + let ty = ghostscope_dwarf::strip_type_aliases(ty); + Ok(matches!( + ty, + DwarfType::PointerType { .. } | DwarfType::ArrayType { .. } + )) + } + + pub(super) fn dwarf_integer_comparison_expr( + &mut self, + expr: &Expr, + ) -> Option { + if let Expr::Cast { target_type, .. } = expr { + return self + .resolve_cast_target_type(target_type) + .ok() + .and_then(|ty| ghostscope_dwarf::c_integer_comparison_type(&ty)); + } + + if let Ok(Some(var)) = self.query_dwarf_for_complex_expr(expr) { + if let Some(ref ty) = var.dwarf_type { + return ghostscope_dwarf::c_integer_comparison_type(ty); + } + } + None + } + + fn integer_comparison_plan_for_exprs( + &mut self, + left: &Expr, + right: &Expr, + ) -> Option { + let left_ty = self.dwarf_integer_comparison_expr(left); + let right_ty = self.dwarf_integer_comparison_expr(right); + if left_ty.is_none() && right_ty.is_none() { + return None; + } + + Some(ghostscope_dwarf::usual_c_arithmetic_comparison_plan( + left_ty.unwrap_or_else(CIntegerComparisonType::signed_i64), + right_ty.unwrap_or_else(CIntegerComparisonType::signed_i64), + )) + } + + pub(in crate::ebpf) fn unsigned_ordering_width_for_exprs( + &mut self, + left: &Expr, + right: &Expr, + ) -> Option { + let plan = self.integer_comparison_plan_for_exprs(left, right)?; + if plan.is_unsigned { + Some((plan.size * 8) as u32) + } else { + None + } + } + + pub(in crate::ebpf) fn unsigned_shift_width_for_expr(&mut self, expr: &Expr) -> Option { + let c_type = self.dwarf_integer_comparison_expr(expr)?.promoted(); + if c_type.is_unsigned { + Some((c_type.size * 8) as u32) + } else { + None + } + } + + /// Ensure that when an expression refers to a DWARF-backed variable (not via address-of), + /// the variable's DWARF type is a pointer or array (decays to pointer for memcmp/strncmp). + pub(super) fn ensure_dwarf_pointer_arg(&mut self, e: &Expr, where_ctx: &str) -> Result<()> { + // Allow explicit address-of forms (&expr), which purposefully produce a pointer + if matches!(e, Expr::AddressOf(_)) { + return Ok(()); + } + if let Some((ptr_side, _)) = self.pointer_arithmetic_parts_expanding_aliases(e)? { + if matches!(&ptr_side, Expr::AddressOf(_)) + || self + .is_dwarf_pointer_or_array_arg(&ptr_side) + .unwrap_or(false) + { + return Ok(()); + } + } + if self.is_dynamic_pointer_arithmetic_expr(e)? + || self.expands_to_nonliteral_pointer_arithmetic(e)? + { + return Ok(()); + } + match self.query_dwarf_for_complex_expr(e) { + Ok(Some(var)) => { + let Some(ty) = var.dwarf_type.as_ref() else { + return Err(CodeGenError::TypeError(format!( + "{where_ctx}: DWARF variable has no type information" + ))); + }; + let ty = ghostscope_dwarf::strip_type_aliases(ty); + if !matches!( + ty, + DwarfType::PointerType { .. } | DwarfType::ArrayType { .. } + ) { + return Err(CodeGenError::TypeError(format!( + "{where_ctx}: only pointer or array DWARF variables are supported" + ))); + } + Ok(()) + } + // No DWARF info or analyzer missing: allow script-level pointer values + Ok(None) | Err(_) => match self.compile_expr(e) { + Ok(BasicValueEnum::PointerValue(_)) => Ok(()), + _ => Err(CodeGenError::TypeError(format!( + "{where_ctx}: expression is not a pointer" + ))), + }, + } + } + + fn is_dynamic_pointer_arithmetic_expr(&mut self, expr: &Expr) -> Result { + use crate::script::ast::BinaryOp as BO; + use crate::script::ast::Expr as E; + + let E::BinaryOp { left, op, right } = expr else { + return Ok(false); + }; + + match op { + BO::Add => Ok(self.is_dynamic_indexable_pointer_base(left)? + || self.is_dynamic_indexable_pointer_base(right)?), + BO::Subtract => self.is_dynamic_indexable_pointer_base(left), + _ => Ok(false), + } + } + + fn is_dynamic_indexable_pointer_base(&mut self, expr: &Expr) -> Result { + if matches!(expr, Expr::AddressOf(_)) { + return Ok(true); + } + + if self.cast_index_base(expr)?.is_some() { + return Ok(true); + } + + if self + .query_dwarf_for_complex_expr(expr) + .ok() + .flatten() + .and_then(|var| var.dwarf_type) + .is_some_and(|ty| ghostscope_dwarf::is_c_pointer_or_array_type(&ty)) + { + return Ok(true); + } + + let expanded = self.expand_alias_variable_expr(expr)?; + if matches!(expanded, Expr::AddressOf(_)) { + return Ok(true); + } + let Some((base_expr, _static_index)) = + self.pointer_arithmetic_parts_expanding_aliases(&expanded)? + else { + return Ok(false); + }; + + Ok(self + .query_dwarf_for_complex_expr(&base_expr) + .ok() + .flatten() + .and_then(|var| var.dwarf_type) + .is_some_and(|ty| ghostscope_dwarf::is_c_pointer_or_array_type(&ty))) + } + + pub(super) fn expands_to_nonliteral_pointer_arithmetic(&mut self, expr: &Expr) -> Result { + let expanded = self.expand_alias_variable_expr(expr)?; + self.is_nonliteral_pointer_arithmetic_expr(&expanded) + } + + fn is_nonliteral_pointer_arithmetic_expr(&mut self, expr: &Expr) -> Result { + use crate::script::ast::BinaryOp as BO; + use crate::script::ast::Expr as E; + + let E::BinaryOp { left, op, right } = expr else { + return Ok(false); + }; + + match op { + BO::Add => { + let left_is_ptr = self.is_dynamic_indexable_pointer_base(left)?; + let right_is_ptr = self.is_dynamic_indexable_pointer_base(right)?; + let left_is_literal = Self::integer_literal_value(left).is_some(); + let right_is_literal = Self::integer_literal_value(right).is_some(); + + if (left_is_ptr && !right_is_ptr && !right_is_literal) + || (right_is_ptr && !left_is_ptr && !left_is_literal) + { + return Ok(true); + } + + Ok(self.expands_to_nonliteral_pointer_arithmetic(left)? + || self.expands_to_nonliteral_pointer_arithmetic(right)?) + } + BO::Subtract => { + let left_is_ptr = self.is_dynamic_indexable_pointer_base(left)?; + let right_is_literal = Self::integer_literal_value(right).is_some(); + + if left_is_ptr && !right_is_literal { + return Ok(true); + } + + self.expands_to_nonliteral_pointer_arithmetic(left) + } + _ => Ok(false), + } + } + + /// Resolve an expression to an i64 pointer value. Accepts integer (address) and pointer values; + /// falls back to DWARF evaluation for complex expressions. + pub(crate) fn resolve_ptr_i64_from_expr( + &mut self, + e: &Expr, + ) -> Result> { + self.resolve_runtime_address_from_expr(e) + .map(|address| address.value) + } + + pub(crate) fn resolve_runtime_address_from_expr( + &mut self, + e: &Expr, + ) -> Result> { + let mut visited = std::collections::HashSet::new(); + self.resolve_runtime_address_from_expr_internal(e, &mut visited, 0) + } + + fn resolve_runtime_address_from_expr_internal( + &mut self, + e: &Expr, + visited: &mut std::collections::HashSet, + depth: usize, + ) -> Result> { + use crate::script::ast::BinaryOp as BO; + use crate::script::ast::Expr as E; + use inkwell::values::BasicValueEnum::*; + const MAX_DEPTH: usize = 64; + if depth > MAX_DEPTH { + return Err(CodeGenError::TypeError( + "alias expansion depth exceeded (cycle?)".into(), + )); + } + if let E::Cast { expr, target_type } = e { + let target_type_info = self.resolve_cast_target_type(target_type)?; + if Self::cast_pointer_target_type(&target_type_info).is_some() { + return self.cast_source_pointer_value(expr); + } + return self.cast_source_memory_address(expr); + } + // Alias variable indirection: resolve its target expression first + if let E::Variable(name) = e { + if self.alias_variable_exists(name) { + if !visited.insert(name.clone()) { + return Err(CodeGenError::TypeError(format!( + "alias cycle detected for '{name}'" + ))); + } + if let Some(target) = self.get_alias_variable(name) { + let r = self.resolve_runtime_address_from_expr_internal( + &target, + visited, + depth + 1, + ); + visited.remove(name); + return r; + } + } + } + // Special-case: explicit address-of must yield a pointer-sized address + if let E::AddressOf(inner) = e { + // Support alias variables transparently: &alias -> address of aliased DWARF expr + let resolved_inner: &E = if let E::Variable(name) = inner.as_ref() { + if self.alias_variable_exists(name) { + // Owned target for query + if let Some(target) = self.get_alias_variable(name) { + if let Some(var) = self.query_dwarf_for_complex_expr(&target)? { + let status_ptr = if self.condition_context_active { + Some(self.get_or_create_cond_error_global()) + } else { + None + }; + let pc_address = self.get_compile_time_context()?.pc_address; + return self.variable_read_plan_to_runtime_address( + &var, pc_address, status_ptr, + ); + } else { + return Err(CodeGenError::TypeError( + "cannot take address of unresolved expression".into(), + )); + } + } else { + return Err(CodeGenError::TypeError( + "cannot take address of unresolved expression".into(), + )); + } + } else { + inner.as_ref() + } + } else { + inner.as_ref() + }; + + if let E::ArrayAccess(array_expr, index_expr) = resolved_inner { + if let Some(element_lvalue) = + self.compile_dynamic_array_element_address(array_expr, index_expr)? + { + return Ok(element_lvalue.address); + } + } + + if let Some(lvalue) = self.dynamic_lvalue_address_and_type(resolved_inner)? { + return Ok(lvalue.address); + } + + if let Some(var) = self.query_dwarf_for_complex_expr(resolved_inner)? { + let status_ptr = if self.condition_context_active { + Some(self.get_or_create_cond_error_global()) + } else { + None + }; + let pc_address = self.get_compile_time_context()?.pc_address; + return self.variable_read_plan_to_runtime_address(&var, pc_address, status_ptr); + } else { + return Err(CodeGenError::TypeError( + "cannot take address of unresolved expression".into(), + )); + } + } + + if let Some(address) = self.dynamic_pointer_arithmetic_address(e)? { + return Ok(address); + } + + if let Some((ptr_side, index)) = self.pointer_arithmetic_parts_expanding_aliases(e)? { + if matches!(&ptr_side, E::AddressOf(_)) { + let base = + self.resolve_runtime_address_from_expr_internal(&ptr_side, visited, depth + 1)?; + let off = self.context.i64_type().const_int(index as u64, false); + let value = self + .builder + .build_int_add(base.value, off, "ptr_add") + .map_err(|err| CodeGenError::Builder(err.to_string()))?; + return Ok(base.with_value(value)); + } else if let Some((element_info, base_address)) = self.cast_index_base(&ptr_side)? { + let index_value = self.context.i64_type().const_int(index as u64, true); + return self + .dynamic_lvalue_from_indexable_base( + element_info, + base_address, + index_value, + "cast_ptr_add", + ) + .map(|lvalue| lvalue.address); + } else if let Some(var) = self.query_dwarf_for_complex_expr(&ptr_side)? { + if var.dwarf_type.is_some() { + let pointed_plan = var + .plan_pointer_element_index(index) + .map_err(|err| CodeGenError::DwarfError(err.to_string()))?; + let status_ptr = if self.condition_context_active { + Some(self.get_or_create_cond_error_global()) + } else { + None + }; + let pc_address = self.get_compile_time_context()?.pc_address; + return self.variable_read_plan_to_runtime_address( + &pointed_plan, + pc_address, + status_ptr, + ); + } + } + } + + // Support constant-offset addressing: (alias_expr + K) or (K + alias_expr) + if let E::BinaryOp { left, op, right } = e { + if matches!(op, BO::Add) { + // alias + K + if let Some(k) = Self::integer_literal_value(right) { + if let Ok(base) = + self.resolve_runtime_address_from_expr_internal(left, visited, depth + 1) + { + let off = self.context.i64_type().const_int(k as u64, false); + let value = self + .builder + .build_int_add(base.value, off, "ptr_add") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(base.with_value(value)); + } + } + // K + alias + if let Some(k) = Self::integer_literal_value(left) { + if let Ok(base) = + self.resolve_runtime_address_from_expr_internal(right, visited, depth + 1) + { + let off = self.context.i64_type().const_int(k as u64, false); + let value = self + .builder + .build_int_add(base.value, off, "ptr_add") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(base.with_value(value)); + } + } + } else if matches!(op, BO::Subtract) { + if let Some(k) = Self::integer_literal_value(right) { + if let Ok(base) = + self.resolve_runtime_address_from_expr_internal(left, visited, depth + 1) + { + let off = self + .context + .i64_type() + .const_int(k.wrapping_neg() as u64, false); + let value = self + .builder + .build_int_add(base.value, off, "ptr_sub") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok(base.with_value(value)); + } + } + } + } + // Prefer DWARF-based address resolution first so that array/aggregate + // expressions decay to their base address rather than loading values. + if let Ok(Some(var)) = self.query_dwarf_for_complex_expr(e) { + if let Some(dty) = var.dwarf_type.as_ref() { + let dty = ghostscope_dwarf::strip_type_aliases(dty); + match dty { + DwarfType::PointerType { .. } => { + let pc_address = self.get_compile_time_context()?.pc_address; + let val_any = + self.variable_read_plan_to_llvm_value(&var, pc_address, None)?; + match val_any { + IntValue(iv) => Ok(RuntimeAddress::available(iv, self.context)), + PointerValue(pv) => self + .builder + .build_ptr_to_int(pv, self.context.i64_type(), "ptr_as_i64") + .map(|value| RuntimeAddress::available(value, self.context)) + .map_err(|e| CodeGenError::Builder(e.to_string())), + _ => Err(CodeGenError::TypeError( + "DWARF value is not pointer/integer".into(), + )), + } + } + DwarfType::ArrayType { .. } => { + // Use the base address of the array as pointer + let status_ptr = if self.condition_context_active { + Some(self.get_or_create_cond_error_global()) + } else { + None + }; + let pc_address = self.get_compile_time_context()?.pc_address; + self.variable_read_plan_to_runtime_address(&var, pc_address, status_ptr) + } + _ => Err(CodeGenError::TypeError( + "DWARF value is not pointer/array".into(), + )), + } + } else { + let status_ptr = if self.condition_context_active { + Some(self.get_or_create_cond_error_global()) + } else { + None + }; + let pc_address = self.get_compile_time_context()?.pc_address; + self.variable_read_plan_to_runtime_address(&var, pc_address, status_ptr) + } + } else { + // No DWARF-backed address and not an address-of/alias+const: reject script-level pointers. + Err(CodeGenError::TypeError( + "expression is not a pointer/address".into(), + )) + } + } + + fn dynamic_pointer_arithmetic_address( + &mut self, + expr: &Expr, + ) -> Result>> { + use crate::script::ast::BinaryOp as BO; + use crate::script::ast::Expr as E; + + let E::BinaryOp { left, op, right } = expr else { + return Ok(None); + }; + + match op { + BO::Add => { + if let Some(address) = self.dynamic_raw_address_candidate(left, right, false)? { + return Ok(Some(address)); + } + if let Some(address) = self.dynamic_raw_address_candidate(right, left, false)? { + return Ok(Some(address)); + } + if let Some(address) = self.dynamic_index_address_candidate(left, right)? { + return Ok(Some(address)); + } + self.dynamic_index_address_candidate(right, left) + } + BO::Subtract => { + if let Some(address) = self.dynamic_raw_address_candidate(left, right, true)? { + return Ok(Some(address)); + } + let negative_right = E::BinaryOp { + left: Box::new(E::Int(0)), + op: BO::Subtract, + right: right.clone(), + }; + self.dynamic_index_address_candidate(left, &negative_right) + } + _ => Ok(None), + } + } + + fn dynamic_raw_address_candidate( + &mut self, + base_expr: &Expr, + offset_expr: &Expr, + subtract: bool, + ) -> Result>> { + if Self::integer_literal_value(offset_expr).is_some() { + return Ok(None); + } + + let expanded_base = self.expand_alias_variable_expr(base_expr)?; + if !matches!(expanded_base, Expr::AddressOf(_)) { + return Ok(None); + } + + let base_address = self.resolve_runtime_address_from_expr(&expanded_base)?; + let offset = match self.compile_expr(offset_expr)? { + BasicValueEnum::IntValue(value) => { + self.normalize_int_to_i64(value, "dynamic_raw_offset_i64")? + } + _ => { + return Err(CodeGenError::TypeError( + "raw address offset expression must compile to an integer".to_string(), + )) + } + }; + let offset = if subtract { + self.builder + .build_int_neg(offset, "dynamic_raw_offset_neg") + .map_err(|err| CodeGenError::Builder(err.to_string()))? + } else { + offset + }; + let address = self + .builder + .build_int_add(base_address.value, offset, "dynamic_raw_address") + .map_err(|err| CodeGenError::Builder(err.to_string()))?; + Ok(Some(base_address.with_value(address))) + } + + fn dynamic_index_address_candidate( + &mut self, + base_expr: &Expr, + index_expr: &Expr, + ) -> Result>> { + match self.compile_dynamic_array_element_address(base_expr, index_expr) { + Ok(Some(element_lvalue)) => Ok(Some(element_lvalue.address)), + Ok(None) => Ok(None), + Err(CodeGenError::VariableNotFound(_)) + | Err(CodeGenError::VariableNotInScope(_)) + | Err(CodeGenError::TypeError(_)) => Ok(None), + Err(err) => Err(err), + } + } +} diff --git a/ghostscope-compiler/src/ebpf/expression/special_vars.rs b/ghostscope-compiler/src/ebpf/expression/special_vars.rs new file mode 100644 index 00000000..b2719a47 --- /dev/null +++ b/ghostscope-compiler/src/ebpf/expression/special_vars.rs @@ -0,0 +1,223 @@ +use crate::ebpf::context::{CodeGenError, EbpfContext, Result}; +use crate::ebpf::expression_plan::SpecialVarPlan; +use inkwell::values::{BasicValueEnum, IntValue}; +use inkwell::AddressSpace; +impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { + pub(crate) fn get_host_pid_tid_values(&mut self) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { + let i32_type = self.context.i32_type(); + let i64_type = self.context.i64_type(); + + // bpf_get_current_pid_tgid() returns: + // - high 32 bits: TGID (process ID / getpid() view) + // - low 32 bits: PID (thread ID / gettid() view) + let host_pid_tgid = self.get_current_pid_tgid()?; + let host_tid = self + .builder + .build_and( + host_pid_tgid, + i64_type.const_int(0xFFFF_FFFF, false), + "host_tid", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let host_pid = self + .builder + .build_right_shift( + host_pid_tgid, + i64_type.const_int(32, false), + false, + "host_pid", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + let host_pid_i32 = self + .builder + .build_int_truncate(host_pid, i32_type, "host_pid_i32") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let host_tid_i32 = self + .builder + .build_int_truncate(host_tid, i32_type, "host_tid_i32") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + Ok((host_pid_i32, host_tid_i32)) + } + + pub(crate) fn get_special_pid_tid_values( + &mut self, + ) -> Result<(IntValue<'ctx>, IntValue<'ctx>)> { + const BPF_FUNC_GET_NS_CURRENT_PID_TGID: u64 = 120; + const BPF_PIDNS_INFO_SIZE: u64 = 8; // struct { u32 pid; u32 tgid; } + + let i32_type = self.context.i32_type(); + let i64_type = self.context.i64_type(); + let (host_pid_i32, host_tid_i32) = self.get_host_pid_tid_values()?; + + let ns_spec = if let Some(crate::PidFilterSpec::NamespaceTgid { pid_ns, .. }) = + self.compile_options.pid_filter_spec + { + pid_ns.helper_dev_inode() + } else { + self.compile_options + .special_pid_ns + .and_then(|pid_ns| pid_ns.helper_dev_inode()) + }; + let Some((pid_ns_dev, pid_ns_inode)) = ns_spec else { + let host_pid = self + .builder + .build_int_z_extend(host_pid_i32, i64_type, "selected_host_pid") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let host_tid = self + .builder + .build_int_z_extend(host_tid_i32, i64_type, "selected_host_tid") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + return Ok((host_pid, host_tid)); + }; + + let ptr_type = self.context.ptr_type(AddressSpace::default()); + let key_arr_ty = i32_type.array_type(4); + let key_alloca = self.pm_key_alloca.ok_or_else(|| { + CodeGenError::LLVMError("pm_key not allocated in entry block".to_string()) + })?; + // Reuse entry-allocated stack key storage: helper only needs first 8 bytes. + self.builder + .build_store(key_alloca, key_arr_ty.const_zero()) + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let pidns_info_ptr = self + .builder + .build_bit_cast(key_alloca, ptr_type, "special_pidns_info_ptr") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let helper_args = [ + i64_type.const_int(pid_ns_dev, false).into(), + i64_type.const_int(pid_ns_inode, false).into(), + pidns_info_ptr, + i64_type.const_int(BPF_PIDNS_INFO_SIZE, false).into(), + ]; + let helper_ret = self.create_bpf_helper_call( + BPF_FUNC_GET_NS_CURRENT_PID_TGID, + &helper_args, + i64_type.into(), + "special_ns_pid_tgid_ret", + )?; + let helper_ret = match helper_ret { + BasicValueEnum::IntValue(v) => v, + _ => { + return Err(CodeGenError::LLVMError( + "bpf_get_ns_current_pid_tgid did not return integer".to_string(), + )) + } + }; + + let helper_ok = self + .builder + .build_int_compare( + inkwell::IntPredicate::EQ, + helper_ret, + i64_type.const_zero(), + "special_ns_helper_ok", + ) + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + // SAFETY: key_alloca temporarily holds the two-field pid namespace helper + // result, so [0, 0] addresses the pid field. + let ns_pid_ptr = unsafe { + self.builder.build_gep( + key_arr_ty, + key_alloca, + &[i32_type.const_zero(), i32_type.const_zero()], + "special_ns_pid_ptr", + ) + } + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + // SAFETY: key_alloca temporarily holds the two-field pid namespace helper + // result, so [0, 1] addresses the tgid field. + let ns_tgid_ptr = unsafe { + self.builder.build_gep( + key_arr_ty, + key_alloca, + &[i32_type.const_zero(), i32_type.const_int(1, false)], + "special_ns_tgid_ptr", + ) + } + .map_err(|e| CodeGenError::LLVMError(e.to_string()))?; + + let ns_pid = self + .builder + .build_load(i32_type, ns_pid_ptr, "special_ns_pid") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + let ns_tgid = self + .builder + .build_load(i32_type, ns_tgid_ptr, "special_ns_tgid") + .map_err(|e| CodeGenError::LLVMError(e.to_string()))? + .into_int_value(); + + let selected_pid_i32 = self + .builder + .build_select(helper_ok, ns_tgid, host_pid_i32, "selected_pid_i32") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + let selected_tid_i32 = self + .builder + .build_select(helper_ok, ns_pid, host_tid_i32, "selected_tid_i32") + .map_err(|e| CodeGenError::Builder(e.to_string()))? + .into_int_value(); + + let selected_pid = self + .builder + .build_int_z_extend(selected_pid_i32, i64_type, "selected_pid") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + let selected_tid = self + .builder + .build_int_z_extend(selected_tid_i32, i64_type, "selected_tid") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + + Ok((selected_pid, selected_tid)) + } + + /// Handle special variables like $pid, $tid, etc. + pub fn handle_special_variable(&mut self, name: &str) -> Result> { + match self.plan_special_variable(name)? { + SpecialVarPlan::Pid => { + let (pid, _tid) = self.get_special_pid_tid_values()?; + Ok(pid.into()) + } + SpecialVarPlan::Tid => { + let (_pid, tid) = self.get_special_pid_tid_values()?; + Ok(tid.into()) + } + SpecialVarPlan::HostPid => { + let (host_pid, _host_tid) = self.get_host_pid_tid_values()?; + let host_pid = self + .builder + .build_int_z_extend(host_pid, self.context.i64_type(), "selected_host_pid") + .map_err(|e| CodeGenError::Builder(e.to_string()))?; + Ok(host_pid.into()) + } + SpecialVarPlan::InputPid => { + let input_pid = self.compile_options.input_pid.ok_or_else(|| { + CodeGenError::NotImplemented( + "Special variable '$input_pid' is only available in -p mode".to_string(), + ) + })?; + Ok(self + .context + .i64_type() + .const_int(input_pid as u64, false) + .into()) + } + SpecialVarPlan::Timestamp => { + // Use BPF helper to get current timestamp + let ts = self.get_current_timestamp()?; + Ok(ts.into()) + } + SpecialVarPlan::Pc => self.load_special_register_value(16), + SpecialVarPlan::Sp => self.load_special_register_value(7), + } + } + + fn load_special_register_value(&mut self, dwarf_reg: u16) -> Result> { + let pt_regs = self.get_pt_regs_parameter()?; + self.load_register_value(dwarf_reg, pt_regs) + } +} diff --git a/ghostscope-compiler/src/script/parser.rs b/ghostscope-compiler/src/script/parser.rs index ebcc5212..042df3cc 100644 --- a/ghostscope-compiler/src/script/parser.rs +++ b/ghostscope-compiler/src/script/parser.rs @@ -3,12 +3,17 @@ use pest::Parser; use pest::RuleType; use pest_derive::Parser; -use crate::script::ast::{ - infer_type, BacktraceStatement, BinaryOp, Expr, PrintStatement, Program, Statement, - TracePattern, +use crate::script::ast::Program; +use tracing::debug; + +mod diagnostics; +mod expr; +mod statement; + +use diagnostics::{ + detect_backtrace_depth_argument, detect_unclosed_print_string, detect_unknown_keyword, }; -use crate::script::format_validator::FormatValidator; -use tracing::{debug, info}; +use statement::parse_statement; #[derive(Parser)] #[grammar = "script/grammar.pest"] @@ -100,2913 +105,5 @@ pub fn parse(input: &str) -> Result { Ok(program) } -// Best-effort heuristic: if a line contains a print statement with an opening quote -// but no closing quote before arguments, give a clearer error. -fn detect_unclosed_print_string(input: &str) -> Option { - for (i, raw_line) in input.lines().enumerate() { - let line = raw_line.trim_start(); - if !line.contains("print ") && !line.starts_with("print") { - continue; - } - // Toggle on '"' to detect unclosed string; ignore escaped quotes for simplicity - let mut open = false; - for ch in line.chars() { - if ch == '"' { - open = !open; - } - } - if open { - // Common case: missing closing quote before comma and arguments - if line.contains(',') { - return Some(format!( - "Unclosed string literal in print at line {}. Did you forget a closing \"\" before ',' and arguments?", - i + 1 - )); - } else { - return Some(format!( - "Unclosed string literal in print at line {}.", - i + 1 - )); - } - } - } - None -} - -fn detect_backtrace_depth_argument(input: &str) -> Option { - fn boundary_before(line: &str, idx: usize) -> bool { - idx == 0 - || line[..idx] - .chars() - .next_back() - .is_some_and(|ch| ch.is_whitespace() || matches!(ch, '{' | ';' | '}')) - } - - for (line_idx, raw_line) in input.lines().enumerate() { - let line = raw_line.split("//").next().unwrap_or(raw_line); - for command in ["bt", "backtrace"] { - for (idx, _) in line.match_indices(command) { - if !boundary_before(line, idx) { - continue; - } - let after = &line[idx + command.len()..]; - if !after.starts_with(char::is_whitespace) { - continue; - } - let arg = after.trim_start(); - if arg.starts_with("depth") - || arg.chars().next().is_some_and(|ch| ch.is_ascii_digit()) - { - return Some(format!( - "bt depth is no longer a script option at line {}. Set the global limit with --backtrace-depth or [ebpf] backtrace_depth = N.", - line_idx + 1 - )); - } - } - } - } - None -} - -// Try to detect lines that start with an unknown/misspelled keyword and suggest known ones. -fn detect_unknown_keyword(input: &str) -> Option { - // Suggest only currently supported top-level keywords. - const SUGGEST: &[&str] = &["trace", "print", "if", "else", "let"]; - // Valid statement starters that should not be flagged as unknown - const SUPPORTED_HEADS: &[&str] = &["trace", "print", "if", "else", "let", "backtrace", "bt"]; - // Builtin call names allowed at expression head - const BUILTIN_CALLS: &[&str] = &["memcmp", "strncmp", "starts_with", "hex", "cast"]; - - // Helper: simple Levenshtein distance (small strings, few keywords) - fn levenshtein(a: &str, b: &str) -> usize { - let (n, m) = (a.len(), b.len()); - let mut dp = vec![0usize; (n + 1) * (m + 1)]; - let idx = |i: usize, j: usize| i * (m + 1) + j; - for i in 0..=n { - dp[idx(i, 0)] = i; - } - for j in 0..=m { - dp[idx(0, j)] = j; - } - let ac: Vec = a.chars().collect(); - let bc: Vec = b.chars().collect(); - for i in 1..=n { - for j in 1..=m { - let cost = if ac[i - 1] == bc[j - 1] { 0 } else { 1 }; - let del = dp[idx(i - 1, j)] + 1; - let ins = dp[idx(i, j - 1)] + 1; - let sub = dp[idx(i - 1, j - 1)] + cost; - dp[idx(i, j)] = del.min(ins).min(sub); - } - } - dp[idx(n, m)] - } - - // Helper: check a slice for a command-like unknown keyword - fn check_slice(slice: &str, line_no_1based: usize) -> Option { - let mut s = slice.trim_start(); - if s.is_empty() || s.starts_with("//") { - return None; - } - - // If this slice begins with an if/else-if header, jump inside the condition - if let Some(rest) = s.strip_prefix("if") { - if rest.starts_with(char::is_whitespace) { - s = rest.trim_start(); - } - } else if let Some(rest) = s.strip_prefix("else") { - let rest = rest.trim_start(); - if let Some(rest2) = rest.strip_prefix("if") { - if rest2.starts_with(char::is_whitespace) { - s = rest2.trim_start(); - } - } else { - // 'else { ... }' — nothing to inspect here - } - } - // Keywords must start with a letter or underscore; skip numeric heads - let mut iter = s.chars(); - let first = iter.next()?; - if !(first.is_ascii_alphabetic() || first == '_') { - return None; - } - let mut token = String::new(); - token.push(first); - for ch in iter { - if ch.is_ascii_alphanumeric() || ch == '_' { - token.push(ch); - } else { - break; - } - } - if token.is_empty() { - return None; - } - if SUPPORTED_HEADS.iter().any(|k| *k == token) { - return None; - } - let rest_untrimmed = &s[token.len()..]; - let rest = rest_untrimmed.trim_start(); - if rest.starts_with('=') || rest.starts_with('[') || rest.starts_with('.') { - // likely an expression starting with identifier - return None; - } - // Allow builtin calls as expression statements - if BUILTIN_CALLS.iter().any(|k| *k == token) && rest.starts_with('(') { - return None; - } - if rest.starts_with('(') - || rest.starts_with('{') - || rest.starts_with('"') - || rest_untrimmed.starts_with(char::is_whitespace) - { - // If it looks like a call (token + '('), include builtin calls in suggestion candidates - let candidates: Vec<&str> = if rest.starts_with('(') { - let mut v = Vec::new(); - v.extend_from_slice(SUGGEST); - v.extend_from_slice(BUILTIN_CALLS); - v - } else { - SUGGEST.to_vec() - }; - let mut suggestions: Vec<(&str, usize)> = candidates - .iter() - .map(|&k| (k, levenshtein(&token, k))) - .collect(); - suggestions.sort_by_key(|&(_, d)| d); - if let Some((cand, dist)) = suggestions.first().copied() { - if dist <= 2 { - return Some(format!( - "Unknown keyword '{token}' at line {line_no_1based}. Did you mean '{cand}'?" - )); - } - } - return Some(format!( - "Unknown keyword '{token}' at line {}. Expected one of: {}", - line_no_1based, - SUGGEST.join(", ") - )); - } - None - } - - for (i, raw_line) in input.lines().enumerate() { - let line = raw_line; - // Scan potential statement starts: at line start, and right after '{', ';', '}', '(', ',' (outside strings) - let mut quote_open = false; - let mut positions: Vec = vec![0]; // include start-of-line - for (idx, ch) in line.char_indices() { - if ch == '"' { - quote_open = !quote_open; - } - if !quote_open && (ch == '{' || ch == ';' || ch == '}' || ch == '(' || ch == ',') { - let next = idx + ch.len_utf8(); - if next < line.len() { - positions.push(next); - } - } - } - for &pos in &positions { - if let Some(msg) = check_slice(&line[pos..], i + 1) { - return Some(msg); - } - } - } - None -} - -fn parse_backtrace_stmt(pair: Pair) -> Result { - let mut stmt = BacktraceStatement::default(); - - for arg in pair.into_inner() { - if arg.as_rule() == Rule::backtrace_flag { - match arg.as_str() { - "raw" => stmt.raw = true, - "full" => stmt.full = true, - "inline" => stmt.inline = true, - "noinline" => stmt.inline = false, - other => { - return Err(ParseError::SyntaxError(format!( - "Unknown bt option '{other}'" - ))) - } - } - } - } - - Ok(stmt) -} - -fn parse_statement(pair: Pair) -> Result { - debug!( - "parse_statement: {:?} = '{}'", - pair.as_rule(), - pair.as_str().trim() - ); - let inner = pair - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - debug!( - "parse_statement inner: {:?} = '{}'", - inner.as_rule(), - inner.as_str().trim() - ); - - match inner.as_rule() { - Rule::trace_stmt => { - let mut inner_pairs = inner.into_inner(); - let pattern_pair = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; - let pattern = parse_trace_pattern(pattern_pair)?; - - let mut body = Vec::new(); - for stmt_pair in inner_pairs { - // Disallow nested trace statements (trace is top-level only) - if stmt_pair.as_rule() == Rule::statement { - let mut peek = stmt_pair.clone().into_inner(); - if let Some(first) = peek.next() { - if first.as_rule() == Rule::trace_stmt { - return Err(ParseError::SyntaxError( - "'trace' cannot be nested; it is only allowed at the top level" - .to_string(), - )); - } - } - } - let stmt = parse_statement(stmt_pair)?; - body.push(stmt); - } - - Ok(Statement::TracePoint { pattern, body }) - } - Rule::print_stmt => { - let print_content = inner - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - let print_stmt = parse_print_content(print_content)?; - Ok(Statement::Print(print_stmt)) - } - Rule::backtrace_stmt => Ok(Statement::Backtrace(parse_backtrace_stmt(inner)?)), - Rule::assign_stmt => { - // Friendly error for immutable variables (no assignment supported) - let mut it = inner.into_inner(); - let name = it - .next() - .ok_or(ParseError::InvalidExpression)? - .as_str() - .to_string(); - // consume rhs expr - let _ = it.next(); - Err(ParseError::TypeError(format!( - "Assignment is not supported: variables are immutable. Use 'let {name} = ...' to bind once." - ))) - } - Rule::expr_stmt => { - let expr = inner - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - let parsed_expr = parse_expr(expr)?; - - // Check expression type to ensure consistent operation types - if let Err(err) = infer_type(&parsed_expr) { - return Err(ParseError::TypeError(err)); - } - - Ok(Statement::Expr(parsed_expr)) - } - Rule::var_decl_stmt => { - let mut inner_pairs = inner.into_inner(); - let name = inner_pairs - .next() - .ok_or(ParseError::InvalidExpression)? - .as_str() - .to_string(); - let expr = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; - let parsed_expr = parse_expr(expr)?; - - // Check expression type to ensure consistent operation types - if let Err(err) = infer_type(&parsed_expr) { - return Err(ParseError::TypeError(err)); - } - - if is_alias_expr(&parsed_expr) { - Ok(Statement::AliasDeclaration { - name, - target: parsed_expr, - }) - } else { - Ok(Statement::VarDeclaration { - name, - value: parsed_expr, - }) - } - } - Rule::if_stmt => { - debug!("Parsing if_stmt"); - let mut inner_pairs = inner.into_inner(); - let condition_pair = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; - debug!( - "if_stmt condition_pair: {:?} = '{}'", - condition_pair.as_rule(), - condition_pair.as_str().trim() - ); - let condition = parse_condition(condition_pair)?; - - // Parse then body statements - let mut then_body = Vec::new(); - let mut else_body = None; - - for pair in inner_pairs { - match pair.as_rule() { - Rule::statement => { - then_body.push(parse_statement(pair)?); - } - Rule::else_clause => { - else_body = Some(Box::new(parse_else_clause(pair)?)); - break; - } - _ => return Err(ParseError::UnexpectedToken(pair.as_rule())), - } - } - - Ok(Statement::If { - condition, - then_body, - else_body, - }) - } - _ => Err(ParseError::UnexpectedToken(inner.as_rule())), - } -} - -fn parse_expr(pair: Pair) -> Result { - match pair.as_rule() { - Rule::expr => { - let inner = pair - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - parse_logical_or(inner) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -/// Determine if an expression should be treated as a DWARF alias binding. -/// This is a purely syntactic check (parser phase) and does not consult DWARF. -fn integer_literal_value(e: &Expr) -> Option { - use crate::script::ast::BinaryOp as BO; - use crate::script::ast::Expr as E; - - match e { - E::Int(value) => Some(*value), - E::BinaryOp { - left, - op: BO::Add, - right, - } => integer_literal_value(left)?.checked_add(integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::Subtract, - right, - } => integer_literal_value(left)?.checked_sub(integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::Multiply, - right, - } => integer_literal_value(left)?.checked_mul(integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::Divide, - right, - } => integer_literal_value(left)?.checked_div(integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::Modulo, - right, - } => integer_literal_value(left)?.checked_rem(integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::BitAnd, - right, - } => Some(integer_literal_value(left)? & integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::BitXor, - right, - } => Some(integer_literal_value(left)? ^ integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::BitOr, - right, - } => Some(integer_literal_value(left)? | integer_literal_value(right)?), - E::BinaryOp { - left, - op: BO::ShiftLeft, - right, - } => { - let shift = u32::try_from(integer_literal_value(right)?).ok()?; - integer_literal_value(left)?.checked_shl(shift) - } - E::BinaryOp { - left, - op: BO::ShiftRight, - right, - } => { - let shift = u32::try_from(integer_literal_value(right)?).ok()?; - integer_literal_value(left)?.checked_shr(shift) - } - E::UnaryBitNot(inner) => Some(!integer_literal_value(inner)?), - _ => None, - } -} - -fn is_alias_expr(e: &Expr) -> bool { - use crate::script::ast::BinaryOp as BO; - use crate::script::ast::Expr as E; - match e { - E::AddressOf(_) => true, - // Constant offset on top of an alias-eligible expression - E::BinaryOp { - left, - op: BO::Add, - right, - } => { - (is_alias_expr(left) && integer_literal_value(right).is_some()) - || (is_alias_expr(right) && integer_literal_value(left).is_some()) - } - _ => false, - } -} - -fn parse_logical_or(pair: Pair) -> Result { - match pair.as_rule() { - Rule::logical_or => { - let mut pairs = pair.into_inner(); - let first = pairs.next().ok_or(ParseError::InvalidExpression)?; - let mut left = parse_logical_and(first)?; - - for chunk in chunks_of_two(pairs) { - if chunk.len() != 2 { - return Err(ParseError::InvalidExpression); - } - if chunk[0].as_rule() != Rule::or_op { - return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); - } - let right = parse_logical_and(chunk[1].clone())?; - let expr = Expr::BinaryOp { - left: Box::new(left), - op: BinaryOp::LogicalOr, - right: Box::new(right), - }; - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - left = expr; - } - Ok(left) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_logical_and(pair: Pair) -> Result { - match pair.as_rule() { - Rule::logical_and => { - let mut pairs = pair.into_inner(); - let first = pairs.next().ok_or(ParseError::InvalidExpression)?; - let mut left = parse_bitwise_or(first)?; - - for chunk in chunks_of_two(pairs) { - if chunk.len() != 2 { - return Err(ParseError::InvalidExpression); - } - if chunk[0].as_rule() != Rule::and_op { - return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); - } - let right = parse_bitwise_or(chunk[1].clone())?; - let expr = Expr::BinaryOp { - left: Box::new(left), - op: BinaryOp::LogicalAnd, - right: Box::new(right), - }; - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - left = expr; - } - Ok(left) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_bitwise_or(pair: Pair) -> Result { - match pair.as_rule() { - Rule::bitwise_or => { - let mut pairs = pair.into_inner(); - let first = pairs.next().ok_or(ParseError::InvalidExpression)?; - let mut left = parse_bitwise_xor(first)?; - - for chunk in chunks_of_two(pairs) { - if chunk.len() != 2 { - return Err(ParseError::InvalidExpression); - } - if chunk[0].as_rule() != Rule::bit_or_op { - return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); - } - let right = parse_bitwise_xor(chunk[1].clone())?; - let expr = Expr::BinaryOp { - left: Box::new(left), - op: BinaryOp::BitOr, - right: Box::new(right), - }; - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - left = expr; - } - Ok(left) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_bitwise_xor(pair: Pair) -> Result { - match pair.as_rule() { - Rule::bitwise_xor => { - let mut pairs = pair.into_inner(); - let first = pairs.next().ok_or(ParseError::InvalidExpression)?; - let mut left = parse_bitwise_and(first)?; - - for chunk in chunks_of_two(pairs) { - if chunk.len() != 2 { - return Err(ParseError::InvalidExpression); - } - if chunk[0].as_rule() != Rule::bit_xor_op { - return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); - } - let right = parse_bitwise_and(chunk[1].clone())?; - let expr = Expr::BinaryOp { - left: Box::new(left), - op: BinaryOp::BitXor, - right: Box::new(right), - }; - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - left = expr; - } - Ok(left) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_bitwise_and(pair: Pair) -> Result { - match pair.as_rule() { - Rule::bitwise_and => { - let mut pairs = pair.into_inner(); - let first = pairs.next().ok_or(ParseError::InvalidExpression)?; - let mut left = parse_equality(first)?; - - for chunk in chunks_of_two(pairs) { - if chunk.len() != 2 { - return Err(ParseError::InvalidExpression); - } - if chunk[0].as_rule() != Rule::bit_and_op { - return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); - } - let right = parse_equality(chunk[1].clone())?; - let expr = Expr::BinaryOp { - left: Box::new(left), - op: BinaryOp::BitAnd, - right: Box::new(right), - }; - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - left = expr; - } - Ok(left) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_equality(pair: Pair) -> Result { - match pair.as_rule() { - Rule::equality => { - let mut pairs = pair.into_inner(); - let first = pairs.next().ok_or(ParseError::InvalidExpression)?; - let mut left = parse_relational(first)?; - - for chunk in chunks_of_two(pairs) { - if chunk.len() != 2 { - return Err(ParseError::InvalidExpression); - } - if chunk[0].as_rule() != Rule::eq_op { - return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); - } - let op = match chunk[0].as_str() { - "==" => BinaryOp::Equal, - "!=" => BinaryOp::NotEqual, - _ => return Err(ParseError::UnexpectedToken(chunk[0].as_rule())), - }; - let right = parse_relational(chunk[1].clone())?; - let expr = Expr::BinaryOp { - left: Box::new(left), - op, - right: Box::new(right), - }; - // Type check literals only - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - left = expr; - } - Ok(left) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_relational(pair: Pair) -> Result { - match pair.as_rule() { - Rule::relational => { - let mut pairs = pair.into_inner(); - let first = pairs.next().ok_or(ParseError::InvalidExpression)?; - let mut left = parse_shift(first)?; - - for chunk in chunks_of_two(pairs) { - if chunk.len() != 2 { - return Err(ParseError::InvalidExpression); - } - if chunk[0].as_rule() != Rule::rel_op { - return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); - } - let op = match chunk[0].as_str() { - "<" => BinaryOp::LessThan, - "<=" => BinaryOp::LessEqual, - ">" => BinaryOp::GreaterThan, - ">=" => BinaryOp::GreaterEqual, - _ => return Err(ParseError::UnexpectedToken(chunk[0].as_rule())), - }; - let right = parse_shift(chunk[1].clone())?; - let expr = Expr::BinaryOp { - left: Box::new(left), - op, - right: Box::new(right), - }; - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - left = expr; - } - Ok(left) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_shift(pair: Pair) -> Result { - match pair.as_rule() { - Rule::shift => { - let mut pairs = pair.into_inner(); - let first = pairs.next().ok_or(ParseError::InvalidExpression)?; - let mut left = parse_additive(first)?; - - for chunk in chunks_of_two(pairs) { - if chunk.len() != 2 { - return Err(ParseError::InvalidExpression); - } - let op = match chunk[0].as_str() { - "<<" => BinaryOp::ShiftLeft, - ">>" => BinaryOp::ShiftRight, - _ => return Err(ParseError::UnexpectedToken(chunk[0].as_rule())), - }; - let right = parse_additive(chunk[1].clone())?; - let expr = Expr::BinaryOp { - left: Box::new(left), - op, - right: Box::new(right), - }; - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - left = expr; - } - Ok(left) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_additive(pair: Pair) -> Result { - match pair.as_rule() { - Rule::additive => { - let mut pairs = pair.into_inner(); - let first = pairs.next().ok_or(ParseError::InvalidExpression)?; - let mut left = parse_term(first)?; - - for chunk in chunks_of_two(pairs) { - if chunk.len() != 2 { - return Err(ParseError::InvalidExpression); - } - let op = match chunk[0].as_str() { - "+" => BinaryOp::Add, - "-" => BinaryOp::Subtract, - _ => return Err(ParseError::UnexpectedToken(chunk[0].as_rule())), - }; - let right = parse_term(chunk[1].clone())?; - let expr = Expr::BinaryOp { - left: Box::new(left), - op, - right: Box::new(right), - }; - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - left = expr; - } - Ok(left) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_condition(pair: Pair) -> Result { - debug!( - "parse_condition: {:?} = '{}'", - pair.as_rule(), - pair.as_str().trim() - ); - match pair.as_rule() { - Rule::condition => { - // Condition now accepts a full expression (equality/relational/additive/etc.) - let inner_expr_pair = pair - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - let expr = parse_expr(inner_expr_pair)?; - // Basic type check of the resulting expression - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - Ok(expr) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_else_clause(pair: Pair) -> Result { - let inner = pair - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - match inner.as_rule() { - Rule::if_stmt => { - // Directly parse if statement for else if - debug!("Parsing else if statement"); - let mut inner_pairs = inner.into_inner(); - let condition_pair = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; - debug!( - "else if condition_pair: {:?} = '{}'", - condition_pair.as_rule(), - condition_pair.as_str().trim() - ); - let condition = parse_condition(condition_pair)?; - - // Parse then body statements - let mut then_body = Vec::new(); - let mut else_body = None; - - for pair in inner_pairs { - match pair.as_rule() { - Rule::statement => { - then_body.push(parse_statement(pair)?); - } - Rule::else_clause => { - else_body = Some(Box::new(parse_else_clause(pair)?)); - break; - } - _ => return Err(ParseError::UnexpectedToken(pair.as_rule())), - } - } - - Ok(Statement::If { - condition, - then_body, - else_body, - }) - } - _ => { - // Parse else block statements - let mut else_body = Vec::new(); - for node in inner.into_inner() { - match node.as_rule() { - Rule::statement => { - else_body.push(parse_statement(node)?); - } - // Some grammars flatten block children to concrete statements (e.g., print_stmt) - Rule::print_stmt => { - let content = node - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - let pr = parse_print_content(content)?; - else_body.push(Statement::Print(pr)); - } - _ => return Err(ParseError::UnexpectedToken(node.as_rule())), - } - } - Ok(Statement::Block(else_body)) - } - } -} - -fn parse_term(pair: Pair) -> Result { - match pair.as_rule() { - Rule::term => { - let mut pairs = pair.into_inner(); - let first = pairs.next().ok_or(ParseError::InvalidExpression)?; - let mut left = parse_unary(first)?; - - for chunk in chunks_of_two(pairs) { - if chunk.len() != 2 { - return Err(ParseError::InvalidExpression); - } - - let op = match chunk[0].as_str() { - "*" => BinaryOp::Multiply, - "/" => BinaryOp::Divide, - "%" => BinaryOp::Modulo, - _ => return Err(ParseError::UnexpectedToken(chunk[0].as_rule())), - }; - - let right = parse_unary(chunk[1].clone())?; - - // Check type consistency for binary operations - let expr = Expr::BinaryOp { - left: Box::new(left), - op, - right: Box::new(right), - }; - - // Only check type consistency for literals here - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - - left = expr; - } - - Ok(left) - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_unary(pair: Pair) -> Result { - match pair.as_rule() { - Rule::unary => { - let mut inner = pair.into_inner(); - let first = inner.next().ok_or(ParseError::InvalidExpression)?; - match first.as_rule() { - Rule::factor => parse_factor(first), - // '-' ~ unary - Rule::neg_unary => { - let u = first - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - let right = parse_unary(u)?; - let expr = Expr::BinaryOp { - left: Box::new(Expr::Int(0)), - op: BinaryOp::Subtract, - right: Box::new(right), - }; - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - Ok(expr) - } - // '!' ~ unary - Rule::not_unary => { - let u = first - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - let right = parse_unary(u)?; - Ok(Expr::UnaryNot(Box::new(right))) - } - Rule::bit_not_unary => { - let u = first - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - let right = parse_unary(u)?; - let expr = Expr::UnaryBitNot(Box::new(right)); - if let Err(err) = infer_type(&expr) { - return Err(ParseError::TypeError(err)); - } - Ok(expr) - } - _ => Err(ParseError::UnexpectedToken(first.as_rule())), - } - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_factor(pair: Pair) -> Result { - match pair.as_rule() { - Rule::factor => { - let inner = pair - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - match inner.as_rule() { - Rule::memcmp_call => parse_builtin_call(inner), - Rule::strncmp_call => parse_builtin_call(inner), - Rule::starts_with_call => parse_builtin_call(inner), - Rule::hex_call => parse_builtin_call(inner), - Rule::postfix_access => parse_postfix_access(inner), - Rule::cast_call => parse_cast_call(inner), - Rule::chain_access => parse_chain_access(inner), - Rule::pointer_deref => parse_pointer_deref(inner), - Rule::address_of => parse_address_of(inner), - Rule::int => match inner.as_str().parse::() { - Ok(value) => Ok(Expr::Int(value)), - Err(_) => Err(ParseError::TypeError( - "invalid decimal integer literal".to_string(), - )), - }, - Rule::hex_int => { - // strip 0x and parse as hex - let s = inner.as_str(); - match i64::from_str_radix(&s[2..], 16) { - Ok(v) => Ok(Expr::Int(v)), - Err(_) => Err(ParseError::TypeError( - "invalid hex integer literal".to_string(), - )), - } - } - Rule::oct_int => { - let s = inner.as_str(); - match i64::from_str_radix(&s[2..], 8) { - Ok(v) => Ok(Expr::Int(v)), - Err(_) => Err(ParseError::TypeError( - "invalid octal integer literal".to_string(), - )), - } - } - Rule::bin_int => { - let s = inner.as_str(); - match i64::from_str_radix(&s[2..], 2) { - Ok(v) => Ok(Expr::Int(v)), - Err(_) => Err(ParseError::TypeError( - "invalid binary integer literal".to_string(), - )), - } - } - // Floats are not supported by scripts/runtime; reject early with friendly error - Rule::float => Err(ParseError::TypeError( - "float literals are not supported".to_string(), - )), - Rule::string => { - // Remove quotes at the beginning and end - let raw_value = inner.as_str(); - let value = &raw_value[1..raw_value.len() - 1]; - Ok(Expr::String(value.to_string())) - } - Rule::bool => { - let val = inner.as_str() == "true"; - Ok(Expr::Bool(val)) - } - Rule::identifier => { - let name = inner.as_str().to_string(); - Ok(Expr::Variable(name)) - } - Rule::array_access => parse_array_access(inner), - Rule::member_access => parse_member_access(inner), - Rule::special_var => { - let var_name = inner.as_str().to_string(); - Ok(Expr::SpecialVar(var_name)) - } - Rule::expr => parse_expr(inner), - _ => Err(ParseError::UnexpectedToken(inner.as_rule())), - } - } - _ => Err(ParseError::UnexpectedToken(pair.as_rule())), - } -} - -fn parse_builtin_call(pair: Pair) -> Result { - // pair is memcmp_call / strncmp_call / starts_with_call / hex_call - let rule = pair.as_rule(); - let mut it = pair.into_inner(); - // First token inside is the function name as identifier within the rule text; easier approach: use rule to select - match rule { - Rule::memcmp_call => { - // grammar: memcmp("(" expr "," expr ["," expr] ")") - let mut nodes: Vec<_> = it.collect(); - if nodes.len() < 2 || nodes.len() > 3 { - return Err(ParseError::InvalidExpression); - } - let a_expr = parse_expr(nodes.remove(0))?; - let b_expr = parse_expr(nodes.remove(0))?; - - // Disallow obviously invalid types early - if matches!(a_expr, Expr::Bool(_)) || matches!(b_expr, Expr::Bool(_)) { - return Err(ParseError::TypeError( - "memcmp pointer arguments cannot be boolean; use an address or hex(...)" - .to_string(), - )); - } - if matches!(a_expr, Expr::String(_)) || matches!(b_expr, Expr::String(_)) { - return Err(ParseError::TypeError( - "memcmp does not accept string literals; use strncmp for strings".to_string(), - )); - } - - // Helper to get hex length (bytes) - let hex_len = |e: &Expr| -> Option { - if let Expr::BuiltinCall { name, args } = e { - if name == "hex" { - if let Some(Expr::String(s)) = args.first() { - return Some(s.len() / 2); - } - } - } - None - }; - - let n_expr = if let Some(n_node) = nodes.first() { - // With explicit len: reuse previous literal checks - let n_expr = parse_expr(n_node.clone())?; - if matches!(n_expr, Expr::Bool(_)) { - return Err(ParseError::TypeError( - "memcmp length must be an integer or expression, not boolean".to_string(), - )); - } - let literal_len_opt: Option = match &n_expr { - Expr::Int(n) => Some(*n as isize), - Expr::BinaryOp { - left, - op: BinaryOp::Subtract, - right, - } => { - if matches!(left.as_ref(), Expr::Int(0)) { - if let Expr::Int(k) = right.as_ref() { - Some(-(*k as isize)) - } else { - None - } - } else { - None - } - } - _ => None, - }; - if let Some(n) = literal_len_opt { - if n < 0 { - return Err(ParseError::TypeError( - "memcmp length must be non-negative".to_string(), - )); - } - let l = n as usize; - if let Some(la) = hex_len(&a_expr) { - if l > la { - return Err(ParseError::TypeError(format!( - "memcmp length ({l}) exceeds hex pattern size on left side ({la} bytes)" - ))); - } - } - if let Some(lb) = hex_len(&b_expr) { - if l > lb { - return Err(ParseError::TypeError(format!( - "memcmp length ({l}) exceeds hex pattern size on right side ({lb} bytes)" - ))); - } - } - } - n_expr - } else { - // No len provided: allow only when at least one side is hex(...) - let la = hex_len(&a_expr); - let lb = hex_len(&b_expr); - match (la, lb) { - (Some(l), None) | (None, Some(l)) => Expr::Int(l as i64), - (Some(la), Some(lb)) => { - if la != lb { - return Err(ParseError::TypeError( - "memcmp hex operands have different sizes; provide explicit len" - .to_string(), - )); - } - Expr::Int(la as i64) - } - _ => { - return Err(ParseError::TypeError( - "memcmp without len requires at least one hex(...) operand".to_string(), - )) - } - } - }; - - // Constant folding: memcmp(hex(...), hex(...), N) - let as_hex = |e: &Expr| -> Option { - if let Expr::BuiltinCall { name, args } = e { - if name == "hex" { - if let Some(Expr::String(s)) = args.first() { - return Some(s.clone()); - } - } - } - None - }; - - if let (Some(h1), Some(h2), Expr::Int(n)) = (as_hex(&a_expr), as_hex(&b_expr), &n_expr) - { - // Safe hex -> bytes (sanitized earlier to hex digits only) - fn hex_to_bytes(s: &str) -> std::result::Result, ParseError> { - let mut out = Vec::with_capacity(s.len() / 2); - let bytes = s.as_bytes(); - let mut i = 0; - while i + 1 < bytes.len() { - let h = bytes[i] as char; - let l = bytes[i + 1] as char; - let hv = h - .to_digit(16) - .ok_or_else(|| ParseError::TypeError("invalid hex digit".to_string()))? - as u8; - let lv = l - .to_digit(16) - .ok_or_else(|| ParseError::TypeError("invalid hex digit".to_string()))? - as u8; - out.push((hv << 4) | lv); - i += 2; - } - Ok(out) - } - - let v1 = hex_to_bytes(&h1)?; - let v2 = hex_to_bytes(&h2)?; - let ln = (*n).max(0) as usize; - let eq = v1.iter().take(ln).eq(v2.iter().take(ln)); - return Ok(Expr::Bool(eq)); - } - - Ok(Expr::BuiltinCall { - name: "memcmp".to_string(), - args: vec![a_expr, b_expr, n_expr], - }) - } - Rule::strncmp_call => { - // grammar: strncmp("(" expr "," expr "," expr ")") - let arg0 = parse_expr(it.next().ok_or(ParseError::InvalidExpression)?)?; - let arg1 = parse_expr(it.next().ok_or(ParseError::InvalidExpression)?)?; - let n_expr_parsed = parse_expr(it.next().ok_or(ParseError::InvalidExpression)?)?; - let literal_len_opt: Option = match &n_expr_parsed { - Expr::Int(n) => Some(*n as isize), - Expr::BinaryOp { - left, - op: BinaryOp::Subtract, - right, - } => { - if matches!(left.as_ref(), Expr::Int(0)) { - if let Expr::Int(k) = right.as_ref() { - Some(-(*k as isize)) - } else { - None - } - } else { - None - } - } - _ => None, - }; - if literal_len_opt.is_some_and(|n| n < 0) { - return Err(ParseError::TypeError( - "strncmp third argument must be non-negative".to_string(), - )); - } - // Optional constant fold when both sides are string literals - if let (Expr::String(a), Expr::String(b), Expr::Int(n_val)) = - (&arg0, &arg1, &n_expr_parsed) - { - let ln = (*n_val).max(0) as usize; - let eq = a - .as_bytes() - .iter() - .take(ln) - .eq(b.as_bytes().iter().take(ln)); - return Ok(Expr::Bool(eq)); - } - Ok(Expr::BuiltinCall { - name: "strncmp".to_string(), - args: vec![arg0, arg1, n_expr_parsed], - }) - } - Rule::starts_with_call => { - // grammar: starts_with("(" expr "," expr ")") - let arg0 = parse_expr(it.next().ok_or(ParseError::InvalidExpression)?)?; - let arg1 = parse_expr(it.next().ok_or(ParseError::InvalidExpression)?)?; - // Constant fold when both are string literals - if let (Expr::String(a), Expr::String(b)) = (&arg0, &arg1) { - return Ok(Expr::Bool(a.as_bytes().starts_with(b.as_bytes()))); - } - Ok(Expr::BuiltinCall { - name: "starts_with".to_string(), - args: vec![arg0, arg1], - }) - } - Rule::hex_call => { - // grammar: hex("HEX...") - // Validate at parse time: allow only hex digits with optional whitespace separators. - let lit_node = it.next().ok_or(ParseError::InvalidExpression)?; - if lit_node.as_rule() != Rule::string { - return Err(ParseError::TypeError( - "hex expects a string literal".to_string(), - )); - } - let raw = lit_node.as_str(); - let inner = &raw[1..raw.len() - 1]; - let mut sanitized = String::with_capacity(inner.len()); - for ch in inner.chars() { - if ch.is_ascii_hexdigit() { - sanitized.push(ch); - } else if ch == ' ' { - // allow spaces as separators (tabs not allowed) - continue; - } else { - return Err(ParseError::TypeError(format!( - "hex literal contains non-hex character: '{ch}'" - ))); - } - } - if sanitized.len() % 2 == 1 { - return Err(ParseError::TypeError( - "hex literal must contain an even number of hex digits".to_string(), - )); - } - Ok(Expr::BuiltinCall { - name: "hex".to_string(), - // Store sanitized hex-only string; codegen will convert to bytes - args: vec![Expr::String(sanitized)], - }) - } - _ => Err(ParseError::UnexpectedToken(rule)), - } -} - -fn parse_cast_call(pair: Pair) -> Result { - let mut inner = pair.into_inner(); - let expr_pair = inner.next().ok_or(ParseError::InvalidExpression)?; - let type_pair = inner.next().ok_or(ParseError::InvalidExpression)?; - let raw_type = type_pair.as_str(); - let target_type = raw_type - .strip_prefix('"') - .and_then(|s| s.strip_suffix('"')) - .ok_or_else(|| ParseError::SyntaxError("cast target type must be a string".to_string()))? - .to_string(); - - Ok(Expr::Cast { - expr: Box::new(parse_expr(expr_pair)?), - target_type, - }) -} - -fn parse_postfix_access(pair: Pair) -> Result { - let mut inner = pair.into_inner(); - let base = inner.next().ok_or(ParseError::InvalidExpression)?; - let mut expr = match base.as_rule() { - Rule::postfix_base => { - let base_inner = base - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - match base_inner.as_rule() { - Rule::cast_call => parse_cast_call(base_inner)?, - Rule::special_var => Expr::SpecialVar(base_inner.as_str().to_string()), - Rule::identifier => Expr::Variable(base_inner.as_str().to_string()), - Rule::expr => parse_expr(base_inner)?, - _ => return Err(ParseError::UnexpectedToken(base_inner.as_rule())), - } - } - _ => return Err(ParseError::UnexpectedToken(base.as_rule())), - }; - - for suffix in inner { - let suffix_inner = suffix - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - match suffix_inner.as_rule() { - Rule::member_suffix => { - let field = suffix_inner - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)? - .as_str() - .to_string(); - expr = Expr::MemberAccess(Box::new(expr), field); - } - Rule::index_suffix => { - let index_pair = suffix_inner - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - let parsed_index = parse_expr(index_pair)?; - let parsed_index = integer_literal_value(&parsed_index) - .map(Expr::Int) - .unwrap_or(parsed_index); - expr = Expr::ArrayAccess(Box::new(expr), Box::new(parsed_index)); - } - _ => return Err(ParseError::UnexpectedToken(suffix_inner.as_rule())), - } - } - - Ok(expr) -} - -fn parse_trace_pattern(pair: Pair) -> Result { - let inner = pair - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - - match inner.as_rule() { - Rule::module_hex_address => { - let mut parts = inner.into_inner(); - let module = parts - .next() - .ok_or(ParseError::InvalidExpression)? - .as_str() - .to_string(); - let hex = parts.next().ok_or(ParseError::InvalidExpression)?.as_str(); - let addr = match u64::from_str_radix(&hex[2..], 16) { - Ok(v) => v, - Err(_) => { - return Err(ParseError::SyntaxError(format!( - "module-qualified address '{hex}' is invalid or too large for u64" - ))) - } - }; - Ok(TracePattern::AddressInModule { - module, - address: addr, - }) - } - Rule::hex_address => { - let addr_str = inner.as_str(); - // Remove "0x" prefix and parse as hex - let addr_hex = &addr_str[2..]; - let addr = match u64::from_str_radix(addr_hex, 16) { - Ok(v) => v, - Err(_) => { - return Err(ParseError::SyntaxError(format!( - "address '{addr_str}' is invalid or too large for u64" - ))) - } - }; - Ok(TracePattern::Address(addr)) - } - Rule::wildcard_pattern => { - let pattern = inner.as_str().to_string(); - Ok(TracePattern::Wildcard(pattern)) - } - Rule::function_name => { - let func_name = inner - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)? - .as_str() - .to_string(); - Ok(TracePattern::FunctionName(func_name)) - } - Rule::source_line => { - let mut parts = inner.into_inner(); - let file_path = parts - .next() - .ok_or(ParseError::InvalidExpression)? - .as_str() - .to_string(); - let line_pair = parts.next().ok_or(ParseError::InvalidExpression)?; - let line_number = line_pair - .as_str() - .parse::() - .map_err(|_| ParseError::InvalidExpression)?; - Ok(TracePattern::SourceLine { - file_path, - line_number, - }) - } - _ => Err(ParseError::UnexpectedToken(inner.as_rule())), - } -} - -fn parse_print_content(pair: Pair) -> Result { - info!( - "parse_print_content: rule={:?} text=\"{}\"", - pair.as_rule(), - pair.as_str().trim() - ); - // Flatten any nested print_content nodes into a single list of children - fn collect_flattened<'a>(p: Pair<'a, Rule>, out: &mut Vec>) { - if p.as_rule() == Rule::print_content { - for c in p.into_inner() { - collect_flattened(c, out); - } - } else { - out.push(p); - } - } - - let mut flat: Vec> = Vec::new(); - collect_flattened(pair, &mut flat); - info!( - "parse_print_content: flat_rules=[{}]", - flat.iter() - .map(|p| format!("{:?}", p.as_rule())) - .collect::>() - .join(", ") - ); - if flat.is_empty() { - return Err(ParseError::InvalidExpression); - } - - // Prefer an explicit format_expr if present - if let Some(fmt_idx) = flat.iter().position(|p| p.as_rule() == Rule::format_expr) { - let fmt_pair = flat.remove(fmt_idx); - info!("parse_print_content: branch=format_expr"); - let mut inner_pairs = fmt_pair.into_inner(); - let format_string = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; - let format_content = &format_string.as_str()[1..format_string.as_str().len() - 1]; - let mut args = Vec::new(); - for arg_pair in inner_pairs { - args.push(parse_expr(arg_pair)?); - } - info!( - "parse_print_content: fmt='{}' argc={}", - format_content, - args.len() - ); - FormatValidator::validate_format_arguments(format_content, &args)?; - return Ok(PrintStatement::Formatted { - format: format_content.to_string(), - args, - }); - } - - // Else, if first is a string and followed by one or more exprs, treat as flattened format - if flat[0].as_rule() == Rule::string && flat.len() >= 2 { - info!("parse_print_content: branch=flattened_string_with_args"); - let content_quoted = flat[0].as_str(); - let content = &content_quoted[1..content_quoted.len() - 1]; - let mut args = Vec::new(); - for p in flat.iter().skip(1) { - if p.as_rule() != Rule::expr { - return Err(ParseError::UnexpectedToken(p.as_rule())); - } - args.push(parse_expr(p.clone())?); - } - info!("parse_print_content: fmt='{}' argc={}", content, args.len()); - FormatValidator::validate_format_arguments(content, &args)?; - return Ok(PrintStatement::Formatted { - format: content.to_string(), - args, - }); - } - - // Single string or single expr - match flat[0].as_rule() { - Rule::string => { - info!("parse_print_content: branch=plain_string"); - let content = flat[0].as_str(); - let content = &content[1..content.len() - 1]; - Ok(PrintStatement::String(content.to_string())) - } - Rule::expr => { - info!("parse_print_content: branch=complex_variable"); - let expr = parse_expr(flat[0].clone())?; - Ok(PrintStatement::ComplexVariable(expr)) - } - other => { - info!("parse_print_content: branch=unexpected rule={:?}", other); - Err(ParseError::UnexpectedToken(other)) - } - } -} - -// Parse complex variable expressions (person.name, arr[0], etc.) -fn parse_complex_variable(pair: Pair) -> Result { - debug!( - "parse_complex_variable: {:?} = \"{}\"", - pair.as_rule(), - pair.as_str().trim() - ); - - let inner = pair - .into_inner() - .next() - .ok_or(ParseError::InvalidExpression)?; - match inner.as_rule() { - Rule::chain_access => parse_chain_access(inner), - Rule::array_access => parse_array_access(inner), - Rule::member_access => parse_member_access(inner), - Rule::pointer_deref => parse_pointer_deref(inner), - Rule::address_of => parse_address_of(inner), - _ => Err(ParseError::UnexpectedToken(inner.as_rule())), - } -} - -// Parse chain access: person.name.first -fn parse_chain_access(pair: Pair) -> Result { - let mut chain: Vec = Vec::new(); - let mut opt_index: Option = None; - for inner_pair in pair.into_inner() { - match inner_pair.as_rule() { - Rule::identifier => { - chain.push(inner_pair.as_str().to_string()); - } - Rule::expr => { - // Array tail index can be a literal or a runtime expression. - let parsed = parse_expr(inner_pair)?; - opt_index = Some( - integer_literal_value(&parsed) - .map(Expr::Int) - .unwrap_or(parsed), - ); - } - _ => {} - } - } - - if chain.is_empty() { - return Err(ParseError::InvalidExpression); - } - - // Build base expression from the chain identifiers - let mut expr = Expr::Variable(chain[0].clone()); - for seg in &chain[1..] { - expr = Expr::MemberAccess(Box::new(expr), seg.clone()); - } - - // If there's a trailing index, convert to ArrayAccess on the built base - if let Some(idx) = opt_index { - expr = Expr::ArrayAccess(Box::new(expr), Box::new(idx)); - } - - Ok(expr) -} - -// Parse array access: arr[index] -fn parse_array_access(pair: Pair) -> Result { - let mut inner_pairs = pair.into_inner(); - let array_name = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; - let index_expr = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; - - let _array_expr = Box::new(Expr::Variable(array_name.as_str().to_string())); - let parsed_index = parse_expr(index_expr)?; - let parsed_index = integer_literal_value(&parsed_index) - .map(Expr::Int) - .unwrap_or(parsed_index); - - // Build base array access expression - let mut expr = Expr::ArrayAccess( - Box::new(Expr::Variable(array_name.as_str().to_string())), - Box::new(parsed_index), - ); - - // Consume trailing .field segments if present - for next in inner_pairs { - // Any remaining tokens are member identifiers - let m = next.as_str().to_string(); - expr = Expr::MemberAccess(Box::new(expr), m); - } - - Ok(expr) -} - -// Parse member access: person.name -fn parse_member_access(pair: Pair) -> Result { - let mut parts = pair.into_inner(); - let base = parts - .next() - .ok_or(ParseError::InvalidExpression)? - .as_str() - .to_string(); - - // Collect all subsequent identifiers after the base - let mut tail: Vec = Vec::new(); - for p in parts { - tail.push(p.as_str().to_string()); - } - - // If there is only one member, keep MemberAccess for simplicity. - // For multi-level chains like a.b.c, normalize to ChainAccess([a, b, c]) - match tail.len() { - 0 => Err(ParseError::InvalidExpression), - 1 => Ok(Expr::MemberAccess( - Box::new(Expr::Variable(base)), - tail.remove(0), - )), - _ => { - let mut chain = Vec::with_capacity(1 + tail.len()); - chain.push(base); - chain.extend(tail); - Ok(Expr::ChainAccess(chain)) - } - } -} - -// Parse pointer dereference: *ptr -fn parse_pointer_deref(pair: Pair) -> Result { - let mut inner = pair.into_inner(); - let target = inner.next().ok_or(ParseError::InvalidExpression)?; - let parsed = match target.as_rule() { - Rule::expr => parse_expr(target)?, - Rule::postfix_access => parse_postfix_access(target)?, - Rule::cast_call => parse_cast_call(target)?, - Rule::complex_variable => parse_complex_variable(target)?, - Rule::special_var => Expr::SpecialVar(target.as_str().to_string()), - Rule::identifier => Expr::Variable(target.as_str().to_string()), - _ => return Err(ParseError::UnexpectedToken(target.as_rule())), - }; - // Early normalization: *(&x) => x - match parsed { - Expr::AddressOf(inner_expr) => Ok(*inner_expr), - other => Ok(Expr::PointerDeref(Box::new(other))), - } -} - -// Parse address-of: &expr -fn parse_address_of(pair: Pair) -> Result { - let mut inner = pair.into_inner(); - let target = inner.next().ok_or(ParseError::InvalidExpression)?; - let parsed = match target.as_rule() { - Rule::expr => parse_expr(target)?, - Rule::postfix_access => parse_postfix_access(target)?, - Rule::cast_call => parse_cast_call(target)?, - Rule::complex_variable => parse_complex_variable(target)?, - Rule::special_var => Expr::SpecialVar(target.as_str().to_string()), - Rule::identifier => Expr::Variable(target.as_str().to_string()), - _ => return Err(ParseError::UnexpectedToken(target.as_rule())), - }; - // Early normalization: &(*p) => p - match parsed { - Expr::PointerDeref(inner_expr) => Ok(*inner_expr), - other => Ok(Expr::AddressOf(Box::new(other))), - } -} - #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_memcmp_builtin_in_if_should_succeed() { - let script = r#" -trace foo { - if memcmp(&buf[0], &buf[1], 16) { print "EQ"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_cast_member_and_index_access() { - let script = r#" -trace foo { - print cast($arg0, "struct request *").id; - print cast($arg1, "u32 *")[2]; - print *cast($arg1, "u32 *"); - print &cast($arg0, "struct request *").id; -} -"#; - let program = parse(script).expect("parse should succeed"); - let Statement::TracePoint { body, .. } = &program.statements[0] else { - panic!("expected trace point"); - }; - assert!(matches!( - &body[0], - Statement::Print(PrintStatement::ComplexVariable(Expr::MemberAccess(obj, field))) - if field == "id" && matches!(obj.as_ref(), Expr::Cast { .. }) - )); - assert!(matches!( - &body[1], - Statement::Print(PrintStatement::ComplexVariable(Expr::ArrayAccess(base, index))) - if matches!(base.as_ref(), Expr::Cast { .. }) - && matches!(index.as_ref(), Expr::Int(2)) - )); - assert!(matches!( - &body[2], - Statement::Print(PrintStatement::ComplexVariable(Expr::PointerDeref(inner))) - if matches!(inner.as_ref(), Expr::Cast { .. }) - )); - assert!(matches!( - &body[3], - Statement::Print(PrintStatement::ComplexVariable(Expr::AddressOf(inner))) - if matches!( - inner.as_ref(), - Expr::MemberAccess(obj, field) - if field == "id" && matches!(obj.as_ref(), Expr::Cast { .. }) - ) - )); - } - - #[test] - fn parse_memcmp_with_dynamic_len() { - let script = r#" -trace foo { - let n = 10; - if memcmp(&buf[0], &buf[0], n) { print "OK"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_if_else_with_flattened_format_and_star_len() { - // else branch contains a flattened format print with {:s.*} and two args - let script = r#" -trace src/http/ngx_http_request.c:1845 { - if strncmp(host.data, "ghostscope", 10) { - print "We got the request {}", *r; - } else { - print "The other hostname is {:s.*}", host.len, host.data; - } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_memcmp_len_zero_and_negative() { - let script = r#" -trace foo { - if memcmp(&p[0], &q[0], 0) { print "Z0"; } - let k = -5; - if memcmp(&p[0], &q[0], k) { print "NEG"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_numeric_literals_hex_oct_bin_and_memcmp_usage() { - let script = r#" -trace foo { - let a = 0x10; // 16 - let b = 0o755; // 493 - let c = 0b1010; // 10 - // use in memcmp length - if memcmp(&buf[0], &buf[0], 0x20) { print "H"; } - if memcmp(&buf[0], &buf[0], 0o40) { print "O"; } - if memcmp(&buf[0], &buf[0], 0b100000) { print "B"; } - // use numeric literal as pointer address for second arg - if memcmp(&buf[0], 0x7fff0000, 16) { print "P"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_memcmp_hex_builtin() { - let script = r#" -trace foo { - if memcmp(&buf[0], hex("504F"), 2) { print "OK"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_memcmp_with_numeric_pointers_and_len_bases() { - let script = r#" -trace foo { - let n = 0x10; - if memcmp(0x1000, 0x2000, n) { print "NP"; } - if memcmp(0o4000, 0b1000000000000, 0o20) { print "NP2"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_hex_with_non_hex_char_should_fail() { - let script = r#" -trace foo { - if memcmp(&buf[0], hex("G0"), 1) { print "X"; } -} -"#; - let r = parse(script); - match r { - Ok(_) => panic!("expected parse error for non-hex char"), - Err(ParseError::TypeError(msg)) => { - assert!( - msg.contains("hex literal contains non-hex character"), - "unexpected msg: {msg}" - ); - } - Err(e) => panic!("unexpected error variant: {e:?}"), - } - } - - #[test] - fn parse_hex_with_odd_digits_should_fail() { - let script = r#" -trace foo { - if memcmp(&buf[0], hex("123"), 1) { print "X"; } -} -"#; - let r = parse(script); - match r { - Ok(_) => panic!("expected parse error for odd-length hex"), - Err(ParseError::TypeError(msg)) => { - assert!( - msg.contains("even number of hex digits"), - "unexpected msg: {msg}" - ); - } - Err(e) => panic!("unexpected error variant: {e:?}"), - } - } - - #[test] - fn parse_hex_with_spaces_should_succeed() { - let script = r#" -trace foo { - if memcmp(&buf[0], hex("4c 49 42 5f"), 4) { print "OK"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_alias_declaration_address_of_and_member_access() { - let script = r#" -trace foo { - let p = &buf[0]; - let s = obj.field; -} -"#; - let prog = parse(script).expect("parse ok"); - let stmt0 = prog.statements.first().expect("trace"); - match stmt0 { - Statement::TracePoint { body, .. } => { - // Only the address-of form should be alias; member access is a value binding - assert!(matches!(body[0], Statement::AliasDeclaration { .. })); - assert!(matches!(body[1], Statement::VarDeclaration { .. })); - } - other => panic!("expected TracePoint, got {other:?}"), - } - } - - #[test] - fn parse_alias_declaration_with_constant_offset() { - let script = r#" -trace foo { - let p = &arr[0] + 16; - let q = 32 + &arr[0]; -} -"#; - let prog = parse(script).expect("parse ok"); - let stmt0 = prog.statements.first().expect("trace"); - match stmt0 { - Statement::TracePoint { body, .. } => { - assert!(matches!(body[0], Statement::AliasDeclaration { .. })); - assert!(matches!(body[1], Statement::AliasDeclaration { .. })); - } - other => panic!("expected TracePoint, got {other:?}"), - } - } - - #[test] - fn parse_member_access_scalar_not_alias() { - let script = r#" -trace foo { - let level = record.level; -} -"#; - let prog = parse(script).expect("parse ok"); - let stmt0 = prog.statements.first().expect("trace"); - match stmt0 { - Statement::TracePoint { body, .. } => { - assert!(matches!(body[0], Statement::VarDeclaration { .. })); - } - other => panic!("expected TracePoint, got {other:?}"), - } - } - - #[test] - fn parse_memcmp_rejects_string_literal() { - let script = r#" -trace foo { - if memcmp(&buf[0], "PO", 2) { print "X"; } -} -"#; - let r = parse(script); - assert!( - matches!(r, Err(ParseError::TypeError(ref msg)) if msg.contains("memcmp does not accept string literals")), - "expected type error, got: {r:?}" - ); - } - - #[test] - fn parse_memcmp_rejects_bool_args_and_len() { - // Bool as pointer argument - let s1 = r#" -trace foo { if memcmp(true, hex("00"), 1) { print "X"; } } -"#; - let r1 = parse(s1); - assert!(r1.is_err()); - - // Bool as length - let s2 = r#" -trace foo { if memcmp(&p[0], hex("00"), false) { print "X"; } } -"#; - let r2 = parse(s2); - assert!( - matches!(r2, Err(ParseError::TypeError(ref msg)) if msg.contains("length must be")), - "unexpected: {r2:?}" - ); - } - - #[test] - fn parse_strncmp_constant_folds_on_two_literals() { - // equal for first 2 bytes - let s = r#" -trace foo { - if strncmp("abc", "abd", 2) { print "T"; } else { print "F"; } -} -"#; - let prog = parse(s).expect("parse ok"); - // Walk down to the If condition and ensure it became a Bool(true) - let stmt0 = prog.statements.first().expect("one trace"); - match stmt0 { - Statement::TracePoint { body, .. } => match &body[0] { - Statement::If { condition, .. } => { - assert!(matches!(condition, Expr::Bool(true))); - } - other => panic!("expected If, got {other:?}"), - }, - other => panic!("expected TracePoint, got {other:?}"), - } - } - - #[test] - fn parse_strncmp_requires_one_string_side_error() { - let s = r#" -trace foo { - if strncmp(1, 2, 1) { print "X"; } -} -"#; - let r = parse(s); - // Parser now accepts generic expr, so error will occur in compiler stage; ensure parse ok here - assert!( - r.is_ok(), - "parse should succeed; semantic error in compiler" - ); - } - - #[test] - fn parse_memcmp_constant_folds_on_two_hex() { - let s = r#" -trace foo { - if memcmp(hex("504f"), hex("504F"), 2) { print "EQ"; } else { print "NE"; } -} -"#; - let prog = parse(s).expect("parse ok"); - let stmt0 = prog.statements.first().expect("one trace"); - match stmt0 { - Statement::TracePoint { body, .. } => match &body[0] { - Statement::If { condition, .. } => assert!(matches!(condition, Expr::Bool(true))), - other => panic!("expected If, got {other:?}"), - }, - other => panic!("expected TracePoint, got {other:?}"), - } - - // Mismatch without explicit len but equal sizes - let s2 = r#" -trace foo { - if memcmp(hex("504f"), hex("514f")) { print "EQ"; } else { print "NE"; } -} -"#; - let prog2 = parse(s2).expect("parse ok"); - let stmt02 = prog2.statements.first().expect("one trace"); - match stmt02 { - Statement::TracePoint { body, .. } => match &body[0] { - Statement::If { condition, .. } => assert!(matches!(condition, Expr::Bool(false))), - other => panic!("expected If, got {other:?}"), - }, - other => panic!("expected TracePoint, got {other:?}"), - } - } - - #[test] - fn parse_starts_with_constant_folds_on_two_literals() { - let s = r#" -trace foo { - if starts_with("abcdef", "abc") { print "T"; } else { print "F"; } -} -"#; - let prog = parse(s).expect("parse ok"); - let stmt0 = prog.statements.first().expect("one trace"); - match stmt0 { - Statement::TracePoint { body, .. } => match &body[0] { - Statement::If { condition, .. } => assert!(matches!(condition, Expr::Bool(true))), - other => panic!("expected If, got {other:?}"), - }, - other => panic!("expected TracePoint, got {other:?}"), - } - - let s2 = r#" -trace foo { - if starts_with("ab", "abc") { print "T"; } else { print "F"; } -} -"#; - let prog2 = parse(s2).expect("parse ok"); - let stmt02 = prog2.statements.first().expect("one trace"); - match stmt02 { - Statement::TracePoint { body, .. } => match &body[0] { - Statement::If { condition, .. } => assert!(matches!(condition, Expr::Bool(false))), - other => panic!("expected If, got {other:?}"), - }, - other => panic!("expected TracePoint, got {other:?}"), - } - } - - #[test] - fn parse_memcmp_hex_len_exceeds_left_should_fail() { - // hex has 2 bytes, len=3 should error on left side - let script = r#" -trace foo { - if memcmp(hex("504f"), &buf[0], 3) { print "X"; } -} -"#; - let r = parse(script); - match r { - Ok(_) => panic!("expected parse error for len > hex(left) size"), - Err(ParseError::TypeError(msg)) => { - assert!( - msg.contains("exceeds hex pattern size on left side"), - "unexpected msg: {msg}" - ); - } - Err(e) => panic!("unexpected error variant: {e:?}"), - } - } - - #[test] - fn parse_memcmp_hex_len_exceeds_right_should_fail() { - // hex has 2 bytes, len=5 should error on right side - let script = r#" -trace foo { - if memcmp(&buf[0], hex("50 4f"), 5) { print "X"; } -} -"#; - let r = parse(script); - match r { - Ok(_) => panic!("expected parse error for len > hex(right) size"), - Err(ParseError::TypeError(msg)) => { - assert!( - msg.contains("exceeds hex pattern size on right side"), - "unexpected msg: {msg}" - ); - } - Err(e) => panic!("unexpected error variant: {e:?}"), - } - } - - #[test] - fn parse_memcmp_hex_negative_len_should_fail() { - let script = r#" -trace foo { - if memcmp(&buf[0], hex("50 4f"), -1) { print "X"; } -} -"#; - let r = parse(script); - match r { - Ok(_) => panic!("expected parse error for negative len"), - Err(ParseError::TypeError(msg)) => { - assert!( - msg.contains("length must be non-negative"), - "unexpected msg: {msg}" - ); - } - Err(e) => panic!("unexpected error variant: {e:?}"), - } - } - - #[test] - fn parse_memcmp_hex_len_equal_should_succeed() { - // hex has 4 bytes, len=4 OK - let script = r#" -trace foo { - if memcmp(&buf[0], hex("de ad be ef"), 4) { print "OK"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_memcmp_hex_infers_len_left_should_succeed() { - let script = r#" -trace foo { - if memcmp(hex("50 4f"), &buf[0]) { print "OK"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_memcmp_hex_infers_len_right_should_succeed() { - let script = r#" -trace foo { - if memcmp(&buf[0], hex("de ad be ef")) { print "OK"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_assignment_is_rejected_with_friendly_message() { - let script = r#" -trace foo { - let a = 1; - a = 2; -} -"#; - let r = parse(script); - match r { - Ok(_) => panic!("expected assignment error for immutable variables"), - Err(ParseError::TypeError(msg)) => { - assert!( - msg.contains("Assignment is not supported"), - "unexpected msg: {msg}" - ); - } - Err(e) => panic!("unexpected error variant: {e:?}"), - } - } - - #[test] - fn parse_starts_with_accepts_two_exprs() { - // Both sides are expr (identifiers); grammar should accept - let script = r#" -trace foo { - if starts_with(name, s) { print "OK"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_strncmp_accepts_two_exprs_and_len() { - let script = r#" -trace foo { - if strncmp(lhs, rhs, 3) { print "EQ"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_strncmp_negative_len_rejected() { - // Negative literal lengths are rejected early. - let script = r#" -trace foo { - if strncmp(lhs, rhs, -1) { print "X"; } -} -"#; - let r = parse(script); - assert!(r.is_err(), "expected parse error for negative length"); - if let Err(ParseError::TypeError(msg)) = r { - assert!(msg.contains("non-negative"), "unexpected msg: {msg}"); - } - } - - #[test] - fn parse_strncmp_accepts_nonliteral_len() { - let script = r#" -trace foo { - let n = 3; - if strncmp(lhs, rhs, n) { print "X"; } -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_memcmp_missing_len_without_hex_should_fail() { - let script = r#" -trace foo { - if memcmp(&buf[0], &buf[1]) { print "OK"; } -} -"#; - let r = parse(script); - assert!( - r.is_err(), - "expected parse error for missing len without hex" - ); - } - - #[test] - fn parse_memcmp_both_hex_mismatch_should_fail() { - let script = r#" -trace foo { - if memcmp(hex("50"), hex("504f")) { print "OK"; } -} -"#; - let r = parse(script); - match r { - Ok(_) => panic!("expected parse error for mismatched hex sizes"), - Err(ParseError::TypeError(msg)) => { - assert!(msg.contains("different sizes"), "unexpected msg: {msg}"); - } - Err(e) => panic!("unexpected error variant: {e:?}"), - } - } - - #[test] - fn parse_format_static_len_bases_in_prints() { - // Validate that static length .N supports 0x/0o/0b in formatted prints - let script = r#" -trace foo { - print "HX={:x.0x10}", buf; - print "HS={:s.0o20}", buf; - print "HB={:X.0b1000}", buf; -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_trace_patterns_function_line_address_wildcard() { - // Function name - let s1 = r#"trace main { print "OK"; }"#; - assert!(parse(s1).is_ok()); - - // Source line with path and hyphen - let s2 = r#"trace /tmp/test-file.c:42 { print "L"; }"#; - assert!(parse(s2).is_ok()); - - // Hex address - let s3 = r#"trace 0x401234 { print "A"; }"#; - assert!(parse(s3).is_ok()); - - // Wildcard - let s4 = r#"trace printf* { print "W"; }"#; - assert!(parse(s4).is_ok()); - - // Module-qualified address - let s5 = r#"trace /lib/x86_64-linux-gnu/libc.so.6:0x1234 { print "M"; }"#; - assert!(parse(s5).is_ok()); - } - - #[test] - fn parse_identifiers_can_start_with_underscore() { - let function = r#"trace __UpdateTicketInformation { print "OK"; }"#; - assert!(parse(function).is_ok()); - - let wildcard = r#"trace __builtin_* { print "W"; }"#; - assert!(parse(wildcard).is_ok()); - - let script = r#" -trace _start { - let _ticket = __dwarf_value; - print _ticket; -} -"#; - assert!(parse(script).is_ok()); - } - - #[test] - fn parse_module_hex_address_overflow_should_error() { - // Address exceeds u64 (17 hex digits) -> parse error, not 0 fallback - let s = r#"trace libfoo.so:0x10000000000000000 { print "X"; }"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => assert!(msg.contains("too large for u64")), - other => panic!("expected friendly SyntaxError, got {other:?}"), - } - } - - #[test] - fn parse_hex_address_overflow_should_error() { - let s = r#"trace 0x10000000000000000 { print "X"; }"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => assert!(msg.contains("too large for u64")), - other => panic!("expected friendly SyntaxError, got {other:?}"), - } - } - - #[test] - fn parse_special_variables_basic() { - // $pid/$tid/$host_pid/$input_pid/$timestamp in expressions and prints - let script = r#" -trace foo { - if $pid == 123 && $tid != 0 && $host_pid != 0 && $input_pid == 123 { print "PID_TID"; } - print $timestamp; - print "P:{} T:{} HP:{} IN:{} TS:{}", $pid, $tid, $host_pid, $input_pid, $timestamp; -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_chain_and_array_access() { - // Member/chain and array tail index - let script = r#" -trace foo { - print person.name.first; - print arr[0]; - // Supported: top-level array access with trailing member - print ifaces[0].mtu; -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_pointer_and_address_of() { - let script = r#" -trace foo { - print *ptr; - print &var; - print *(arr_ptr); -} -"#; - let r = parse(script); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_nested_trace_is_rejected() { - let s = r#" -trace foo { - trace bar { print "X"; } -} -"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => assert!(msg.contains("cannot be nested")), - other => panic!("expected SyntaxError for nested trace, got {other:?}"), - } - } - - #[test] - fn parse_float_literal_is_rejected() { - let s = r#" -trace foo { - let x = 1.23; -} -"#; - let r = parse(s); - match r { - Err(ParseError::TypeError(msg)) => { - assert!(msg.contains("float literals are not supported")) - } - other => panic!("expected TypeError for float literal, got {other:?}"), - } - } - - #[test] - fn parse_unclosed_print_string_reports_friendly_error() { - let bad = r#" -trace foo { - print "Unclosed {}, value -} -"#; - let r = parse(bad); - match r { - Err(ParseError::SyntaxError(msg)) => assert!(msg.contains("Unclosed string literal")), - other => panic!("expected SyntaxError, got {other:?}"), - } - } - - #[test] - fn parse_array_index_accepts_dynamic_expr() { - // Dynamic index on top-level array - let s1 = r#" -trace foo { - print arr[i]; -} -"#; - let r1 = parse(s1).expect("dynamic top-level index should parse"); - match r1.statements.first().expect("trace") { - Statement::TracePoint { body, .. } => match &body[0] { - Statement::Print(PrintStatement::ComplexVariable(Expr::ArrayAccess(_, index))) => { - assert!(matches!(index.as_ref(), Expr::Variable(name) if name == "i")) - } - other => panic!("unexpected first print body: {other:?}"), - }, - other => panic!("expected TracePoint, got {other:?}"), - } - - // Dynamic index at chain tail - let s2 = r#" -trace foo { - print obj.arr[i - (i / 0x8) * 0x8]; -} -"#; - let r2 = parse(s2).expect("dynamic chain index should parse"); - match r2.statements.first().expect("trace") { - Statement::TracePoint { body, .. } => match &body[0] { - Statement::Print(PrintStatement::ComplexVariable(Expr::ArrayAccess(_, index))) => { - assert!(matches!(index.as_ref(), Expr::BinaryOp { .. })) - } - other => panic!("unexpected first print body: {other:?}"), - }, - other => panic!("expected TracePoint, got {other:?}"), - } - } - - #[test] - fn parse_integer_modulo_and_bitwise_ops() { - let script = r#" -trace foo { - let value = 0x1 | 0x2 ^ 0x3 & 0x4 << 0x1 + 0x2 % 0x3; - let inverse = ~value; -} -"#; - let prog = parse(script).expect("integer and bitwise ops should parse"); - let Statement::TracePoint { body, .. } = prog.statements.first().expect("trace") else { - panic!("expected trace point"); - }; - let Statement::VarDeclaration { value, .. } = &body[0] else { - panic!("expected var declaration"); - }; - let Expr::BinaryOp { op, left, right } = value else { - panic!("expected bitwise-or root"); - }; - assert_eq!(*op, BinaryOp::BitOr); - assert!(matches!(left.as_ref(), Expr::Int(1))); - assert!(matches!( - right.as_ref(), - Expr::BinaryOp { - op: BinaryOp::BitXor, - .. - } - )); - assert!(matches!( - &body[1], - Statement::VarDeclaration { - value: Expr::UnaryBitNot(_), - .. - } - )); - } - - #[test] - fn parse_array_index_accepts_constant_negative_literal() { - let script = r#" -trace foo { - print arr[-0x1]; - print obj.arr[0b10 - 0x3]; -} -"#; - let prog = parse(script).expect("parse ok"); - let trace = prog.statements.first().expect("trace"); - match trace { - Statement::TracePoint { body, .. } => { - match &body[0] { - Statement::Print(PrintStatement::ComplexVariable(Expr::ArrayAccess( - _, - index, - ))) => { - assert!(matches!(index.as_ref(), Expr::Int(-1))); - } - other => panic!("unexpected first print body: {other:?}"), - } - match &body[1] { - Statement::Print(PrintStatement::ComplexVariable(Expr::ArrayAccess( - _, - index, - ))) => { - assert!(matches!(index.as_ref(), Expr::Int(-1))); - } - other => panic!("unexpected second print body: {other:?}"), - } - } - other => panic!("expected TracePoint, got {other:?}"), - } - } - - #[test] - fn parse_print_format_arg_mismatch_reports_error() { - // format_expr form - let s1 = r#" -trace foo { - print "A {} {}", x; -} -"#; - let r1 = parse(s1); - match r1 { - Err(ParseError::TypeError(msg)) => { - assert!(msg.contains("expects 2 argument(s)"), "unexpected: {msg}"); - } - other => panic!("expected TypeError from format arg mismatch, got {other:?}"), - } - - // flattened string + args form - let s2 = r#" -trace foo { - print "B {} {}", y; -} -"#; - let r2 = parse(s2); - match r2 { - Err(ParseError::TypeError(msg)) => { - assert!(msg.contains("expects 2 argument(s)")); - } - other => panic!("expected TypeError from format arg mismatch, got {other:?}"), - } - } - - #[test] - fn parse_print_invalid_format_specifier_errors() { - // Missing ':' prefix inside { } - let s1 = r#" -trace foo { print "Bad {x}", 1; } -"#; - let r1 = parse(s1); - match r1 { - Err(ParseError::TypeError(msg)) => { - assert!(msg.contains("Invalid format specifier"), "{msg}"); - } - other => panic!("expected TypeError, got {other:?}"), - } - - // Unsupported conversion {:q} - let s2 = r#" -trace foo { print "Bad {:q}", 1; } -"#; - let r2 = parse(s2); - match r2 { - Err(ParseError::TypeError(msg)) => { - assert!(msg.contains("Unsupported format conversion"), "{msg}"); - } - other => panic!("expected TypeError, got {other:?}"), - } - } - - #[test] - fn parse_hex_with_tab_is_rejected() { - let s = r#" -trace foo { - if memcmp(&buf[0], hex("50\t4f"), 2) { print "X"; } -} -"#; - let r = parse(s); - match r { - Err(ParseError::TypeError(msg)) => { - assert!(msg.contains("non-hex character"), "{msg}"); - } - other => panic!("expected TypeError for tab in hex literal, got {other:?}"), - } - } - - #[test] - fn parse_starts_with_constant_folds_on_literals() { - let s = r#" -trace foo { - if starts_with("abcdef", "abc") { print "T"; } else { print "F"; } -} -"#; - let prog = parse(s).expect("parse ok"); - let stmt0 = prog.statements.first().expect("trace"); - match stmt0 { - Statement::TracePoint { body, .. } => match &body[0] { - Statement::If { condition, .. } => { - assert!(matches!(condition, Expr::Bool(true))); - } - other => panic!("expected If, got {other:?}"), - }, - other => panic!("expected TracePoint, got {other:?}"), - } - } - - #[test] - fn parse_backtrace_and_bt_statements() { - let s = r#" - trace foo { - backtrace; - bt; - bt raw; - bt full noinline; - } - "#; - let program = parse(s).expect("parse ok"); - let Statement::TracePoint { body, .. } = &program.statements[0] else { - panic!("expected trace"); - }; - assert_eq!(body.len(), 4); - match &body[2] { - Statement::Backtrace(bt) => { - assert!(bt.raw); - assert!(bt.inline); - } - other => panic!("expected backtrace, got {other:?}"), - } - match &body[3] { - Statement::Backtrace(bt) => { - assert!(bt.full); - assert!(!bt.inline); - } - other => panic!("expected backtrace, got {other:?}"), - } - } - - #[test] - fn parse_backtrace_rejects_named_depth_option() { - let s = r#" - trace foo { - bt depth=8; - } - "#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!(msg.contains("no longer a script option"), "{msg}"); - assert!(msg.contains("--backtrace-depth"), "{msg}"); - } - other => panic!("expected SyntaxError, got {other:?}"), - } - } - - #[test] - fn parse_backtrace_rejects_positional_depth_option() { - let s = r#" - trace foo { - bt 4 raw; - } - "#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!(msg.contains("no longer a script option"), "{msg}"); - } - other => panic!("expected SyntaxError, got {other:?}"), - } - } - - #[test] - fn parse_print_capture_len_suffix() { - // {:s.name$} uses capture; does not consume extra arg - let s = r#" -trace foo { - let n = 3; - print "tail={:s.n$}", p; -} -"#; - let r = parse(s); - assert!(r.is_ok(), "parse failed: {:?}", r.err()); - } - - #[test] - fn parse_unknown_keyword_inside_trace_suggests_print() { - let s = r#" -trace foo { - pront "hello"; -} -"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!( - msg.contains("Unknown keyword 'pront'"), - "unexpected msg: {msg}" - ); - assert!( - msg.contains("Did you mean 'print'"), - "no suggestion in msg: {msg}" - ); - } - other => panic!("expected friendly SyntaxError for unknown keyword, got {other:?}"), - } - } - - #[test] - fn parse_unknown_keyword_same_line_after_brace_suggests_print() { - // Unknown keyword immediately after '{' on the same line - let s = r#"trace foo {pirnt \"sa\";}"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!( - msg.contains("Unknown keyword 'pirnt'"), - "unexpected msg: {msg}" - ); - assert!( - msg.contains("Did you mean 'print'"), - "no suggestion in msg: {msg}" - ); - } - other => { - panic!("expected friendly SyntaxError for same-line unknown keyword, got {other:?}") - } - } - } - - #[test] - fn parse_unknown_top_level_keyword_suggests_trace() { - let s = r#" -traec bar { - print "x"; -} -"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!( - msg.contains("Unknown keyword 'traec'"), - "unexpected msg: {msg}" - ); - assert!( - msg.contains("Did you mean 'trace'"), - "no suggestion in msg: {msg}" - ); - } - other => panic!("expected friendly SyntaxError for unknown keyword, got {other:?}"), - } - } - - #[test] - fn parse_builtin_then_misspelled_keyword_should_point_to_misspell() { - // Ensure builtin calls are not flagged; the real typo should be reported - let s = r#" -trace foo { - starts_with("a", "b"); prnit "oops"; -} -"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!( - msg.contains("prnit"), - "should point to misspelled 'prnit': {msg}" - ); - assert!( - !msg.contains("starts_with"), - "should not flag builtin call: {msg}" - ); - } - other => panic!("expected friendly SyntaxError for misspelled print, got {other:?}"), - } - } - - #[test] - fn parse_misspelled_builtin_suggests_starts_with() { - // Misspelled builtin should suggest the correct builtin name - let s = r#" -trace foo { - starst_with("a", "b"); -} -"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!(msg.contains("Unknown keyword 'starst_with'"), "{msg}"); - assert!(msg.contains("Did you mean 'starts_with'"), "{msg}"); - } - other => panic!("expected friendly suggestion for misspelled builtin, got {other:?}"), - } - } - - #[test] - fn parse_misspelled_builtin_suggests_memcmp() { - let s = r#" -trace foo { - memcpm(&buf[0], &buf[1], 16); -} -"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!(msg.contains("Unknown keyword 'memcpm'"), "{msg}"); - assert!(msg.contains("Did you mean 'memcmp'"), "{msg}"); - } - other => panic!("expected friendly suggestion for misspelled builtin, got {other:?}"), - } - } - - #[test] - fn parse_if_condition_misspelled_builtin_suggests() { - let s = r#" -trace foo { - if starst_with("a", "b") { print "ok"; } -} -"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!(msg.contains("Unknown keyword 'starst_with'"), "{msg}"); - assert!(msg.contains("Did you mean 'starts_with'"), "{msg}"); - } - other => panic!("expected friendly suggestion inside if(), got {other:?}"), - } - } - - #[test] - fn parse_else_if_condition_misspelled_builtin_suggests() { - let s = r#" -trace foo { - if 1 { print "a"; } else if starst_with("a", "b") { print "b"; } -} -"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!(msg.contains("Unknown keyword 'starst_with'"), "{msg}"); - assert!(msg.contains("Did you mean 'starts_with'"), "{msg}"); - } - other => panic!("expected friendly suggestion inside else if(), got {other:?}"), - } - } - #[test] - fn parse_unknown_keyword_generic_expected_list() { - let s = r#" -foobarbaz { - print "x"; -} -"#; - let r = parse(s); - match r { - Err(ParseError::SyntaxError(msg)) => { - assert!( - msg.contains("Unknown keyword 'foobarbaz'"), - "unexpected msg: {msg}" - ); - assert!( - msg.contains("Expected one of"), - "missing expected list in msg: {msg}" - ); - } - other => panic!("expected friendly SyntaxError with expected list, got {other:?}"), - } - } -} +mod tests; diff --git a/ghostscope-compiler/src/script/parser/diagnostics.rs b/ghostscope-compiler/src/script/parser/diagnostics.rs new file mode 100644 index 00000000..ec812569 --- /dev/null +++ b/ghostscope-compiler/src/script/parser/diagnostics.rs @@ -0,0 +1,214 @@ +// Best-effort heuristic: if a line contains a print statement with an opening quote +// but no closing quote before arguments, give a clearer error. +pub(super) fn detect_unclosed_print_string(input: &str) -> Option { + for (i, raw_line) in input.lines().enumerate() { + let line = raw_line.trim_start(); + if !line.contains("print ") && !line.starts_with("print") { + continue; + } + // Toggle on '"' to detect unclosed string; ignore escaped quotes for simplicity + let mut open = false; + for ch in line.chars() { + if ch == '"' { + open = !open; + } + } + if open { + // Common case: missing closing quote before comma and arguments + if line.contains(',') { + return Some(format!( + "Unclosed string literal in print at line {}. Did you forget a closing \"\" before ',' and arguments?", + i + 1 + )); + } else { + return Some(format!( + "Unclosed string literal in print at line {}.", + i + 1 + )); + } + } + } + None +} + +pub(super) fn detect_backtrace_depth_argument(input: &str) -> Option { + fn boundary_before(line: &str, idx: usize) -> bool { + idx == 0 + || line[..idx] + .chars() + .next_back() + .is_some_and(|ch| ch.is_whitespace() || matches!(ch, '{' | ';' | '}')) + } + + for (line_idx, raw_line) in input.lines().enumerate() { + let line = raw_line.split("//").next().unwrap_or(raw_line); + for command in ["bt", "backtrace"] { + for (idx, _) in line.match_indices(command) { + if !boundary_before(line, idx) { + continue; + } + let after = &line[idx + command.len()..]; + if !after.starts_with(char::is_whitespace) { + continue; + } + let arg = after.trim_start(); + if arg.starts_with("depth") + || arg.chars().next().is_some_and(|ch| ch.is_ascii_digit()) + { + return Some(format!( + "bt depth is no longer a script option at line {}. Set the global limit with --backtrace-depth or [ebpf] backtrace_depth = N.", + line_idx + 1 + )); + } + } + } + } + None +} + +// Try to detect lines that start with an unknown/misspelled keyword and suggest known ones. +pub(super) fn detect_unknown_keyword(input: &str) -> Option { + // Suggest only currently supported top-level keywords. + const SUGGEST: &[&str] = &["trace", "print", "if", "else", "let"]; + // Valid statement starters that should not be flagged as unknown + const SUPPORTED_HEADS: &[&str] = &["trace", "print", "if", "else", "let", "backtrace", "bt"]; + // Builtin call names allowed at expression head + const BUILTIN_CALLS: &[&str] = &["memcmp", "strncmp", "starts_with", "hex", "cast"]; + + // Helper: simple Levenshtein distance (small strings, few keywords) + fn levenshtein(a: &str, b: &str) -> usize { + let (n, m) = (a.len(), b.len()); + let mut dp = vec![0usize; (n + 1) * (m + 1)]; + let idx = |i: usize, j: usize| i * (m + 1) + j; + for i in 0..=n { + dp[idx(i, 0)] = i; + } + for j in 0..=m { + dp[idx(0, j)] = j; + } + let ac: Vec = a.chars().collect(); + let bc: Vec = b.chars().collect(); + for i in 1..=n { + for j in 1..=m { + let cost = if ac[i - 1] == bc[j - 1] { 0 } else { 1 }; + let del = dp[idx(i - 1, j)] + 1; + let ins = dp[idx(i, j - 1)] + 1; + let sub = dp[idx(i - 1, j - 1)] + cost; + dp[idx(i, j)] = del.min(ins).min(sub); + } + } + dp[idx(n, m)] + } + + // Helper: check a slice for a command-like unknown keyword + fn check_slice(slice: &str, line_no_1based: usize) -> Option { + let mut s = slice.trim_start(); + if s.is_empty() || s.starts_with("//") { + return None; + } + + // If this slice begins with an if/else-if header, jump inside the condition + if let Some(rest) = s.strip_prefix("if") { + if rest.starts_with(char::is_whitespace) { + s = rest.trim_start(); + } + } else if let Some(rest) = s.strip_prefix("else") { + let rest = rest.trim_start(); + if let Some(rest2) = rest.strip_prefix("if") { + if rest2.starts_with(char::is_whitespace) { + s = rest2.trim_start(); + } + } else { + // 'else { ... }' — nothing to inspect here + } + } + // Keywords must start with a letter or underscore; skip numeric heads + let mut iter = s.chars(); + let first = iter.next()?; + if !(first.is_ascii_alphabetic() || first == '_') { + return None; + } + let mut token = String::new(); + token.push(first); + for ch in iter { + if ch.is_ascii_alphanumeric() || ch == '_' { + token.push(ch); + } else { + break; + } + } + if token.is_empty() { + return None; + } + if SUPPORTED_HEADS.iter().any(|k| *k == token) { + return None; + } + let rest_untrimmed = &s[token.len()..]; + let rest = rest_untrimmed.trim_start(); + if rest.starts_with('=') || rest.starts_with('[') || rest.starts_with('.') { + // likely an expression starting with identifier + return None; + } + // Allow builtin calls as expression statements + if BUILTIN_CALLS.iter().any(|k| *k == token) && rest.starts_with('(') { + return None; + } + if rest.starts_with('(') + || rest.starts_with('{') + || rest.starts_with('"') + || rest_untrimmed.starts_with(char::is_whitespace) + { + // If it looks like a call (token + '('), include builtin calls in suggestion candidates + let candidates: Vec<&str> = if rest.starts_with('(') { + let mut v = Vec::new(); + v.extend_from_slice(SUGGEST); + v.extend_from_slice(BUILTIN_CALLS); + v + } else { + SUGGEST.to_vec() + }; + let mut suggestions: Vec<(&str, usize)> = candidates + .iter() + .map(|&k| (k, levenshtein(&token, k))) + .collect(); + suggestions.sort_by_key(|&(_, d)| d); + if let Some((cand, dist)) = suggestions.first().copied() { + if dist <= 2 { + return Some(format!( + "Unknown keyword '{token}' at line {line_no_1based}. Did you mean '{cand}'?" + )); + } + } + return Some(format!( + "Unknown keyword '{token}' at line {}. Expected one of: {}", + line_no_1based, + SUGGEST.join(", ") + )); + } + None + } + + for (i, raw_line) in input.lines().enumerate() { + let line = raw_line; + // Scan potential statement starts: at line start, and right after '{', ';', '}', '(', ',' (outside strings) + let mut quote_open = false; + let mut positions: Vec = vec![0]; // include start-of-line + for (idx, ch) in line.char_indices() { + if ch == '"' { + quote_open = !quote_open; + } + if !quote_open && (ch == '{' || ch == ';' || ch == '}' || ch == '(' || ch == ',') { + let next = idx + ch.len_utf8(); + if next < line.len() { + positions.push(next); + } + } + } + for &pos in &positions { + if let Some(msg) = check_slice(&line[pos..], i + 1) { + return Some(msg); + } + } + } + None +} diff --git a/ghostscope-compiler/src/script/parser/expr.rs b/ghostscope-compiler/src/script/parser/expr.rs new file mode 100644 index 00000000..57581acd --- /dev/null +++ b/ghostscope-compiler/src/script/parser/expr.rs @@ -0,0 +1,1070 @@ +use pest::iterators::Pair; + +use crate::script::ast::{infer_type, BinaryOp, Expr}; +use tracing::debug; + +use super::{chunks_of_two, ParseError, Result, Rule}; + +pub(super) fn parse_expr(pair: Pair) -> Result { + match pair.as_rule() { + Rule::expr => { + let inner = pair + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + parse_logical_or(inner) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +/// Determine if an expression should be treated as a DWARF alias binding. +/// This is a purely syntactic check (parser phase) and does not consult DWARF. +fn integer_literal_value(e: &Expr) -> Option { + use crate::script::ast::BinaryOp as BO; + use crate::script::ast::Expr as E; + + match e { + E::Int(value) => Some(*value), + E::BinaryOp { + left, + op: BO::Add, + right, + } => integer_literal_value(left)?.checked_add(integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::Subtract, + right, + } => integer_literal_value(left)?.checked_sub(integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::Multiply, + right, + } => integer_literal_value(left)?.checked_mul(integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::Divide, + right, + } => integer_literal_value(left)?.checked_div(integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::Modulo, + right, + } => integer_literal_value(left)?.checked_rem(integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::BitAnd, + right, + } => Some(integer_literal_value(left)? & integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::BitXor, + right, + } => Some(integer_literal_value(left)? ^ integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::BitOr, + right, + } => Some(integer_literal_value(left)? | integer_literal_value(right)?), + E::BinaryOp { + left, + op: BO::ShiftLeft, + right, + } => { + let shift = u32::try_from(integer_literal_value(right)?).ok()?; + integer_literal_value(left)?.checked_shl(shift) + } + E::BinaryOp { + left, + op: BO::ShiftRight, + right, + } => { + let shift = u32::try_from(integer_literal_value(right)?).ok()?; + integer_literal_value(left)?.checked_shr(shift) + } + E::UnaryBitNot(inner) => Some(!integer_literal_value(inner)?), + _ => None, + } +} +pub(super) fn is_alias_expr(e: &Expr) -> bool { + use crate::script::ast::BinaryOp as BO; + use crate::script::ast::Expr as E; + match e { + E::AddressOf(_) => true, + // Constant offset on top of an alias-eligible expression + E::BinaryOp { + left, + op: BO::Add, + right, + } => { + (is_alias_expr(left) && integer_literal_value(right).is_some()) + || (is_alias_expr(right) && integer_literal_value(left).is_some()) + } + _ => false, + } +} +fn parse_logical_or(pair: Pair) -> Result { + match pair.as_rule() { + Rule::logical_or => { + let mut pairs = pair.into_inner(); + let first = pairs.next().ok_or(ParseError::InvalidExpression)?; + let mut left = parse_logical_and(first)?; + + for chunk in chunks_of_two(pairs) { + if chunk.len() != 2 { + return Err(ParseError::InvalidExpression); + } + if chunk[0].as_rule() != Rule::or_op { + return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); + } + let right = parse_logical_and(chunk[1].clone())?; + let expr = Expr::BinaryOp { + left: Box::new(left), + op: BinaryOp::LogicalOr, + right: Box::new(right), + }; + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + left = expr; + } + Ok(left) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_logical_and(pair: Pair) -> Result { + match pair.as_rule() { + Rule::logical_and => { + let mut pairs = pair.into_inner(); + let first = pairs.next().ok_or(ParseError::InvalidExpression)?; + let mut left = parse_bitwise_or(first)?; + + for chunk in chunks_of_two(pairs) { + if chunk.len() != 2 { + return Err(ParseError::InvalidExpression); + } + if chunk[0].as_rule() != Rule::and_op { + return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); + } + let right = parse_bitwise_or(chunk[1].clone())?; + let expr = Expr::BinaryOp { + left: Box::new(left), + op: BinaryOp::LogicalAnd, + right: Box::new(right), + }; + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + left = expr; + } + Ok(left) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_bitwise_or(pair: Pair) -> Result { + match pair.as_rule() { + Rule::bitwise_or => { + let mut pairs = pair.into_inner(); + let first = pairs.next().ok_or(ParseError::InvalidExpression)?; + let mut left = parse_bitwise_xor(first)?; + + for chunk in chunks_of_two(pairs) { + if chunk.len() != 2 { + return Err(ParseError::InvalidExpression); + } + if chunk[0].as_rule() != Rule::bit_or_op { + return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); + } + let right = parse_bitwise_xor(chunk[1].clone())?; + let expr = Expr::BinaryOp { + left: Box::new(left), + op: BinaryOp::BitOr, + right: Box::new(right), + }; + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + left = expr; + } + Ok(left) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_bitwise_xor(pair: Pair) -> Result { + match pair.as_rule() { + Rule::bitwise_xor => { + let mut pairs = pair.into_inner(); + let first = pairs.next().ok_or(ParseError::InvalidExpression)?; + let mut left = parse_bitwise_and(first)?; + + for chunk in chunks_of_two(pairs) { + if chunk.len() != 2 { + return Err(ParseError::InvalidExpression); + } + if chunk[0].as_rule() != Rule::bit_xor_op { + return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); + } + let right = parse_bitwise_and(chunk[1].clone())?; + let expr = Expr::BinaryOp { + left: Box::new(left), + op: BinaryOp::BitXor, + right: Box::new(right), + }; + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + left = expr; + } + Ok(left) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_bitwise_and(pair: Pair) -> Result { + match pair.as_rule() { + Rule::bitwise_and => { + let mut pairs = pair.into_inner(); + let first = pairs.next().ok_or(ParseError::InvalidExpression)?; + let mut left = parse_equality(first)?; + + for chunk in chunks_of_two(pairs) { + if chunk.len() != 2 { + return Err(ParseError::InvalidExpression); + } + if chunk[0].as_rule() != Rule::bit_and_op { + return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); + } + let right = parse_equality(chunk[1].clone())?; + let expr = Expr::BinaryOp { + left: Box::new(left), + op: BinaryOp::BitAnd, + right: Box::new(right), + }; + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + left = expr; + } + Ok(left) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_equality(pair: Pair) -> Result { + match pair.as_rule() { + Rule::equality => { + let mut pairs = pair.into_inner(); + let first = pairs.next().ok_or(ParseError::InvalidExpression)?; + let mut left = parse_relational(first)?; + + for chunk in chunks_of_two(pairs) { + if chunk.len() != 2 { + return Err(ParseError::InvalidExpression); + } + if chunk[0].as_rule() != Rule::eq_op { + return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); + } + let op = match chunk[0].as_str() { + "==" => BinaryOp::Equal, + "!=" => BinaryOp::NotEqual, + _ => return Err(ParseError::UnexpectedToken(chunk[0].as_rule())), + }; + let right = parse_relational(chunk[1].clone())?; + let expr = Expr::BinaryOp { + left: Box::new(left), + op, + right: Box::new(right), + }; + // Type check literals only + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + left = expr; + } + Ok(left) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_relational(pair: Pair) -> Result { + match pair.as_rule() { + Rule::relational => { + let mut pairs = pair.into_inner(); + let first = pairs.next().ok_or(ParseError::InvalidExpression)?; + let mut left = parse_shift(first)?; + + for chunk in chunks_of_two(pairs) { + if chunk.len() != 2 { + return Err(ParseError::InvalidExpression); + } + if chunk[0].as_rule() != Rule::rel_op { + return Err(ParseError::UnexpectedToken(chunk[0].as_rule())); + } + let op = match chunk[0].as_str() { + "<" => BinaryOp::LessThan, + "<=" => BinaryOp::LessEqual, + ">" => BinaryOp::GreaterThan, + ">=" => BinaryOp::GreaterEqual, + _ => return Err(ParseError::UnexpectedToken(chunk[0].as_rule())), + }; + let right = parse_shift(chunk[1].clone())?; + let expr = Expr::BinaryOp { + left: Box::new(left), + op, + right: Box::new(right), + }; + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + left = expr; + } + Ok(left) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_shift(pair: Pair) -> Result { + match pair.as_rule() { + Rule::shift => { + let mut pairs = pair.into_inner(); + let first = pairs.next().ok_or(ParseError::InvalidExpression)?; + let mut left = parse_additive(first)?; + + for chunk in chunks_of_two(pairs) { + if chunk.len() != 2 { + return Err(ParseError::InvalidExpression); + } + let op = match chunk[0].as_str() { + "<<" => BinaryOp::ShiftLeft, + ">>" => BinaryOp::ShiftRight, + _ => return Err(ParseError::UnexpectedToken(chunk[0].as_rule())), + }; + let right = parse_additive(chunk[1].clone())?; + let expr = Expr::BinaryOp { + left: Box::new(left), + op, + right: Box::new(right), + }; + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + left = expr; + } + Ok(left) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_additive(pair: Pair) -> Result { + match pair.as_rule() { + Rule::additive => { + let mut pairs = pair.into_inner(); + let first = pairs.next().ok_or(ParseError::InvalidExpression)?; + let mut left = parse_term(first)?; + + for chunk in chunks_of_two(pairs) { + if chunk.len() != 2 { + return Err(ParseError::InvalidExpression); + } + let op = match chunk[0].as_str() { + "+" => BinaryOp::Add, + "-" => BinaryOp::Subtract, + _ => return Err(ParseError::UnexpectedToken(chunk[0].as_rule())), + }; + let right = parse_term(chunk[1].clone())?; + let expr = Expr::BinaryOp { + left: Box::new(left), + op, + right: Box::new(right), + }; + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + left = expr; + } + Ok(left) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +pub(super) fn parse_condition(pair: Pair) -> Result { + debug!( + "parse_condition: {:?} = '{}'", + pair.as_rule(), + pair.as_str().trim() + ); + match pair.as_rule() { + Rule::condition => { + // Condition now accepts a full expression (equality/relational/additive/etc.) + let inner_expr_pair = pair + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + let expr = parse_expr(inner_expr_pair)?; + // Basic type check of the resulting expression + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + Ok(expr) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_term(pair: Pair) -> Result { + match pair.as_rule() { + Rule::term => { + let mut pairs = pair.into_inner(); + let first = pairs.next().ok_or(ParseError::InvalidExpression)?; + let mut left = parse_unary(first)?; + + for chunk in chunks_of_two(pairs) { + if chunk.len() != 2 { + return Err(ParseError::InvalidExpression); + } + + let op = match chunk[0].as_str() { + "*" => BinaryOp::Multiply, + "/" => BinaryOp::Divide, + "%" => BinaryOp::Modulo, + _ => return Err(ParseError::UnexpectedToken(chunk[0].as_rule())), + }; + + let right = parse_unary(chunk[1].clone())?; + + // Check type consistency for binary operations + let expr = Expr::BinaryOp { + left: Box::new(left), + op, + right: Box::new(right), + }; + + // Only check type consistency for literals here + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + + left = expr; + } + + Ok(left) + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_unary(pair: Pair) -> Result { + match pair.as_rule() { + Rule::unary => { + let mut inner = pair.into_inner(); + let first = inner.next().ok_or(ParseError::InvalidExpression)?; + match first.as_rule() { + Rule::factor => parse_factor(first), + // '-' ~ unary + Rule::neg_unary => { + let u = first + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + let right = parse_unary(u)?; + let expr = Expr::BinaryOp { + left: Box::new(Expr::Int(0)), + op: BinaryOp::Subtract, + right: Box::new(right), + }; + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + Ok(expr) + } + // '!' ~ unary + Rule::not_unary => { + let u = first + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + let right = parse_unary(u)?; + Ok(Expr::UnaryNot(Box::new(right))) + } + Rule::bit_not_unary => { + let u = first + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + let right = parse_unary(u)?; + let expr = Expr::UnaryBitNot(Box::new(right)); + if let Err(err) = infer_type(&expr) { + return Err(ParseError::TypeError(err)); + } + Ok(expr) + } + _ => Err(ParseError::UnexpectedToken(first.as_rule())), + } + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_factor(pair: Pair) -> Result { + match pair.as_rule() { + Rule::factor => { + let inner = pair + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + match inner.as_rule() { + Rule::memcmp_call => parse_builtin_call(inner), + Rule::strncmp_call => parse_builtin_call(inner), + Rule::starts_with_call => parse_builtin_call(inner), + Rule::hex_call => parse_builtin_call(inner), + Rule::postfix_access => parse_postfix_access(inner), + Rule::cast_call => parse_cast_call(inner), + Rule::chain_access => parse_chain_access(inner), + Rule::pointer_deref => parse_pointer_deref(inner), + Rule::address_of => parse_address_of(inner), + Rule::int => match inner.as_str().parse::() { + Ok(value) => Ok(Expr::Int(value)), + Err(_) => Err(ParseError::TypeError( + "invalid decimal integer literal".to_string(), + )), + }, + Rule::hex_int => { + // strip 0x and parse as hex + let s = inner.as_str(); + match i64::from_str_radix(&s[2..], 16) { + Ok(v) => Ok(Expr::Int(v)), + Err(_) => Err(ParseError::TypeError( + "invalid hex integer literal".to_string(), + )), + } + } + Rule::oct_int => { + let s = inner.as_str(); + match i64::from_str_radix(&s[2..], 8) { + Ok(v) => Ok(Expr::Int(v)), + Err(_) => Err(ParseError::TypeError( + "invalid octal integer literal".to_string(), + )), + } + } + Rule::bin_int => { + let s = inner.as_str(); + match i64::from_str_radix(&s[2..], 2) { + Ok(v) => Ok(Expr::Int(v)), + Err(_) => Err(ParseError::TypeError( + "invalid binary integer literal".to_string(), + )), + } + } + // Floats are not supported by scripts/runtime; reject early with friendly error + Rule::float => Err(ParseError::TypeError( + "float literals are not supported".to_string(), + )), + Rule::string => { + // Remove quotes at the beginning and end + let raw_value = inner.as_str(); + let value = &raw_value[1..raw_value.len() - 1]; + Ok(Expr::String(value.to_string())) + } + Rule::bool => { + let val = inner.as_str() == "true"; + Ok(Expr::Bool(val)) + } + Rule::identifier => { + let name = inner.as_str().to_string(); + Ok(Expr::Variable(name)) + } + Rule::array_access => parse_array_access(inner), + Rule::member_access => parse_member_access(inner), + Rule::special_var => { + let var_name = inner.as_str().to_string(); + Ok(Expr::SpecialVar(var_name)) + } + Rule::expr => parse_expr(inner), + _ => Err(ParseError::UnexpectedToken(inner.as_rule())), + } + } + _ => Err(ParseError::UnexpectedToken(pair.as_rule())), + } +} +fn parse_builtin_call(pair: Pair) -> Result { + // pair is memcmp_call / strncmp_call / starts_with_call / hex_call + let rule = pair.as_rule(); + let mut it = pair.into_inner(); + // First token inside is the function name as identifier within the rule text; easier approach: use rule to select + match rule { + Rule::memcmp_call => { + // grammar: memcmp("(" expr "," expr ["," expr] ")") + let mut nodes: Vec<_> = it.collect(); + if nodes.len() < 2 || nodes.len() > 3 { + return Err(ParseError::InvalidExpression); + } + let a_expr = parse_expr(nodes.remove(0))?; + let b_expr = parse_expr(nodes.remove(0))?; + + // Disallow obviously invalid types early + if matches!(a_expr, Expr::Bool(_)) || matches!(b_expr, Expr::Bool(_)) { + return Err(ParseError::TypeError( + "memcmp pointer arguments cannot be boolean; use an address or hex(...)" + .to_string(), + )); + } + if matches!(a_expr, Expr::String(_)) || matches!(b_expr, Expr::String(_)) { + return Err(ParseError::TypeError( + "memcmp does not accept string literals; use strncmp for strings".to_string(), + )); + } + + // Helper to get hex length (bytes) + let hex_len = |e: &Expr| -> Option { + if let Expr::BuiltinCall { name, args } = e { + if name == "hex" { + if let Some(Expr::String(s)) = args.first() { + return Some(s.len() / 2); + } + } + } + None + }; + + let n_expr = if let Some(n_node) = nodes.first() { + // With explicit len: reuse previous literal checks + let n_expr = parse_expr(n_node.clone())?; + if matches!(n_expr, Expr::Bool(_)) { + return Err(ParseError::TypeError( + "memcmp length must be an integer or expression, not boolean".to_string(), + )); + } + let literal_len_opt: Option = match &n_expr { + Expr::Int(n) => Some(*n as isize), + Expr::BinaryOp { + left, + op: BinaryOp::Subtract, + right, + } => { + if matches!(left.as_ref(), Expr::Int(0)) { + if let Expr::Int(k) = right.as_ref() { + Some(-(*k as isize)) + } else { + None + } + } else { + None + } + } + _ => None, + }; + if let Some(n) = literal_len_opt { + if n < 0 { + return Err(ParseError::TypeError( + "memcmp length must be non-negative".to_string(), + )); + } + let l = n as usize; + if let Some(la) = hex_len(&a_expr) { + if l > la { + return Err(ParseError::TypeError(format!( + "memcmp length ({l}) exceeds hex pattern size on left side ({la} bytes)" + ))); + } + } + if let Some(lb) = hex_len(&b_expr) { + if l > lb { + return Err(ParseError::TypeError(format!( + "memcmp length ({l}) exceeds hex pattern size on right side ({lb} bytes)" + ))); + } + } + } + n_expr + } else { + // No len provided: allow only when at least one side is hex(...) + let la = hex_len(&a_expr); + let lb = hex_len(&b_expr); + match (la, lb) { + (Some(l), None) | (None, Some(l)) => Expr::Int(l as i64), + (Some(la), Some(lb)) => { + if la != lb { + return Err(ParseError::TypeError( + "memcmp hex operands have different sizes; provide explicit len" + .to_string(), + )); + } + Expr::Int(la as i64) + } + _ => { + return Err(ParseError::TypeError( + "memcmp without len requires at least one hex(...) operand".to_string(), + )) + } + } + }; + + // Constant folding: memcmp(hex(...), hex(...), N) + let as_hex = |e: &Expr| -> Option { + if let Expr::BuiltinCall { name, args } = e { + if name == "hex" { + if let Some(Expr::String(s)) = args.first() { + return Some(s.clone()); + } + } + } + None + }; + + if let (Some(h1), Some(h2), Expr::Int(n)) = (as_hex(&a_expr), as_hex(&b_expr), &n_expr) + { + // Safe hex -> bytes (sanitized earlier to hex digits only) + fn hex_to_bytes(s: &str) -> std::result::Result, ParseError> { + let mut out = Vec::with_capacity(s.len() / 2); + let bytes = s.as_bytes(); + let mut i = 0; + while i + 1 < bytes.len() { + let h = bytes[i] as char; + let l = bytes[i + 1] as char; + let hv = h + .to_digit(16) + .ok_or_else(|| ParseError::TypeError("invalid hex digit".to_string()))? + as u8; + let lv = l + .to_digit(16) + .ok_or_else(|| ParseError::TypeError("invalid hex digit".to_string()))? + as u8; + out.push((hv << 4) | lv); + i += 2; + } + Ok(out) + } + + let v1 = hex_to_bytes(&h1)?; + let v2 = hex_to_bytes(&h2)?; + let ln = (*n).max(0) as usize; + let eq = v1.iter().take(ln).eq(v2.iter().take(ln)); + return Ok(Expr::Bool(eq)); + } + + Ok(Expr::BuiltinCall { + name: "memcmp".to_string(), + args: vec![a_expr, b_expr, n_expr], + }) + } + Rule::strncmp_call => { + // grammar: strncmp("(" expr "," expr "," expr ")") + let arg0 = parse_expr(it.next().ok_or(ParseError::InvalidExpression)?)?; + let arg1 = parse_expr(it.next().ok_or(ParseError::InvalidExpression)?)?; + let n_expr_parsed = parse_expr(it.next().ok_or(ParseError::InvalidExpression)?)?; + let literal_len_opt: Option = match &n_expr_parsed { + Expr::Int(n) => Some(*n as isize), + Expr::BinaryOp { + left, + op: BinaryOp::Subtract, + right, + } => { + if matches!(left.as_ref(), Expr::Int(0)) { + if let Expr::Int(k) = right.as_ref() { + Some(-(*k as isize)) + } else { + None + } + } else { + None + } + } + _ => None, + }; + if literal_len_opt.is_some_and(|n| n < 0) { + return Err(ParseError::TypeError( + "strncmp third argument must be non-negative".to_string(), + )); + } + // Optional constant fold when both sides are string literals + if let (Expr::String(a), Expr::String(b), Expr::Int(n_val)) = + (&arg0, &arg1, &n_expr_parsed) + { + let ln = (*n_val).max(0) as usize; + let eq = a + .as_bytes() + .iter() + .take(ln) + .eq(b.as_bytes().iter().take(ln)); + return Ok(Expr::Bool(eq)); + } + Ok(Expr::BuiltinCall { + name: "strncmp".to_string(), + args: vec![arg0, arg1, n_expr_parsed], + }) + } + Rule::starts_with_call => { + // grammar: starts_with("(" expr "," expr ")") + let arg0 = parse_expr(it.next().ok_or(ParseError::InvalidExpression)?)?; + let arg1 = parse_expr(it.next().ok_or(ParseError::InvalidExpression)?)?; + // Constant fold when both are string literals + if let (Expr::String(a), Expr::String(b)) = (&arg0, &arg1) { + return Ok(Expr::Bool(a.as_bytes().starts_with(b.as_bytes()))); + } + Ok(Expr::BuiltinCall { + name: "starts_with".to_string(), + args: vec![arg0, arg1], + }) + } + Rule::hex_call => { + // grammar: hex("HEX...") + // Validate at parse time: allow only hex digits with optional whitespace separators. + let lit_node = it.next().ok_or(ParseError::InvalidExpression)?; + if lit_node.as_rule() != Rule::string { + return Err(ParseError::TypeError( + "hex expects a string literal".to_string(), + )); + } + let raw = lit_node.as_str(); + let inner = &raw[1..raw.len() - 1]; + let mut sanitized = String::with_capacity(inner.len()); + for ch in inner.chars() { + if ch.is_ascii_hexdigit() { + sanitized.push(ch); + } else if ch == ' ' { + // allow spaces as separators (tabs not allowed) + continue; + } else { + return Err(ParseError::TypeError(format!( + "hex literal contains non-hex character: '{ch}'" + ))); + } + } + if sanitized.len() % 2 == 1 { + return Err(ParseError::TypeError( + "hex literal must contain an even number of hex digits".to_string(), + )); + } + Ok(Expr::BuiltinCall { + name: "hex".to_string(), + // Store sanitized hex-only string; codegen will convert to bytes + args: vec![Expr::String(sanitized)], + }) + } + _ => Err(ParseError::UnexpectedToken(rule)), + } +} +fn parse_cast_call(pair: Pair) -> Result { + let mut inner = pair.into_inner(); + let expr_pair = inner.next().ok_or(ParseError::InvalidExpression)?; + let type_pair = inner.next().ok_or(ParseError::InvalidExpression)?; + let raw_type = type_pair.as_str(); + let target_type = raw_type + .strip_prefix('"') + .and_then(|s| s.strip_suffix('"')) + .ok_or_else(|| ParseError::SyntaxError("cast target type must be a string".to_string()))? + .to_string(); + + Ok(Expr::Cast { + expr: Box::new(parse_expr(expr_pair)?), + target_type, + }) +} +fn parse_postfix_access(pair: Pair) -> Result { + let mut inner = pair.into_inner(); + let base = inner.next().ok_or(ParseError::InvalidExpression)?; + let mut expr = match base.as_rule() { + Rule::postfix_base => { + let base_inner = base + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + match base_inner.as_rule() { + Rule::cast_call => parse_cast_call(base_inner)?, + Rule::special_var => Expr::SpecialVar(base_inner.as_str().to_string()), + Rule::identifier => Expr::Variable(base_inner.as_str().to_string()), + Rule::expr => parse_expr(base_inner)?, + _ => return Err(ParseError::UnexpectedToken(base_inner.as_rule())), + } + } + _ => return Err(ParseError::UnexpectedToken(base.as_rule())), + }; + + for suffix in inner { + let suffix_inner = suffix + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + match suffix_inner.as_rule() { + Rule::member_suffix => { + let field = suffix_inner + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)? + .as_str() + .to_string(); + expr = Expr::MemberAccess(Box::new(expr), field); + } + Rule::index_suffix => { + let index_pair = suffix_inner + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + let parsed_index = parse_expr(index_pair)?; + let parsed_index = integer_literal_value(&parsed_index) + .map(Expr::Int) + .unwrap_or(parsed_index); + expr = Expr::ArrayAccess(Box::new(expr), Box::new(parsed_index)); + } + _ => return Err(ParseError::UnexpectedToken(suffix_inner.as_rule())), + } + } + + Ok(expr) +} +// Parse complex variable expressions (person.name, arr[0], etc.) +fn parse_complex_variable(pair: Pair) -> Result { + debug!( + "parse_complex_variable: {:?} = \"{}\"", + pair.as_rule(), + pair.as_str().trim() + ); + + let inner = pair + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + match inner.as_rule() { + Rule::chain_access => parse_chain_access(inner), + Rule::array_access => parse_array_access(inner), + Rule::member_access => parse_member_access(inner), + Rule::pointer_deref => parse_pointer_deref(inner), + Rule::address_of => parse_address_of(inner), + _ => Err(ParseError::UnexpectedToken(inner.as_rule())), + } +} +// Parse chain access: person.name.first +fn parse_chain_access(pair: Pair) -> Result { + let mut chain: Vec = Vec::new(); + let mut opt_index: Option = None; + for inner_pair in pair.into_inner() { + match inner_pair.as_rule() { + Rule::identifier => { + chain.push(inner_pair.as_str().to_string()); + } + Rule::expr => { + // Array tail index can be a literal or a runtime expression. + let parsed = parse_expr(inner_pair)?; + opt_index = Some( + integer_literal_value(&parsed) + .map(Expr::Int) + .unwrap_or(parsed), + ); + } + _ => {} + } + } + + if chain.is_empty() { + return Err(ParseError::InvalidExpression); + } + + // Build base expression from the chain identifiers + let mut expr = Expr::Variable(chain[0].clone()); + for seg in &chain[1..] { + expr = Expr::MemberAccess(Box::new(expr), seg.clone()); + } + + // If there's a trailing index, convert to ArrayAccess on the built base + if let Some(idx) = opt_index { + expr = Expr::ArrayAccess(Box::new(expr), Box::new(idx)); + } + + Ok(expr) +} +// Parse array access: arr[index] +fn parse_array_access(pair: Pair) -> Result { + let mut inner_pairs = pair.into_inner(); + let array_name = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; + let index_expr = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; + + let _array_expr = Box::new(Expr::Variable(array_name.as_str().to_string())); + let parsed_index = parse_expr(index_expr)?; + let parsed_index = integer_literal_value(&parsed_index) + .map(Expr::Int) + .unwrap_or(parsed_index); + + // Build base array access expression + let mut expr = Expr::ArrayAccess( + Box::new(Expr::Variable(array_name.as_str().to_string())), + Box::new(parsed_index), + ); + + // Consume trailing .field segments if present + for next in inner_pairs { + // Any remaining tokens are member identifiers + let m = next.as_str().to_string(); + expr = Expr::MemberAccess(Box::new(expr), m); + } + + Ok(expr) +} +// Parse member access: person.name +fn parse_member_access(pair: Pair) -> Result { + let mut parts = pair.into_inner(); + let base = parts + .next() + .ok_or(ParseError::InvalidExpression)? + .as_str() + .to_string(); + + // Collect all subsequent identifiers after the base + let mut tail: Vec = Vec::new(); + for p in parts { + tail.push(p.as_str().to_string()); + } + + // If there is only one member, keep MemberAccess for simplicity. + // For multi-level chains like a.b.c, normalize to ChainAccess([a, b, c]) + match tail.len() { + 0 => Err(ParseError::InvalidExpression), + 1 => Ok(Expr::MemberAccess( + Box::new(Expr::Variable(base)), + tail.remove(0), + )), + _ => { + let mut chain = Vec::with_capacity(1 + tail.len()); + chain.push(base); + chain.extend(tail); + Ok(Expr::ChainAccess(chain)) + } + } +} +// Parse pointer dereference: *ptr +fn parse_pointer_deref(pair: Pair) -> Result { + let mut inner = pair.into_inner(); + let target = inner.next().ok_or(ParseError::InvalidExpression)?; + let parsed = match target.as_rule() { + Rule::expr => parse_expr(target)?, + Rule::postfix_access => parse_postfix_access(target)?, + Rule::cast_call => parse_cast_call(target)?, + Rule::complex_variable => parse_complex_variable(target)?, + Rule::special_var => Expr::SpecialVar(target.as_str().to_string()), + Rule::identifier => Expr::Variable(target.as_str().to_string()), + _ => return Err(ParseError::UnexpectedToken(target.as_rule())), + }; + // Early normalization: *(&x) => x + match parsed { + Expr::AddressOf(inner_expr) => Ok(*inner_expr), + other => Ok(Expr::PointerDeref(Box::new(other))), + } +} +// Parse address-of: &expr +fn parse_address_of(pair: Pair) -> Result { + let mut inner = pair.into_inner(); + let target = inner.next().ok_or(ParseError::InvalidExpression)?; + let parsed = match target.as_rule() { + Rule::expr => parse_expr(target)?, + Rule::postfix_access => parse_postfix_access(target)?, + Rule::cast_call => parse_cast_call(target)?, + Rule::complex_variable => parse_complex_variable(target)?, + Rule::special_var => Expr::SpecialVar(target.as_str().to_string()), + Rule::identifier => Expr::Variable(target.as_str().to_string()), + _ => return Err(ParseError::UnexpectedToken(target.as_rule())), + }; + // Early normalization: &(*p) => p + match parsed { + Expr::PointerDeref(inner_expr) => Ok(*inner_expr), + other => Ok(Expr::AddressOf(Box::new(other))), + } +} diff --git a/ghostscope-compiler/src/script/parser/statement.rs b/ghostscope-compiler/src/script/parser/statement.rs new file mode 100644 index 00000000..a131d499 --- /dev/null +++ b/ghostscope-compiler/src/script/parser/statement.rs @@ -0,0 +1,405 @@ +use pest::iterators::Pair; + +use crate::script::ast::{infer_type, BacktraceStatement, PrintStatement, Statement, TracePattern}; +use crate::script::format_validator::FormatValidator; +use tracing::{debug, info}; + +use super::expr::{is_alias_expr, parse_condition, parse_expr}; +use super::{ParseError, Result, Rule}; + +fn parse_backtrace_stmt(pair: Pair) -> Result { + let mut stmt = BacktraceStatement::default(); + + for arg in pair.into_inner() { + if arg.as_rule() == Rule::backtrace_flag { + match arg.as_str() { + "raw" => stmt.raw = true, + "full" => stmt.full = true, + "inline" => stmt.inline = true, + "noinline" => stmt.inline = false, + other => { + return Err(ParseError::SyntaxError(format!( + "Unknown bt option '{other}'" + ))) + } + } + } + } + + Ok(stmt) +} +pub(super) fn parse_statement(pair: Pair) -> Result { + debug!( + "parse_statement: {:?} = '{}'", + pair.as_rule(), + pair.as_str().trim() + ); + let inner = pair + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + debug!( + "parse_statement inner: {:?} = '{}'", + inner.as_rule(), + inner.as_str().trim() + ); + + match inner.as_rule() { + Rule::trace_stmt => { + let mut inner_pairs = inner.into_inner(); + let pattern_pair = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; + let pattern = parse_trace_pattern(pattern_pair)?; + + let mut body = Vec::new(); + for stmt_pair in inner_pairs { + // Disallow nested trace statements (trace is top-level only) + if stmt_pair.as_rule() == Rule::statement { + let mut peek = stmt_pair.clone().into_inner(); + if let Some(first) = peek.next() { + if first.as_rule() == Rule::trace_stmt { + return Err(ParseError::SyntaxError( + "'trace' cannot be nested; it is only allowed at the top level" + .to_string(), + )); + } + } + } + let stmt = parse_statement(stmt_pair)?; + body.push(stmt); + } + + Ok(Statement::TracePoint { pattern, body }) + } + Rule::print_stmt => { + let print_content = inner + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + let print_stmt = parse_print_content(print_content)?; + Ok(Statement::Print(print_stmt)) + } + Rule::backtrace_stmt => Ok(Statement::Backtrace(parse_backtrace_stmt(inner)?)), + Rule::assign_stmt => { + // Friendly error for immutable variables (no assignment supported) + let mut it = inner.into_inner(); + let name = it + .next() + .ok_or(ParseError::InvalidExpression)? + .as_str() + .to_string(); + // consume rhs expr + let _ = it.next(); + Err(ParseError::TypeError(format!( + "Assignment is not supported: variables are immutable. Use 'let {name} = ...' to bind once." + ))) + } + Rule::expr_stmt => { + let expr = inner + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + let parsed_expr = parse_expr(expr)?; + + // Check expression type to ensure consistent operation types + if let Err(err) = infer_type(&parsed_expr) { + return Err(ParseError::TypeError(err)); + } + + Ok(Statement::Expr(parsed_expr)) + } + Rule::var_decl_stmt => { + let mut inner_pairs = inner.into_inner(); + let name = inner_pairs + .next() + .ok_or(ParseError::InvalidExpression)? + .as_str() + .to_string(); + let expr = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; + let parsed_expr = parse_expr(expr)?; + + // Check expression type to ensure consistent operation types + if let Err(err) = infer_type(&parsed_expr) { + return Err(ParseError::TypeError(err)); + } + + if is_alias_expr(&parsed_expr) { + Ok(Statement::AliasDeclaration { + name, + target: parsed_expr, + }) + } else { + Ok(Statement::VarDeclaration { + name, + value: parsed_expr, + }) + } + } + Rule::if_stmt => { + debug!("Parsing if_stmt"); + let mut inner_pairs = inner.into_inner(); + let condition_pair = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; + debug!( + "if_stmt condition_pair: {:?} = '{}'", + condition_pair.as_rule(), + condition_pair.as_str().trim() + ); + let condition = parse_condition(condition_pair)?; + + // Parse then body statements + let mut then_body = Vec::new(); + let mut else_body = None; + + for pair in inner_pairs { + match pair.as_rule() { + Rule::statement => { + then_body.push(parse_statement(pair)?); + } + Rule::else_clause => { + else_body = Some(Box::new(parse_else_clause(pair)?)); + break; + } + _ => return Err(ParseError::UnexpectedToken(pair.as_rule())), + } + } + + Ok(Statement::If { + condition, + then_body, + else_body, + }) + } + _ => Err(ParseError::UnexpectedToken(inner.as_rule())), + } +} +fn parse_else_clause(pair: Pair) -> Result { + let inner = pair + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + match inner.as_rule() { + Rule::if_stmt => { + // Directly parse if statement for else if + debug!("Parsing else if statement"); + let mut inner_pairs = inner.into_inner(); + let condition_pair = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; + debug!( + "else if condition_pair: {:?} = '{}'", + condition_pair.as_rule(), + condition_pair.as_str().trim() + ); + let condition = parse_condition(condition_pair)?; + + // Parse then body statements + let mut then_body = Vec::new(); + let mut else_body = None; + + for pair in inner_pairs { + match pair.as_rule() { + Rule::statement => { + then_body.push(parse_statement(pair)?); + } + Rule::else_clause => { + else_body = Some(Box::new(parse_else_clause(pair)?)); + break; + } + _ => return Err(ParseError::UnexpectedToken(pair.as_rule())), + } + } + + Ok(Statement::If { + condition, + then_body, + else_body, + }) + } + _ => { + // Parse else block statements + let mut else_body = Vec::new(); + for node in inner.into_inner() { + match node.as_rule() { + Rule::statement => { + else_body.push(parse_statement(node)?); + } + // Some grammars flatten block children to concrete statements (e.g., print_stmt) + Rule::print_stmt => { + let content = node + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + let pr = parse_print_content(content)?; + else_body.push(Statement::Print(pr)); + } + _ => return Err(ParseError::UnexpectedToken(node.as_rule())), + } + } + Ok(Statement::Block(else_body)) + } + } +} +fn parse_trace_pattern(pair: Pair) -> Result { + let inner = pair + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)?; + + match inner.as_rule() { + Rule::module_hex_address => { + let mut parts = inner.into_inner(); + let module = parts + .next() + .ok_or(ParseError::InvalidExpression)? + .as_str() + .to_string(); + let hex = parts.next().ok_or(ParseError::InvalidExpression)?.as_str(); + let addr = match u64::from_str_radix(&hex[2..], 16) { + Ok(v) => v, + Err(_) => { + return Err(ParseError::SyntaxError(format!( + "module-qualified address '{hex}' is invalid or too large for u64" + ))) + } + }; + Ok(TracePattern::AddressInModule { + module, + address: addr, + }) + } + Rule::hex_address => { + let addr_str = inner.as_str(); + // Remove "0x" prefix and parse as hex + let addr_hex = &addr_str[2..]; + let addr = match u64::from_str_radix(addr_hex, 16) { + Ok(v) => v, + Err(_) => { + return Err(ParseError::SyntaxError(format!( + "address '{addr_str}' is invalid or too large for u64" + ))) + } + }; + Ok(TracePattern::Address(addr)) + } + Rule::wildcard_pattern => { + let pattern = inner.as_str().to_string(); + Ok(TracePattern::Wildcard(pattern)) + } + Rule::function_name => { + let func_name = inner + .into_inner() + .next() + .ok_or(ParseError::InvalidExpression)? + .as_str() + .to_string(); + Ok(TracePattern::FunctionName(func_name)) + } + Rule::source_line => { + let mut parts = inner.into_inner(); + let file_path = parts + .next() + .ok_or(ParseError::InvalidExpression)? + .as_str() + .to_string(); + let line_pair = parts.next().ok_or(ParseError::InvalidExpression)?; + let line_number = line_pair + .as_str() + .parse::() + .map_err(|_| ParseError::InvalidExpression)?; + Ok(TracePattern::SourceLine { + file_path, + line_number, + }) + } + _ => Err(ParseError::UnexpectedToken(inner.as_rule())), + } +} +fn parse_print_content(pair: Pair) -> Result { + info!( + "parse_print_content: rule={:?} text=\"{}\"", + pair.as_rule(), + pair.as_str().trim() + ); + // Flatten any nested print_content nodes into a single list of children + fn collect_flattened<'a>(p: Pair<'a, Rule>, out: &mut Vec>) { + if p.as_rule() == Rule::print_content { + for c in p.into_inner() { + collect_flattened(c, out); + } + } else { + out.push(p); + } + } + + let mut flat: Vec> = Vec::new(); + collect_flattened(pair, &mut flat); + info!( + "parse_print_content: flat_rules=[{}]", + flat.iter() + .map(|p| format!("{:?}", p.as_rule())) + .collect::>() + .join(", ") + ); + if flat.is_empty() { + return Err(ParseError::InvalidExpression); + } + + // Prefer an explicit format_expr if present + if let Some(fmt_idx) = flat.iter().position(|p| p.as_rule() == Rule::format_expr) { + let fmt_pair = flat.remove(fmt_idx); + info!("parse_print_content: branch=format_expr"); + let mut inner_pairs = fmt_pair.into_inner(); + let format_string = inner_pairs.next().ok_or(ParseError::InvalidExpression)?; + let format_content = &format_string.as_str()[1..format_string.as_str().len() - 1]; + let mut args = Vec::new(); + for arg_pair in inner_pairs { + args.push(parse_expr(arg_pair)?); + } + info!( + "parse_print_content: fmt='{}' argc={}", + format_content, + args.len() + ); + FormatValidator::validate_format_arguments(format_content, &args)?; + return Ok(PrintStatement::Formatted { + format: format_content.to_string(), + args, + }); + } + + // Else, if first is a string and followed by one or more exprs, treat as flattened format + if flat[0].as_rule() == Rule::string && flat.len() >= 2 { + info!("parse_print_content: branch=flattened_string_with_args"); + let content_quoted = flat[0].as_str(); + let content = &content_quoted[1..content_quoted.len() - 1]; + let mut args = Vec::new(); + for p in flat.iter().skip(1) { + if p.as_rule() != Rule::expr { + return Err(ParseError::UnexpectedToken(p.as_rule())); + } + args.push(parse_expr(p.clone())?); + } + info!("parse_print_content: fmt='{}' argc={}", content, args.len()); + FormatValidator::validate_format_arguments(content, &args)?; + return Ok(PrintStatement::Formatted { + format: content.to_string(), + args, + }); + } + + // Single string or single expr + match flat[0].as_rule() { + Rule::string => { + info!("parse_print_content: branch=plain_string"); + let content = flat[0].as_str(); + let content = &content[1..content.len() - 1]; + Ok(PrintStatement::String(content.to_string())) + } + Rule::expr => { + info!("parse_print_content: branch=complex_variable"); + let expr = parse_expr(flat[0].clone())?; + Ok(PrintStatement::ComplexVariable(expr)) + } + other => { + info!("parse_print_content: branch=unexpected rule={:?}", other); + Err(ParseError::UnexpectedToken(other)) + } + } +} diff --git a/ghostscope-compiler/src/script/parser/tests.rs b/ghostscope-compiler/src/script/parser/tests.rs new file mode 100644 index 00000000..08993c00 --- /dev/null +++ b/ghostscope-compiler/src/script/parser/tests.rs @@ -0,0 +1,1198 @@ +use super::{parse, ParseError}; +use crate::script::ast::{BinaryOp, Expr, PrintStatement, Statement}; + +#[test] +fn parse_memcmp_builtin_in_if_should_succeed() { + let script = r#" +trace foo { + if memcmp(&buf[0], &buf[1], 16) { print "EQ"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_cast_member_and_index_access() { + let script = r#" +trace foo { + print cast($arg0, "struct request *").id; + print cast($arg1, "u32 *")[2]; + print *cast($arg1, "u32 *"); + print &cast($arg0, "struct request *").id; +} +"#; + let program = parse(script).expect("parse should succeed"); + let Statement::TracePoint { body, .. } = &program.statements[0] else { + panic!("expected trace point"); + }; + assert!(matches!( + &body[0], + Statement::Print(PrintStatement::ComplexVariable(Expr::MemberAccess(obj, field))) + if field == "id" && matches!(obj.as_ref(), Expr::Cast { .. }) + )); + assert!(matches!( + &body[1], + Statement::Print(PrintStatement::ComplexVariable(Expr::ArrayAccess(base, index))) + if matches!(base.as_ref(), Expr::Cast { .. }) + && matches!(index.as_ref(), Expr::Int(2)) + )); + assert!(matches!( + &body[2], + Statement::Print(PrintStatement::ComplexVariable(Expr::PointerDeref(inner))) + if matches!(inner.as_ref(), Expr::Cast { .. }) + )); + assert!(matches!( + &body[3], + Statement::Print(PrintStatement::ComplexVariable(Expr::AddressOf(inner))) + if matches!( + inner.as_ref(), + Expr::MemberAccess(obj, field) + if field == "id" && matches!(obj.as_ref(), Expr::Cast { .. }) + ) + )); +} + +#[test] +fn parse_memcmp_with_dynamic_len() { + let script = r#" +trace foo { + let n = 10; + if memcmp(&buf[0], &buf[0], n) { print "OK"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_if_else_with_flattened_format_and_star_len() { + // else branch contains a flattened format print with {:s.*} and two args + let script = r#" +trace src/http/ngx_http_request.c:1845 { + if strncmp(host.data, "ghostscope", 10) { + print "We got the request {}", *r; + } else { + print "The other hostname is {:s.*}", host.len, host.data; + } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_memcmp_len_zero_and_negative() { + let script = r#" +trace foo { + if memcmp(&p[0], &q[0], 0) { print "Z0"; } + let k = -5; + if memcmp(&p[0], &q[0], k) { print "NEG"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_numeric_literals_hex_oct_bin_and_memcmp_usage() { + let script = r#" +trace foo { + let a = 0x10; // 16 + let b = 0o755; // 493 + let c = 0b1010; // 10 + // use in memcmp length + if memcmp(&buf[0], &buf[0], 0x20) { print "H"; } + if memcmp(&buf[0], &buf[0], 0o40) { print "O"; } + if memcmp(&buf[0], &buf[0], 0b100000) { print "B"; } + // use numeric literal as pointer address for second arg + if memcmp(&buf[0], 0x7fff0000, 16) { print "P"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_memcmp_hex_builtin() { + let script = r#" +trace foo { + if memcmp(&buf[0], hex("504F"), 2) { print "OK"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_memcmp_with_numeric_pointers_and_len_bases() { + let script = r#" +trace foo { + let n = 0x10; + if memcmp(0x1000, 0x2000, n) { print "NP"; } + if memcmp(0o4000, 0b1000000000000, 0o20) { print "NP2"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_hex_with_non_hex_char_should_fail() { + let script = r#" +trace foo { + if memcmp(&buf[0], hex("G0"), 1) { print "X"; } +} +"#; + let r = parse(script); + match r { + Ok(_) => panic!("expected parse error for non-hex char"), + Err(ParseError::TypeError(msg)) => { + assert!( + msg.contains("hex literal contains non-hex character"), + "unexpected msg: {msg}" + ); + } + Err(e) => panic!("unexpected error variant: {e:?}"), + } +} + +#[test] +fn parse_hex_with_odd_digits_should_fail() { + let script = r#" +trace foo { + if memcmp(&buf[0], hex("123"), 1) { print "X"; } +} +"#; + let r = parse(script); + match r { + Ok(_) => panic!("expected parse error for odd-length hex"), + Err(ParseError::TypeError(msg)) => { + assert!( + msg.contains("even number of hex digits"), + "unexpected msg: {msg}" + ); + } + Err(e) => panic!("unexpected error variant: {e:?}"), + } +} + +#[test] +fn parse_hex_with_spaces_should_succeed() { + let script = r#" +trace foo { + if memcmp(&buf[0], hex("4c 49 42 5f"), 4) { print "OK"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_alias_declaration_address_of_and_member_access() { + let script = r#" +trace foo { + let p = &buf[0]; + let s = obj.field; +} +"#; + let prog = parse(script).expect("parse ok"); + let stmt0 = prog.statements.first().expect("trace"); + match stmt0 { + Statement::TracePoint { body, .. } => { + // Only the address-of form should be alias; member access is a value binding + assert!(matches!(body[0], Statement::AliasDeclaration { .. })); + assert!(matches!(body[1], Statement::VarDeclaration { .. })); + } + other => panic!("expected TracePoint, got {other:?}"), + } +} + +#[test] +fn parse_alias_declaration_with_constant_offset() { + let script = r#" +trace foo { + let p = &arr[0] + 16; + let q = 32 + &arr[0]; +} +"#; + let prog = parse(script).expect("parse ok"); + let stmt0 = prog.statements.first().expect("trace"); + match stmt0 { + Statement::TracePoint { body, .. } => { + assert!(matches!(body[0], Statement::AliasDeclaration { .. })); + assert!(matches!(body[1], Statement::AliasDeclaration { .. })); + } + other => panic!("expected TracePoint, got {other:?}"), + } +} + +#[test] +fn parse_member_access_scalar_not_alias() { + let script = r#" +trace foo { + let level = record.level; +} +"#; + let prog = parse(script).expect("parse ok"); + let stmt0 = prog.statements.first().expect("trace"); + match stmt0 { + Statement::TracePoint { body, .. } => { + assert!(matches!(body[0], Statement::VarDeclaration { .. })); + } + other => panic!("expected TracePoint, got {other:?}"), + } +} + +#[test] +fn parse_memcmp_rejects_string_literal() { + let script = r#" +trace foo { + if memcmp(&buf[0], "PO", 2) { print "X"; } +} +"#; + let r = parse(script); + assert!( + matches!(r, Err(ParseError::TypeError(ref msg)) if msg.contains("memcmp does not accept string literals")), + "expected type error, got: {r:?}" + ); +} + +#[test] +fn parse_memcmp_rejects_bool_args_and_len() { + // Bool as pointer argument + let s1 = r#" +trace foo { if memcmp(true, hex("00"), 1) { print "X"; } } +"#; + let r1 = parse(s1); + assert!(r1.is_err()); + + // Bool as length + let s2 = r#" +trace foo { if memcmp(&p[0], hex("00"), false) { print "X"; } } +"#; + let r2 = parse(s2); + assert!( + matches!(r2, Err(ParseError::TypeError(ref msg)) if msg.contains("length must be")), + "unexpected: {r2:?}" + ); +} + +#[test] +fn parse_strncmp_constant_folds_on_two_literals() { + // equal for first 2 bytes + let s = r#" +trace foo { + if strncmp("abc", "abd", 2) { print "T"; } else { print "F"; } +} +"#; + let prog = parse(s).expect("parse ok"); + // Walk down to the If condition and ensure it became a Bool(true) + let stmt0 = prog.statements.first().expect("one trace"); + match stmt0 { + Statement::TracePoint { body, .. } => match &body[0] { + Statement::If { condition, .. } => { + assert!(matches!(condition, Expr::Bool(true))); + } + other => panic!("expected If, got {other:?}"), + }, + other => panic!("expected TracePoint, got {other:?}"), + } +} + +#[test] +fn parse_strncmp_requires_one_string_side_error() { + let s = r#" +trace foo { + if strncmp(1, 2, 1) { print "X"; } +} +"#; + let r = parse(s); + // Parser now accepts generic expr, so error will occur in compiler stage; ensure parse ok here + assert!( + r.is_ok(), + "parse should succeed; semantic error in compiler" + ); +} + +#[test] +fn parse_memcmp_constant_folds_on_two_hex() { + let s = r#" +trace foo { + if memcmp(hex("504f"), hex("504F"), 2) { print "EQ"; } else { print "NE"; } +} +"#; + let prog = parse(s).expect("parse ok"); + let stmt0 = prog.statements.first().expect("one trace"); + match stmt0 { + Statement::TracePoint { body, .. } => match &body[0] { + Statement::If { condition, .. } => assert!(matches!(condition, Expr::Bool(true))), + other => panic!("expected If, got {other:?}"), + }, + other => panic!("expected TracePoint, got {other:?}"), + } + + // Mismatch without explicit len but equal sizes + let s2 = r#" +trace foo { + if memcmp(hex("504f"), hex("514f")) { print "EQ"; } else { print "NE"; } +} +"#; + let prog2 = parse(s2).expect("parse ok"); + let stmt02 = prog2.statements.first().expect("one trace"); + match stmt02 { + Statement::TracePoint { body, .. } => match &body[0] { + Statement::If { condition, .. } => assert!(matches!(condition, Expr::Bool(false))), + other => panic!("expected If, got {other:?}"), + }, + other => panic!("expected TracePoint, got {other:?}"), + } +} + +#[test] +fn parse_starts_with_constant_folds_on_two_literals() { + let s = r#" +trace foo { + if starts_with("abcdef", "abc") { print "T"; } else { print "F"; } +} +"#; + let prog = parse(s).expect("parse ok"); + let stmt0 = prog.statements.first().expect("one trace"); + match stmt0 { + Statement::TracePoint { body, .. } => match &body[0] { + Statement::If { condition, .. } => assert!(matches!(condition, Expr::Bool(true))), + other => panic!("expected If, got {other:?}"), + }, + other => panic!("expected TracePoint, got {other:?}"), + } + + let s2 = r#" +trace foo { + if starts_with("ab", "abc") { print "T"; } else { print "F"; } +} +"#; + let prog2 = parse(s2).expect("parse ok"); + let stmt02 = prog2.statements.first().expect("one trace"); + match stmt02 { + Statement::TracePoint { body, .. } => match &body[0] { + Statement::If { condition, .. } => assert!(matches!(condition, Expr::Bool(false))), + other => panic!("expected If, got {other:?}"), + }, + other => panic!("expected TracePoint, got {other:?}"), + } +} + +#[test] +fn parse_memcmp_hex_len_exceeds_left_should_fail() { + // hex has 2 bytes, len=3 should error on left side + let script = r#" +trace foo { + if memcmp(hex("504f"), &buf[0], 3) { print "X"; } +} +"#; + let r = parse(script); + match r { + Ok(_) => panic!("expected parse error for len > hex(left) size"), + Err(ParseError::TypeError(msg)) => { + assert!( + msg.contains("exceeds hex pattern size on left side"), + "unexpected msg: {msg}" + ); + } + Err(e) => panic!("unexpected error variant: {e:?}"), + } +} + +#[test] +fn parse_memcmp_hex_len_exceeds_right_should_fail() { + // hex has 2 bytes, len=5 should error on right side + let script = r#" +trace foo { + if memcmp(&buf[0], hex("50 4f"), 5) { print "X"; } +} +"#; + let r = parse(script); + match r { + Ok(_) => panic!("expected parse error for len > hex(right) size"), + Err(ParseError::TypeError(msg)) => { + assert!( + msg.contains("exceeds hex pattern size on right side"), + "unexpected msg: {msg}" + ); + } + Err(e) => panic!("unexpected error variant: {e:?}"), + } +} + +#[test] +fn parse_memcmp_hex_negative_len_should_fail() { + let script = r#" +trace foo { + if memcmp(&buf[0], hex("50 4f"), -1) { print "X"; } +} +"#; + let r = parse(script); + match r { + Ok(_) => panic!("expected parse error for negative len"), + Err(ParseError::TypeError(msg)) => { + assert!( + msg.contains("length must be non-negative"), + "unexpected msg: {msg}" + ); + } + Err(e) => panic!("unexpected error variant: {e:?}"), + } +} + +#[test] +fn parse_memcmp_hex_len_equal_should_succeed() { + // hex has 4 bytes, len=4 OK + let script = r#" +trace foo { + if memcmp(&buf[0], hex("de ad be ef"), 4) { print "OK"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_memcmp_hex_infers_len_left_should_succeed() { + let script = r#" +trace foo { + if memcmp(hex("50 4f"), &buf[0]) { print "OK"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_memcmp_hex_infers_len_right_should_succeed() { + let script = r#" +trace foo { + if memcmp(&buf[0], hex("de ad be ef")) { print "OK"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_assignment_is_rejected_with_friendly_message() { + let script = r#" +trace foo { + let a = 1; + a = 2; +} +"#; + let r = parse(script); + match r { + Ok(_) => panic!("expected assignment error for immutable variables"), + Err(ParseError::TypeError(msg)) => { + assert!( + msg.contains("Assignment is not supported"), + "unexpected msg: {msg}" + ); + } + Err(e) => panic!("unexpected error variant: {e:?}"), + } +} + +#[test] +fn parse_starts_with_accepts_two_exprs() { + // Both sides are expr (identifiers); grammar should accept + let script = r#" +trace foo { + if starts_with(name, s) { print "OK"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_strncmp_accepts_two_exprs_and_len() { + let script = r#" +trace foo { + if strncmp(lhs, rhs, 3) { print "EQ"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_strncmp_negative_len_rejected() { + // Negative literal lengths are rejected early. + let script = r#" +trace foo { + if strncmp(lhs, rhs, -1) { print "X"; } +} +"#; + let r = parse(script); + assert!(r.is_err(), "expected parse error for negative length"); + if let Err(ParseError::TypeError(msg)) = r { + assert!(msg.contains("non-negative"), "unexpected msg: {msg}"); + } +} + +#[test] +fn parse_strncmp_accepts_nonliteral_len() { + let script = r#" +trace foo { + let n = 3; + if strncmp(lhs, rhs, n) { print "X"; } +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_memcmp_missing_len_without_hex_should_fail() { + let script = r#" +trace foo { + if memcmp(&buf[0], &buf[1]) { print "OK"; } +} +"#; + let r = parse(script); + assert!( + r.is_err(), + "expected parse error for missing len without hex" + ); +} + +#[test] +fn parse_memcmp_both_hex_mismatch_should_fail() { + let script = r#" +trace foo { + if memcmp(hex("50"), hex("504f")) { print "OK"; } +} +"#; + let r = parse(script); + match r { + Ok(_) => panic!("expected parse error for mismatched hex sizes"), + Err(ParseError::TypeError(msg)) => { + assert!(msg.contains("different sizes"), "unexpected msg: {msg}"); + } + Err(e) => panic!("unexpected error variant: {e:?}"), + } +} + +#[test] +fn parse_format_static_len_bases_in_prints() { + // Validate that static length .N supports 0x/0o/0b in formatted prints + let script = r#" +trace foo { + print "HX={:x.0x10}", buf; + print "HS={:s.0o20}", buf; + print "HB={:X.0b1000}", buf; +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_trace_patterns_function_line_address_wildcard() { + // Function name + let s1 = r#"trace main { print "OK"; }"#; + assert!(parse(s1).is_ok()); + + // Source line with path and hyphen + let s2 = r#"trace /tmp/test-file.c:42 { print "L"; }"#; + assert!(parse(s2).is_ok()); + + // Hex address + let s3 = r#"trace 0x401234 { print "A"; }"#; + assert!(parse(s3).is_ok()); + + // Wildcard + let s4 = r#"trace printf* { print "W"; }"#; + assert!(parse(s4).is_ok()); + + // Module-qualified address + let s5 = r#"trace /lib/x86_64-linux-gnu/libc.so.6:0x1234 { print "M"; }"#; + assert!(parse(s5).is_ok()); +} + +#[test] +fn parse_identifiers_can_start_with_underscore() { + let function = r#"trace __UpdateTicketInformation { print "OK"; }"#; + assert!(parse(function).is_ok()); + + let wildcard = r#"trace __builtin_* { print "W"; }"#; + assert!(parse(wildcard).is_ok()); + + let script = r#" +trace _start { + let _ticket = __dwarf_value; + print _ticket; +} +"#; + assert!(parse(script).is_ok()); +} + +#[test] +fn parse_module_hex_address_overflow_should_error() { + // Address exceeds u64 (17 hex digits) -> parse error, not 0 fallback + let s = r#"trace libfoo.so:0x10000000000000000 { print "X"; }"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => assert!(msg.contains("too large for u64")), + other => panic!("expected friendly SyntaxError, got {other:?}"), + } +} + +#[test] +fn parse_hex_address_overflow_should_error() { + let s = r#"trace 0x10000000000000000 { print "X"; }"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => assert!(msg.contains("too large for u64")), + other => panic!("expected friendly SyntaxError, got {other:?}"), + } +} + +#[test] +fn parse_special_variables_basic() { + // $pid/$tid/$host_pid/$input_pid/$timestamp in expressions and prints + let script = r#" +trace foo { + if $pid == 123 && $tid != 0 && $host_pid != 0 && $input_pid == 123 { print "PID_TID"; } + print $timestamp; + print "P:{} T:{} HP:{} IN:{} TS:{}", $pid, $tid, $host_pid, $input_pid, $timestamp; +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_chain_and_array_access() { + // Member/chain and array tail index + let script = r#" +trace foo { + print person.name.first; + print arr[0]; + // Supported: top-level array access with trailing member + print ifaces[0].mtu; +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_pointer_and_address_of() { + let script = r#" +trace foo { + print *ptr; + print &var; + print *(arr_ptr); +} +"#; + let r = parse(script); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_nested_trace_is_rejected() { + let s = r#" +trace foo { + trace bar { print "X"; } +} +"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => assert!(msg.contains("cannot be nested")), + other => panic!("expected SyntaxError for nested trace, got {other:?}"), + } +} + +#[test] +fn parse_float_literal_is_rejected() { + let s = r#" +trace foo { + let x = 1.23; +} +"#; + let r = parse(s); + match r { + Err(ParseError::TypeError(msg)) => { + assert!(msg.contains("float literals are not supported")) + } + other => panic!("expected TypeError for float literal, got {other:?}"), + } +} + +#[test] +fn parse_unclosed_print_string_reports_friendly_error() { + let bad = r#" +trace foo { + print "Unclosed {}, value +} +"#; + let r = parse(bad); + match r { + Err(ParseError::SyntaxError(msg)) => assert!(msg.contains("Unclosed string literal")), + other => panic!("expected SyntaxError, got {other:?}"), + } +} + +#[test] +fn parse_array_index_accepts_dynamic_expr() { + // Dynamic index on top-level array + let s1 = r#" +trace foo { + print arr[i]; +} +"#; + let r1 = parse(s1).expect("dynamic top-level index should parse"); + match r1.statements.first().expect("trace") { + Statement::TracePoint { body, .. } => match &body[0] { + Statement::Print(PrintStatement::ComplexVariable(Expr::ArrayAccess(_, index))) => { + assert!(matches!(index.as_ref(), Expr::Variable(name) if name == "i")) + } + other => panic!("unexpected first print body: {other:?}"), + }, + other => panic!("expected TracePoint, got {other:?}"), + } + + // Dynamic index at chain tail + let s2 = r#" +trace foo { + print obj.arr[i - (i / 0x8) * 0x8]; +} +"#; + let r2 = parse(s2).expect("dynamic chain index should parse"); + match r2.statements.first().expect("trace") { + Statement::TracePoint { body, .. } => match &body[0] { + Statement::Print(PrintStatement::ComplexVariable(Expr::ArrayAccess(_, index))) => { + assert!(matches!(index.as_ref(), Expr::BinaryOp { .. })) + } + other => panic!("unexpected first print body: {other:?}"), + }, + other => panic!("expected TracePoint, got {other:?}"), + } +} + +#[test] +fn parse_integer_modulo_and_bitwise_ops() { + let script = r#" +trace foo { + let value = 0x1 | 0x2 ^ 0x3 & 0x4 << 0x1 + 0x2 % 0x3; + let inverse = ~value; +} +"#; + let prog = parse(script).expect("integer and bitwise ops should parse"); + let Statement::TracePoint { body, .. } = prog.statements.first().expect("trace") else { + panic!("expected trace point"); + }; + let Statement::VarDeclaration { value, .. } = &body[0] else { + panic!("expected var declaration"); + }; + let Expr::BinaryOp { op, left, right } = value else { + panic!("expected bitwise-or root"); + }; + assert_eq!(*op, BinaryOp::BitOr); + assert!(matches!(left.as_ref(), Expr::Int(1))); + assert!(matches!( + right.as_ref(), + Expr::BinaryOp { + op: BinaryOp::BitXor, + .. + } + )); + assert!(matches!( + &body[1], + Statement::VarDeclaration { + value: Expr::UnaryBitNot(_), + .. + } + )); +} + +#[test] +fn parse_array_index_accepts_constant_negative_literal() { + let script = r#" +trace foo { + print arr[-0x1]; + print obj.arr[0b10 - 0x3]; +} +"#; + let prog = parse(script).expect("parse ok"); + let trace = prog.statements.first().expect("trace"); + match trace { + Statement::TracePoint { body, .. } => { + match &body[0] { + Statement::Print(PrintStatement::ComplexVariable(Expr::ArrayAccess(_, index))) => { + assert!(matches!(index.as_ref(), Expr::Int(-1))); + } + other => panic!("unexpected first print body: {other:?}"), + } + match &body[1] { + Statement::Print(PrintStatement::ComplexVariable(Expr::ArrayAccess(_, index))) => { + assert!(matches!(index.as_ref(), Expr::Int(-1))); + } + other => panic!("unexpected second print body: {other:?}"), + } + } + other => panic!("expected TracePoint, got {other:?}"), + } +} + +#[test] +fn parse_print_format_arg_mismatch_reports_error() { + // format_expr form + let s1 = r#" +trace foo { + print "A {} {}", x; +} +"#; + let r1 = parse(s1); + match r1 { + Err(ParseError::TypeError(msg)) => { + assert!(msg.contains("expects 2 argument(s)"), "unexpected: {msg}"); + } + other => panic!("expected TypeError from format arg mismatch, got {other:?}"), + } + + // flattened string + args form + let s2 = r#" +trace foo { + print "B {} {}", y; +} +"#; + let r2 = parse(s2); + match r2 { + Err(ParseError::TypeError(msg)) => { + assert!(msg.contains("expects 2 argument(s)")); + } + other => panic!("expected TypeError from format arg mismatch, got {other:?}"), + } +} + +#[test] +fn parse_print_invalid_format_specifier_errors() { + // Missing ':' prefix inside { } + let s1 = r#" +trace foo { print "Bad {x}", 1; } +"#; + let r1 = parse(s1); + match r1 { + Err(ParseError::TypeError(msg)) => { + assert!(msg.contains("Invalid format specifier"), "{msg}"); + } + other => panic!("expected TypeError, got {other:?}"), + } + + // Unsupported conversion {:q} + let s2 = r#" +trace foo { print "Bad {:q}", 1; } +"#; + let r2 = parse(s2); + match r2 { + Err(ParseError::TypeError(msg)) => { + assert!(msg.contains("Unsupported format conversion"), "{msg}"); + } + other => panic!("expected TypeError, got {other:?}"), + } +} + +#[test] +fn parse_hex_with_tab_is_rejected() { + let s = r#" +trace foo { + if memcmp(&buf[0], hex("50\t4f"), 2) { print "X"; } +} +"#; + let r = parse(s); + match r { + Err(ParseError::TypeError(msg)) => { + assert!(msg.contains("non-hex character"), "{msg}"); + } + other => panic!("expected TypeError for tab in hex literal, got {other:?}"), + } +} + +#[test] +fn parse_starts_with_constant_folds_on_literals() { + let s = r#" +trace foo { + if starts_with("abcdef", "abc") { print "T"; } else { print "F"; } +} +"#; + let prog = parse(s).expect("parse ok"); + let stmt0 = prog.statements.first().expect("trace"); + match stmt0 { + Statement::TracePoint { body, .. } => match &body[0] { + Statement::If { condition, .. } => { + assert!(matches!(condition, Expr::Bool(true))); + } + other => panic!("expected If, got {other:?}"), + }, + other => panic!("expected TracePoint, got {other:?}"), + } +} + +#[test] +fn parse_backtrace_and_bt_statements() { + let s = r#" + trace foo { + backtrace; + bt; + bt raw; + bt full noinline; + } + "#; + let program = parse(s).expect("parse ok"); + let Statement::TracePoint { body, .. } = &program.statements[0] else { + panic!("expected trace"); + }; + assert_eq!(body.len(), 4); + match &body[2] { + Statement::Backtrace(bt) => { + assert!(bt.raw); + assert!(bt.inline); + } + other => panic!("expected backtrace, got {other:?}"), + } + match &body[3] { + Statement::Backtrace(bt) => { + assert!(bt.full); + assert!(!bt.inline); + } + other => panic!("expected backtrace, got {other:?}"), + } +} + +#[test] +fn parse_backtrace_rejects_named_depth_option() { + let s = r#" + trace foo { + bt depth=8; + } + "#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!(msg.contains("no longer a script option"), "{msg}"); + assert!(msg.contains("--backtrace-depth"), "{msg}"); + } + other => panic!("expected SyntaxError, got {other:?}"), + } +} + +#[test] +fn parse_backtrace_rejects_positional_depth_option() { + let s = r#" + trace foo { + bt 4 raw; + } + "#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!(msg.contains("no longer a script option"), "{msg}"); + } + other => panic!("expected SyntaxError, got {other:?}"), + } +} + +#[test] +fn parse_print_capture_len_suffix() { + // {:s.name$} uses capture; does not consume extra arg + let s = r#" +trace foo { + let n = 3; + print "tail={:s.n$}", p; +} +"#; + let r = parse(s); + assert!(r.is_ok(), "parse failed: {:?}", r.err()); +} + +#[test] +fn parse_unknown_keyword_inside_trace_suggests_print() { + let s = r#" +trace foo { + pront "hello"; +} +"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!( + msg.contains("Unknown keyword 'pront'"), + "unexpected msg: {msg}" + ); + assert!( + msg.contains("Did you mean 'print'"), + "no suggestion in msg: {msg}" + ); + } + other => panic!("expected friendly SyntaxError for unknown keyword, got {other:?}"), + } +} + +#[test] +fn parse_unknown_keyword_same_line_after_brace_suggests_print() { + // Unknown keyword immediately after '{' on the same line + let s = r#"trace foo {pirnt \"sa\";}"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!( + msg.contains("Unknown keyword 'pirnt'"), + "unexpected msg: {msg}" + ); + assert!( + msg.contains("Did you mean 'print'"), + "no suggestion in msg: {msg}" + ); + } + other => { + panic!("expected friendly SyntaxError for same-line unknown keyword, got {other:?}") + } + } +} + +#[test] +fn parse_unknown_top_level_keyword_suggests_trace() { + let s = r#" +traec bar { + print "x"; +} +"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!( + msg.contains("Unknown keyword 'traec'"), + "unexpected msg: {msg}" + ); + assert!( + msg.contains("Did you mean 'trace'"), + "no suggestion in msg: {msg}" + ); + } + other => panic!("expected friendly SyntaxError for unknown keyword, got {other:?}"), + } +} + +#[test] +fn parse_builtin_then_misspelled_keyword_should_point_to_misspell() { + // Ensure builtin calls are not flagged; the real typo should be reported + let s = r#" +trace foo { + starts_with("a", "b"); prnit "oops"; +} +"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!( + msg.contains("prnit"), + "should point to misspelled 'prnit': {msg}" + ); + assert!( + !msg.contains("starts_with"), + "should not flag builtin call: {msg}" + ); + } + other => panic!("expected friendly SyntaxError for misspelled print, got {other:?}"), + } +} + +#[test] +fn parse_misspelled_builtin_suggests_starts_with() { + // Misspelled builtin should suggest the correct builtin name + let s = r#" +trace foo { + starst_with("a", "b"); +} +"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!(msg.contains("Unknown keyword 'starst_with'"), "{msg}"); + assert!(msg.contains("Did you mean 'starts_with'"), "{msg}"); + } + other => panic!("expected friendly suggestion for misspelled builtin, got {other:?}"), + } +} + +#[test] +fn parse_misspelled_builtin_suggests_memcmp() { + let s = r#" +trace foo { + memcpm(&buf[0], &buf[1], 16); +} +"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!(msg.contains("Unknown keyword 'memcpm'"), "{msg}"); + assert!(msg.contains("Did you mean 'memcmp'"), "{msg}"); + } + other => panic!("expected friendly suggestion for misspelled builtin, got {other:?}"), + } +} + +#[test] +fn parse_if_condition_misspelled_builtin_suggests() { + let s = r#" +trace foo { + if starst_with("a", "b") { print "ok"; } +} +"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!(msg.contains("Unknown keyword 'starst_with'"), "{msg}"); + assert!(msg.contains("Did you mean 'starts_with'"), "{msg}"); + } + other => panic!("expected friendly suggestion inside if(), got {other:?}"), + } +} + +#[test] +fn parse_else_if_condition_misspelled_builtin_suggests() { + let s = r#" +trace foo { + if 1 { print "a"; } else if starst_with("a", "b") { print "b"; } +} +"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!(msg.contains("Unknown keyword 'starst_with'"), "{msg}"); + assert!(msg.contains("Did you mean 'starts_with'"), "{msg}"); + } + other => panic!("expected friendly suggestion inside else if(), got {other:?}"), + } +} +#[test] +fn parse_unknown_keyword_generic_expected_list() { + let s = r#" +foobarbaz { + print "x"; +} +"#; + let r = parse(s); + match r { + Err(ParseError::SyntaxError(msg)) => { + assert!( + msg.contains("Unknown keyword 'foobarbaz'"), + "unexpected msg: {msg}" + ); + assert!( + msg.contains("Expected one of"), + "missing expected list in msg: {msg}" + ); + } + other => panic!("expected friendly SyntaxError with expected list, got {other:?}"), + } +} diff --git a/ghostscope-dwarf/src/analyzer/cache.rs b/ghostscope-dwarf/src/analyzer/cache.rs new file mode 100644 index 00000000..dd59c1a9 --- /dev/null +++ b/ghostscope-dwarf/src/analyzer/cache.rs @@ -0,0 +1,69 @@ +use crate::semantics::PcContext; +use std::collections::{HashMap, VecDeque}; +use std::path::{Path, PathBuf}; + +const PC_CONTEXT_CACHE_MAX_ENTRIES: usize = 8192; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct PcContextCacheKey { + module_path: PathBuf, + address: u64, +} + +#[derive(Debug)] +pub(super) struct PcContextCache { + entries: HashMap>, + insertion_order: VecDeque, + len: usize, + max_entries: usize, +} + +impl Default for PcContextCache { + fn default() -> Self { + Self { + entries: HashMap::new(), + insertion_order: VecDeque::new(), + len: 0, + max_entries: PC_CONTEXT_CACHE_MAX_ENTRIES, + } + } +} + +impl PcContextCache { + pub(super) fn get(&self, module_path: &Path, address: u64) -> Option { + self.entries + .get(module_path) + .and_then(|entries| entries.get(&address)) + .cloned() + } + + pub(super) fn insert(&mut self, module_path: PathBuf, address: u64, context: PcContext) { + if self.max_entries == 0 { + return; + } + + let key = PcContextCacheKey { + module_path, + address, + }; + let module_entries = self.entries.entry(key.module_path.clone()).or_default(); + if module_entries.insert(address, context).is_none() { + self.insertion_order.push_back(key.clone()); + self.len += 1; + } + + while self.len > self.max_entries { + let Some(expired) = self.insertion_order.pop_front() else { + break; + }; + if let Some(module_entries) = self.entries.get_mut(&expired.module_path) { + if module_entries.remove(&expired.address).is_some() { + self.len -= 1; + } + if module_entries.is_empty() { + self.entries.remove(&expired.module_path); + } + } + } + } +} diff --git a/ghostscope-dwarf/src/analyzer/mod.rs b/ghostscope-dwarf/src/analyzer/mod.rs index a397052e..8f0e2745 100644 --- a/ghostscope-dwarf/src/analyzer/mod.rs +++ b/ghostscope-dwarf/src/analyzer/mod.rs @@ -7,103 +7,29 @@ use crate::{ }, loader::ExplicitDebugFile, objfile::LoadedObjfile, - semantics::{CompactUnwindRow, CompactUnwindTable, PcContext, VisibleVariable}, + semantics::{CompactUnwindRow, CompactUnwindTable, PcContext}, }; use ghostscope_debuginfod::DebuginfodClient; use object::{Object, ObjectSection}; -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::{Arc, RwLock}; +mod cache; mod module_resolution; mod plan_global; mod plan_pc; mod source_resolution; +#[cfg(test)] +mod tests; mod type_lookup; +mod types; +use cache::PcContextCache; pub use module_resolution::ModuleDefaultPolicy; pub use source_resolution::{SourceLineAddressSearch, SourceLineQuerySearch}; pub use type_lookup::TypeLookupAmbiguity; - -#[cfg(test)] -use crate::{ - core::{AddressExpr, Availability, Provenance, VariableLocation}, - semantics::VariableReadPlan, -}; - -/// Events emitted during module loading process -#[derive(Debug, Clone)] -pub enum ModuleLoadingEvent { - /// Module discovered during process scanning - Discovered { - module_path: String, - current: usize, - total: usize, - }, - /// Module loading started - LoadingStarted { - module_path: String, - current: usize, - total: usize, - }, - /// Module loading completed successfully - LoadingCompleted { - module_path: String, - stats: ModuleLoadingStats, - current: usize, - total: usize, - }, - /// Module loading failed - LoadingFailed { - module_path: String, - error: String, - current: usize, - total: usize, - }, -} - -/// Statistics for a loaded module -#[derive(Debug, Clone)] -pub struct ModuleLoadingStats { - pub functions: usize, - pub variables: usize, - pub types: usize, - pub debug_info_source: DebugInfoSource, - pub load_time_ms: u64, - pub parse_time_ms: u64, - pub index_time_ms: u64, - pub module_total_time_ms: u64, -} - -/// Rich query result for a single address within a module. -#[derive(Debug, Clone)] -pub struct AddressQueryResult { - pub module_path: PathBuf, - pub address: u64, - pub source_file: Option, - pub source_line: Option, - pub source_column: Option, - pub function_name: Option, - pub is_inline: Option, - pub variables: Vec, - pub parameters: Vec, -} - -/// Runtime mapping metadata for a loaded module. -#[derive(Debug, Clone)] -pub struct LoadedModuleRuntimeInfo { - pub module_path: PathBuf, - pub loaded_address: Option, - pub load_bias: Option, - pub size: u64, -} - -/// Rich query result for a function lookup across modules. -#[derive(Debug, Clone)] -pub struct FunctionQueryResult { - pub function_name: String, - pub addresses: Vec, -} +pub use types::*; /// DWARF analyzer - unified entry point for all DWARF analysis #[derive(Debug)] @@ -116,72 +42,6 @@ pub struct DwarfAnalyzer { pc_context_cache: RwLock, } -const PC_CONTEXT_CACHE_MAX_ENTRIES: usize = 8192; - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct PcContextCacheKey { - module_path: PathBuf, - address: u64, -} - -#[derive(Debug)] -struct PcContextCache { - entries: HashMap>, - insertion_order: VecDeque, - len: usize, - max_entries: usize, -} - -impl Default for PcContextCache { - fn default() -> Self { - Self { - entries: HashMap::new(), - insertion_order: VecDeque::new(), - len: 0, - max_entries: PC_CONTEXT_CACHE_MAX_ENTRIES, - } - } -} - -impl PcContextCache { - fn get(&self, module_path: &Path, address: u64) -> Option { - self.entries - .get(module_path) - .and_then(|entries| entries.get(&address)) - .cloned() - } - - fn insert(&mut self, module_path: PathBuf, address: u64, context: PcContext) { - if self.max_entries == 0 { - return; - } - - let key = PcContextCacheKey { - module_path, - address, - }; - let module_entries = self.entries.entry(key.module_path.clone()).or_default(); - if module_entries.insert(address, context).is_none() { - self.insertion_order.push_back(key.clone()); - self.len += 1; - } - - while self.len > self.max_entries { - let Some(expired) = self.insertion_order.pop_front() else { - break; - }; - if let Some(module_entries) = self.entries.get_mut(&expired.module_path) { - if module_entries.remove(&expired.address).is_some() { - self.len -= 1; - } - if module_entries.is_empty() { - self.entries.remove(&expired.module_path); - } - } - } - } -} - impl DwarfAnalyzer { fn build_address_query_result( &self, @@ -1497,234 +1357,3 @@ impl DwarfAnalyzer { Ok(grouped) } } - -/// -/// Module statistics compatible with ghostscope-binary -#[derive(Debug, Clone)] -pub struct ModuleStats { - pub total_modules: usize, - pub executable_modules: usize, - pub library_modules: usize, - pub total_symbols: usize, - pub modules_with_debug_info: usize, -} - -/// Main executable information -#[derive(Debug, Clone)] -pub struct MainExecutableInfo { - pub path: String, -} - -/// Statistics for debugging and monitoring -#[derive(Debug, Clone)] -pub struct AnalyzerStats { - pub pid: u32, - pub module_count: usize, - pub total_functions: usize, - pub total_variables: usize, - pub total_line_headers: usize, -} - -/// Shared library information (compatible with ghostscope-ui) -#[derive(Debug, Clone)] -pub struct SharedLibraryInfo { - pub from_address: u64, // Starting address in memory - pub to_address: u64, // Ending address in memory - pub symbols_read: bool, // Whether symbols were successfully read - pub debug_info_available: bool, // Whether debug information is available - pub library_path: String, // Full path to the library file - pub size: u64, // Size of the library in memory - pub debug_file_path: Option, // Path to separate debug file (if via .gnu_debuglink) -} - -/// Executable file information (for "info file" command) -#[derive(Debug, Clone)] -pub struct ExecutableFileInfo { - pub file_path: String, - pub file_type: String, - pub entry_point: Option, - pub has_symbols: bool, - pub has_debug_info: bool, - pub debug_file_path: Option, - pub text_section: Option, - pub data_section: Option, - pub mode_description: String, -} - -/// Section information for executable files -#[derive(Debug, Clone)] -pub struct SectionInfo { - pub start_address: u64, - pub end_address: u64, - pub size: u64, -} - -/// Simple file information compatible with ghostscope-binary -#[derive(Debug, Clone)] -pub struct SimpleFileInfo { - pub full_path: String, - pub basename: String, - pub directory: String, -} - -#[cfg(test)] -mod tests { - use super::*; - - fn global_plan(name: &str, address: u64) -> VariableReadPlan { - VariableReadPlan { - name: name.to_string(), - type_name: "int".to_string(), - access_path: crate::VariableAccessPath::default(), - module_path: None, - dwarf_type: Some(crate::TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }), - declaration: None, - type_id: None, - location: VariableLocation::Address(AddressExpr::constant(address)), - availability: Availability::Available, - scope_depth: 0, - is_parameter: false, - is_artificial: false, - pc_range: None, - inline_context: None, - provenance: Provenance::Synthesized { - detail: "test".to_string(), - }, - } - } - - fn visible_var(name: &str, scope_depth: usize) -> VisibleVariable { - VisibleVariable { - name: name.to_string(), - type_name: "int".to_string(), - dwarf_type: Some(crate::TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }), - declaration: None, - type_id: None, - location: VariableLocation::RegisterValue { dwarf_reg: 0 }, - availability: Availability::Available, - scope_depth, - is_parameter: false, - is_artificial: false, - } - } - - fn diagnostic( - name: &str, - scope_depth: usize, - detail: &str, - ) -> crate::semantics::VariableQueryDiagnostic { - crate::semantics::VariableQueryDiagnostic { - pc: 0x1234, - name: Some(name.to_string()), - scope_depth, - availability: Availability::Unsupported(crate::UnsupportedReason::ExpressionShape { - detail: detail.to_string(), - }), - detail: detail.to_string(), - } - } - - #[test] - fn variable_selection_rejects_inner_diagnostic_over_outer_match() { - let err = DwarfAnalyzer::select_visible_variable_by_name( - 0x1234, - "state", - vec![visible_var("state", 1)], - &[diagnostic("state", 2, "DW_OP_bad is unsupported")], - ) - .expect_err("inner unavailable variable should block outer fallback"); - - assert!(err.to_string().contains("Unavailable variable 'state'")); - assert!(err.to_string().contains("DW_OP_bad is unsupported")); - } - - #[test] - fn variable_selection_keeps_inner_match_over_outer_diagnostic() { - let selected = DwarfAnalyzer::select_visible_variable_by_name( - 0x1234, - "state", - vec![visible_var("state", 2)], - &[diagnostic("state", 1, "outer variable is unavailable")], - ) - .expect("outer diagnostic should not block inner match") - .expect("inner match should be returned"); - - assert_eq!(selected.name, "state"); - assert_eq!(selected.scope_depth, 2); - } - - #[test] - fn global_plan_selection_rejects_ambiguous_matches() { - let err = DwarfAnalyzer::select_unambiguous_global_plan( - "state", - vec![ - (PathBuf::from("/tmp/a"), global_plan("state", 0x1000)), - (PathBuf::from("/tmp/b"), global_plan("state", 0x2000)), - ], - ) - .expect_err("multiple global candidates should be ambiguous"); - - assert!(err.to_string().contains("Ambiguous global 'state'")); - assert!(err.to_string().contains("2 matches")); - } - - #[test] - fn global_plan_selection_accepts_single_match() { - let selected = DwarfAnalyzer::select_unambiguous_global_plan( - "state", - vec![(PathBuf::from("/tmp/a"), global_plan("state", 0x1000))], - ) - .expect("single global candidate should be accepted") - .expect("single global candidate should be returned"); - - assert_eq!(selected.0, PathBuf::from("/tmp/a")); - assert_eq!(selected.1.name, "state"); - } - - #[test] - fn global_plan_selection_prefers_current_module_match() { - let selected = DwarfAnalyzer::select_global_plan_with_preferred_module( - "state", - Path::new("/tmp/current"), - vec![ - (PathBuf::from("/tmp/other"), global_plan("state", 0x2000)), - (PathBuf::from("/tmp/current"), global_plan("state", 0x1000)), - ], - ) - .expect("current module candidate should be accepted") - .expect("current module candidate should be returned"); - - assert_eq!(selected.0, PathBuf::from("/tmp/current")); - assert_eq!( - selected.1.location, - VariableLocation::Address(AddressExpr::constant(0x1000)) - ); - } - - #[test] - fn global_plan_selection_rejects_ambiguous_current_module_matches() { - let err = DwarfAnalyzer::select_global_plan_with_preferred_module( - "state", - Path::new("/tmp/current"), - vec![ - (PathBuf::from("/tmp/current"), global_plan("state", 0x1000)), - (PathBuf::from("/tmp/current"), global_plan("state", 0x1004)), - (PathBuf::from("/tmp/other"), global_plan("state", 0x2000)), - ], - ) - .expect_err("duplicate current-module candidates should be ambiguous"); - - assert!(err.to_string().contains("Ambiguous global 'state'")); - assert!(err.to_string().contains("2 matches")); - assert!(err.to_string().contains("/tmp/current")); - assert!(!err.to_string().contains("/tmp/other")); - } -} diff --git a/ghostscope-dwarf/src/analyzer/tests.rs b/ghostscope-dwarf/src/analyzer/tests.rs new file mode 100644 index 00000000..64092a39 --- /dev/null +++ b/ghostscope-dwarf/src/analyzer/tests.rs @@ -0,0 +1,163 @@ +use super::DwarfAnalyzer; +use crate::{ + core::{AddressExpr, Availability, Provenance, VariableLocation}, + semantics::{VariableReadPlan, VisibleVariable}, +}; +use std::path::{Path, PathBuf}; + +fn global_plan(name: &str, address: u64) -> VariableReadPlan { + VariableReadPlan { + name: name.to_string(), + type_name: "int".to_string(), + access_path: crate::VariableAccessPath::default(), + module_path: None, + dwarf_type: Some(crate::TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }), + declaration: None, + type_id: None, + location: VariableLocation::Address(AddressExpr::constant(address)), + availability: Availability::Available, + scope_depth: 0, + is_parameter: false, + is_artificial: false, + pc_range: None, + inline_context: None, + provenance: Provenance::Synthesized { + detail: "test".to_string(), + }, + } +} + +fn visible_var(name: &str, scope_depth: usize) -> VisibleVariable { + VisibleVariable { + name: name.to_string(), + type_name: "int".to_string(), + dwarf_type: Some(crate::TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }), + declaration: None, + type_id: None, + location: VariableLocation::RegisterValue { dwarf_reg: 0 }, + availability: Availability::Available, + scope_depth, + is_parameter: false, + is_artificial: false, + } +} + +fn diagnostic( + name: &str, + scope_depth: usize, + detail: &str, +) -> crate::semantics::VariableQueryDiagnostic { + crate::semantics::VariableQueryDiagnostic { + pc: 0x1234, + name: Some(name.to_string()), + scope_depth, + availability: Availability::Unsupported(crate::UnsupportedReason::ExpressionShape { + detail: detail.to_string(), + }), + detail: detail.to_string(), + } +} + +#[test] +fn variable_selection_rejects_inner_diagnostic_over_outer_match() { + let err = DwarfAnalyzer::select_visible_variable_by_name( + 0x1234, + "state", + vec![visible_var("state", 1)], + &[diagnostic("state", 2, "DW_OP_bad is unsupported")], + ) + .expect_err("inner unavailable variable should block outer fallback"); + + assert!(err.to_string().contains("Unavailable variable 'state'")); + assert!(err.to_string().contains("DW_OP_bad is unsupported")); +} + +#[test] +fn variable_selection_keeps_inner_match_over_outer_diagnostic() { + let selected = DwarfAnalyzer::select_visible_variable_by_name( + 0x1234, + "state", + vec![visible_var("state", 2)], + &[diagnostic("state", 1, "outer variable is unavailable")], + ) + .expect("outer diagnostic should not block inner match") + .expect("inner match should be returned"); + + assert_eq!(selected.name, "state"); + assert_eq!(selected.scope_depth, 2); +} + +#[test] +fn global_plan_selection_rejects_ambiguous_matches() { + let err = DwarfAnalyzer::select_unambiguous_global_plan( + "state", + vec![ + (PathBuf::from("/tmp/a"), global_plan("state", 0x1000)), + (PathBuf::from("/tmp/b"), global_plan("state", 0x2000)), + ], + ) + .expect_err("multiple global candidates should be ambiguous"); + + assert!(err.to_string().contains("Ambiguous global 'state'")); + assert!(err.to_string().contains("2 matches")); +} + +#[test] +fn global_plan_selection_accepts_single_match() { + let selected = DwarfAnalyzer::select_unambiguous_global_plan( + "state", + vec![(PathBuf::from("/tmp/a"), global_plan("state", 0x1000))], + ) + .expect("single global candidate should be accepted") + .expect("single global candidate should be returned"); + + assert_eq!(selected.0, PathBuf::from("/tmp/a")); + assert_eq!(selected.1.name, "state"); +} + +#[test] +fn global_plan_selection_prefers_current_module_match() { + let selected = DwarfAnalyzer::select_global_plan_with_preferred_module( + "state", + Path::new("/tmp/current"), + vec![ + (PathBuf::from("/tmp/other"), global_plan("state", 0x2000)), + (PathBuf::from("/tmp/current"), global_plan("state", 0x1000)), + ], + ) + .expect("current module candidate should be accepted") + .expect("current module candidate should be returned"); + + assert_eq!(selected.0, PathBuf::from("/tmp/current")); + assert_eq!( + selected.1.location, + VariableLocation::Address(AddressExpr::constant(0x1000)) + ); +} + +#[test] +fn global_plan_selection_rejects_ambiguous_current_module_matches() { + let err = DwarfAnalyzer::select_global_plan_with_preferred_module( + "state", + Path::new("/tmp/current"), + vec![ + (PathBuf::from("/tmp/current"), global_plan("state", 0x1000)), + (PathBuf::from("/tmp/current"), global_plan("state", 0x1004)), + (PathBuf::from("/tmp/other"), global_plan("state", 0x2000)), + ], + ) + .expect_err("duplicate current-module candidates should be ambiguous"); + + assert!(err.to_string().contains("Ambiguous global 'state'")); + assert!(err.to_string().contains("2 matches")); + assert!(err.to_string().contains("/tmp/current")); + assert!(!err.to_string().contains("/tmp/other")); +} diff --git a/ghostscope-dwarf/src/analyzer/types.rs b/ghostscope-dwarf/src/analyzer/types.rs new file mode 100644 index 00000000..959baf7b --- /dev/null +++ b/ghostscope-dwarf/src/analyzer/types.rs @@ -0,0 +1,144 @@ +use crate::{core::DebugInfoSource, semantics::VisibleVariable}; +use std::path::PathBuf; + +/// Events emitted during module loading process +#[derive(Debug, Clone)] +pub enum ModuleLoadingEvent { + /// Module discovered during process scanning + Discovered { + module_path: String, + current: usize, + total: usize, + }, + /// Module loading started + LoadingStarted { + module_path: String, + current: usize, + total: usize, + }, + /// Module loading completed successfully + LoadingCompleted { + module_path: String, + stats: ModuleLoadingStats, + current: usize, + total: usize, + }, + /// Module loading failed + LoadingFailed { + module_path: String, + error: String, + current: usize, + total: usize, + }, +} + +/// Statistics for a loaded module +#[derive(Debug, Clone)] +pub struct ModuleLoadingStats { + pub functions: usize, + pub variables: usize, + pub types: usize, + pub debug_info_source: DebugInfoSource, + pub load_time_ms: u64, + pub parse_time_ms: u64, + pub index_time_ms: u64, + pub module_total_time_ms: u64, +} + +/// Rich query result for a single address within a module. +#[derive(Debug, Clone)] +pub struct AddressQueryResult { + pub module_path: PathBuf, + pub address: u64, + pub source_file: Option, + pub source_line: Option, + pub source_column: Option, + pub function_name: Option, + pub is_inline: Option, + pub variables: Vec, + pub parameters: Vec, +} + +/// Runtime mapping metadata for a loaded module. +#[derive(Debug, Clone)] +pub struct LoadedModuleRuntimeInfo { + pub module_path: PathBuf, + pub loaded_address: Option, + pub load_bias: Option, + pub size: u64, +} + +/// Rich query result for a function lookup across modules. +#[derive(Debug, Clone)] +pub struct FunctionQueryResult { + pub function_name: String, + pub addresses: Vec, +} + +/// Module statistics compatible with ghostscope-binary +#[derive(Debug, Clone)] +pub struct ModuleStats { + pub total_modules: usize, + pub executable_modules: usize, + pub library_modules: usize, + pub total_symbols: usize, + pub modules_with_debug_info: usize, +} + +/// Main executable information +#[derive(Debug, Clone)] +pub struct MainExecutableInfo { + pub path: String, +} + +/// Statistics for debugging and monitoring +#[derive(Debug, Clone)] +pub struct AnalyzerStats { + pub pid: u32, + pub module_count: usize, + pub total_functions: usize, + pub total_variables: usize, + pub total_line_headers: usize, +} + +/// Shared library information (compatible with ghostscope-ui) +#[derive(Debug, Clone)] +pub struct SharedLibraryInfo { + pub from_address: u64, // Starting address in memory + pub to_address: u64, // Ending address in memory + pub symbols_read: bool, // Whether symbols were successfully read + pub debug_info_available: bool, // Whether debug information is available + pub library_path: String, // Full path to the library file + pub size: u64, // Size of the library in memory + pub debug_file_path: Option, // Path to separate debug file (if via .gnu_debuglink) +} + +/// Executable file information (for "info file" command) +#[derive(Debug, Clone)] +pub struct ExecutableFileInfo { + pub file_path: String, + pub file_type: String, + pub entry_point: Option, + pub has_symbols: bool, + pub has_debug_info: bool, + pub debug_file_path: Option, + pub text_section: Option, + pub data_section: Option, + pub mode_description: String, +} + +/// Section information for executable files +#[derive(Debug, Clone)] +pub struct SectionInfo { + pub start_address: u64, + pub end_address: u64, + pub size: u64, +} + +/// Simple file information compatible with ghostscope-binary +#[derive(Debug, Clone)] +pub struct SimpleFileInfo { + pub full_path: String, + pub basename: String, + pub directory: String, +} diff --git a/ghostscope-dwarf/src/dwarf_expr/lower.rs b/ghostscope-dwarf/src/dwarf_expr/lower/mod.rs similarity index 77% rename from ghostscope-dwarf/src/dwarf_expr/lower.rs rename to ghostscope-dwarf/src/dwarf_expr/lower/mod.rs index 63876000..5025cf7b 100644 --- a/ghostscope-dwarf/src/dwarf_expr/lower.rs +++ b/ghostscope-dwarf/src/dwarf_expr/lower/mod.rs @@ -1618,610 +1618,4 @@ impl ExpressionEvaluator { } #[cfg(test)] -mod tests { - use super::ExpressionEvaluator; - use crate::core::{ - AddressExpr, Availability, CfaResult, DirectValueResult, LocationResult, MemoryAccessSize, - ParsedLocation, PieceLocation, PieceResult, PlanExprOp, RawExpressionResult, - VariableLocation, - }; - use gimli::constants; - use gimli::RunTimeEndian; - - fn test_encoding() -> gimli::Encoding { - gimli::Encoding { - format: gimli::Format::Dwarf32, - version: 5, - address_size: 8, - } - } - - fn encode_uleb(mut value: u64) -> Vec { - let mut out = Vec::new(); - loop { - let mut byte = (value & 0x7f) as u8; - value >>= 7; - if value != 0 { - byte |= 0x80; - } - out.push(byte); - if value == 0 { - break; - } - } - out - } - - #[test] - fn converts_register_address_result_to_semantic_location() { - let result = RawExpressionResult::MemoryLocation(LocationResult::RegisterAddress { - register: 6, - offset: Some(-16), - size: None, - }); - - assert_eq!( - ExpressionEvaluator::variable_location_from_result(&result), - VariableLocation::RegisterAddress { - dwarf_reg: 6, - offset: -16 - } - ); - } - - #[test] - fn converts_absolute_address_value_as_rebasable_value() { - let result = RawExpressionResult::DirectValue(DirectValueResult::AbsoluteAddress(0x1234)); - - assert_eq!( - ExpressionEvaluator::variable_location_from_result(&result), - VariableLocation::AbsoluteAddressValue(AddressExpr::constant(0x1234)) - ); - } - - #[test] - fn converts_composite_result_to_semantic_pieces() { - let result = RawExpressionResult::Composite(vec![PieceResult { - location: RawExpressionResult::DirectValue(DirectValueResult::RegisterValue(0)), - size: 4, - bit_offset: Some(32), - }]); - - assert_eq!( - ExpressionEvaluator::variable_location_from_result(&result), - VariableLocation::Pieces(vec![PieceLocation { - bit_offset: 32, - bit_size: 32, - location: Box::new(VariableLocation::RegisterValue { dwarf_reg: 0 }), - }]) - ); - } - - fn encode_sleb(mut value: i64) -> Vec { - let mut out = Vec::new(); - loop { - let mut byte = (value as u8) & 0x7f; - value >>= 7; - let sign_bit_set = (byte & 0x40) != 0; - let done = (value == 0 && !sign_bit_set) || (value == -1 && sign_bit_set); - if !done { - byte |= 0x80; - } - out.push(byte); - if done { - break; - } - } - out - } - - fn parse_test_expr(bytes: &[u8]) -> anyhow::Result { - ExpressionEvaluator::parse_expression_with_context( - bytes, - RunTimeEndian::Little, - test_encoding(), - None, - None, - 0, - None, - None, - None, - 0, - ) - } - - fn lower_test_expr(bytes: &[u8]) -> anyhow::Result { - let raw = parse_test_expr(bytes)?; - Ok(ExpressionEvaluator::parsed_location_from_result(&raw)) - } - - fn addr_expr(address: u64) -> Vec { - let mut bytes = vec![constants::DW_OP_addr.0]; - bytes.extend(address.to_le_bytes()); - bytes - } - - fn regx_expr(register: u64) -> Vec { - let mut bytes = vec![constants::DW_OP_regx.0]; - bytes.extend(encode_uleb(register)); - bytes - } - - fn bregx_expr(register: u64, offset: i64) -> Vec { - let mut bytes = vec![constants::DW_OP_bregx.0]; - bytes.extend(encode_uleb(register)); - bytes.extend(encode_sleb(offset)); - bytes - } - - #[test] - fn dwarf_expression_lowering_returns_register_value_parsed_location() { - let parsed = lower_test_expr(&[constants::DW_OP_reg5.0]) - .expect("DW_OP_reg5 should lower to ParsedLocation"); - - assert_eq!( - parsed.location, - VariableLocation::RegisterValue { dwarf_reg: 5 } - ); - assert_eq!(parsed.availability, Availability::Available); - } - - #[test] - fn dwarf_expression_lowering_returns_register_address_parsed_location() { - let parsed = lower_test_expr(&bregx_expr(6, -16)) - .expect("DW_OP_bregx should lower to ParsedLocation"); - - assert_eq!( - parsed.location, - VariableLocation::RegisterAddress { - dwarf_reg: 6, - offset: -16 - } - ); - assert_eq!(parsed.availability, Availability::Available); - } - - #[test] - fn dwarf_expression_lowering_returns_stack_value_parsed_location() { - let parsed = lower_test_expr(&[constants::DW_OP_lit1.0, constants::DW_OP_stack_value.0]) - .expect("DW_OP_stack_value should lower to ParsedLocation"); - - assert_eq!( - parsed.location, - VariableLocation::ComputedValue(vec![PlanExprOp::PushConstant(1)]) - ); - assert_eq!(parsed.availability, Availability::Available); - } - - #[test] - fn dwarf_expression_lowering_marks_mixed_pieces_partially_available() { - let mut bytes = vec![constants::DW_OP_piece.0]; - bytes.extend(encode_uleb(4)); - bytes.push(constants::DW_OP_reg0.0); - bytes.push(constants::DW_OP_piece.0); - bytes.extend(encode_uleb(4)); - - let parsed = lower_test_expr(&bytes) - .expect("mixed DW_OP_piece expression should lower to ParsedLocation"); - - assert_eq!( - parsed.location, - VariableLocation::Pieces(vec![ - PieceLocation { - bit_offset: 0, - bit_size: 32, - location: Box::new(VariableLocation::OptimizedOut), - }, - PieceLocation { - bit_offset: 32, - bit_size: 32, - location: Box::new(VariableLocation::RegisterValue { dwarf_reg: 0 }), - }, - ]) - ); - assert_eq!(parsed.availability, Availability::PartiallyAvailable); - } - - #[test] - fn dwarf_op_supported_coverage_matrix() { - let cases = vec![ - ( - "DW_OP_regN", - vec![constants::DW_OP_reg5.0], - RawExpressionResult::DirectValue(DirectValueResult::RegisterValue(5)), - ), - ( - "DW_OP_regx", - regx_expr(33), - RawExpressionResult::DirectValue(DirectValueResult::RegisterValue(33)), - ), - ( - "DW_OP_bregN", - { - let mut bytes = vec![constants::DW_OP_breg7.0]; - bytes.extend(encode_sleb(8)); - bytes - }, - RawExpressionResult::MemoryLocation(LocationResult::RegisterAddress { - register: 7, - offset: Some(8), - size: None, - }), - ), - ( - "DW_OP_bregx", - bregx_expr(33, -2), - RawExpressionResult::MemoryLocation(LocationResult::RegisterAddress { - register: 33, - offset: Some(-2), - size: None, - }), - ), - ( - "DW_OP_addr", - addr_expr(0x1234), - RawExpressionResult::MemoryLocation(LocationResult::Address(0x1234)), - ), - ( - "DW_OP_stack_value", - vec![constants::DW_OP_lit1.0, constants::DW_OP_stack_value.0], - RawExpressionResult::DirectValue(DirectValueResult::Constant(1)), - ), - ( - "DW_OP_addr stack value", - { - let mut bytes = addr_expr(0x1234); - bytes.push(constants::DW_OP_stack_value.0); - bytes - }, - RawExpressionResult::DirectValue(DirectValueResult::AbsoluteAddress(0x1234)), - ), - ( - "arithmetic stack value subset", - vec![ - constants::DW_OP_lit1.0, - constants::DW_OP_lit2.0, - constants::DW_OP_plus.0, - constants::DW_OP_stack_value.0, - ], - RawExpressionResult::DirectValue(DirectValueResult::ComputedValue { - steps: vec![ - PlanExprOp::PushConstant(1), - PlanExprOp::PushConstant(2), - PlanExprOp::Add, - ], - result_size: MemoryAccessSize::U64, - }), - ), - ( - "DW_OP_implicit_value", - vec![constants::DW_OP_implicit_value.0, 3, 0xaa, 0xbb, 0xcc], - RawExpressionResult::DirectValue(DirectValueResult::ImplicitValue(vec![ - 0xaa, 0xbb, 0xcc, - ])), - ), - ( - "DW_OP_piece split stack values", - { - let mut bytes = vec![constants::DW_OP_breg5.0]; - bytes.extend(encode_sleb(1)); - bytes.push(constants::DW_OP_stack_value.0); - bytes.push(constants::DW_OP_piece.0); - bytes.extend(encode_uleb(4)); - bytes.push(constants::DW_OP_breg5.0); - bytes.extend(encode_sleb(2)); - bytes.push(constants::DW_OP_stack_value.0); - bytes.push(constants::DW_OP_piece.0); - bytes.extend(encode_uleb(4)); - bytes - }, - RawExpressionResult::Composite(vec![ - PieceResult { - location: RawExpressionResult::DirectValue( - DirectValueResult::ComputedValue { - steps: vec![ - PlanExprOp::LoadRegister(5), - PlanExprOp::PushConstant(1), - PlanExprOp::Add, - ], - result_size: MemoryAccessSize::U64, - }, - ), - size: 4, - bit_offset: Some(0), - }, - PieceResult { - location: RawExpressionResult::DirectValue( - DirectValueResult::ComputedValue { - steps: vec![ - PlanExprOp::LoadRegister(5), - PlanExprOp::PushConstant(2), - PlanExprOp::Add, - ], - result_size: MemoryAccessSize::U64, - }, - ), - size: 4, - bit_offset: Some(32), - }, - ]), - ), - ( - "DW_OP_piece empty segment", - { - let mut bytes = vec![constants::DW_OP_piece.0]; - bytes.extend(encode_uleb(4)); - bytes - }, - RawExpressionResult::Composite(vec![PieceResult { - location: RawExpressionResult::Optimized, - size: 4, - bit_offset: Some(0), - }]), - ), - ]; - - for (name, bytes, expected) in cases { - let result = parse_test_expr(&bytes).unwrap_or_else(|error| { - panic!("{name} should parse successfully, bytes={bytes:?}: {error}") - }); - assert_eq!(result, expected, "{name} lowered incorrectly"); - } - } - - #[test] - fn dwarf_op_fbreg_coverage_uses_cfa_provider() { - let get_cfa = |_address| { - Ok(Some(CfaResult::RegisterPlusOffset { - register: 7, - offset: 16, - })) - }; - let mut expr = vec![constants::DW_OP_fbreg.0]; - expr.extend(encode_sleb(4)); - - let result = ExpressionEvaluator::parse_expression_with_context( - &expr, - RunTimeEndian::Little, - test_encoding(), - None, - None, - 0, - Some(&get_cfa), - None, - None, - 0, - ) - .expect("DW_OP_fbreg should parse with a CFA provider"); - - assert_eq!( - result, - RawExpressionResult::MemoryLocation(LocationResult::RegisterAddress { - register: 7, - offset: Some(20), - size: None, - }) - ); - } - - #[test] - fn dwarf_frame_base_reg_lowers_to_register_cfa() { - let cfa = ExpressionEvaluator::raw_expression_result_to_cfa( - RawExpressionResult::DirectValue(DirectValueResult::RegisterValue(6)), - ); - - assert_eq!( - cfa, - Some(CfaResult::RegisterPlusOffset { - register: 6, - offset: 0, - }) - ); - } - - #[test] - fn dwarf_op_unsupported_diagnostic_matrix_names_ops() { - let cases = vec![ - ( - "DW_OP_drop", - vec![ - constants::DW_OP_lit1.0, - constants::DW_OP_drop.0, - constants::DW_OP_stack_value.0, - ], - "DW_OP_drop", - ), - ( - "DW_OP_bit_piece", - { - let mut bytes = vec![constants::DW_OP_lit1.0, constants::DW_OP_bit_piece.0]; - bytes.extend(encode_uleb(8)); - bytes.extend(encode_uleb(0)); - bytes - }, - "DW_OP_bit_piece", - ), - ( - "DW_OP_addrx", - { - let mut bytes = vec![constants::DW_OP_addrx.0]; - bytes.extend(encode_uleb(0)); - bytes - }, - "DW_OP_addrx", - ), - ( - "DW_OP_bra", - vec![ - constants::DW_OP_lit1.0, - constants::DW_OP_bra.0, - 0, - 0, - constants::DW_OP_stack_value.0, - ], - "DW_OP_bra", - ), - ]; - - for (name, bytes, expected_op) in cases { - let error = match parse_test_expr(&bytes) { - Ok(result) => panic!("{name} should be unsupported, got {result:?}"), - Err(error) => error, - }; - let message = error.to_string(); - assert!( - message.contains(expected_op), - "{name} diagnostic should mention {expected_op}, got: {message}" - ); - assert!( - message.contains("unsupported"), - "{name} diagnostic should be explicit, got: {message}" - ); - assert_eq!( - crate::dwarf_expr::ops::unsupported_op_from_error(&error), - Some(expected_op), - "{name} diagnostic should carry a typed unsupported-op cause" - ); - } - } - - #[test] - fn implicit_pointer_to_static_storage_preserves_absolute_address_semantics() { - let result = ExpressionEvaluator::addressable_location_to_pointer_value( - RawExpressionResult::MemoryLocation(LocationResult::Address(0x1234)), - 0x10, - ) - .expect("static address should convert to an implicit pointer value"); - - assert_eq!( - result, - RawExpressionResult::DirectValue(DirectValueResult::AbsoluteAddress(0x1244)) - ); - } - - #[test] - fn implicit_pointer_accepts_target_absolute_address_value() { - let result = ExpressionEvaluator::addressable_location_to_pointer_value( - RawExpressionResult::DirectValue(DirectValueResult::AbsoluteAddress(0x1234)), - 0x10, - ) - .expect("static address values should convert to an implicit pointer value"); - - assert_eq!( - result, - RawExpressionResult::DirectValue(DirectValueResult::AbsoluteAddress(0x1244)) - ); - } - - #[test] - fn multi_op_expression_rejects_invalid_opcode_after_valid_prefix() { - let expr_bytes = [ - constants::DW_OP_lit1.0, - 0xff, - constants::DW_OP_stack_value.0, - ]; - - let error = ExpressionEvaluator::parse_expression_with_context( - &expr_bytes, - RunTimeEndian::Little, - test_encoding(), - None, - None, - 0, - None, - None, - None, - 0, - ) - .expect_err("invalid opcode after a valid prefix must not return a value"); - - assert!( - error.to_string().contains("DWARF expression"), - "unexpected error: {error}" - ); - } - - #[test] - fn multi_op_expression_rejects_unsupported_operation_after_valid_prefix() { - let expr_bytes = [ - constants::DW_OP_lit1.0, - constants::DW_OP_drop.0, - constants::DW_OP_stack_value.0, - ]; - - let error = ExpressionEvaluator::parse_expression_with_context( - &expr_bytes, - RunTimeEndian::Little, - test_encoding(), - None, - None, - 0, - None, - None, - None, - 0, - ) - .expect_err("unsupported operation after a valid prefix must not return a value"); - - assert!( - error.to_string().contains("unsupported"), - "unexpected error: {error}" - ); - } - - #[test] - fn single_fbreg_fast_path_saturates_cfa_offset_addition() { - let get_cfa = |_address| { - Ok(Some(CfaResult::RegisterPlusOffset { - register: 7, - offset: i64::MAX - 3, - })) - }; - - let result = ExpressionEvaluator::parse_expression_with_context( - &[0x91, 0x0a], - RunTimeEndian::Little, - test_encoding(), - None, - None, - 0, - Some(&get_cfa), - None, - None, - 0, - ) - .expect("single DW_OP_fbreg should parse"); - - assert_eq!( - result, - RawExpressionResult::MemoryLocation(LocationResult::RegisterAddress { - register: 7, - offset: Some(i64::MAX), - size: None, - }) - ); - } - - #[test] - fn big_endian_dw_op_addr_preserves_absolute_address() { - let expr_bytes = [0x03, 0, 0, 0, 0, 0, 0, 0x12, 0x34]; - let result = ExpressionEvaluator::parse_expression_with_context( - &expr_bytes, - gimli::RunTimeEndian::Big, - test_encoding(), - None, - None, - 0, - None, - None, - None, - 0, - ) - .expect("big-endian DW_OP_addr should parse"); - - assert_eq!( - result, - RawExpressionResult::MemoryLocation(LocationResult::Address(0x1234)) - ); - } -} +mod tests; diff --git a/ghostscope-dwarf/src/dwarf_expr/lower/tests.rs b/ghostscope-dwarf/src/dwarf_expr/lower/tests.rs new file mode 100644 index 00000000..14632cde --- /dev/null +++ b/ghostscope-dwarf/src/dwarf_expr/lower/tests.rs @@ -0,0 +1,600 @@ +use super::ExpressionEvaluator; +use crate::core::{ + AddressExpr, Availability, CfaResult, DirectValueResult, LocationResult, MemoryAccessSize, + ParsedLocation, PieceLocation, PieceResult, PlanExprOp, RawExpressionResult, VariableLocation, +}; +use gimli::constants; +use gimli::RunTimeEndian; + +fn test_encoding() -> gimli::Encoding { + gimli::Encoding { + format: gimli::Format::Dwarf32, + version: 5, + address_size: 8, + } +} + +fn encode_uleb(mut value: u64) -> Vec { + let mut out = Vec::new(); + loop { + let mut byte = (value & 0x7f) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + out.push(byte); + if value == 0 { + break; + } + } + out +} + +#[test] +fn converts_register_address_result_to_semantic_location() { + let result = RawExpressionResult::MemoryLocation(LocationResult::RegisterAddress { + register: 6, + offset: Some(-16), + size: None, + }); + + assert_eq!( + ExpressionEvaluator::variable_location_from_result(&result), + VariableLocation::RegisterAddress { + dwarf_reg: 6, + offset: -16 + } + ); +} + +#[test] +fn converts_absolute_address_value_as_rebasable_value() { + let result = RawExpressionResult::DirectValue(DirectValueResult::AbsoluteAddress(0x1234)); + + assert_eq!( + ExpressionEvaluator::variable_location_from_result(&result), + VariableLocation::AbsoluteAddressValue(AddressExpr::constant(0x1234)) + ); +} + +#[test] +fn converts_composite_result_to_semantic_pieces() { + let result = RawExpressionResult::Composite(vec![PieceResult { + location: RawExpressionResult::DirectValue(DirectValueResult::RegisterValue(0)), + size: 4, + bit_offset: Some(32), + }]); + + assert_eq!( + ExpressionEvaluator::variable_location_from_result(&result), + VariableLocation::Pieces(vec![PieceLocation { + bit_offset: 32, + bit_size: 32, + location: Box::new(VariableLocation::RegisterValue { dwarf_reg: 0 }), + }]) + ); +} + +fn encode_sleb(mut value: i64) -> Vec { + let mut out = Vec::new(); + loop { + let mut byte = (value as u8) & 0x7f; + value >>= 7; + let sign_bit_set = (byte & 0x40) != 0; + let done = (value == 0 && !sign_bit_set) || (value == -1 && sign_bit_set); + if !done { + byte |= 0x80; + } + out.push(byte); + if done { + break; + } + } + out +} + +fn parse_test_expr(bytes: &[u8]) -> anyhow::Result { + ExpressionEvaluator::parse_expression_with_context( + bytes, + RunTimeEndian::Little, + test_encoding(), + None, + None, + 0, + None, + None, + None, + 0, + ) +} + +fn lower_test_expr(bytes: &[u8]) -> anyhow::Result { + let raw = parse_test_expr(bytes)?; + Ok(ExpressionEvaluator::parsed_location_from_result(&raw)) +} + +fn addr_expr(address: u64) -> Vec { + let mut bytes = vec![constants::DW_OP_addr.0]; + bytes.extend(address.to_le_bytes()); + bytes +} + +fn regx_expr(register: u64) -> Vec { + let mut bytes = vec![constants::DW_OP_regx.0]; + bytes.extend(encode_uleb(register)); + bytes +} + +fn bregx_expr(register: u64, offset: i64) -> Vec { + let mut bytes = vec![constants::DW_OP_bregx.0]; + bytes.extend(encode_uleb(register)); + bytes.extend(encode_sleb(offset)); + bytes +} + +#[test] +fn dwarf_expression_lowering_returns_register_value_parsed_location() { + let parsed = lower_test_expr(&[constants::DW_OP_reg5.0]) + .expect("DW_OP_reg5 should lower to ParsedLocation"); + + assert_eq!( + parsed.location, + VariableLocation::RegisterValue { dwarf_reg: 5 } + ); + assert_eq!(parsed.availability, Availability::Available); +} + +#[test] +fn dwarf_expression_lowering_returns_register_address_parsed_location() { + let parsed = + lower_test_expr(&bregx_expr(6, -16)).expect("DW_OP_bregx should lower to ParsedLocation"); + + assert_eq!( + parsed.location, + VariableLocation::RegisterAddress { + dwarf_reg: 6, + offset: -16 + } + ); + assert_eq!(parsed.availability, Availability::Available); +} + +#[test] +fn dwarf_expression_lowering_returns_stack_value_parsed_location() { + let parsed = lower_test_expr(&[constants::DW_OP_lit1.0, constants::DW_OP_stack_value.0]) + .expect("DW_OP_stack_value should lower to ParsedLocation"); + + assert_eq!( + parsed.location, + VariableLocation::ComputedValue(vec![PlanExprOp::PushConstant(1)]) + ); + assert_eq!(parsed.availability, Availability::Available); +} + +#[test] +fn dwarf_expression_lowering_marks_mixed_pieces_partially_available() { + let mut bytes = vec![constants::DW_OP_piece.0]; + bytes.extend(encode_uleb(4)); + bytes.push(constants::DW_OP_reg0.0); + bytes.push(constants::DW_OP_piece.0); + bytes.extend(encode_uleb(4)); + + let parsed = lower_test_expr(&bytes) + .expect("mixed DW_OP_piece expression should lower to ParsedLocation"); + + assert_eq!( + parsed.location, + VariableLocation::Pieces(vec![ + PieceLocation { + bit_offset: 0, + bit_size: 32, + location: Box::new(VariableLocation::OptimizedOut), + }, + PieceLocation { + bit_offset: 32, + bit_size: 32, + location: Box::new(VariableLocation::RegisterValue { dwarf_reg: 0 }), + }, + ]) + ); + assert_eq!(parsed.availability, Availability::PartiallyAvailable); +} + +#[test] +fn dwarf_op_supported_coverage_matrix() { + let cases = vec![ + ( + "DW_OP_regN", + vec![constants::DW_OP_reg5.0], + RawExpressionResult::DirectValue(DirectValueResult::RegisterValue(5)), + ), + ( + "DW_OP_regx", + regx_expr(33), + RawExpressionResult::DirectValue(DirectValueResult::RegisterValue(33)), + ), + ( + "DW_OP_bregN", + { + let mut bytes = vec![constants::DW_OP_breg7.0]; + bytes.extend(encode_sleb(8)); + bytes + }, + RawExpressionResult::MemoryLocation(LocationResult::RegisterAddress { + register: 7, + offset: Some(8), + size: None, + }), + ), + ( + "DW_OP_bregx", + bregx_expr(33, -2), + RawExpressionResult::MemoryLocation(LocationResult::RegisterAddress { + register: 33, + offset: Some(-2), + size: None, + }), + ), + ( + "DW_OP_addr", + addr_expr(0x1234), + RawExpressionResult::MemoryLocation(LocationResult::Address(0x1234)), + ), + ( + "DW_OP_stack_value", + vec![constants::DW_OP_lit1.0, constants::DW_OP_stack_value.0], + RawExpressionResult::DirectValue(DirectValueResult::Constant(1)), + ), + ( + "DW_OP_addr stack value", + { + let mut bytes = addr_expr(0x1234); + bytes.push(constants::DW_OP_stack_value.0); + bytes + }, + RawExpressionResult::DirectValue(DirectValueResult::AbsoluteAddress(0x1234)), + ), + ( + "arithmetic stack value subset", + vec![ + constants::DW_OP_lit1.0, + constants::DW_OP_lit2.0, + constants::DW_OP_plus.0, + constants::DW_OP_stack_value.0, + ], + RawExpressionResult::DirectValue(DirectValueResult::ComputedValue { + steps: vec![ + PlanExprOp::PushConstant(1), + PlanExprOp::PushConstant(2), + PlanExprOp::Add, + ], + result_size: MemoryAccessSize::U64, + }), + ), + ( + "DW_OP_implicit_value", + vec![constants::DW_OP_implicit_value.0, 3, 0xaa, 0xbb, 0xcc], + RawExpressionResult::DirectValue(DirectValueResult::ImplicitValue(vec![ + 0xaa, 0xbb, 0xcc, + ])), + ), + ( + "DW_OP_piece split stack values", + { + let mut bytes = vec![constants::DW_OP_breg5.0]; + bytes.extend(encode_sleb(1)); + bytes.push(constants::DW_OP_stack_value.0); + bytes.push(constants::DW_OP_piece.0); + bytes.extend(encode_uleb(4)); + bytes.push(constants::DW_OP_breg5.0); + bytes.extend(encode_sleb(2)); + bytes.push(constants::DW_OP_stack_value.0); + bytes.push(constants::DW_OP_piece.0); + bytes.extend(encode_uleb(4)); + bytes + }, + RawExpressionResult::Composite(vec![ + PieceResult { + location: RawExpressionResult::DirectValue(DirectValueResult::ComputedValue { + steps: vec![ + PlanExprOp::LoadRegister(5), + PlanExprOp::PushConstant(1), + PlanExprOp::Add, + ], + result_size: MemoryAccessSize::U64, + }), + size: 4, + bit_offset: Some(0), + }, + PieceResult { + location: RawExpressionResult::DirectValue(DirectValueResult::ComputedValue { + steps: vec![ + PlanExprOp::LoadRegister(5), + PlanExprOp::PushConstant(2), + PlanExprOp::Add, + ], + result_size: MemoryAccessSize::U64, + }), + size: 4, + bit_offset: Some(32), + }, + ]), + ), + ( + "DW_OP_piece empty segment", + { + let mut bytes = vec![constants::DW_OP_piece.0]; + bytes.extend(encode_uleb(4)); + bytes + }, + RawExpressionResult::Composite(vec![PieceResult { + location: RawExpressionResult::Optimized, + size: 4, + bit_offset: Some(0), + }]), + ), + ]; + + for (name, bytes, expected) in cases { + let result = parse_test_expr(&bytes).unwrap_or_else(|error| { + panic!("{name} should parse successfully, bytes={bytes:?}: {error}") + }); + assert_eq!(result, expected, "{name} lowered incorrectly"); + } +} + +#[test] +fn dwarf_op_fbreg_coverage_uses_cfa_provider() { + let get_cfa = |_address| { + Ok(Some(CfaResult::RegisterPlusOffset { + register: 7, + offset: 16, + })) + }; + let mut expr = vec![constants::DW_OP_fbreg.0]; + expr.extend(encode_sleb(4)); + + let result = ExpressionEvaluator::parse_expression_with_context( + &expr, + RunTimeEndian::Little, + test_encoding(), + None, + None, + 0, + Some(&get_cfa), + None, + None, + 0, + ) + .expect("DW_OP_fbreg should parse with a CFA provider"); + + assert_eq!( + result, + RawExpressionResult::MemoryLocation(LocationResult::RegisterAddress { + register: 7, + offset: Some(20), + size: None, + }) + ); +} + +#[test] +fn dwarf_frame_base_reg_lowers_to_register_cfa() { + let cfa = ExpressionEvaluator::raw_expression_result_to_cfa(RawExpressionResult::DirectValue( + DirectValueResult::RegisterValue(6), + )); + + assert_eq!( + cfa, + Some(CfaResult::RegisterPlusOffset { + register: 6, + offset: 0, + }) + ); +} + +#[test] +fn dwarf_op_unsupported_diagnostic_matrix_names_ops() { + let cases = vec![ + ( + "DW_OP_drop", + vec![ + constants::DW_OP_lit1.0, + constants::DW_OP_drop.0, + constants::DW_OP_stack_value.0, + ], + "DW_OP_drop", + ), + ( + "DW_OP_bit_piece", + { + let mut bytes = vec![constants::DW_OP_lit1.0, constants::DW_OP_bit_piece.0]; + bytes.extend(encode_uleb(8)); + bytes.extend(encode_uleb(0)); + bytes + }, + "DW_OP_bit_piece", + ), + ( + "DW_OP_addrx", + { + let mut bytes = vec![constants::DW_OP_addrx.0]; + bytes.extend(encode_uleb(0)); + bytes + }, + "DW_OP_addrx", + ), + ( + "DW_OP_bra", + vec![ + constants::DW_OP_lit1.0, + constants::DW_OP_bra.0, + 0, + 0, + constants::DW_OP_stack_value.0, + ], + "DW_OP_bra", + ), + ]; + + for (name, bytes, expected_op) in cases { + let error = match parse_test_expr(&bytes) { + Ok(result) => panic!("{name} should be unsupported, got {result:?}"), + Err(error) => error, + }; + let message = error.to_string(); + assert!( + message.contains(expected_op), + "{name} diagnostic should mention {expected_op}, got: {message}" + ); + assert!( + message.contains("unsupported"), + "{name} diagnostic should be explicit, got: {message}" + ); + assert_eq!( + crate::dwarf_expr::ops::unsupported_op_from_error(&error), + Some(expected_op), + "{name} diagnostic should carry a typed unsupported-op cause" + ); + } +} + +#[test] +fn implicit_pointer_to_static_storage_preserves_absolute_address_semantics() { + let result = ExpressionEvaluator::addressable_location_to_pointer_value( + RawExpressionResult::MemoryLocation(LocationResult::Address(0x1234)), + 0x10, + ) + .expect("static address should convert to an implicit pointer value"); + + assert_eq!( + result, + RawExpressionResult::DirectValue(DirectValueResult::AbsoluteAddress(0x1244)) + ); +} + +#[test] +fn implicit_pointer_accepts_target_absolute_address_value() { + let result = ExpressionEvaluator::addressable_location_to_pointer_value( + RawExpressionResult::DirectValue(DirectValueResult::AbsoluteAddress(0x1234)), + 0x10, + ) + .expect("static address values should convert to an implicit pointer value"); + + assert_eq!( + result, + RawExpressionResult::DirectValue(DirectValueResult::AbsoluteAddress(0x1244)) + ); +} + +#[test] +fn multi_op_expression_rejects_invalid_opcode_after_valid_prefix() { + let expr_bytes = [ + constants::DW_OP_lit1.0, + 0xff, + constants::DW_OP_stack_value.0, + ]; + + let error = ExpressionEvaluator::parse_expression_with_context( + &expr_bytes, + RunTimeEndian::Little, + test_encoding(), + None, + None, + 0, + None, + None, + None, + 0, + ) + .expect_err("invalid opcode after a valid prefix must not return a value"); + + assert!( + error.to_string().contains("DWARF expression"), + "unexpected error: {error}" + ); +} + +#[test] +fn multi_op_expression_rejects_unsupported_operation_after_valid_prefix() { + let expr_bytes = [ + constants::DW_OP_lit1.0, + constants::DW_OP_drop.0, + constants::DW_OP_stack_value.0, + ]; + + let error = ExpressionEvaluator::parse_expression_with_context( + &expr_bytes, + RunTimeEndian::Little, + test_encoding(), + None, + None, + 0, + None, + None, + None, + 0, + ) + .expect_err("unsupported operation after a valid prefix must not return a value"); + + assert!( + error.to_string().contains("unsupported"), + "unexpected error: {error}" + ); +} + +#[test] +fn single_fbreg_fast_path_saturates_cfa_offset_addition() { + let get_cfa = |_address| { + Ok(Some(CfaResult::RegisterPlusOffset { + register: 7, + offset: i64::MAX - 3, + })) + }; + + let result = ExpressionEvaluator::parse_expression_with_context( + &[0x91, 0x0a], + RunTimeEndian::Little, + test_encoding(), + None, + None, + 0, + Some(&get_cfa), + None, + None, + 0, + ) + .expect("single DW_OP_fbreg should parse"); + + assert_eq!( + result, + RawExpressionResult::MemoryLocation(LocationResult::RegisterAddress { + register: 7, + offset: Some(i64::MAX), + size: None, + }) + ); +} + +#[test] +fn big_endian_dw_op_addr_preserves_absolute_address() { + let expr_bytes = [0x03, 0, 0, 0, 0, 0, 0, 0x12, 0x34]; + let result = ExpressionEvaluator::parse_expression_with_context( + &expr_bytes, + gimli::RunTimeEndian::Big, + test_encoding(), + None, + None, + 0, + None, + None, + None, + 0, + ) + .expect("big-endian DW_OP_addr should parse"); + + assert_eq!( + result, + RawExpressionResult::MemoryLocation(LocationResult::Address(0x1234)) + ); +} diff --git a/ghostscope-dwarf/src/parser/fast_parser.rs b/ghostscope-dwarf/src/parser/fast_parser/mod.rs similarity index 63% rename from ghostscope-dwarf/src/parser/fast_parser.rs rename to ghostscope-dwarf/src/parser/fast_parser/mod.rs index a5559549..0fa73475 100644 --- a/ghostscope-dwarf/src/parser/fast_parser.rs +++ b/ghostscope-dwarf/src/parser/fast_parser/mod.rs @@ -1430,865 +1430,4 @@ impl<'a> DwarfParser<'a> { } #[cfg(test)] -mod tests { - use super::*; - use crate::binary::dwarf_reader_from_arc; - use gimli::write::{ - Address, AttributeValue as WriteAttributeValue, DebugInfoRef as WriteDebugInfoRef, - Dwarf as WriteDwarf, EndianVec, Expression as WriteExpression, LineProgram, Sections, Unit, - }; - use gimli::{Format, LittleEndian}; - use object::{Object, ObjectSection}; - use std::os::unix::fs::PermissionsExt; - use std::path::{Path, PathBuf}; - use std::process::Command; - use std::sync::Arc; - use tempfile::TempPath; - - fn test_line_entry(address: u64) -> LineEntry { - LineEntry { - address, - end_address: None, - file_path: "/src/main.c".to_string(), - file_index: 1, - compilation_unit: Arc::from("main.c"), - line: 10, - column: 0, - is_stmt: true, - prologue_end: false, - } - } - - fn write_dwarf_to_read(mut dwarf: WriteDwarf) -> gimli::Dwarf { - let mut sections = Sections::new(EndianVec::new(LittleEndian)); - dwarf.write(&mut sections).unwrap(); - - let dwarf_sections: gimli::DwarfSections> = gimli::DwarfSections::load(|id| { - Ok::<_, gimli::Error>( - sections - .get(id) - .map(|section| section.slice().to_vec()) - .unwrap_or_default(), - ) - }) - .unwrap(); - - dwarf_sections - .borrow(|section| dwarf_reader_from_arc(Arc::<[u8]>::from(section.as_slice()))) - } - - fn dwarf_with_debug_info_bytes(debug_info: Vec) -> gimli::Dwarf { - gimli::Dwarf::load(|id| { - let data = if id.name() == ".debug_info" { - debug_info.clone() - } else { - Vec::new() - }; - Ok::<_, gimli::Error>(dwarf_reader_from_arc(Arc::<[u8]>::from(data))) - }) - .unwrap() - } - - #[test] - fn flush_pending_line_entries_keeps_equal_end_bounded() { - let mut out = Vec::new(); - let mut pending = vec![test_line_entry(0x1000)]; - - DwarfParser::flush_pending_line_entries(&mut out, &mut pending, Some(0x1000)); - - assert!(pending.is_empty()); - assert_eq!(out.len(), 1); - assert_eq!(out[0].end_address, Some(0x1000)); - assert!(!out[0].contains_address(0x1000)); - assert!(!out[0].contains_address(0x1001)); - } - - #[test] - fn parse_line_info_propagates_unit_header_errors() { - let dwarf = dwarf_with_debug_info_bytes(vec![0x0b, 0x00, 0x00, 0x00, 0x04]); - let parser = DwarfParser { dwarf: &dwarf }; - - let err = match parser.parse_line_info("malformed") { - Ok(_) => panic!("malformed unit header must not be treated as end-of-units"), - Err(err) => err, - }; - - assert!( - err.downcast_ref::().is_some(), - "expected gimli parse error, got: {err}" - ); - } - - #[test] - fn parse_debug_info_propagates_unit_header_errors() { - let dwarf = dwarf_with_debug_info_bytes(vec![0x0b, 0x00, 0x00, 0x00, 0x04]); - let parser = DwarfParser { dwarf: &dwarf }; - - let err = match parser.parse_debug_info("malformed") { - Ok(_) => panic!("malformed unit header must not be treated as end-of-units"), - Err(err) => err, - }; - - assert!( - err.downcast_ref::().is_some(), - "expected gimli parse error, got: {err}" - ); - } - - fn build_variable_index_fixture() -> gimli::Dwarf { - let encoding = gimli::Encoding { - format: Format::Dwarf32, - version: 4, - address_size: 8, - }; - - let mut dwarf = WriteDwarf::new(); - let unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); - let unit = dwarf.units.get_mut(unit_id); - let root = unit.root(); - - let global_id = unit.add(root, gimli::constants::DW_TAG_variable); - let global = unit.get_mut(global_id); - global.set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"real_global".to_vec()), - ); - global.set( - gimli::constants::DW_AT_external, - WriteAttributeValue::Flag(true), - ); - let mut global_loc = WriteExpression::new(); - global_loc.op_addr(Address::Constant(0x401000)); - global.set( - gimli::constants::DW_AT_location, - WriteAttributeValue::Exprloc(global_loc), - ); - - let subprogram_id = unit.add(root, gimli::constants::DW_TAG_subprogram); - unit.get_mut(subprogram_id).set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"touch".to_vec()), - ); - - let local_static_id = unit.add(subprogram_id, gimli::constants::DW_TAG_variable); - let local_static = unit.get_mut(local_static_id); - local_static.set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"local_static".to_vec()), - ); - let mut local_static_loc = WriteExpression::new(); - local_static_loc.op_addr(Address::Constant(0x402000)); - local_static.set( - gimli::constants::DW_AT_location, - WriteAttributeValue::Exprloc(local_static_loc), - ); - - let local_ptr_id = unit.add(subprogram_id, gimli::constants::DW_TAG_variable); - let local_ptr = unit.get_mut(local_ptr_id); - local_ptr.set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"p".to_vec()), - ); - let mut local_ptr_loc = WriteExpression::new(); - local_ptr_loc.op_addr(Address::Constant(0x403000)); - local_ptr_loc.op(gimli::constants::DW_OP_stack_value); - local_ptr.set( - gimli::constants::DW_AT_location, - WriteAttributeValue::Exprloc(local_ptr_loc), - ); - - let mut sections = Sections::new(EndianVec::new(LittleEndian)); - dwarf.write(&mut sections).unwrap(); - - let dwarf_sections: gimli::DwarfSections> = gimli::DwarfSections::load(|id| { - Ok::<_, gimli::Error>( - sections - .get(id) - .map(|section| section.slice().to_vec()) - .unwrap_or_default(), - ) - }) - .unwrap(); - - dwarf_sections - .borrow(|section| dwarf_reader_from_arc(Arc::<[u8]>::from(section.as_slice()))) - } - - fn build_inline_origin_fixture() -> gimli::Dwarf { - let encoding = gimli::Encoding { - format: Format::Dwarf32, - version: 4, - address_size: 8, - }; - - let mut dwarf = WriteDwarf::new(); - let unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); - let unit = dwarf.units.get_mut(unit_id); - let root = unit.root(); - - let abstract_id = unit.add(root, gimli::constants::DW_TAG_subprogram); - let abstract_fn = unit.get_mut(abstract_id); - abstract_fn.set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"CGPsend".to_vec()), - ); - abstract_fn.set( - gimli::constants::DW_AT_inline, - WriteAttributeValue::Inline(gimli::DW_INL_inlined), - ); - abstract_fn.set( - gimli::constants::DW_AT_external, - WriteAttributeValue::Flag(true), - ); - - let concrete_id = unit.add(root, gimli::constants::DW_TAG_subprogram); - let concrete_fn = unit.get_mut(concrete_id); - concrete_fn.set( - gimli::constants::DW_AT_abstract_origin, - WriteAttributeValue::UnitRef(abstract_id), - ); - concrete_fn.set( - gimli::constants::DW_AT_low_pc, - WriteAttributeValue::Address(Address::Constant(0x8e97c0)), - ); - concrete_fn.set( - gimli::constants::DW_AT_high_pc, - WriteAttributeValue::Udata(0x420), - ); - - let inlined_id = unit.add(root, gimli::constants::DW_TAG_inlined_subroutine); - unit.get_mut(inlined_id).set( - gimli::constants::DW_AT_abstract_origin, - WriteAttributeValue::UnitRef(abstract_id), - ); - - let mut sections = Sections::new(EndianVec::new(LittleEndian)); - dwarf.write(&mut sections).unwrap(); - - let dwarf_sections: gimli::DwarfSections> = gimli::DwarfSections::load(|id| { - Ok::<_, gimli::Error>( - sections - .get(id) - .map(|section| section.slice().to_vec()) - .unwrap_or_default(), - ) - }) - .unwrap(); - - dwarf_sections - .borrow(|section| dwarf_reader_from_arc(Arc::<[u8]>::from(section.as_slice()))) - } - - fn build_cross_cu_origin_fixture() -> gimli::Dwarf { - let encoding = gimli::Encoding { - format: Format::Dwarf32, - version: 4, - address_size: 8, - }; - - let mut dwarf = WriteDwarf::new(); - - let decl_unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); - let decl_unit = dwarf.units.get_mut(decl_unit_id); - let decl_root = decl_unit.root(); - decl_unit.get_mut(decl_root).set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"decl_unit.c".to_vec()), - ); - - let abstract_fn_id = decl_unit.add(decl_root, gimli::constants::DW_TAG_subprogram); - let abstract_fn = decl_unit.get_mut(abstract_fn_id); - abstract_fn.set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"cross_cu_origin_fn".to_vec()), - ); - abstract_fn.set( - gimli::constants::DW_AT_external, - WriteAttributeValue::Flag(true), - ); - - let spec_type_id = decl_unit.add(decl_root, gimli::constants::DW_TAG_structure_type); - let spec_type = decl_unit.get_mut(spec_type_id); - spec_type.set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"CrossCuOriginType".to_vec()), - ); - spec_type.set( - gimli::constants::DW_AT_declaration, - WriteAttributeValue::Flag(true), - ); - - let concrete_unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); - let concrete_unit = dwarf.units.get_mut(concrete_unit_id); - let concrete_root = concrete_unit.root(); - concrete_unit.get_mut(concrete_root).set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"concrete_unit.c".to_vec()), - ); - - let concrete_fn_id = concrete_unit.add(concrete_root, gimli::constants::DW_TAG_subprogram); - let concrete_fn = concrete_unit.get_mut(concrete_fn_id); - concrete_fn.set( - gimli::constants::DW_AT_abstract_origin, - WriteAttributeValue::DebugInfoRef(WriteDebugInfoRef::Entry( - decl_unit_id, - abstract_fn_id, - )), - ); - concrete_fn.set( - gimli::constants::DW_AT_low_pc, - WriteAttributeValue::Address(Address::Constant(0x501000)), - ); - concrete_fn.set( - gimli::constants::DW_AT_high_pc, - WriteAttributeValue::Udata(0x40), - ); - - let concrete_type_id = - concrete_unit.add(concrete_root, gimli::constants::DW_TAG_structure_type); - concrete_unit.get_mut(concrete_type_id).set( - gimli::constants::DW_AT_specification, - WriteAttributeValue::DebugInfoRef(WriteDebugInfoRef::Entry(decl_unit_id, spec_type_id)), - ); - - write_dwarf_to_read(dwarf) - } - - fn build_multi_cu_shared_function_fixture() -> gimli::Dwarf { - let encoding = gimli::Encoding { - format: Format::Dwarf32, - version: 4, - address_size: 8, - }; - - let mut dwarf = WriteDwarf::new(); - - for (cu_name, low_pc) in [("unit_one.rs", 0x401000_u64), ("unit_two.rs", 0x402000_u64)] { - let unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); - let unit = dwarf.units.get_mut(unit_id); - let root = unit.root(); - unit.get_mut(root).set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(cu_name.as_bytes().to_vec()), - ); - - let subprogram_id = unit.add(root, gimli::constants::DW_TAG_subprogram); - let subprogram = unit.get_mut(subprogram_id); - subprogram.set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"shared".to_vec()), - ); - subprogram.set( - gimli::constants::DW_AT_low_pc, - WriteAttributeValue::Address(Address::Constant(low_pc)), - ); - subprogram.set( - gimli::constants::DW_AT_high_pc, - WriteAttributeValue::Udata(0x20), - ); - } - - let mut sections = Sections::new(EndianVec::new(LittleEndian)); - dwarf.write(&mut sections).unwrap(); - - let dwarf_sections: gimli::DwarfSections> = gimli::DwarfSections::load(|id| { - Ok::<_, gimli::Error>( - sections - .get(id) - .map(|section| section.slice().to_vec()) - .unwrap_or_default(), - ) - }) - .unwrap(); - - dwarf_sections - .borrow(|section| dwarf_reader_from_arc(Arc::<[u8]>::from(section.as_slice()))) - } - - fn build_cu_body_lookup_fixture() -> gimli::Dwarf { - let encoding = gimli::Encoding { - format: Format::Dwarf32, - version: 4, - address_size: 8, - }; - - let mut dwarf = WriteDwarf::new(); - let unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); - let unit = dwarf.units.get_mut(unit_id); - let root = unit.root(); - unit.get_mut(root).set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"body_lookup_unit".to_vec()), - ); - - let subprogram_id = unit.add(root, gimli::constants::DW_TAG_subprogram); - let subprogram = unit.get_mut(subprogram_id); - subprogram.set( - gimli::constants::DW_AT_name, - WriteAttributeValue::String(b"body_lookup".to_vec()), - ); - subprogram.set( - gimli::constants::DW_AT_low_pc, - WriteAttributeValue::Address(Address::Constant(0x401000)), - ); - subprogram.set( - gimli::constants::DW_AT_high_pc, - WriteAttributeValue::Udata(0x40), - ); - - let mut sections = Sections::new(EndianVec::new(LittleEndian)); - dwarf.write(&mut sections).unwrap(); - - let dwarf_sections: gimli::DwarfSections> = gimli::DwarfSections::load(|id| { - Ok::<_, gimli::Error>( - sections - .get(id) - .map(|section| section.slice().to_vec()) - .unwrap_or_default(), - ) - }) - .unwrap(); - - dwarf_sections - .borrow(|section| dwarf_reader_from_arc(Arc::<[u8]>::from(section.as_slice()))) - } - - fn clang_available() -> bool { - Command::new("clang") - .arg("--version") - .status() - .map(|status| status.success()) - .unwrap_or(false) - } - - fn compile_inline_callsite_fixture_with_clang_dwarf5() -> anyhow::Result { - let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .ok_or_else(|| anyhow::anyhow!("ghostscope-dwarf has no workspace parent"))? - .to_path_buf(); - let source = workspace_root - .join("e2e-tests/tests/fixtures/inline_callsite_program/inline_callsite_program.c"); - let binary = tempfile::Builder::new() - .prefix("ghostscope-fast-parser-") - .tempfile()? - .into_temp_path(); - let binary_path = binary.to_path_buf(); - - let compile_output = Command::new("clang") - .args(["-Wall", "-Wextra", "-gdwarf-5", "-O3"]) - .arg("-o") - .arg(&binary_path) - .arg(&source) - .output() - .map_err(|e| anyhow::anyhow!("Failed to run clang for {}: {}", source.display(), e))?; - - if compile_output.status.success() { - Ok(binary) - } else { - let stderr = String::from_utf8_lossy(&compile_output.stderr); - Err(anyhow::anyhow!( - "Failed to compile {} with clang -gdwarf-5 -O3: {}", - source.display(), - stderr - )) - } - } - - fn load_dwarf_from_binary(path: &Path) -> anyhow::Result> { - let bytes = std::fs::read(path) - .map_err(|e| anyhow::anyhow!("Failed to read {}: {}", path.display(), e))?; - let object = object::File::parse(&*bytes) - .map_err(|e| anyhow::anyhow!("Failed to parse {}: {}", path.display(), e))?; - let dwarf = gimli::Dwarf::load(|id| { - let section_data = object - .section_by_name(id.name()) - .and_then(|section| section.uncompressed_data().ok()) - .map(|data| data.into_owned()) - .unwrap_or_default(); - Ok::<_, gimli::Error>(dwarf_reader_from_arc(Arc::<[u8]>::from(section_data))) - })?; - Ok(dwarf) - } - - fn read_uleb128(input: &[u8], offset: &mut usize) -> anyhow::Result { - let mut value = 0_u64; - let mut shift = 0_u32; - loop { - let byte = *input - .get(*offset) - .ok_or_else(|| anyhow::anyhow!("Unexpected EOF while reading ULEB128"))?; - *offset += 1; - let low_bits = u64::from(byte & 0x7f); - anyhow::ensure!( - shift < 64 && !(shift == 63 && low_bits > 1), - "ULEB128 value exceeds u64" - ); - value |= low_bits << shift; - if byte & 0x80 == 0 { - return Ok(value); - } - shift += 7; - } - } - - #[test] - fn read_uleb128_rejects_values_that_overflow_u64() { - let overflow = [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x02]; - let mut offset = 0; - let err = read_uleb128(&overflow, &mut offset).expect_err("overflow should be rejected"); - assert!( - err.to_string().contains("exceeds u64"), - "unexpected overflow error: {err}" - ); - } - - fn patch_inlined_subroutine_low_pc_to_entry_pc(abbrev: &mut [u8]) -> anyhow::Result { - let mut offset = 0; - let mut patched = 0; - - while offset < abbrev.len() { - let code = read_uleb128(abbrev, &mut offset)?; - if code == 0 { - continue; - } - - let tag = read_uleb128(abbrev, &mut offset)?; - let _has_children = *abbrev - .get(offset) - .ok_or_else(|| anyhow::anyhow!("Missing abbrev children byte"))?; - offset += 1; - - loop { - let name_offset = offset; - let name = read_uleb128(abbrev, &mut offset)?; - let form = read_uleb128(abbrev, &mut offset)?; - if name == 0 && form == 0 { - break; - } - - let is_addrx_form = form == u64::from(gimli::constants::DW_FORM_addrx.0) - || form == u64::from(gimli::constants::DW_FORM_addrx1.0) - || form == u64::from(gimli::constants::DW_FORM_addrx2.0) - || form == u64::from(gimli::constants::DW_FORM_addrx3.0) - || form == u64::from(gimli::constants::DW_FORM_addrx4.0); - if tag == u64::from(gimli::constants::DW_TAG_inlined_subroutine.0) - && name == u64::from(gimli::constants::DW_AT_low_pc.0) - && is_addrx_form - { - *abbrev - .get_mut(name_offset) - .ok_or_else(|| anyhow::anyhow!("Invalid abbrev attribute offset"))? = - gimli::constants::DW_AT_entry_pc.0 as u8; - patched += 1; - } - } - } - - Ok(patched) - } - - fn rewrite_inline_fixture_entry_pc_attr(input_path: &Path) -> anyhow::Result { - let mut bytes = std::fs::read(input_path) - .map_err(|e| anyhow::anyhow!("Failed to read {}: {}", input_path.display(), e))?; - let (abbrev_offset, abbrev_size) = { - let object = object::File::parse(&*bytes) - .map_err(|e| anyhow::anyhow!("Failed to parse {}: {}", input_path.display(), e))?; - let section = object.section_by_name(".debug_abbrev").ok_or_else(|| { - anyhow::anyhow!("{} is missing .debug_abbrev", input_path.display()) - })?; - section.file_range().ok_or_else(|| { - anyhow::anyhow!( - "{} has no file range for .debug_abbrev", - input_path.display() - ) - })? - }; - - let patched = patch_inlined_subroutine_low_pc_to_entry_pc( - &mut bytes[abbrev_offset as usize..(abbrev_offset + abbrev_size) as usize], - )?; - anyhow::ensure!( - patched > 0, - "Expected to patch at least one inline low_pc abbrev in {}", - input_path.display() - ); - - let output = tempfile::Builder::new() - .prefix(".ghostscope-fast-parser-patched-") - .tempfile()? - .into_temp_path(); - std::fs::write(&output, &bytes)?; - let perms = std::fs::metadata(input_path)?.permissions().mode(); - std::fs::set_permissions(&output, std::fs::Permissions::from_mode(perms))?; - Ok(output) - } - - fn has_inline_entry_pc_debug_addr_index(dwarf: &gimli::Dwarf) -> bool { - let mut units = dwarf.units(); - while let Ok(Some(header)) = units.next() { - let Ok(unit) = dwarf.unit(header) else { - continue; - }; - let mut entries = unit.entries(); - while let Ok(Some(entry)) = entries.next_dfs() { - if entry.tag() != gimli::constants::DW_TAG_inlined_subroutine { - continue; - } - if let Some(attr) = entry.attr(gimli::constants::DW_AT_entry_pc) { - if matches!(attr.value(), gimli::AttributeValue::DebugAddrIndex(_)) { - return true; - } - } - } - } - false - } - - #[test] - fn parse_debug_info_skips_stack_value_address_locals_from_global_index() { - let dwarf = build_variable_index_fixture(); - let parser = DwarfParser { dwarf: &dwarf }; - - let result = parser.parse_debug_info("synthetic").unwrap(); - let real_global = result - .lightweight_index - .find_variables_by_name("real_global"); - let local_static = result - .lightweight_index - .find_variables_by_name("local_static"); - let optimized_local = result.lightweight_index.find_variables_by_name("p"); - - assert_eq!( - real_global.len(), - 1, - "real global should remain indexed: {real_global:?}" - ); - assert_eq!( - local_static.len(), - 1, - "function-scoped static with real storage should remain indexed: {local_static:?}" - ); - assert!( - optimized_local.is_empty(), - "address-valued optimized local must not be indexed as a global: {optimized_local:?}" - ); - } - - #[test] - fn parse_debug_info_keeps_concrete_abstract_origin_subprogram_non_inline() { - // Regression scenario: - // GCC/Clang can emit all three DIE shapes for one logical function: - // 1. an abstract DW_TAG_subprogram marked DW_AT_inline, - // 2. a concrete out-of-line DW_TAG_subprogram with DW_AT_abstract_origin, - // 3. one or more DW_TAG_inlined_subroutine instances. - // - // The bug was that merge_from_origin copied the abstract function's - // inline attribute from (1) onto (2) and downstream code treated that - // as if the concrete body were an inline instance. - // Once that happened, the concrete body was routed through the inline - // address-selection path and could pick the wrong cold-partition PC. - // - // This test keeps the synthetic DIE graph minimal and asserts the parser - // preserves the intended split: - // - abstract definition stays inline - // - concrete out-of-line body stays non-inline - // - inlined_subroutine instance stays inline - let dwarf = build_inline_origin_fixture(); - let parser = DwarfParser { dwarf: &dwarf }; - - let result = parser.parse_debug_info("synthetic").unwrap(); - let entries = result - .lightweight_index - .find_dies_by_function_name("CGPsend"); - - let concrete_entries: Vec<_> = entries - .iter() - .copied() - .filter(|entry| { - entry.function_kind() == crate::core::FunctionDieKind::ConcreteSubprogram - }) - .collect(); - let abstract_entries: Vec<_> = entries - .iter() - .copied() - .filter(|entry| { - entry.function_kind() == crate::core::FunctionDieKind::AbstractSubprogram - }) - .collect(); - let inlined_entries: Vec<_> = entries - .iter() - .copied() - .filter(|entry| entry.function_kind() == crate::core::FunctionDieKind::InlineInstance) - .collect(); - - assert_eq!( - concrete_entries.len(), - 1, - "concrete out-of-line subprogram should stay non-inline: {entries:?}" - ); - assert_eq!( - abstract_entries.len(), - 1, - "only the abstract inline definition should carry the inline flag: {entries:?}" - ); - assert_eq!( - inlined_entries.len(), - 1, - "expected one inlined subroutine instance: {entries:?}" - ); - assert!( - inlined_entries[0].is_inline_instance(), - "DW_TAG_inlined_subroutine must remain an inline instance: {entries:?}" - ); - assert!( - !concrete_entries[0].flags.has_inline_attribute, - "concrete out-of-line body should not inherit the abstract inline attribute: {entries:?}" - ); - assert!( - abstract_entries[0].flags.has_inline_attribute, - "abstract definition should retain its original DW_AT_inline attribute: {entries:?}" - ); - } - - #[test] - fn parse_debug_info_resolves_cross_cu_origin_names_for_index_entries() { - let dwarf = build_cross_cu_origin_fixture(); - let parser = DwarfParser { dwarf: &dwarf }; - - let result = parser.parse_debug_info("synthetic").unwrap(); - let function_entries = result - .lightweight_index - .find_dies_by_function_name("cross_cu_origin_fn"); - assert!( - function_entries.iter().any(|entry| { - entry.function_kind() == crate::core::FunctionDieKind::ConcreteSubprogram - && entry.representative_addr == Some(0x501000) - }), - "concrete function should inherit its name through a cross-CU origin: {function_entries:?}" - ); - - let mut has_concrete_type = false; - result - .lightweight_index - .for_each_type_map_entry(|name, entry_base, indices| { - if name != "CrossCuOriginType" { - return; - } - has_concrete_type |= indices.iter().any(|local_idx| { - result - .lightweight_index - .entry(entry_base + *local_idx) - .is_some_and(|entry| !entry.flags.is_type_declaration) - }); - }); - - assert!( - has_concrete_type, - "concrete type should inherit its name through a cross-CU specification" - ); - } - - #[test] - fn parse_debug_info_keeps_sharded_name_lookup_without_duplicate_keys() { - let dwarf = build_multi_cu_shared_function_fixture(); - let parser = DwarfParser { dwarf: &dwarf }; - - let result = parser.parse_debug_info("synthetic").unwrap(); - let entries = result - .lightweight_index - .find_dies_by_function_name("shared"); - let names = result.lightweight_index.get_function_names(); - - assert_eq!( - result.functions_count, 1, - "unique function-name count should stay deduplicated across shards" - ); - assert_eq!( - entries.len(), - 2, - "function lookup should fan out across CU shards: {entries:?}" - ); - assert_eq!( - names - .iter() - .filter(|name| name.as_str() == "shared") - .count(), - 1, - "function name listing should not duplicate shard-local keys: {names:?}" - ); - } - - #[test] - fn parse_debug_info_builds_fallback_cu_map_for_function_body_addresses() { - let dwarf = build_cu_body_lookup_fixture(); - let parser = DwarfParser { dwarf: &dwarf }; - - let result = parser.parse_debug_info("synthetic").unwrap(); - let entry = result - .lightweight_index - .find_function_by_address(0x401020, |entry| { - let header = dwarf.unit_header(entry.unit_offset).ok()?; - let unit = dwarf.unit(header).ok()?; - let die = unit.entry(entry.die_offset).ok()?; - crate::parser::RangeExtractor::extract_all_ranges(&die, &unit, &dwarf).ok() - }) - .expect("function body address should resolve through fallback CU map"); - - assert_eq!(entry.name.as_ref(), "body_lookup"); - assert_eq!( - result.lightweight_index.find_cu_by_address(0x401020), - Some(entry.unit_offset), - "fallback CU map should cover addresses inside the function body, not just the representative address" - ); - } - - #[test] - fn parse_debug_info_resolves_debug_addr_index_entry_pc_for_inline_instances() { - if !clang_available() { - eprintln!("Skipping fast_parser DWARF5 entry_pc regression: clang is unavailable"); - return; - } - - let patched_binary = { - let binary = compile_inline_callsite_fixture_with_clang_dwarf5() - .expect("clang dwarf5 inline fixture should compile"); - rewrite_inline_fixture_entry_pc_attr(binary.as_ref()) - .expect("inline fixture abbrev should rewrite low_pc addrx into entry_pc addrx") - }; - let dwarf = load_dwarf_from_binary(patched_binary.as_ref()) - .expect("compiled inline fixture should load as DWARF"); - assert!( - has_inline_entry_pc_debug_addr_index(&dwarf), - "patched inline fixture should expose an inlined_subroutine DW_AT_entry_pc via DW_FORM_addrx/.debug_addr" - ); - - let parser = DwarfParser { dwarf: &dwarf }; - let result = parser - .parse_debug_info(patched_binary.to_string_lossy().as_ref()) - .expect("fast parser should index patched clang dwarf5 inline fixture"); - - let inline_entries: Vec<_> = ["add3", "consume_state"] - .into_iter() - .flat_map(|name| { - result - .lightweight_index - .find_dies_by_function_name(name) - .iter() - .copied() - .filter(|entry| { - entry.function_kind() == crate::core::FunctionDieKind::InlineInstance - }) - .collect::>() - }) - .collect(); - - assert!( - !inline_entries.is_empty(), - "expected inline entries for clang dwarf5 inline fixture" - ); - assert!( - inline_entries.iter().any(|entry| entry.entry_pc.is_some()), - "fast parser should resolve DW_FORM_addrx-backed DW_AT_entry_pc values for inline instances: {inline_entries:?}" - ); - } -} +mod tests; diff --git a/ghostscope-dwarf/src/parser/fast_parser/tests.rs b/ghostscope-dwarf/src/parser/fast_parser/tests.rs new file mode 100644 index 00000000..eae33389 --- /dev/null +++ b/ghostscope-dwarf/src/parser/fast_parser/tests.rs @@ -0,0 +1,848 @@ +use super::*; +use crate::binary::dwarf_reader_from_arc; +use gimli::write::{ + Address, AttributeValue as WriteAttributeValue, DebugInfoRef as WriteDebugInfoRef, + Dwarf as WriteDwarf, EndianVec, Expression as WriteExpression, LineProgram, Sections, Unit, +}; +use gimli::{Format, LittleEndian}; +use object::{Object, ObjectSection}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; +use tempfile::TempPath; + +fn test_line_entry(address: u64) -> LineEntry { + LineEntry { + address, + end_address: None, + file_path: "/src/main.c".to_string(), + file_index: 1, + compilation_unit: Arc::from("main.c"), + line: 10, + column: 0, + is_stmt: true, + prologue_end: false, + } +} + +fn write_dwarf_to_read(mut dwarf: WriteDwarf) -> gimli::Dwarf { + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + dwarf.write(&mut sections).unwrap(); + + let dwarf_sections: gimli::DwarfSections> = gimli::DwarfSections::load(|id| { + Ok::<_, gimli::Error>( + sections + .get(id) + .map(|section| section.slice().to_vec()) + .unwrap_or_default(), + ) + }) + .unwrap(); + + dwarf_sections.borrow(|section| dwarf_reader_from_arc(Arc::<[u8]>::from(section.as_slice()))) +} + +fn dwarf_with_debug_info_bytes(debug_info: Vec) -> gimli::Dwarf { + gimli::Dwarf::load(|id| { + let data = if id.name() == ".debug_info" { + debug_info.clone() + } else { + Vec::new() + }; + Ok::<_, gimli::Error>(dwarf_reader_from_arc(Arc::<[u8]>::from(data))) + }) + .unwrap() +} + +#[test] +fn flush_pending_line_entries_keeps_equal_end_bounded() { + let mut out = Vec::new(); + let mut pending = vec![test_line_entry(0x1000)]; + + DwarfParser::flush_pending_line_entries(&mut out, &mut pending, Some(0x1000)); + + assert!(pending.is_empty()); + assert_eq!(out.len(), 1); + assert_eq!(out[0].end_address, Some(0x1000)); + assert!(!out[0].contains_address(0x1000)); + assert!(!out[0].contains_address(0x1001)); +} + +#[test] +fn parse_line_info_propagates_unit_header_errors() { + let dwarf = dwarf_with_debug_info_bytes(vec![0x0b, 0x00, 0x00, 0x00, 0x04]); + let parser = DwarfParser { dwarf: &dwarf }; + + let err = match parser.parse_line_info("malformed") { + Ok(_) => panic!("malformed unit header must not be treated as end-of-units"), + Err(err) => err, + }; + + assert!( + err.downcast_ref::().is_some(), + "expected gimli parse error, got: {err}" + ); +} + +#[test] +fn parse_debug_info_propagates_unit_header_errors() { + let dwarf = dwarf_with_debug_info_bytes(vec![0x0b, 0x00, 0x00, 0x00, 0x04]); + let parser = DwarfParser { dwarf: &dwarf }; + + let err = match parser.parse_debug_info("malformed") { + Ok(_) => panic!("malformed unit header must not be treated as end-of-units"), + Err(err) => err, + }; + + assert!( + err.downcast_ref::().is_some(), + "expected gimli parse error, got: {err}" + ); +} + +fn build_variable_index_fixture() -> gimli::Dwarf { + let encoding = gimli::Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 8, + }; + + let mut dwarf = WriteDwarf::new(); + let unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); + let unit = dwarf.units.get_mut(unit_id); + let root = unit.root(); + + let global_id = unit.add(root, gimli::constants::DW_TAG_variable); + let global = unit.get_mut(global_id); + global.set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"real_global".to_vec()), + ); + global.set( + gimli::constants::DW_AT_external, + WriteAttributeValue::Flag(true), + ); + let mut global_loc = WriteExpression::new(); + global_loc.op_addr(Address::Constant(0x401000)); + global.set( + gimli::constants::DW_AT_location, + WriteAttributeValue::Exprloc(global_loc), + ); + + let subprogram_id = unit.add(root, gimli::constants::DW_TAG_subprogram); + unit.get_mut(subprogram_id).set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"touch".to_vec()), + ); + + let local_static_id = unit.add(subprogram_id, gimli::constants::DW_TAG_variable); + let local_static = unit.get_mut(local_static_id); + local_static.set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"local_static".to_vec()), + ); + let mut local_static_loc = WriteExpression::new(); + local_static_loc.op_addr(Address::Constant(0x402000)); + local_static.set( + gimli::constants::DW_AT_location, + WriteAttributeValue::Exprloc(local_static_loc), + ); + + let local_ptr_id = unit.add(subprogram_id, gimli::constants::DW_TAG_variable); + let local_ptr = unit.get_mut(local_ptr_id); + local_ptr.set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"p".to_vec()), + ); + let mut local_ptr_loc = WriteExpression::new(); + local_ptr_loc.op_addr(Address::Constant(0x403000)); + local_ptr_loc.op(gimli::constants::DW_OP_stack_value); + local_ptr.set( + gimli::constants::DW_AT_location, + WriteAttributeValue::Exprloc(local_ptr_loc), + ); + + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + dwarf.write(&mut sections).unwrap(); + + let dwarf_sections: gimli::DwarfSections> = gimli::DwarfSections::load(|id| { + Ok::<_, gimli::Error>( + sections + .get(id) + .map(|section| section.slice().to_vec()) + .unwrap_or_default(), + ) + }) + .unwrap(); + + dwarf_sections.borrow(|section| dwarf_reader_from_arc(Arc::<[u8]>::from(section.as_slice()))) +} + +fn build_inline_origin_fixture() -> gimli::Dwarf { + let encoding = gimli::Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 8, + }; + + let mut dwarf = WriteDwarf::new(); + let unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); + let unit = dwarf.units.get_mut(unit_id); + let root = unit.root(); + + let abstract_id = unit.add(root, gimli::constants::DW_TAG_subprogram); + let abstract_fn = unit.get_mut(abstract_id); + abstract_fn.set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"CGPsend".to_vec()), + ); + abstract_fn.set( + gimli::constants::DW_AT_inline, + WriteAttributeValue::Inline(gimli::DW_INL_inlined), + ); + abstract_fn.set( + gimli::constants::DW_AT_external, + WriteAttributeValue::Flag(true), + ); + + let concrete_id = unit.add(root, gimli::constants::DW_TAG_subprogram); + let concrete_fn = unit.get_mut(concrete_id); + concrete_fn.set( + gimli::constants::DW_AT_abstract_origin, + WriteAttributeValue::UnitRef(abstract_id), + ); + concrete_fn.set( + gimli::constants::DW_AT_low_pc, + WriteAttributeValue::Address(Address::Constant(0x8e97c0)), + ); + concrete_fn.set( + gimli::constants::DW_AT_high_pc, + WriteAttributeValue::Udata(0x420), + ); + + let inlined_id = unit.add(root, gimli::constants::DW_TAG_inlined_subroutine); + unit.get_mut(inlined_id).set( + gimli::constants::DW_AT_abstract_origin, + WriteAttributeValue::UnitRef(abstract_id), + ); + + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + dwarf.write(&mut sections).unwrap(); + + let dwarf_sections: gimli::DwarfSections> = gimli::DwarfSections::load(|id| { + Ok::<_, gimli::Error>( + sections + .get(id) + .map(|section| section.slice().to_vec()) + .unwrap_or_default(), + ) + }) + .unwrap(); + + dwarf_sections.borrow(|section| dwarf_reader_from_arc(Arc::<[u8]>::from(section.as_slice()))) +} + +fn build_cross_cu_origin_fixture() -> gimli::Dwarf { + let encoding = gimli::Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 8, + }; + + let mut dwarf = WriteDwarf::new(); + + let decl_unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); + let decl_unit = dwarf.units.get_mut(decl_unit_id); + let decl_root = decl_unit.root(); + decl_unit.get_mut(decl_root).set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"decl_unit.c".to_vec()), + ); + + let abstract_fn_id = decl_unit.add(decl_root, gimli::constants::DW_TAG_subprogram); + let abstract_fn = decl_unit.get_mut(abstract_fn_id); + abstract_fn.set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"cross_cu_origin_fn".to_vec()), + ); + abstract_fn.set( + gimli::constants::DW_AT_external, + WriteAttributeValue::Flag(true), + ); + + let spec_type_id = decl_unit.add(decl_root, gimli::constants::DW_TAG_structure_type); + let spec_type = decl_unit.get_mut(spec_type_id); + spec_type.set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"CrossCuOriginType".to_vec()), + ); + spec_type.set( + gimli::constants::DW_AT_declaration, + WriteAttributeValue::Flag(true), + ); + + let concrete_unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); + let concrete_unit = dwarf.units.get_mut(concrete_unit_id); + let concrete_root = concrete_unit.root(); + concrete_unit.get_mut(concrete_root).set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"concrete_unit.c".to_vec()), + ); + + let concrete_fn_id = concrete_unit.add(concrete_root, gimli::constants::DW_TAG_subprogram); + let concrete_fn = concrete_unit.get_mut(concrete_fn_id); + concrete_fn.set( + gimli::constants::DW_AT_abstract_origin, + WriteAttributeValue::DebugInfoRef(WriteDebugInfoRef::Entry(decl_unit_id, abstract_fn_id)), + ); + concrete_fn.set( + gimli::constants::DW_AT_low_pc, + WriteAttributeValue::Address(Address::Constant(0x501000)), + ); + concrete_fn.set( + gimli::constants::DW_AT_high_pc, + WriteAttributeValue::Udata(0x40), + ); + + let concrete_type_id = + concrete_unit.add(concrete_root, gimli::constants::DW_TAG_structure_type); + concrete_unit.get_mut(concrete_type_id).set( + gimli::constants::DW_AT_specification, + WriteAttributeValue::DebugInfoRef(WriteDebugInfoRef::Entry(decl_unit_id, spec_type_id)), + ); + + write_dwarf_to_read(dwarf) +} + +fn build_multi_cu_shared_function_fixture() -> gimli::Dwarf { + let encoding = gimli::Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 8, + }; + + let mut dwarf = WriteDwarf::new(); + + for (cu_name, low_pc) in [("unit_one.rs", 0x401000_u64), ("unit_two.rs", 0x402000_u64)] { + let unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); + let unit = dwarf.units.get_mut(unit_id); + let root = unit.root(); + unit.get_mut(root).set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(cu_name.as_bytes().to_vec()), + ); + + let subprogram_id = unit.add(root, gimli::constants::DW_TAG_subprogram); + let subprogram = unit.get_mut(subprogram_id); + subprogram.set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"shared".to_vec()), + ); + subprogram.set( + gimli::constants::DW_AT_low_pc, + WriteAttributeValue::Address(Address::Constant(low_pc)), + ); + subprogram.set( + gimli::constants::DW_AT_high_pc, + WriteAttributeValue::Udata(0x20), + ); + } + + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + dwarf.write(&mut sections).unwrap(); + + let dwarf_sections: gimli::DwarfSections> = gimli::DwarfSections::load(|id| { + Ok::<_, gimli::Error>( + sections + .get(id) + .map(|section| section.slice().to_vec()) + .unwrap_or_default(), + ) + }) + .unwrap(); + + dwarf_sections.borrow(|section| dwarf_reader_from_arc(Arc::<[u8]>::from(section.as_slice()))) +} + +fn build_cu_body_lookup_fixture() -> gimli::Dwarf { + let encoding = gimli::Encoding { + format: Format::Dwarf32, + version: 4, + address_size: 8, + }; + + let mut dwarf = WriteDwarf::new(); + let unit_id = dwarf.units.add(Unit::new(encoding, LineProgram::none())); + let unit = dwarf.units.get_mut(unit_id); + let root = unit.root(); + unit.get_mut(root).set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"body_lookup_unit".to_vec()), + ); + + let subprogram_id = unit.add(root, gimli::constants::DW_TAG_subprogram); + let subprogram = unit.get_mut(subprogram_id); + subprogram.set( + gimli::constants::DW_AT_name, + WriteAttributeValue::String(b"body_lookup".to_vec()), + ); + subprogram.set( + gimli::constants::DW_AT_low_pc, + WriteAttributeValue::Address(Address::Constant(0x401000)), + ); + subprogram.set( + gimli::constants::DW_AT_high_pc, + WriteAttributeValue::Udata(0x40), + ); + + let mut sections = Sections::new(EndianVec::new(LittleEndian)); + dwarf.write(&mut sections).unwrap(); + + let dwarf_sections: gimli::DwarfSections> = gimli::DwarfSections::load(|id| { + Ok::<_, gimli::Error>( + sections + .get(id) + .map(|section| section.slice().to_vec()) + .unwrap_or_default(), + ) + }) + .unwrap(); + + dwarf_sections.borrow(|section| dwarf_reader_from_arc(Arc::<[u8]>::from(section.as_slice()))) +} + +fn clang_available() -> bool { + Command::new("clang") + .arg("--version") + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +fn compile_inline_callsite_fixture_with_clang_dwarf5() -> anyhow::Result { + let workspace_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or_else(|| anyhow::anyhow!("ghostscope-dwarf has no workspace parent"))? + .to_path_buf(); + let source = workspace_root + .join("e2e-tests/tests/fixtures/inline_callsite_program/inline_callsite_program.c"); + let binary = tempfile::Builder::new() + .prefix("ghostscope-fast-parser-") + .tempfile()? + .into_temp_path(); + let binary_path = binary.to_path_buf(); + + let compile_output = Command::new("clang") + .args(["-Wall", "-Wextra", "-gdwarf-5", "-O3"]) + .arg("-o") + .arg(&binary_path) + .arg(&source) + .output() + .map_err(|e| anyhow::anyhow!("Failed to run clang for {}: {}", source.display(), e))?; + + if compile_output.status.success() { + Ok(binary) + } else { + let stderr = String::from_utf8_lossy(&compile_output.stderr); + Err(anyhow::anyhow!( + "Failed to compile {} with clang -gdwarf-5 -O3: {}", + source.display(), + stderr + )) + } +} + +fn load_dwarf_from_binary(path: &Path) -> anyhow::Result> { + let bytes = std::fs::read(path) + .map_err(|e| anyhow::anyhow!("Failed to read {}: {}", path.display(), e))?; + let object = object::File::parse(&*bytes) + .map_err(|e| anyhow::anyhow!("Failed to parse {}: {}", path.display(), e))?; + let dwarf = gimli::Dwarf::load(|id| { + let section_data = object + .section_by_name(id.name()) + .and_then(|section| section.uncompressed_data().ok()) + .map(|data| data.into_owned()) + .unwrap_or_default(); + Ok::<_, gimli::Error>(dwarf_reader_from_arc(Arc::<[u8]>::from(section_data))) + })?; + Ok(dwarf) +} + +fn read_uleb128(input: &[u8], offset: &mut usize) -> anyhow::Result { + let mut value = 0_u64; + let mut shift = 0_u32; + loop { + let byte = *input + .get(*offset) + .ok_or_else(|| anyhow::anyhow!("Unexpected EOF while reading ULEB128"))?; + *offset += 1; + let low_bits = u64::from(byte & 0x7f); + anyhow::ensure!( + shift < 64 && !(shift == 63 && low_bits > 1), + "ULEB128 value exceeds u64" + ); + value |= low_bits << shift; + if byte & 0x80 == 0 { + return Ok(value); + } + shift += 7; + } +} + +#[test] +fn read_uleb128_rejects_values_that_overflow_u64() { + let overflow = [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x02]; + let mut offset = 0; + let err = read_uleb128(&overflow, &mut offset).expect_err("overflow should be rejected"); + assert!( + err.to_string().contains("exceeds u64"), + "unexpected overflow error: {err}" + ); +} + +fn patch_inlined_subroutine_low_pc_to_entry_pc(abbrev: &mut [u8]) -> anyhow::Result { + let mut offset = 0; + let mut patched = 0; + + while offset < abbrev.len() { + let code = read_uleb128(abbrev, &mut offset)?; + if code == 0 { + continue; + } + + let tag = read_uleb128(abbrev, &mut offset)?; + let _has_children = *abbrev + .get(offset) + .ok_or_else(|| anyhow::anyhow!("Missing abbrev children byte"))?; + offset += 1; + + loop { + let name_offset = offset; + let name = read_uleb128(abbrev, &mut offset)?; + let form = read_uleb128(abbrev, &mut offset)?; + if name == 0 && form == 0 { + break; + } + + let is_addrx_form = form == u64::from(gimli::constants::DW_FORM_addrx.0) + || form == u64::from(gimli::constants::DW_FORM_addrx1.0) + || form == u64::from(gimli::constants::DW_FORM_addrx2.0) + || form == u64::from(gimli::constants::DW_FORM_addrx3.0) + || form == u64::from(gimli::constants::DW_FORM_addrx4.0); + if tag == u64::from(gimli::constants::DW_TAG_inlined_subroutine.0) + && name == u64::from(gimli::constants::DW_AT_low_pc.0) + && is_addrx_form + { + *abbrev + .get_mut(name_offset) + .ok_or_else(|| anyhow::anyhow!("Invalid abbrev attribute offset"))? = + gimli::constants::DW_AT_entry_pc.0 as u8; + patched += 1; + } + } + } + + Ok(patched) +} + +fn rewrite_inline_fixture_entry_pc_attr(input_path: &Path) -> anyhow::Result { + let mut bytes = std::fs::read(input_path) + .map_err(|e| anyhow::anyhow!("Failed to read {}: {}", input_path.display(), e))?; + let (abbrev_offset, abbrev_size) = { + let object = object::File::parse(&*bytes) + .map_err(|e| anyhow::anyhow!("Failed to parse {}: {}", input_path.display(), e))?; + let section = object + .section_by_name(".debug_abbrev") + .ok_or_else(|| anyhow::anyhow!("{} is missing .debug_abbrev", input_path.display()))?; + section.file_range().ok_or_else(|| { + anyhow::anyhow!( + "{} has no file range for .debug_abbrev", + input_path.display() + ) + })? + }; + + let patched = patch_inlined_subroutine_low_pc_to_entry_pc( + &mut bytes[abbrev_offset as usize..(abbrev_offset + abbrev_size) as usize], + )?; + anyhow::ensure!( + patched > 0, + "Expected to patch at least one inline low_pc abbrev in {}", + input_path.display() + ); + + let output = tempfile::Builder::new() + .prefix(".ghostscope-fast-parser-patched-") + .tempfile()? + .into_temp_path(); + std::fs::write(&output, &bytes)?; + let perms = std::fs::metadata(input_path)?.permissions().mode(); + std::fs::set_permissions(&output, std::fs::Permissions::from_mode(perms))?; + Ok(output) +} + +fn has_inline_entry_pc_debug_addr_index(dwarf: &gimli::Dwarf) -> bool { + let mut units = dwarf.units(); + while let Ok(Some(header)) = units.next() { + let Ok(unit) = dwarf.unit(header) else { + continue; + }; + let mut entries = unit.entries(); + while let Ok(Some(entry)) = entries.next_dfs() { + if entry.tag() != gimli::constants::DW_TAG_inlined_subroutine { + continue; + } + if let Some(attr) = entry.attr(gimli::constants::DW_AT_entry_pc) { + if matches!(attr.value(), gimli::AttributeValue::DebugAddrIndex(_)) { + return true; + } + } + } + } + false +} + +#[test] +fn parse_debug_info_skips_stack_value_address_locals_from_global_index() { + let dwarf = build_variable_index_fixture(); + let parser = DwarfParser { dwarf: &dwarf }; + + let result = parser.parse_debug_info("synthetic").unwrap(); + let real_global = result + .lightweight_index + .find_variables_by_name("real_global"); + let local_static = result + .lightweight_index + .find_variables_by_name("local_static"); + let optimized_local = result.lightweight_index.find_variables_by_name("p"); + + assert_eq!( + real_global.len(), + 1, + "real global should remain indexed: {real_global:?}" + ); + assert_eq!( + local_static.len(), + 1, + "function-scoped static with real storage should remain indexed: {local_static:?}" + ); + assert!( + optimized_local.is_empty(), + "address-valued optimized local must not be indexed as a global: {optimized_local:?}" + ); +} + +#[test] +fn parse_debug_info_keeps_concrete_abstract_origin_subprogram_non_inline() { + // Regression scenario: + // GCC/Clang can emit all three DIE shapes for one logical function: + // 1. an abstract DW_TAG_subprogram marked DW_AT_inline, + // 2. a concrete out-of-line DW_TAG_subprogram with DW_AT_abstract_origin, + // 3. one or more DW_TAG_inlined_subroutine instances. + // + // The bug was that merge_from_origin copied the abstract function's + // inline attribute from (1) onto (2) and downstream code treated that + // as if the concrete body were an inline instance. + // Once that happened, the concrete body was routed through the inline + // address-selection path and could pick the wrong cold-partition PC. + // + // This test keeps the synthetic DIE graph minimal and asserts the parser + // preserves the intended split: + // - abstract definition stays inline + // - concrete out-of-line body stays non-inline + // - inlined_subroutine instance stays inline + let dwarf = build_inline_origin_fixture(); + let parser = DwarfParser { dwarf: &dwarf }; + + let result = parser.parse_debug_info("synthetic").unwrap(); + let entries = result + .lightweight_index + .find_dies_by_function_name("CGPsend"); + + let concrete_entries: Vec<_> = entries + .iter() + .copied() + .filter(|entry| entry.function_kind() == crate::core::FunctionDieKind::ConcreteSubprogram) + .collect(); + let abstract_entries: Vec<_> = entries + .iter() + .copied() + .filter(|entry| entry.function_kind() == crate::core::FunctionDieKind::AbstractSubprogram) + .collect(); + let inlined_entries: Vec<_> = entries + .iter() + .copied() + .filter(|entry| entry.function_kind() == crate::core::FunctionDieKind::InlineInstance) + .collect(); + + assert_eq!( + concrete_entries.len(), + 1, + "concrete out-of-line subprogram should stay non-inline: {entries:?}" + ); + assert_eq!( + abstract_entries.len(), + 1, + "only the abstract inline definition should carry the inline flag: {entries:?}" + ); + assert_eq!( + inlined_entries.len(), + 1, + "expected one inlined subroutine instance: {entries:?}" + ); + assert!( + inlined_entries[0].is_inline_instance(), + "DW_TAG_inlined_subroutine must remain an inline instance: {entries:?}" + ); + assert!( + !concrete_entries[0].flags.has_inline_attribute, + "concrete out-of-line body should not inherit the abstract inline attribute: {entries:?}" + ); + assert!( + abstract_entries[0].flags.has_inline_attribute, + "abstract definition should retain its original DW_AT_inline attribute: {entries:?}" + ); +} + +#[test] +fn parse_debug_info_resolves_cross_cu_origin_names_for_index_entries() { + let dwarf = build_cross_cu_origin_fixture(); + let parser = DwarfParser { dwarf: &dwarf }; + + let result = parser.parse_debug_info("synthetic").unwrap(); + let function_entries = result + .lightweight_index + .find_dies_by_function_name("cross_cu_origin_fn"); + assert!( + function_entries.iter().any(|entry| { + entry.function_kind() == crate::core::FunctionDieKind::ConcreteSubprogram + && entry.representative_addr == Some(0x501000) + }), + "concrete function should inherit its name through a cross-CU origin: {function_entries:?}" + ); + + let mut has_concrete_type = false; + result + .lightweight_index + .for_each_type_map_entry(|name, entry_base, indices| { + if name != "CrossCuOriginType" { + return; + } + has_concrete_type |= indices.iter().any(|local_idx| { + result + .lightweight_index + .entry(entry_base + *local_idx) + .is_some_and(|entry| !entry.flags.is_type_declaration) + }); + }); + + assert!( + has_concrete_type, + "concrete type should inherit its name through a cross-CU specification" + ); +} + +#[test] +fn parse_debug_info_keeps_sharded_name_lookup_without_duplicate_keys() { + let dwarf = build_multi_cu_shared_function_fixture(); + let parser = DwarfParser { dwarf: &dwarf }; + + let result = parser.parse_debug_info("synthetic").unwrap(); + let entries = result + .lightweight_index + .find_dies_by_function_name("shared"); + let names = result.lightweight_index.get_function_names(); + + assert_eq!( + result.functions_count, 1, + "unique function-name count should stay deduplicated across shards" + ); + assert_eq!( + entries.len(), + 2, + "function lookup should fan out across CU shards: {entries:?}" + ); + assert_eq!( + names + .iter() + .filter(|name| name.as_str() == "shared") + .count(), + 1, + "function name listing should not duplicate shard-local keys: {names:?}" + ); +} + +#[test] +fn parse_debug_info_builds_fallback_cu_map_for_function_body_addresses() { + let dwarf = build_cu_body_lookup_fixture(); + let parser = DwarfParser { dwarf: &dwarf }; + + let result = parser.parse_debug_info("synthetic").unwrap(); + let entry = result + .lightweight_index + .find_function_by_address(0x401020, |entry| { + let header = dwarf.unit_header(entry.unit_offset).ok()?; + let unit = dwarf.unit(header).ok()?; + let die = unit.entry(entry.die_offset).ok()?; + crate::parser::RangeExtractor::extract_all_ranges(&die, &unit, &dwarf).ok() + }) + .expect("function body address should resolve through fallback CU map"); + + assert_eq!(entry.name.as_ref(), "body_lookup"); + assert_eq!( + result.lightweight_index.find_cu_by_address(0x401020), + Some(entry.unit_offset), + "fallback CU map should cover addresses inside the function body, not just the representative address" + ); +} + +#[test] +fn parse_debug_info_resolves_debug_addr_index_entry_pc_for_inline_instances() { + if !clang_available() { + eprintln!("Skipping fast_parser DWARF5 entry_pc regression: clang is unavailable"); + return; + } + + let patched_binary = { + let binary = compile_inline_callsite_fixture_with_clang_dwarf5() + .expect("clang dwarf5 inline fixture should compile"); + rewrite_inline_fixture_entry_pc_attr(binary.as_ref()) + .expect("inline fixture abbrev should rewrite low_pc addrx into entry_pc addrx") + }; + let dwarf = load_dwarf_from_binary(patched_binary.as_ref()) + .expect("compiled inline fixture should load as DWARF"); + assert!( + has_inline_entry_pc_debug_addr_index(&dwarf), + "patched inline fixture should expose an inlined_subroutine DW_AT_entry_pc via DW_FORM_addrx/.debug_addr" + ); + + let parser = DwarfParser { dwarf: &dwarf }; + let result = parser + .parse_debug_info(patched_binary.to_string_lossy().as_ref()) + .expect("fast parser should index patched clang dwarf5 inline fixture"); + + let inline_entries: Vec<_> = ["add3", "consume_state"] + .into_iter() + .flat_map(|name| { + result + .lightweight_index + .find_dies_by_function_name(name) + .iter() + .copied() + .filter(|entry| { + entry.function_kind() == crate::core::FunctionDieKind::InlineInstance + }) + .collect::>() + }) + .collect(); + + assert!( + !inline_entries.is_empty(), + "expected inline entries for clang dwarf5 inline fixture" + ); + assert!( + inline_entries.iter().any(|entry| entry.entry_pc.is_some()), + "fast parser should resolve DW_FORM_addrx-backed DW_AT_entry_pc values for inline instances: {inline_entries:?}" + ); +} diff --git a/ghostscope-dwarf/src/semantics/variable_plan.rs b/ghostscope-dwarf/src/semantics/variable_plan/mod.rs similarity index 57% rename from ghostscope-dwarf/src/semantics/variable_plan.rs rename to ghostscope-dwarf/src/semantics/variable_plan/mod.rs index 5c4c2496..1b1a2ea3 100644 --- a/ghostscope-dwarf/src/semantics/variable_plan.rs +++ b/ghostscope-dwarf/src/semantics/variable_plan/mod.rs @@ -1173,901 +1173,4 @@ pub struct VariablePlan { } #[cfg(test)] -mod tests { - use super::*; - use crate::core::{AddressExpr, EntryValueCase, MemoryAccessSize, TargetArch}; - use crate::StructMember; - - fn capabilities(regular_uprobe: bool) -> RuntimeCapabilities { - RuntimeCapabilities { - regular_uprobe, - sleepable_uprobe: false, - uprobe_multi: false, - copy_from_user_task: false, - max_bpf_stack_bytes: 512, - bounded_loops: true, - arch: TargetArch::X86_64, - } - } - - fn read_plan(location: VariableLocation) -> VariableReadPlan { - VariableReadPlan { - name: "value".to_string(), - type_name: "int".to_string(), - access_path: VariableAccessPath::default(), - module_path: None, - dwarf_type: None, - declaration: None, - type_id: None, - location, - availability: Availability::Available, - scope_depth: 0, - is_parameter: false, - is_artificial: false, - pc_range: None, - inline_context: None, - provenance: Provenance::DirectDie, - } - } - - fn typed_read_plan(location: VariableLocation, dwarf_type: TypeInfo) -> VariableReadPlan { - VariableReadPlan { - type_name: dwarf_type.type_name(), - dwarf_type: Some(dwarf_type), - ..read_plan(location) - } - } - - #[test] - fn lvalue_address_plan_accepts_address_locations_without_type_info() { - let plan = read_plan(VariableLocation::RegisterAddress { - dwarf_reg: 6, - offset: -16, - }); - - let lvalue = plan.lvalue_address_plan(); - - assert_eq!( - lvalue, - LvalueAddressPlan::Address { - address: PlannedAddress { - kind: PlannedAddressKind::RegisterOffset { - dwarf_reg: 6, - offset: -16 - }, - origin: AddressOrigin::RuntimeDerived, - } - } - ); - } - - #[test] - fn lvalue_address_plan_rejects_value_backed_locations() { - let plan = read_plan(VariableLocation::RegisterValue { dwarf_reg: 0 }); - - let lvalue = plan.lvalue_address_plan(); - - assert!(matches!( - lvalue, - LvalueAddressPlan::Unavailable { - availability: Availability::Unsupported(UnsupportedReason::AddressClass { .. }) - } - )); - match lvalue { - LvalueAddressPlan::Unavailable { - availability: Availability::Unsupported(UnsupportedReason::AddressClass { detail }), - } => { - assert!(detail.contains("value-backed")); - } - other => panic!("unexpected lvalue availability: {other:?}"), - } - } - - #[test] - fn lvalue_address_plan_rejects_absolute_address_values() { - let plan = read_plan(VariableLocation::AbsoluteAddressValue( - AddressExpr::constant(0x2000), - )); - - let lvalue = plan.lvalue_address_plan(); - - match lvalue { - LvalueAddressPlan::Unavailable { - availability: Availability::Unsupported(UnsupportedReason::AddressClass { detail }), - } => { - assert!(detail.contains("value-backed")); - } - other => panic!("unexpected lvalue availability: {other:?}"), - } - } - - #[test] - fn lvalue_address_plan_rejects_piece_locations() { - let plan = read_plan(VariableLocation::Pieces(vec![PieceLocation { - bit_offset: 0, - bit_size: 32, - location: Box::new(VariableLocation::RegisterValue { dwarf_reg: 0 }), - }])); - - let lvalue = plan.lvalue_address_plan(); - - match lvalue { - LvalueAddressPlan::Unavailable { - availability: - Availability::Unsupported(UnsupportedReason::ExpressionShape { detail }), - } => { - assert!(detail.contains("split variable pieces")); - } - other => panic!("unexpected lvalue availability: {other:?}"), - } - } - - #[test] - fn lvalue_address_plan_preserves_optimized_out_availability() { - let plan = VariableReadPlan { - availability: Availability::OptimizedOut, - ..read_plan(VariableLocation::OptimizedOut) - }; - - let lvalue = plan.lvalue_address_plan(); - - assert_eq!( - lvalue, - LvalueAddressPlan::Unavailable { - availability: Availability::OptimizedOut - } - ); - } - - #[test] - fn register_value_lowers_without_runtime_requirements() { - let plan = read_plan(VariableLocation::RegisterValue { dwarf_reg: 0 }); - let lowering = plan.bpf_lowering_plan(&capabilities(false)); - - assert_eq!(lowering.kind, VariableLoweringKind::DirectValue); - assert_eq!(lowering.availability, Availability::Available); - assert!(lowering.requirements.is_empty()); - } - - #[test] - fn memory_location_requires_user_memory_read() { - let plan = read_plan(VariableLocation::Address(AddressExpr::constant(0x1000))); - let lowering = plan.bpf_lowering_plan(&capabilities(false)); - - assert_eq!(lowering.kind, VariableLoweringKind::UserMemoryRead); - assert_eq!( - lowering.availability, - Availability::Requires(RuntimeRequirement::UserMemoryRead) - ); - assert_eq!( - lowering.requirements, - vec![RuntimeRequirement::UserMemoryRead] - ); - } - - #[test] - fn memory_location_is_available_with_regular_uprobe() { - let plan = read_plan(VariableLocation::Address(AddressExpr::constant(0x1000))); - let lowering = plan.bpf_lowering_plan(&capabilities(true)); - - assert_eq!(lowering.kind, VariableLoweringKind::UserMemoryRead); - assert_eq!(lowering.availability, Availability::Available); - assert_eq!(lowering.helper_mode, HelperMode::ProbeReadUser); - assert_eq!(lowering.verifier_risk, VerifierRisk::Low); - assert!(lowering.required_registers.is_empty()); - } - - #[test] - fn materialization_plan_preserves_link_time_address_origin() { - let plan = read_plan(VariableLocation::Address(AddressExpr::constant(0x1000))); - let materialized = plan.materialization_plan(&capabilities(true)); - - match materialized.materialization { - VariableMaterialization::UserMemoryRead { address } => { - assert_eq!(address.origin, AddressOrigin::LinkTime); - assert_eq!(address.constant_link_time_address(), Some(0x1000)); - assert_eq!( - address.kind, - PlannedAddressKind::Constant { address: 0x1000 } - ); - } - other => panic!("unexpected materialization: {other:?}"), - } - } - - #[test] - fn materialization_plan_preserves_module_path_origin() { - let mut plan = read_plan(VariableLocation::Address(AddressExpr::constant(0x1000))); - plan.module_path = Some(PathBuf::from("/tmp/libstate.so")); - - let materialized = plan.materialization_plan(&capabilities(true)); - - assert_eq!( - materialized.module_path, - Some(PathBuf::from("/tmp/libstate.so")) - ); - } - - #[test] - fn materialization_plan_converts_register_address_to_address_kind() { - let plan = read_plan(VariableLocation::RegisterAddress { - dwarf_reg: 6, - offset: -16, - }); - let materialized = plan.materialization_plan(&capabilities(true)); - - match materialized.materialization { - VariableMaterialization::UserMemoryRead { address } => { - assert_eq!(address.origin, AddressOrigin::RuntimeDerived); - assert_eq!( - address.kind, - PlannedAddressKind::RegisterOffset { - dwarf_reg: 6, - offset: -16 - } - ); - } - other => panic!("unexpected materialization: {other:?}"), - } - } - - #[test] - fn materialization_plan_marks_static_base_before_deref() { - let plan = read_plan(VariableLocation::ComputedAddress(vec![ - PlanExprOp::PushConstant(0x3000), - PlanExprOp::Dereference { - size: MemoryAccessSize::U64, - }, - PlanExprOp::PushConstant(16), - PlanExprOp::Add, - ])); - let materialized = plan.materialization_plan(&capabilities(true)); - - match materialized.materialization { - VariableMaterialization::UserMemoryRead { address } => { - assert_eq!(address.origin, AddressOrigin::LinkTimeBase); - match &address.kind { - PlannedAddressKind::RuntimeComputed { expr } => { - assert_eq!(expr.kind(), RuntimeComputedKind::Address); - } - other => panic!("unexpected address kind: {other:?}"), - } - let (base, tail) = address - .link_time_base_and_runtime_tail() - .expect("link-time base"); - assert_eq!(base, 0x3000); - assert_eq!(tail.len(), 3); - } - other => panic!("unexpected materialization: {other:?}"), - } - } - - #[test] - fn materialization_plan_preserves_arithmetic_before_first_deref() { - let plan = read_plan(VariableLocation::ComputedAddress(vec![ - PlanExprOp::PushConstant(0x3000), - PlanExprOp::PushConstant(8), - PlanExprOp::Add, - PlanExprOp::Dereference { - size: MemoryAccessSize::U64, - }, - ])); - let materialized = plan.materialization_plan(&capabilities(true)); - - match materialized.materialization { - VariableMaterialization::UserMemoryRead { address } => { - assert_eq!(address.origin, AddressOrigin::LinkTimeBase); - let (base, tail) = address - .link_time_base_and_runtime_tail() - .expect("link-time base"); - assert_eq!(base, 0x3000); - assert_eq!( - tail, - &[ - PlanExprOp::PushConstant(8), - PlanExprOp::Add, - PlanExprOp::Dereference { - size: MemoryAccessSize::U64, - }, - ] - ); - } - other => panic!("unexpected materialization: {other:?}"), - } - } - - #[test] - fn materialization_plan_keeps_absolute_address_value_direct() { - let plan = read_plan(VariableLocation::AbsoluteAddressValue( - AddressExpr::constant(0x2000), - )); - let materialized = plan.materialization_plan(&capabilities(false)); - - match materialized.materialization { - VariableMaterialization::DirectValue { - value: - PlannedValue::AddressValue { - address: - PlannedAddress { - origin: AddressOrigin::LinkTime, - kind: PlannedAddressKind::Constant { address: 0x2000 }, - .. - }, - size: MemoryAccessSize::U64, - }, - } => {} - VariableMaterialization::DirectValue { value } => { - panic!("unexpected direct value: {value:?}"); - } - other => panic!("unexpected materialization: {other:?}"), - } - } - - #[test] - fn materialization_plan_converts_constant_direct_value() { - let plan = read_plan(VariableLocation::ComputedValue(vec![ - PlanExprOp::PushConstant(42), - ])); - let materialized = plan.materialization_plan(&capabilities(false)); - - match materialized.materialization { - VariableMaterialization::DirectValue { - value: - PlannedValue::Constant { - value: 42, - size: MemoryAccessSize::U64, - }, - } => {} - other => panic!("unexpected materialization: {other:?}"), - } - } - - #[test] - fn materialization_plan_records_direct_value_size_from_type() { - let byte_type = TypeInfo::BaseType { - name: "uint8_t".to_string(), - size: 1, - encoding: gimli::constants::DW_ATE_unsigned.0 as u16, - }; - let plan = typed_read_plan( - VariableLocation::ComputedValue(vec![ - PlanExprOp::LoadRegister(0), - PlanExprOp::PushConstant(1), - PlanExprOp::Add, - ]), - byte_type, - ); - let materialized = plan.materialization_plan(&capabilities(false)); - - match materialized.materialization { - VariableMaterialization::DirectValue { - value: - PlannedValue::RuntimeComputed { - ref expr, - result_size: MemoryAccessSize::U8, - .. - }, - } => { - assert_eq!(expr.kind(), RuntimeComputedKind::Value); - } - other => panic!("unexpected materialization: {other:?}"), - } - } - - #[test] - fn materialization_plan_converts_register_direct_value() { - let plan = read_plan(VariableLocation::RegisterValue { dwarf_reg: 6 }); - let materialized = plan.materialization_plan(&capabilities(false)); - - match materialized.materialization { - VariableMaterialization::DirectValue { - value: - PlannedValue::RegisterValue { - dwarf_reg: 6, - size: MemoryAccessSize::U64, - }, - } => {} - other => panic!("unexpected materialization: {other:?}"), - } - } - - #[test] - fn materialization_plan_surfaces_piece_locations_without_first_piece_fallback() { - let plan = read_plan(VariableLocation::Pieces(vec![PieceLocation { - bit_offset: 0, - bit_size: 32, - location: Box::new(VariableLocation::RegisterValue { dwarf_reg: 0 }), - }])); - let materialized = plan.materialization_plan(&capabilities(true)); - - match materialized.materialization { - VariableMaterialization::Composite { pieces } => { - assert_eq!(pieces.len(), 1); - } - other => panic!("unexpected materialization: {other:?}"), - } - } - - #[test] - fn absolute_address_value_lowers_without_user_memory_read() { - let plan = read_plan(VariableLocation::AbsoluteAddressValue( - AddressExpr::constant(0x1000), - )); - let lowering = plan.bpf_lowering_plan(&capabilities(false)); - - assert_eq!(lowering.kind, VariableLoweringKind::DirectValue); - assert_eq!(lowering.availability, Availability::Available); - assert!(lowering.requirements.is_empty()); - } - - #[test] - fn memory_location_prefers_copy_from_user_task_when_available() { - let mut capabilities = capabilities(false); - capabilities.sleepable_uprobe = true; - capabilities.copy_from_user_task = true; - let plan = read_plan(VariableLocation::Address(AddressExpr::constant(0x1000))); - let lowering = plan.bpf_lowering_plan(&capabilities); - - assert_eq!(lowering.availability, Availability::Available); - assert_eq!(lowering.helper_mode, HelperMode::CopyFromUserTask); - } - - #[test] - fn register_address_records_required_register() { - let plan = read_plan(VariableLocation::RegisterAddress { - dwarf_reg: 6, - offset: -16, - }); - let lowering = plan.bpf_lowering_plan(&capabilities(true)); - - assert_eq!(lowering.required_registers, vec![6]); - assert_eq!(lowering.estimated_stack_bytes, 8); - } - - #[test] - fn entry_value_steps_surface_caller_frame_and_memory_requirements() { - let plan = read_plan(VariableLocation::ComputedValue(vec![ - PlanExprOp::EntryValueLookup { - caller_pc_steps: vec![ - PlanExprOp::LoadRegister(7), - PlanExprOp::Dereference { - size: MemoryAccessSize::U64, - }, - ], - cases: vec![EntryValueCase { - caller_return_pc: 0x10, - value_steps: vec![PlanExprOp::LoadRegister(5)], - }], - }, - ])); - let lowering = plan.bpf_lowering_plan(&capabilities(true)); - - assert_eq!(lowering.availability, Availability::Available); - assert_eq!( - lowering.requirements, - vec![ - RuntimeRequirement::CallerFrame, - RuntimeRequirement::UserMemoryRead - ] - ); - assert_eq!(lowering.required_registers, vec![5, 7]); - assert_eq!(lowering.verifier_risk, VerifierRisk::RequiresBoundedLoops); - } - - #[test] - fn stack_budget_excess_reports_unsupported_availability() { - let mut capabilities = capabilities(true); - capabilities.max_bpf_stack_bytes = 16; - let plan = read_plan(VariableLocation::ComputedValue(vec![ - PlanExprOp::PushConstant(1); - 8 - ])); - let lowering = plan.bpf_lowering_plan(&capabilities); - - assert!(matches!( - lowering.availability, - Availability::Unsupported(UnsupportedReason::ExpressionShape { .. }) - )); - assert_eq!( - lowering.verifier_risk, - VerifierRisk::StackBudgetExceeded { - estimated: 64, - max: 16, - } - ); - } - - #[test] - fn field_access_adds_member_offset_and_type() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let plan = typed_read_plan( - VariableLocation::RegisterAddress { - dwarf_reg: 6, - offset: -32, - }, - TypeInfo::StructType { - name: "Request".to_string(), - size: 16, - members: vec![StructMember { - name: "fd".to_string(), - member_type: int_type.clone(), - offset: 12, - bit_offset: None, - bit_size: None, - }], - }, - ); - - let access = VariableAccessPath::fields(["fd"]); - let planned = plan.plan_access_path(&access).expect("field access"); - - assert_eq!(planned.name, "value.fd"); - assert_eq!(planned.access_path, access); - assert_eq!(planned.dwarf_type, Some(int_type)); - assert_eq!( - planned.location, - VariableLocation::RegisterAddress { - dwarf_reg: 6, - offset: -20, - } - ); - assert_eq!( - planned - .materialization_plan(&capabilities(true)) - .access_path - .segments, - vec![VariableAccessSegment::Field("fd".to_string())] - ); - } - - #[test] - fn field_access_unknown_member_reports_known_members() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let plan = typed_read_plan( - VariableLocation::Address(AddressExpr::constant(0x1000)), - TypeInfo::StructType { - name: "Request".to_string(), - size: 8, - members: vec![ - StructMember { - name: "fd".to_string(), - member_type: int_type.clone(), - offset: 0, - bit_offset: None, - bit_size: None, - }, - StructMember { - name: "flags".to_string(), - member_type: int_type, - offset: 4, - bit_offset: None, - bit_size: None, - }, - ], - }, - ); - - let err = plan - .plan_access_path(&VariableAccessPath::fields(["missing"])) - .expect_err("unknown member should fail"); - - assert_eq!( - err.to_string(), - "Unknown member 'missing' in struct 'Request' (known members: fd, flags)" - ); - } - - #[test] - fn field_access_folds_constant_address_offsets() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let plan = typed_read_plan( - VariableLocation::Address(AddressExpr::constant(0x1000)), - TypeInfo::StructType { - name: "Request".to_string(), - size: 16, - members: vec![StructMember { - name: "fd".to_string(), - member_type: int_type, - offset: 12, - bit_offset: None, - bit_size: None, - }], - }, - ); - - let planned = plan - .plan_access_path(&VariableAccessPath::fields(["fd"])) - .expect("field access"); - - assert_eq!( - planned.location, - VariableLocation::Address(AddressExpr::constant(0x100c)) - ); - } - - #[test] - fn field_access_rejects_value_backed_aggregates() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let struct_type = TypeInfo::StructType { - name: "Pair".to_string(), - size: 8, - members: vec![StructMember { - name: "b".to_string(), - member_type: int_type, - offset: 4, - bit_offset: None, - bit_size: None, - }], - }; - let access = VariableAccessPath::fields(["b"]); - - for location in [ - VariableLocation::AbsoluteAddressValue(AddressExpr::constant(0x1000)), - VariableLocation::RegisterValue { dwarf_reg: 0 }, - VariableLocation::ComputedValue(vec![PlanExprOp::LoadRegister(0)]), - ] { - let plan = typed_read_plan(location, struct_type.clone()); - let err = plan - .plan_access_path(&access) - .expect_err("value-backed aggregate field access should fail"); - assert!( - err.downcast_ref::() - .is_some_and(PlanError::is_value_backed_aggregate_access), - "unexpected error: {err}" - ); - } - } - - #[test] - fn array_index_rejects_value_backed_aggregates() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let array_type = TypeInfo::ArrayType { - element_type: Box::new(int_type), - element_count: Some(2), - total_size: Some(8), - }; - let access = VariableAccessPath::new(vec![VariableAccessSegment::ArrayIndex(1)]); - - for location in [ - VariableLocation::AbsoluteAddressValue(AddressExpr::constant(0x1000)), - VariableLocation::RegisterValue { dwarf_reg: 0 }, - VariableLocation::ComputedValue(vec![PlanExprOp::LoadRegister(0)]), - ] { - let plan = typed_read_plan(location, array_type.clone()); - let err = plan - .plan_access_path(&access) - .expect_err("value-backed aggregate array access should fail"); - assert!( - err.downcast_ref::() - .is_some_and(PlanError::is_value_backed_aggregate_access), - "unexpected error: {err}" - ); - } - } - - #[test] - fn pointer_field_access_dereferences_then_offsets() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let struct_type = TypeInfo::StructType { - name: "Node".to_string(), - size: 16, - members: vec![StructMember { - name: "value".to_string(), - member_type: int_type, - offset: 8, - bit_offset: None, - bit_size: None, - }], - }; - let plan = typed_read_plan( - VariableLocation::RegisterValue { dwarf_reg: 5 }, - TypeInfo::PointerType { - target_type: Box::new(struct_type), - size: 8, - }, - ); - - let access = VariableAccessPath::fields(["value"]); - let planned = plan.plan_access_path(&access).expect("pointer field"); - - assert_eq!( - planned.location, - VariableLocation::ComputedAddress(vec![ - PlanExprOp::LoadRegister(5), - PlanExprOp::PushConstant(8), - PlanExprOp::Add, - ]) - ); - } - - #[test] - fn pointer_field_access_from_absolute_address_value_rebases_memory_location() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let struct_type = TypeInfo::StructType { - name: "Node".to_string(), - size: 16, - members: vec![StructMember { - name: "value".to_string(), - member_type: int_type, - offset: 8, - bit_offset: None, - bit_size: None, - }], - }; - let plan = typed_read_plan( - VariableLocation::AbsoluteAddressValue(AddressExpr::constant(0x1000)), - TypeInfo::PointerType { - target_type: Box::new(struct_type), - size: 8, - }, - ); - - let planned = plan - .plan_access_path(&VariableAccessPath::fields(["value"])) - .expect("pointer field"); - - assert_eq!( - planned.location, - VariableLocation::Address(AddressExpr::constant(0x1008)) - ); - } - - #[test] - fn pointer_field_access_from_computed_value_uses_value_as_address() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let struct_type = TypeInfo::StructType { - name: "Node".to_string(), - size: 16, - members: vec![StructMember { - name: "value".to_string(), - member_type: int_type, - offset: 8, - bit_offset: None, - bit_size: None, - }], - }; - let plan = typed_read_plan( - VariableLocation::ComputedValue(vec![PlanExprOp::PushConstant(0x2000)]), - TypeInfo::PointerType { - target_type: Box::new(struct_type), - size: 8, - }, - ); - - let planned = plan - .plan_access_path(&VariableAccessPath::fields(["value"])) - .expect("pointer field"); - - assert_eq!( - planned.location, - VariableLocation::ComputedAddress(vec![ - PlanExprOp::PushConstant(0x2000), - PlanExprOp::PushConstant(8), - PlanExprOp::Add, - ]) - ); - } - - #[test] - fn pointer_element_index_is_planned_in_dwarf_semantics() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let plan = typed_read_plan( - VariableLocation::RegisterValue { dwarf_reg: 5 }, - TypeInfo::PointerType { - target_type: Box::new(int_type), - size: 8, - }, - ); - - let planned = plan - .plan_pointer_element_index(3) - .expect("pointer element index"); - - assert_eq!(planned.name, "value[3]"); - assert_eq!( - planned.location, - VariableLocation::ComputedAddress(vec![ - PlanExprOp::LoadRegister(5), - PlanExprOp::PushConstant(12), - PlanExprOp::Add, - ]) - ); - } - - #[test] - fn pointer_element_index_rejects_aggregate_arithmetic_with_pointer_error() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let plan = typed_read_plan( - VariableLocation::Address(AddressExpr::constant(0x1000)), - TypeInfo::StructType { - name: "GlobalState".to_string(), - size: 16, - members: vec![StructMember { - name: "counter".to_string(), - member_type: int_type, - offset: 0, - bit_offset: None, - bit_size: None, - }], - }, - ); - - let err = plan - .plan_pointer_element_index(1) - .expect_err("struct arithmetic must be rejected"); - let plan_error = err - .downcast_ref::() - .expect("structured plan error"); - assert!(matches!( - plan_error, - PlanError::InvalidPointerArithmetic { type_name } - if type_name == "struct GlobalState" - )); - } - - #[test] - fn array_index_access_uses_element_stride() { - let int_type = TypeInfo::BaseType { - name: "int".to_string(), - size: 4, - encoding: gimli::constants::DW_ATE_signed.0 as u16, - }; - let plan = typed_read_plan( - VariableLocation::Address(AddressExpr::constant(0x1000)), - TypeInfo::ArrayType { - element_type: Box::new(int_type), - element_count: Some(8), - total_size: Some(32), - }, - ); - - let access = VariableAccessPath::new(vec![VariableAccessSegment::ArrayIndex(3)]); - let planned = plan.plan_access_path(&access).expect("array index"); - - assert_eq!(planned.name, "value[3]"); - assert_eq!( - planned.location, - VariableLocation::Address(AddressExpr::constant(0x100c)) - ); - } -} +mod tests; diff --git a/ghostscope-dwarf/src/semantics/variable_plan/tests.rs b/ghostscope-dwarf/src/semantics/variable_plan/tests.rs new file mode 100644 index 00000000..9aa6a662 --- /dev/null +++ b/ghostscope-dwarf/src/semantics/variable_plan/tests.rs @@ -0,0 +1,895 @@ +use super::*; +use crate::core::{AddressExpr, EntryValueCase, MemoryAccessSize, TargetArch}; +use crate::StructMember; + +fn capabilities(regular_uprobe: bool) -> RuntimeCapabilities { + RuntimeCapabilities { + regular_uprobe, + sleepable_uprobe: false, + uprobe_multi: false, + copy_from_user_task: false, + max_bpf_stack_bytes: 512, + bounded_loops: true, + arch: TargetArch::X86_64, + } +} + +fn read_plan(location: VariableLocation) -> VariableReadPlan { + VariableReadPlan { + name: "value".to_string(), + type_name: "int".to_string(), + access_path: VariableAccessPath::default(), + module_path: None, + dwarf_type: None, + declaration: None, + type_id: None, + location, + availability: Availability::Available, + scope_depth: 0, + is_parameter: false, + is_artificial: false, + pc_range: None, + inline_context: None, + provenance: Provenance::DirectDie, + } +} + +fn typed_read_plan(location: VariableLocation, dwarf_type: TypeInfo) -> VariableReadPlan { + VariableReadPlan { + type_name: dwarf_type.type_name(), + dwarf_type: Some(dwarf_type), + ..read_plan(location) + } +} + +#[test] +fn lvalue_address_plan_accepts_address_locations_without_type_info() { + let plan = read_plan(VariableLocation::RegisterAddress { + dwarf_reg: 6, + offset: -16, + }); + + let lvalue = plan.lvalue_address_plan(); + + assert_eq!( + lvalue, + LvalueAddressPlan::Address { + address: PlannedAddress { + kind: PlannedAddressKind::RegisterOffset { + dwarf_reg: 6, + offset: -16 + }, + origin: AddressOrigin::RuntimeDerived, + } + } + ); +} + +#[test] +fn lvalue_address_plan_rejects_value_backed_locations() { + let plan = read_plan(VariableLocation::RegisterValue { dwarf_reg: 0 }); + + let lvalue = plan.lvalue_address_plan(); + + assert!(matches!( + lvalue, + LvalueAddressPlan::Unavailable { + availability: Availability::Unsupported(UnsupportedReason::AddressClass { .. }) + } + )); + match lvalue { + LvalueAddressPlan::Unavailable { + availability: Availability::Unsupported(UnsupportedReason::AddressClass { detail }), + } => { + assert!(detail.contains("value-backed")); + } + other => panic!("unexpected lvalue availability: {other:?}"), + } +} + +#[test] +fn lvalue_address_plan_rejects_absolute_address_values() { + let plan = read_plan(VariableLocation::AbsoluteAddressValue( + AddressExpr::constant(0x2000), + )); + + let lvalue = plan.lvalue_address_plan(); + + match lvalue { + LvalueAddressPlan::Unavailable { + availability: Availability::Unsupported(UnsupportedReason::AddressClass { detail }), + } => { + assert!(detail.contains("value-backed")); + } + other => panic!("unexpected lvalue availability: {other:?}"), + } +} + +#[test] +fn lvalue_address_plan_rejects_piece_locations() { + let plan = read_plan(VariableLocation::Pieces(vec![PieceLocation { + bit_offset: 0, + bit_size: 32, + location: Box::new(VariableLocation::RegisterValue { dwarf_reg: 0 }), + }])); + + let lvalue = plan.lvalue_address_plan(); + + match lvalue { + LvalueAddressPlan::Unavailable { + availability: Availability::Unsupported(UnsupportedReason::ExpressionShape { detail }), + } => { + assert!(detail.contains("split variable pieces")); + } + other => panic!("unexpected lvalue availability: {other:?}"), + } +} + +#[test] +fn lvalue_address_plan_preserves_optimized_out_availability() { + let plan = VariableReadPlan { + availability: Availability::OptimizedOut, + ..read_plan(VariableLocation::OptimizedOut) + }; + + let lvalue = plan.lvalue_address_plan(); + + assert_eq!( + lvalue, + LvalueAddressPlan::Unavailable { + availability: Availability::OptimizedOut + } + ); +} + +#[test] +fn register_value_lowers_without_runtime_requirements() { + let plan = read_plan(VariableLocation::RegisterValue { dwarf_reg: 0 }); + let lowering = plan.bpf_lowering_plan(&capabilities(false)); + + assert_eq!(lowering.kind, VariableLoweringKind::DirectValue); + assert_eq!(lowering.availability, Availability::Available); + assert!(lowering.requirements.is_empty()); +} + +#[test] +fn memory_location_requires_user_memory_read() { + let plan = read_plan(VariableLocation::Address(AddressExpr::constant(0x1000))); + let lowering = plan.bpf_lowering_plan(&capabilities(false)); + + assert_eq!(lowering.kind, VariableLoweringKind::UserMemoryRead); + assert_eq!( + lowering.availability, + Availability::Requires(RuntimeRequirement::UserMemoryRead) + ); + assert_eq!( + lowering.requirements, + vec![RuntimeRequirement::UserMemoryRead] + ); +} + +#[test] +fn memory_location_is_available_with_regular_uprobe() { + let plan = read_plan(VariableLocation::Address(AddressExpr::constant(0x1000))); + let lowering = plan.bpf_lowering_plan(&capabilities(true)); + + assert_eq!(lowering.kind, VariableLoweringKind::UserMemoryRead); + assert_eq!(lowering.availability, Availability::Available); + assert_eq!(lowering.helper_mode, HelperMode::ProbeReadUser); + assert_eq!(lowering.verifier_risk, VerifierRisk::Low); + assert!(lowering.required_registers.is_empty()); +} + +#[test] +fn materialization_plan_preserves_link_time_address_origin() { + let plan = read_plan(VariableLocation::Address(AddressExpr::constant(0x1000))); + let materialized = plan.materialization_plan(&capabilities(true)); + + match materialized.materialization { + VariableMaterialization::UserMemoryRead { address } => { + assert_eq!(address.origin, AddressOrigin::LinkTime); + assert_eq!(address.constant_link_time_address(), Some(0x1000)); + assert_eq!( + address.kind, + PlannedAddressKind::Constant { address: 0x1000 } + ); + } + other => panic!("unexpected materialization: {other:?}"), + } +} + +#[test] +fn materialization_plan_preserves_module_path_origin() { + let mut plan = read_plan(VariableLocation::Address(AddressExpr::constant(0x1000))); + plan.module_path = Some(PathBuf::from("/tmp/libstate.so")); + + let materialized = plan.materialization_plan(&capabilities(true)); + + assert_eq!( + materialized.module_path, + Some(PathBuf::from("/tmp/libstate.so")) + ); +} + +#[test] +fn materialization_plan_converts_register_address_to_address_kind() { + let plan = read_plan(VariableLocation::RegisterAddress { + dwarf_reg: 6, + offset: -16, + }); + let materialized = plan.materialization_plan(&capabilities(true)); + + match materialized.materialization { + VariableMaterialization::UserMemoryRead { address } => { + assert_eq!(address.origin, AddressOrigin::RuntimeDerived); + assert_eq!( + address.kind, + PlannedAddressKind::RegisterOffset { + dwarf_reg: 6, + offset: -16 + } + ); + } + other => panic!("unexpected materialization: {other:?}"), + } +} + +#[test] +fn materialization_plan_marks_static_base_before_deref() { + let plan = read_plan(VariableLocation::ComputedAddress(vec![ + PlanExprOp::PushConstant(0x3000), + PlanExprOp::Dereference { + size: MemoryAccessSize::U64, + }, + PlanExprOp::PushConstant(16), + PlanExprOp::Add, + ])); + let materialized = plan.materialization_plan(&capabilities(true)); + + match materialized.materialization { + VariableMaterialization::UserMemoryRead { address } => { + assert_eq!(address.origin, AddressOrigin::LinkTimeBase); + match &address.kind { + PlannedAddressKind::RuntimeComputed { expr } => { + assert_eq!(expr.kind(), RuntimeComputedKind::Address); + } + other => panic!("unexpected address kind: {other:?}"), + } + let (base, tail) = address + .link_time_base_and_runtime_tail() + .expect("link-time base"); + assert_eq!(base, 0x3000); + assert_eq!(tail.len(), 3); + } + other => panic!("unexpected materialization: {other:?}"), + } +} + +#[test] +fn materialization_plan_preserves_arithmetic_before_first_deref() { + let plan = read_plan(VariableLocation::ComputedAddress(vec![ + PlanExprOp::PushConstant(0x3000), + PlanExprOp::PushConstant(8), + PlanExprOp::Add, + PlanExprOp::Dereference { + size: MemoryAccessSize::U64, + }, + ])); + let materialized = plan.materialization_plan(&capabilities(true)); + + match materialized.materialization { + VariableMaterialization::UserMemoryRead { address } => { + assert_eq!(address.origin, AddressOrigin::LinkTimeBase); + let (base, tail) = address + .link_time_base_and_runtime_tail() + .expect("link-time base"); + assert_eq!(base, 0x3000); + assert_eq!( + tail, + &[ + PlanExprOp::PushConstant(8), + PlanExprOp::Add, + PlanExprOp::Dereference { + size: MemoryAccessSize::U64, + }, + ] + ); + } + other => panic!("unexpected materialization: {other:?}"), + } +} + +#[test] +fn materialization_plan_keeps_absolute_address_value_direct() { + let plan = read_plan(VariableLocation::AbsoluteAddressValue( + AddressExpr::constant(0x2000), + )); + let materialized = plan.materialization_plan(&capabilities(false)); + + match materialized.materialization { + VariableMaterialization::DirectValue { + value: + PlannedValue::AddressValue { + address: + PlannedAddress { + origin: AddressOrigin::LinkTime, + kind: PlannedAddressKind::Constant { address: 0x2000 }, + .. + }, + size: MemoryAccessSize::U64, + }, + } => {} + VariableMaterialization::DirectValue { value } => { + panic!("unexpected direct value: {value:?}"); + } + other => panic!("unexpected materialization: {other:?}"), + } +} + +#[test] +fn materialization_plan_converts_constant_direct_value() { + let plan = read_plan(VariableLocation::ComputedValue(vec![ + PlanExprOp::PushConstant(42), + ])); + let materialized = plan.materialization_plan(&capabilities(false)); + + match materialized.materialization { + VariableMaterialization::DirectValue { + value: + PlannedValue::Constant { + value: 42, + size: MemoryAccessSize::U64, + }, + } => {} + other => panic!("unexpected materialization: {other:?}"), + } +} + +#[test] +fn materialization_plan_records_direct_value_size_from_type() { + let byte_type = TypeInfo::BaseType { + name: "uint8_t".to_string(), + size: 1, + encoding: gimli::constants::DW_ATE_unsigned.0 as u16, + }; + let plan = typed_read_plan( + VariableLocation::ComputedValue(vec![ + PlanExprOp::LoadRegister(0), + PlanExprOp::PushConstant(1), + PlanExprOp::Add, + ]), + byte_type, + ); + let materialized = plan.materialization_plan(&capabilities(false)); + + match materialized.materialization { + VariableMaterialization::DirectValue { + value: + PlannedValue::RuntimeComputed { + ref expr, + result_size: MemoryAccessSize::U8, + .. + }, + } => { + assert_eq!(expr.kind(), RuntimeComputedKind::Value); + } + other => panic!("unexpected materialization: {other:?}"), + } +} + +#[test] +fn materialization_plan_converts_register_direct_value() { + let plan = read_plan(VariableLocation::RegisterValue { dwarf_reg: 6 }); + let materialized = plan.materialization_plan(&capabilities(false)); + + match materialized.materialization { + VariableMaterialization::DirectValue { + value: + PlannedValue::RegisterValue { + dwarf_reg: 6, + size: MemoryAccessSize::U64, + }, + } => {} + other => panic!("unexpected materialization: {other:?}"), + } +} + +#[test] +fn materialization_plan_surfaces_piece_locations_without_first_piece_fallback() { + let plan = read_plan(VariableLocation::Pieces(vec![PieceLocation { + bit_offset: 0, + bit_size: 32, + location: Box::new(VariableLocation::RegisterValue { dwarf_reg: 0 }), + }])); + let materialized = plan.materialization_plan(&capabilities(true)); + + match materialized.materialization { + VariableMaterialization::Composite { pieces } => { + assert_eq!(pieces.len(), 1); + } + other => panic!("unexpected materialization: {other:?}"), + } +} + +#[test] +fn absolute_address_value_lowers_without_user_memory_read() { + let plan = read_plan(VariableLocation::AbsoluteAddressValue( + AddressExpr::constant(0x1000), + )); + let lowering = plan.bpf_lowering_plan(&capabilities(false)); + + assert_eq!(lowering.kind, VariableLoweringKind::DirectValue); + assert_eq!(lowering.availability, Availability::Available); + assert!(lowering.requirements.is_empty()); +} + +#[test] +fn memory_location_prefers_copy_from_user_task_when_available() { + let mut capabilities = capabilities(false); + capabilities.sleepable_uprobe = true; + capabilities.copy_from_user_task = true; + let plan = read_plan(VariableLocation::Address(AddressExpr::constant(0x1000))); + let lowering = plan.bpf_lowering_plan(&capabilities); + + assert_eq!(lowering.availability, Availability::Available); + assert_eq!(lowering.helper_mode, HelperMode::CopyFromUserTask); +} + +#[test] +fn register_address_records_required_register() { + let plan = read_plan(VariableLocation::RegisterAddress { + dwarf_reg: 6, + offset: -16, + }); + let lowering = plan.bpf_lowering_plan(&capabilities(true)); + + assert_eq!(lowering.required_registers, vec![6]); + assert_eq!(lowering.estimated_stack_bytes, 8); +} + +#[test] +fn entry_value_steps_surface_caller_frame_and_memory_requirements() { + let plan = read_plan(VariableLocation::ComputedValue(vec![ + PlanExprOp::EntryValueLookup { + caller_pc_steps: vec![ + PlanExprOp::LoadRegister(7), + PlanExprOp::Dereference { + size: MemoryAccessSize::U64, + }, + ], + cases: vec![EntryValueCase { + caller_return_pc: 0x10, + value_steps: vec![PlanExprOp::LoadRegister(5)], + }], + }, + ])); + let lowering = plan.bpf_lowering_plan(&capabilities(true)); + + assert_eq!(lowering.availability, Availability::Available); + assert_eq!( + lowering.requirements, + vec![ + RuntimeRequirement::CallerFrame, + RuntimeRequirement::UserMemoryRead + ] + ); + assert_eq!(lowering.required_registers, vec![5, 7]); + assert_eq!(lowering.verifier_risk, VerifierRisk::RequiresBoundedLoops); +} + +#[test] +fn stack_budget_excess_reports_unsupported_availability() { + let mut capabilities = capabilities(true); + capabilities.max_bpf_stack_bytes = 16; + let plan = read_plan(VariableLocation::ComputedValue(vec![ + PlanExprOp::PushConstant(1); + 8 + ])); + let lowering = plan.bpf_lowering_plan(&capabilities); + + assert!(matches!( + lowering.availability, + Availability::Unsupported(UnsupportedReason::ExpressionShape { .. }) + )); + assert_eq!( + lowering.verifier_risk, + VerifierRisk::StackBudgetExceeded { + estimated: 64, + max: 16, + } + ); +} + +#[test] +fn field_access_adds_member_offset_and_type() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let plan = typed_read_plan( + VariableLocation::RegisterAddress { + dwarf_reg: 6, + offset: -32, + }, + TypeInfo::StructType { + name: "Request".to_string(), + size: 16, + members: vec![StructMember { + name: "fd".to_string(), + member_type: int_type.clone(), + offset: 12, + bit_offset: None, + bit_size: None, + }], + }, + ); + + let access = VariableAccessPath::fields(["fd"]); + let planned = plan.plan_access_path(&access).expect("field access"); + + assert_eq!(planned.name, "value.fd"); + assert_eq!(planned.access_path, access); + assert_eq!(planned.dwarf_type, Some(int_type)); + assert_eq!( + planned.location, + VariableLocation::RegisterAddress { + dwarf_reg: 6, + offset: -20, + } + ); + assert_eq!( + planned + .materialization_plan(&capabilities(true)) + .access_path + .segments, + vec![VariableAccessSegment::Field("fd".to_string())] + ); +} + +#[test] +fn field_access_unknown_member_reports_known_members() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let plan = typed_read_plan( + VariableLocation::Address(AddressExpr::constant(0x1000)), + TypeInfo::StructType { + name: "Request".to_string(), + size: 8, + members: vec![ + StructMember { + name: "fd".to_string(), + member_type: int_type.clone(), + offset: 0, + bit_offset: None, + bit_size: None, + }, + StructMember { + name: "flags".to_string(), + member_type: int_type, + offset: 4, + bit_offset: None, + bit_size: None, + }, + ], + }, + ); + + let err = plan + .plan_access_path(&VariableAccessPath::fields(["missing"])) + .expect_err("unknown member should fail"); + + assert_eq!( + err.to_string(), + "Unknown member 'missing' in struct 'Request' (known members: fd, flags)" + ); +} + +#[test] +fn field_access_folds_constant_address_offsets() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let plan = typed_read_plan( + VariableLocation::Address(AddressExpr::constant(0x1000)), + TypeInfo::StructType { + name: "Request".to_string(), + size: 16, + members: vec![StructMember { + name: "fd".to_string(), + member_type: int_type, + offset: 12, + bit_offset: None, + bit_size: None, + }], + }, + ); + + let planned = plan + .plan_access_path(&VariableAccessPath::fields(["fd"])) + .expect("field access"); + + assert_eq!( + planned.location, + VariableLocation::Address(AddressExpr::constant(0x100c)) + ); +} + +#[test] +fn field_access_rejects_value_backed_aggregates() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let struct_type = TypeInfo::StructType { + name: "Pair".to_string(), + size: 8, + members: vec![StructMember { + name: "b".to_string(), + member_type: int_type, + offset: 4, + bit_offset: None, + bit_size: None, + }], + }; + let access = VariableAccessPath::fields(["b"]); + + for location in [ + VariableLocation::AbsoluteAddressValue(AddressExpr::constant(0x1000)), + VariableLocation::RegisterValue { dwarf_reg: 0 }, + VariableLocation::ComputedValue(vec![PlanExprOp::LoadRegister(0)]), + ] { + let plan = typed_read_plan(location, struct_type.clone()); + let err = plan + .plan_access_path(&access) + .expect_err("value-backed aggregate field access should fail"); + assert!( + err.downcast_ref::() + .is_some_and(PlanError::is_value_backed_aggregate_access), + "unexpected error: {err}" + ); + } +} + +#[test] +fn array_index_rejects_value_backed_aggregates() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let array_type = TypeInfo::ArrayType { + element_type: Box::new(int_type), + element_count: Some(2), + total_size: Some(8), + }; + let access = VariableAccessPath::new(vec![VariableAccessSegment::ArrayIndex(1)]); + + for location in [ + VariableLocation::AbsoluteAddressValue(AddressExpr::constant(0x1000)), + VariableLocation::RegisterValue { dwarf_reg: 0 }, + VariableLocation::ComputedValue(vec![PlanExprOp::LoadRegister(0)]), + ] { + let plan = typed_read_plan(location, array_type.clone()); + let err = plan + .plan_access_path(&access) + .expect_err("value-backed aggregate array access should fail"); + assert!( + err.downcast_ref::() + .is_some_and(PlanError::is_value_backed_aggregate_access), + "unexpected error: {err}" + ); + } +} + +#[test] +fn pointer_field_access_dereferences_then_offsets() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let struct_type = TypeInfo::StructType { + name: "Node".to_string(), + size: 16, + members: vec![StructMember { + name: "value".to_string(), + member_type: int_type, + offset: 8, + bit_offset: None, + bit_size: None, + }], + }; + let plan = typed_read_plan( + VariableLocation::RegisterValue { dwarf_reg: 5 }, + TypeInfo::PointerType { + target_type: Box::new(struct_type), + size: 8, + }, + ); + + let access = VariableAccessPath::fields(["value"]); + let planned = plan.plan_access_path(&access).expect("pointer field"); + + assert_eq!( + planned.location, + VariableLocation::ComputedAddress(vec![ + PlanExprOp::LoadRegister(5), + PlanExprOp::PushConstant(8), + PlanExprOp::Add, + ]) + ); +} + +#[test] +fn pointer_field_access_from_absolute_address_value_rebases_memory_location() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let struct_type = TypeInfo::StructType { + name: "Node".to_string(), + size: 16, + members: vec![StructMember { + name: "value".to_string(), + member_type: int_type, + offset: 8, + bit_offset: None, + bit_size: None, + }], + }; + let plan = typed_read_plan( + VariableLocation::AbsoluteAddressValue(AddressExpr::constant(0x1000)), + TypeInfo::PointerType { + target_type: Box::new(struct_type), + size: 8, + }, + ); + + let planned = plan + .plan_access_path(&VariableAccessPath::fields(["value"])) + .expect("pointer field"); + + assert_eq!( + planned.location, + VariableLocation::Address(AddressExpr::constant(0x1008)) + ); +} + +#[test] +fn pointer_field_access_from_computed_value_uses_value_as_address() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let struct_type = TypeInfo::StructType { + name: "Node".to_string(), + size: 16, + members: vec![StructMember { + name: "value".to_string(), + member_type: int_type, + offset: 8, + bit_offset: None, + bit_size: None, + }], + }; + let plan = typed_read_plan( + VariableLocation::ComputedValue(vec![PlanExprOp::PushConstant(0x2000)]), + TypeInfo::PointerType { + target_type: Box::new(struct_type), + size: 8, + }, + ); + + let planned = plan + .plan_access_path(&VariableAccessPath::fields(["value"])) + .expect("pointer field"); + + assert_eq!( + planned.location, + VariableLocation::ComputedAddress(vec![ + PlanExprOp::PushConstant(0x2000), + PlanExprOp::PushConstant(8), + PlanExprOp::Add, + ]) + ); +} + +#[test] +fn pointer_element_index_is_planned_in_dwarf_semantics() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let plan = typed_read_plan( + VariableLocation::RegisterValue { dwarf_reg: 5 }, + TypeInfo::PointerType { + target_type: Box::new(int_type), + size: 8, + }, + ); + + let planned = plan + .plan_pointer_element_index(3) + .expect("pointer element index"); + + assert_eq!(planned.name, "value[3]"); + assert_eq!( + planned.location, + VariableLocation::ComputedAddress(vec![ + PlanExprOp::LoadRegister(5), + PlanExprOp::PushConstant(12), + PlanExprOp::Add, + ]) + ); +} + +#[test] +fn pointer_element_index_rejects_aggregate_arithmetic_with_pointer_error() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let plan = typed_read_plan( + VariableLocation::Address(AddressExpr::constant(0x1000)), + TypeInfo::StructType { + name: "GlobalState".to_string(), + size: 16, + members: vec![StructMember { + name: "counter".to_string(), + member_type: int_type, + offset: 0, + bit_offset: None, + bit_size: None, + }], + }, + ); + + let err = plan + .plan_pointer_element_index(1) + .expect_err("struct arithmetic must be rejected"); + let plan_error = err + .downcast_ref::() + .expect("structured plan error"); + assert!(matches!( + plan_error, + PlanError::InvalidPointerArithmetic { type_name } + if type_name == "struct GlobalState" + )); +} + +#[test] +fn array_index_access_uses_element_stride() { + let int_type = TypeInfo::BaseType { + name: "int".to_string(), + size: 4, + encoding: gimli::constants::DW_ATE_signed.0 as u16, + }; + let plan = typed_read_plan( + VariableLocation::Address(AddressExpr::constant(0x1000)), + TypeInfo::ArrayType { + element_type: Box::new(int_type), + element_count: Some(8), + total_size: Some(32), + }, + ); + + let access = VariableAccessPath::new(vec![VariableAccessSegment::ArrayIndex(3)]); + let planned = plan.plan_access_path(&access).expect("array index"); + + assert_eq!(planned.name, "value[3]"); + assert_eq!( + planned.location, + VariableLocation::Address(AddressExpr::constant(0x100c)) + ); +} diff --git a/ghostscope-process/src/sysmon.rs b/ghostscope-process/src/sysmon.rs deleted file mode 100644 index 1552132a..00000000 --- a/ghostscope-process/src/sysmon.rs +++ /dev/null @@ -1,2945 +0,0 @@ -use crate::{ - module_probe::cookie_for_path, - offsets::{PidOffsetsEntry, ProcessManager}, - pid::{ - resolve_event_pid_for_proc, resolve_proc_pid_for_event, runtime_pid_candidates_for_proc, - PidNamespaceId, - }, - pinned_bpf_maps, - proc_maps::{ - normalize_mapped_module_path, read_proc_maps, should_skip_mapped_module_path, - visit_proc_maps, ModuleIdentity, - }, -}; -use std::collections::{BTreeSet, HashMap}; -use std::ops::ControlFlow; -use std::path::{Path, PathBuf}; -use std::sync::{mpsc, Arc, Mutex}; -use std::thread::{self, JoinHandle}; -use std::time::{Duration, Instant}; -use tracing::{error, info, warn}; - -/// Kind of process lifecycle event -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SysEventKind { - Exec, - Fork, - Exit, - MapChange, -} - -impl SysEventKind { - fn from_u32(v: u32) -> Option { - match v { - 1 => Some(SysEventKind::Exec), - 2 => Some(SysEventKind::Fork), - 3 => Some(SysEventKind::Exit), - 4 => Some(SysEventKind::MapChange), - _ => None, - } - } - - fn as_u32(self) -> u32 { - match self { - SysEventKind::Exec => 1, - SysEventKind::Fork => 2, - SysEventKind::Exit => 3, - SysEventKind::MapChange => 4, - } - } -} - -/// Internal sysmon event selector. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct SysmonEventMask { - pub exec: bool, - pub fork: bool, - pub exit: bool, - pub map_change: bool, -} - -impl SysmonEventMask { - pub fn target_mode() -> Self { - Self { - exec: true, - fork: true, - exit: true, - map_change: false, - } - } - - pub fn target_mode_with_map_changes() -> Self { - Self { - exec: true, - fork: true, - exit: true, - map_change: true, - } - } - - pub fn pid_module_changes() -> Self { - Self { - exec: false, - fork: false, - exit: false, - map_change: true, - } - } - - fn without_map_change(self) -> Self { - Self { - map_change: false, - ..self - } - } - - fn has_lifecycle_events(self) -> bool { - self.exec || self.fork || self.exit - } - - #[cfg(feature = "sysmon-ebpf")] - fn bits(self) -> u32 { - let mut bits = 0u32; - if self.exec { - bits |= SYSMON_EVENT_MASK_EXEC; - } - if self.fork { - bits |= SYSMON_EVENT_MASK_FORK; - } - if self.exit { - bits |= SYSMON_EVENT_MASK_EXIT; - } - if self.map_change { - bits |= SYSMON_EVENT_MASK_MAP_CHANGE; - } - bits - } -} - -impl Default for SysmonEventMask { - fn default() -> Self { - Self::target_mode() - } -} - -/// Raw SysEvent ABI — must match eBPF side exactly -/// ABI note: This layout is mirrored in eBPF at -/// `ghostscope-process/ebpf/sysmon-bpf/src/lib.rs`. We intentionally keep -/// two copies for now to avoid entangling the BPF build with the workspace. -/// Keep repr(C), field order and sizes identical on both sides. Current -/// layout (12 bytes): { tgid: u32, host_tgid: u32, kind: u32 }. -#[repr(C)] -#[derive(Clone, Copy)] -pub struct SysEvent { - /// Runtime TGID in the configured sysmon event namespace when available. - pub tgid: u32, - /// Host/initial-namespace TGID from bpf_get_current_pid_tgid(). - pub host_tgid: u32, - pub kind: u32, // 1=exec,2=fork,3=exit,4=map-change -} - -impl SysEvent { - pub fn event_kind(self) -> Option { - SysEventKind::from_u32(self.kind) - } -} - -const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(150); -const PENDING_MAX_ATTEMPTS: u32 = 20; -const MAP_CHANGE_DEBOUNCE_INTERVAL: Duration = Duration::from_millis(75); -const MODULE_REFRESH_INTERVAL: Duration = Duration::from_millis(250); -const SYSMON_EVENT_QUEUE_CAPACITY: usize = 1024; - -#[cfg(feature = "sysmon-ebpf")] -const SYSMON_EVENT_MASK_EXEC: u32 = 1 << 0; -#[cfg(feature = "sysmon-ebpf")] -const SYSMON_EVENT_MASK_FORK: u32 = 1 << 1; -#[cfg(feature = "sysmon-ebpf")] -const SYSMON_EVENT_MASK_EXIT: u32 = 1 << 2; -#[cfg(feature = "sysmon-ebpf")] -const SYSMON_EVENT_MASK_MAP_CHANGE: u32 = 1 << 3; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum PendingOffsetsKind { - Retry, - MapChangeCandidate, -} - -impl PendingOffsetsKind { - fn keep_for_map_changes_after_retry_exhaustion(self) -> bool { - matches!(self, Self::MapChangeCandidate) - } -} - -#[derive(Debug, Clone)] -pub(crate) struct PendingOffsetsEntry { - target_path: PathBuf, - attempts: u32, - kind: PendingOffsetsKind, - retry_exhausted: bool, - last_poll: Instant, -} - -#[derive(Debug, Clone)] -struct PendingOffsetsDue { - event_pid: u32, - target_path: PathBuf, - attempts: u32, - kind: PendingOffsetsKind, -} - -#[derive(Debug, Default)] -pub(crate) struct PendingOffsets { - entries: HashMap, -} - -impl PendingOffsets { - fn new() -> Self { - Self { - entries: HashMap::new(), - } - } - - fn register(&mut self, pid: u32, target: &Path) { - self.register_with_kind(pid, target, PendingOffsetsKind::Retry); - } - - fn register_map_change_candidate(&mut self, pid: u32, target: &Path) { - self.register_with_kind(pid, target, PendingOffsetsKind::MapChangeCandidate); - } - - fn register_with_kind(&mut self, pid: u32, target: &Path, kind: PendingOffsetsKind) { - let now = Instant::now(); - let last_poll = now.checked_sub(PENDING_POLL_INTERVAL).unwrap_or(now); - self.entries - .entry(pid) - .and_modify(|entry| { - entry.target_path = target.to_path_buf(); - entry.attempts = 0; - entry.kind = kind; - entry.retry_exhausted = false; - entry.last_poll = last_poll; - }) - .or_insert(PendingOffsetsEntry { - target_path: target.to_path_buf(), - attempts: 0, - kind, - retry_exhausted: false, - last_poll, - }); - } - - fn remove(&mut self, pid: u32) { - self.entries.remove(&pid); - } - - fn contains_map_change_candidate(&self, pid: u32, target: &Path) -> bool { - self.entries - .get(&pid) - .map(|entry| { - entry.kind == PendingOffsetsKind::MapChangeCandidate && entry.target_path == target - }) - .unwrap_or(false) - } - - fn mark_retry_exhausted(&mut self, pid: u32) { - if let Some(entry) = self.entries.get_mut(&pid) { - entry.retry_exhausted = true; - } - } - - fn take_due(&mut self) -> Vec { - let mut due = Vec::new(); - let now = Instant::now(); - for (&pid, entry) in self.entries.iter_mut() { - if entry.retry_exhausted { - continue; - } - if now.duration_since(entry.last_poll) >= PENDING_POLL_INTERVAL { - entry.last_poll = now; - entry.attempts = entry.attempts.saturating_add(1); - due.push(PendingOffsetsDue { - event_pid: pid, - target_path: entry.target_path.clone(), - attempts: entry.attempts, - kind: entry.kind, - }); - } - } - due - } -} - -#[derive(Debug, Clone)] -pub(crate) struct PendingMapRefreshEntry { - last_seen: Instant, - event_pid: u32, - host_pid: u32, -} - -#[derive(Debug, Clone, Copy)] -struct PendingMapRefreshDue { - event_pid: u32, - host_pid: u32, - proc_pid: u32, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct PendingMapChangeCandidate { - event_pid: u32, - host_pid: u32, - proc_pid: u32, -} - -#[derive(Debug, Default)] -pub(crate) struct PendingMapRefreshes { - entries: HashMap, -} - -impl PendingMapRefreshes { - fn new() -> Self { - Self { - entries: HashMap::new(), - } - } - - fn register(&mut self, event_pid: u32, host_pid: u32, proc_pid: u32) { - self.entries.insert( - proc_pid, - PendingMapRefreshEntry { - last_seen: Instant::now(), - event_pid, - host_pid, - }, - ); - } - - fn take_due(&mut self) -> Vec { - let now = Instant::now(); - let due: Vec = self - .entries - .iter() - .filter_map(|(&proc_pid, entry)| { - (now.duration_since(entry.last_seen) >= MAP_CHANGE_DEBOUNCE_INTERVAL).then_some( - PendingMapRefreshDue { - event_pid: entry.event_pid, - host_pid: entry.host_pid, - proc_pid, - }, - ) - }) - .collect(); - for entry in &due { - self.entries.remove(&entry.proc_pid); - } - due - } -} - -/// Configuration for sysmon -#[derive(Debug, Clone)] -pub struct SysmonConfig { - /// If set, only attempt offsets prefill for events whose binary/module path matches this target. - pub target_module: Option, - /// Maximum number of entries for the pinned proc offsets map (used when ensuring existence). - pub proc_offsets_max_entries: u32, - /// PerfEventArray per-CPU buffer pages (used when ringbuf is unavailable). - pub perf_page_count: Option, - /// Internal event selector for the sysmon eBPF side. - pub event_mask: SysmonEventMask, - /// Whether map-change events should be emitted before the PID allowlist is populated. - pub map_change_unfiltered: bool, - /// Optional event PID to watch. `None` means system-wide. - pub watched_pid: Option, - /// Optional PID namespace for interpreting `watched_pid`. - pub watched_pid_ns: Option, - /// Optional PID namespace for reporting target-mode event TGIDs. - pub event_pid_ns: Option, - /// Optional `/proc` PID corresponding to `watched_pid`. - pub watched_proc_pid: Option, -} - -impl SysmonConfig { - pub fn new() -> Self { - Self { - target_module: None, - proc_offsets_max_entries: 4096, - perf_page_count: None, - event_mask: SysmonEventMask::target_mode(), - map_change_unfiltered: false, - watched_pid: None, - watched_pid_ns: None, - event_pid_ns: None, - watched_proc_pid: None, - } - } -} - -impl Default for SysmonConfig { - fn default() -> Self { - Self::new() - } -} - -/// Process sysmon — userspace controller that listens for process lifecycle events and -/// performs incremental prefill/cleanup of offsets. -/// -/// Note: The low-level event source (tracepoints via eBPF or kernel proc connector) is pluggable. -/// This initial implementation provides the public API and a background loop stub; the event source -/// integration will be wired subsequently. -pub struct ProcessSysmon { - cfg: SysmonConfig, - mgr: Arc>, // shared manager to compute/prefill offsets - tx: mpsc::SyncSender, - rx: mpsc::Receiver, - pending_offsets: Arc>, - pending_map_refreshes: Arc>, - handle: Option>, -} - -impl core::fmt::Debug for ProcessSysmon { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.write_str("ProcessSysmon{..}") - } -} - -impl ProcessSysmon { - /// Create a new sysmon instance with shared ProcessManager and config. - pub fn new(mgr: Arc>, cfg: SysmonConfig) -> Self { - let (tx, rx) = mpsc::sync_channel(SYSMON_EVENT_QUEUE_CAPACITY); - Self { - cfg, - mgr, - tx, - rx, - pending_offsets: Arc::new(Mutex::new(PendingOffsets::new())), - pending_map_refreshes: Arc::new(Mutex::new(PendingMapRefreshes::new())), - handle: None, - } - } - - /// Start background monitoring thread and return immediately. - /// - /// In the next iteration we will attach eBPF tracepoints (sched_process_exec/exit/fork) - /// and stream events into this channel. For now, we ensure the pinned offsets map exists - /// and keep a placeholder loop that can be extended to consume a real source. - pub fn start(&mut self) { - let _ = - pinned_bpf_maps::ensure_pinned_proc_offsets_exists(self.cfg.proc_offsets_max_entries); - let _ = - pinned_bpf_maps::ensure_pinned_pid_aliases_exists(self.cfg.proc_offsets_max_entries); - let _ = pinned_bpf_maps::ensure_pinned_proc_module_ranges_exist( - self.cfg.proc_offsets_max_entries, - ); - let _ = pinned_bpf_maps::ensure_pinned_allowed_pids_exists(16_384); - - let tx = self.tx.clone(); - let mgr = Arc::clone(&self.mgr); - let pending = Arc::clone(&self.pending_offsets); - let pending_map_refreshes = Arc::clone(&self.pending_map_refreshes); - let cfg = self.cfg.clone(); - - let handle = thread::Builder::new() - .name("gs-sysmon".to_string()) - .spawn(move || { - info!("ProcessSysmon thread started"); - #[cfg(feature = "sysmon-ebpf")] - { - if let Err(e) = run_sysmon_loop(mgr, cfg, pending, pending_map_refreshes, tx) { - error!("Sysmon loop error: {}", e); - } - } - #[cfg(not(feature = "sysmon-ebpf"))] - { - let _ = pending; - let _ = pending_map_refreshes; - let _ = cfg; - warn!("sysmon-ebpf feature is disabled; sysmon is in stub mode"); - loop { - std::thread::sleep(std::time::Duration::from_millis(5000)); - } - } - info!("ProcessSysmon thread exiting"); - }); - match handle { - Ok(h) => self.handle = Some(h), - Err(e) => { - error!("Failed to spawn ProcessSysmon thread: {}", e); - self.handle = None; - } - } - } - - /// Blocking poll (with timeout) for the next system event. - pub fn recv_timeout(&self, timeout: std::time::Duration) -> Option { - match self.rx.recv_timeout(timeout) { - Ok(ev) => Some(ev), - Err(mpsc::RecvTimeoutError::Timeout) => None, - Err(mpsc::RecvTimeoutError::Disconnected) => None, - } - } - - /// Handle one system event: prefill on Exec/Fork, cleanup on Exit. - fn handle_event_with_proc_pid_resolver( - mgr: &Arc>, - target: &Option, - pending: &Arc>, - ev: &SysEvent, - proc_pid_for_event: impl Fn(u32) -> u32, - ) -> anyhow::Result<()> { - let kind = match SysEventKind::from_u32(ev.kind) { - Some(k) => k, - None => { - tracing::warn!( - "Sysmon: invalid event kind {} for pid {}; ignoring", - ev.kind, - ev.tgid - ); - return Ok(()); - } - }; - tracing::trace!("Sysmon event: kind={:?} event_pid={}", kind, ev.tgid); - match kind { - SysEventKind::Exec | SysEventKind::Fork => { - let proc_pid = proc_pid_for_event(ev.tgid); - record_runtime_pid_aliases_for_sys_event(mgr, ev, proc_pid); - if let Some(tpath) = target { - let path = tpath.as_path(); - if crate::util::is_shared_object(path) { - if kind == SysEventKind::Exec && !pid_maps_target_module(proc_pid, path) { - if pid_alive(proc_pid) { - tracing::debug!( - "Sysmon: event pid {} (proc pid {}) does not map target module yet; scheduling retry", - ev.tgid, - proc_pid - ); - if let Ok(mut guard) = pending.lock() { - guard.register_map_change_candidate(ev.tgid, path); - } - } else { - let host_pid = sys_event_host_pid(ev); - if host_pid != ev.tgid && pid_alive(host_pid) { - tracing::debug!( - "Sysmon: event pid {} is not visible in current /proc namespace; scheduling host pid {} retry", - ev.tgid, - host_pid - ); - if let Ok(mut guard) = pending.lock() { - guard.register_map_change_candidate(host_pid, path); - } - } else { - tracing::debug!( - "Sysmon: event pid {} (host pid {}) is not visible in current /proc namespace; skip exec-based target retry", - ev.tgid, - host_pid - ); - } - } - return Ok(()); - } else if let Ok(mut guard) = pending.lock() { - guard.remove(ev.tgid); - } - } else if kind == SysEventKind::Exec { - if let Some(actual) = get_comm_from_proc(proc_pid) { - let expected = truncate_basename_to_comm(path); - if actual.as_bytes() != expected.as_slice() { - tracing::warn!( - "Sysmon: comm mismatch for event pid {} (proc pid {}) (actual='{}', expected='{}'); skip prefill/insert", - ev.tgid, - proc_pid, - actual, - core::str::from_utf8(&expected).unwrap_or("") - ); - return Ok(()); - } - } - } - } - let inserted = - prefill_offsets_for_pid(mgr, ev.tgid, target.as_deref(), &proc_pid_for_event)?; - if inserted { - let host_pid = sys_event_host_pid(ev); - if host_pid != ev.tgid { - let _ = crate::pinned_bpf_maps::insert_allowed_pid(host_pid); - } - } - if kind == SysEventKind::Exec { - if let Some(tpath) = target { - if inserted { - if let Ok(mut guard) = pending.lock() { - guard.remove(ev.tgid); - } - } else if let Ok(mut guard) = pending.lock() { - tracing::debug!( - "Sysmon: event pid {} (proc pid {}) prefill inserted no matching offsets; scheduling retry", - ev.tgid, - proc_pid - ); - guard.register(ev.tgid, tpath.as_path()); - } - } - } - } - SysEventKind::Exit => { - let proc_pid = proc_pid_for_event(ev.tgid); - let host_pid = sys_event_host_pid(ev); - if let Ok(mut guard) = pending.lock() { - guard.remove(ev.tgid); - if host_pid != ev.tgid { - guard.remove(host_pid); - } - } - if let Ok(mut guard) = mgr.lock() { - guard.forget_pid(proc_pid); - if proc_pid != ev.tgid { - guard.forget_pid(ev.tgid); - } - if host_pid != ev.tgid && host_pid != proc_pid { - guard.forget_pid(host_pid); - } - } - let purged = purge_runtime_pid_artifacts(proc_pid, ev.tgid, host_pid); - info!( - "Sysmon: observed exit for event pid {} (host pid {}, proc pid {}) (purged {} entries)", - ev.tgid, - host_pid, - proc_pid, - purged - ); - } - SysEventKind::MapChange => { - tracing::trace!( - "Sysmon: map-change event for pid {} is handled by the debounce queue", - ev.tgid - ); - } - } - Ok(()) - } -} - -fn try_publish_sys_event(tx: &mpsc::SyncSender, ev: SysEvent) -> bool { - match tx.try_send(ev) { - Ok(()) => true, - Err(mpsc::TrySendError::Full(ev)) => { - tracing::trace!( - "Sysmon event queue full; dropping lifecycle notification for pid {} kind {}", - ev.tgid, - ev.kind - ); - false - } - Err(mpsc::TrySendError::Disconnected(ev)) => { - tracing::trace!( - "Sysmon event receiver disconnected; dropping lifecycle notification for pid {} kind {}", - ev.tgid, - ev.kind - ); - false - } - } -} - -fn dispatch_sysmon_event( - mgr: &Arc>, - target: &Option, - pending: &Arc>, - pending_map_refreshes: &Arc>, - proc_pid_for_event: &impl Fn(u32) -> u32, - ev: &SysEvent, -) -> bool { - match SysEventKind::from_u32(ev.kind) { - Some(SysEventKind::MapChange) => { - if let Some(proc_pid) = - proc_pid_for_map_change_event(mgr, target, proc_pid_for_event, ev) - { - let proc_pid = target - .as_deref() - .map(|target_path| { - canonicalize_cached_target_proc_pid(mgr, target_path, proc_pid) - }) - .unwrap_or(proc_pid); - record_runtime_pid_aliases_for_sys_event(mgr, ev, proc_pid); - if let Ok(mut guard) = pending_map_refreshes.lock() { - guard.register(ev.tgid, sys_event_host_pid(ev), proc_pid); - } - true - } else if let Some(target_path) = target.as_deref() { - let candidates = pending - .lock() - .ok() - .map(|guard| { - pending_map_change_candidates(&guard, target_path, proc_pid_for_event, ev) - }) - .unwrap_or_default(); - - if candidates.is_empty() { - tracing::trace!( - "Sysmon: map-change event pid {} (host pid {}) did not resolve to a target /proc pid; skipping per-pid refresh", - ev.tgid, - sys_event_host_pid(ev) - ); - false - } else { - if let Ok(mut guard) = pending_map_refreshes.lock() { - for candidate in &candidates { - guard.register( - candidate.event_pid, - candidate.host_pid, - candidate.proc_pid, - ); - } - } - tracing::trace!( - "Sysmon: queued map-change refresh for {} pending target candidate(s) from event pid {} (host pid {})", - candidates.len(), - ev.tgid, - sys_event_host_pid(ev) - ); - false - } - } else { - tracing::trace!( - "Sysmon: map-change event pid {} (host pid {}) did not resolve to a target /proc pid; skipping per-pid refresh", - ev.tgid, - sys_event_host_pid(ev) - ); - false - } - } - Some(_) => { - match ProcessSysmon::handle_event_with_proc_pid_resolver( - mgr, - target, - pending, - ev, - proc_pid_for_event, - ) { - Ok(()) => true, - Err(e) => { - tracing::debug!( - "Sysmon: handle_event failed for pid {} kind {}: {}", - ev.tgid, - ev.kind, - e - ); - false - } - } - } - None => { - match ProcessSysmon::handle_event_with_proc_pid_resolver( - mgr, - target, - pending, - ev, - proc_pid_for_event, - ) { - Ok(()) => true, - Err(e) => { - tracing::debug!( - "Sysmon: handle_event rejected invalid event for pid {} kind {}: {}", - ev.tgid, - ev.kind, - e - ); - false - } - } - } - } -} - -fn proc_pid_for_map_change_event( - mgr: &Arc>, - target: &Option, - proc_pid_for_event: &impl Fn(u32) -> u32, - ev: &SysEvent, -) -> Option { - let host_pid = sys_event_host_pid(ev); - let mut candidates = Vec::with_capacity(2); - push_unique_pid(&mut candidates, proc_pid_for_event(ev.tgid)); - if host_pid != ev.tgid { - push_unique_pid(&mut candidates, proc_pid_for_event(host_pid)); - } - - for proc_pid in candidates { - if !pid_alive(proc_pid) { - continue; - } - if target.is_some() && is_current_process_pid(proc_pid) { - tracing::trace!( - "Sysmon: ignoring self map-change candidate proc pid {}", - proc_pid - ); - continue; - } - - let Some(target_path) = target.as_deref() else { - return Some(proc_pid); - }; - - if pid_maps_target_module(proc_pid, target_path) - || cached_offsets_exist_for_target_pid(mgr, target_path, proc_pid) - { - return Some(proc_pid); - } - } - - None -} - -fn pending_map_change_candidates( - pending: &PendingOffsets, - target_path: &Path, - proc_pid_for_event: &impl Fn(u32) -> u32, - ev: &SysEvent, -) -> Vec { - let host_pid = sys_event_host_pid(ev); - let mut candidates = Vec::with_capacity(2); - push_pending_map_change_candidate( - &mut candidates, - pending, - target_path, - proc_pid_for_event, - ev.tgid, - host_pid, - ); - if host_pid != ev.tgid { - push_pending_map_change_candidate( - &mut candidates, - pending, - target_path, - proc_pid_for_event, - host_pid, - host_pid, - ); - } - candidates -} - -fn push_pending_map_change_candidate( - candidates: &mut Vec, - pending: &PendingOffsets, - target_path: &Path, - proc_pid_for_event: &impl Fn(u32) -> u32, - event_pid: u32, - host_pid: u32, -) { - if !pending.contains_map_change_candidate(event_pid, target_path) { - return; - } - - let proc_pid = proc_pid_for_event(event_pid); - if !pid_alive(proc_pid) || is_current_process_pid(proc_pid) { - return; - } - - if candidates - .iter() - .any(|candidate| candidate.proc_pid == proc_pid) - { - return; - } - - candidates.push(PendingMapChangeCandidate { - event_pid, - host_pid, - proc_pid, - }); -} - -fn is_current_process_pid(proc_pid: u32) -> bool { - proc_pid == std::process::id() -} - -fn cached_single_target_proc_pid( - mgr: &Arc>, - target_path: &Path, -) -> Option { - let module_path = target_path.to_string_lossy(); - let mut target_pids = BTreeSet::new(); - let guard = mgr.lock().ok()?; - for (pid, _, _, _, _) in guard.cached_offsets_for_module(module_path.as_ref()) { - if !is_current_process_pid(pid) - && pid_alive(pid) - && pid_maps_target_module(pid, target_path) - { - target_pids.insert(pid); - } - } - - if target_pids.len() == 1 { - target_pids.iter().next().copied() - } else { - None - } -} - -fn canonicalize_cached_target_proc_pid( - mgr: &Arc>, - target_path: &Path, - fallback_proc_pid: u32, -) -> u32 { - if pid_alive(fallback_proc_pid) && pid_maps_target_module(fallback_proc_pid, target_path) { - return fallback_proc_pid; - } - - let Some(proc_pid) = cached_single_target_proc_pid(mgr, target_path) else { - return fallback_proc_pid; - }; - - if proc_pid != fallback_proc_pid { - tracing::debug!( - "Sysmon: canonicalized map-change proc pid {} -> {} for target {}", - fallback_proc_pid, - proc_pid, - target_path.display() - ); - } - - proc_pid -} - -fn push_unique_pid(pids: &mut Vec, pid: u32) { - if !pids.contains(&pid) { - pids.push(pid); - } -} - -fn sysmon_proc_pid_resolver( - watched_event_pid: Option, - watched_proc_pid: Option, -) -> impl Fn(u32) -> u32 { - move |event_pid| { - if watched_event_pid == Some(event_pid) { - if let Some(proc_pid) = watched_proc_pid { - return proc_pid; - } - } - - resolve_proc_pid_for_event(event_pid) - } -} - -fn write_pinned_runtime_pid_alias(runtime_pid: u32, proc_pid: u32) { - if runtime_pid == proc_pid { - return; - } - match crate::pinned_bpf_maps::insert_pid_alias(runtime_pid, proc_pid) { - Ok(()) => tracing::trace!( - "Sysmon: inserted PID alias runtime pid {} -> proc pid {}", - runtime_pid, - proc_pid - ), - Err(e) => { - tracing::debug!( - "Sysmon: failed to insert PID alias runtime pid {} -> proc pid {}: {}", - runtime_pid, - proc_pid, - e - ); - } - } -} - -fn record_runtime_pid_alias_for_event( - mgr: &Arc>, - runtime_pid: u32, - proc_pid: u32, -) { - write_pinned_runtime_pid_alias(runtime_pid, proc_pid); - if let Ok(mut guard) = mgr.lock() { - guard.record_runtime_pid_alias(runtime_pid, proc_pid); - } -} - -fn record_runtime_pid_aliases_for_proc_pid_locked(guard: &mut ProcessManager, proc_pid: u32) { - for runtime_pid in runtime_pid_candidates_for_proc(proc_pid) { - write_pinned_runtime_pid_alias(runtime_pid, proc_pid); - guard.record_runtime_pid_alias(runtime_pid, proc_pid); - } -} - -fn record_runtime_pid_aliases_for_proc_pid(mgr: &Arc>, proc_pid: u32) { - if let Ok(mut guard) = mgr.lock() { - record_runtime_pid_aliases_for_proc_pid_locked(&mut guard, proc_pid); - } else { - for runtime_pid in runtime_pid_candidates_for_proc(proc_pid) { - write_pinned_runtime_pid_alias(runtime_pid, proc_pid); - } - } -} - -fn runtime_pid_keys_for_proc_event( - proc_pid: u32, - event_pid: u32, - extra_runtime_pids: impl IntoIterator, -) -> Vec { - let mut keys = BTreeSet::new(); - keys.insert(proc_pid); - keys.insert(event_pid); - for runtime_pid in runtime_pid_candidates_for_proc(proc_pid) { - keys.insert(runtime_pid); - } - for runtime_pid in extra_runtime_pids { - if runtime_pid != 0 { - keys.insert(runtime_pid); - } - } - keys.into_iter().collect() -} - -fn record_runtime_pid_aliases_for_keys( - mgr: &Arc>, - proc_pid: u32, - runtime_pids: &[u32], -) { - for runtime_pid in runtime_pids { - write_pinned_runtime_pid_alias(*runtime_pid, proc_pid); - } - if let Ok(mut guard) = mgr.lock() { - for runtime_pid in runtime_pids { - guard.record_runtime_pid_alias(*runtime_pid, proc_pid); - } - } -} - -fn insert_allowed_runtime_pid_keys(runtime_pids: &[u32]) { - for runtime_pid in runtime_pids { - let _ = crate::pinned_bpf_maps::insert_allowed_pid(*runtime_pid); - } -} - -fn publish_offsets_for_runtime_pid_keys( - proc_pid: u32, - event_pid: u32, - runtime_pids: &[u32], - items: &[(u64, crate::pinned_bpf_maps::ProcModuleOffsetsValue)], - log_context: &str, -) -> anyhow::Result { - use crate::pinned_bpf_maps::{insert_offsets_for_pid, replace_ranges_for_pid}; - - let mut total_inserted = 0usize; - for runtime_pid in runtime_pids { - match insert_offsets_for_pid(*runtime_pid, items) { - Ok(inserted) => { - if inserted == 0 { - tracing::warn!( - "Sysmon: no offsets inserted for {} runtime pid {} (event pid {}, proc pid {}) (entry count={})", - log_context, - runtime_pid, - event_pid, - proc_pid, - items.len() - ); - continue; - } - total_inserted += inserted; - if let Err(e) = replace_ranges_for_pid(*runtime_pid, items) { - tracing::warn!( - "Sysmon: failed to replace module ranges for {} runtime pid {} (event pid {}, proc pid {}): {}", - log_context, - runtime_pid, - event_pid, - proc_pid, - e - ); - } - } - Err(e) => { - tracing::warn!( - "Sysmon: failed to insert offsets for {} runtime pid {} (event pid {}, proc pid {}): {}", - log_context, - runtime_pid, - event_pid, - proc_pid, - e - ); - } - } - } - - Ok(total_inserted) -} - -fn purge_offsets_for_runtime_pid_keys(runtime_pids: &[u32]) -> anyhow::Result { - let mut purged = 0usize; - for runtime_pid in runtime_pids { - purged += crate::pinned_bpf_maps::purge_offsets_for_pid(*runtime_pid)?; - let _ = crate::pinned_bpf_maps::purge_ranges_for_pid(*runtime_pid); - } - Ok(purged) -} - -fn purge_runtime_pid_artifacts(proc_pid: u32, event_pid: u32, host_pid: u32) -> usize { - let runtime_pids = runtime_pid_keys_for_proc_event(proc_pid, event_pid, [host_pid]); - let mut purged_offsets = 0usize; - for runtime_pid in runtime_pids { - if let Ok(purged) = crate::pinned_bpf_maps::purge_offsets_for_pid(runtime_pid) { - purged_offsets += purged; - } - let _ = crate::pinned_bpf_maps::purge_ranges_for_pid(runtime_pid); - let _ = crate::pinned_bpf_maps::remove_allowed_pid(runtime_pid); - if runtime_pid != proc_pid { - let _ = crate::pinned_bpf_maps::remove_pid_alias(runtime_pid); - } - } - purged_offsets -} - -fn sys_event_host_pid(ev: &SysEvent) -> u32 { - if ev.host_tgid != 0 { - ev.host_tgid - } else { - ev.tgid - } -} - -fn record_runtime_pid_aliases_for_sys_event( - mgr: &Arc>, - ev: &SysEvent, - proc_pid: u32, -) { - record_runtime_pid_alias_for_event(mgr, ev.tgid, proc_pid); - let host_pid = sys_event_host_pid(ev); - if host_pid != ev.tgid { - record_runtime_pid_alias_for_event(mgr, host_pid, proc_pid); - } - record_runtime_pid_aliases_for_proc_pid(mgr, proc_pid); -} - -#[cfg(feature = "sysmon-ebpf")] -#[derive(Debug, Clone, Copy)] -enum SysmonAttachBackend { - Raw, - Btf, - Classic, -} - -#[cfg(feature = "sysmon-ebpf")] -impl SysmonAttachBackend { - fn label(self) -> &'static str { - match self { - SysmonAttachBackend::Raw => "raw tracepoint", - SysmonAttachBackend::Btf => "BTF tracepoint", - SysmonAttachBackend::Classic => "classic tracepoint", - } - } -} - -#[cfg(feature = "sysmon-ebpf")] -struct SysmonTracepoint { - event: &'static str, - category: &'static str, - classic_program: &'static str, - raw_program: &'static str, - btf_program: &'static str, -} - -#[cfg(feature = "sysmon-ebpf")] -const SYSMON_TRACEPOINTS: &[SysmonTracepoint] = &[ - SysmonTracepoint { - event: "sched_process_exec", - category: "sched", - classic_program: "sched_process_exec", - raw_program: "raw_sched_process_exec", - btf_program: "btf_sched_process_exec", - }, - SysmonTracepoint { - event: "sched_process_exit", - category: "sched", - classic_program: "sched_process_exit", - raw_program: "raw_sched_process_exit", - btf_program: "btf_sched_process_exit", - }, - SysmonTracepoint { - event: "sched_process_fork", - category: "sched", - classic_program: "sched_process_fork", - raw_program: "raw_sched_process_fork", - btf_program: "btf_sched_process_fork", - }, -]; - -#[cfg(feature = "sysmon-ebpf")] -struct SysmonMapChangeTracepoint { - event: &'static str, - category: &'static str, - program: &'static str, -} - -#[cfg(feature = "sysmon-ebpf")] -const SYSMON_MAP_CHANGE_TRACEPOINTS: &[SysmonMapChangeTracepoint] = &[ - SysmonMapChangeTracepoint { - event: "sys_exit_mmap", - category: "syscalls", - program: "sys_exit_mmap", - }, - SysmonMapChangeTracepoint { - event: "sys_exit_mprotect", - category: "syscalls", - program: "sys_exit_mprotect", - }, - SysmonMapChangeTracepoint { - event: "sys_exit_munmap", - category: "syscalls", - program: "sys_exit_munmap", - }, - SysmonMapChangeTracepoint { - event: "sys_exit_mremap", - category: "syscalls", - program: "sys_exit_mremap", - }, -]; - -#[cfg(feature = "sysmon-ebpf")] -fn load_sysmon_bpf(obj: &[u8], use_verbose: bool) -> anyhow::Result { - use aya::{EbpfLoader, VerifierLogLevel}; - - let mut loader = EbpfLoader::new(); - if use_verbose { - loader.verifier_log_level(VerifierLogLevel::VERBOSE | VerifierLogLevel::STATS); - tracing::info!("Sysmon verifier logs: VERBOSE (debug build/log)"); - } else { - loader.verifier_log_level(VerifierLogLevel::DEBUG | VerifierLogLevel::STATS); - tracing::info!("Sysmon verifier logs: DEBUG (release/info)"); - } - - let pin_dir = crate::pinned_bpf_maps::proc_offsets_pin_dir()?; - loader.map_pin_path( - crate::pinned_bpf_maps::ALLOWED_PIDS_MAP_NAME, - pin_dir.join(crate::pinned_bpf_maps::ALLOWED_PIDS_MAP_NAME), - ); - loader.map_pin_path( - crate::pinned_bpf_maps::TARGET_EXEC_COMM_MAP_NAME, - pin_dir.join(crate::pinned_bpf_maps::TARGET_EXEC_COMM_MAP_NAME), - ); - loader.map_pin_path( - crate::pinned_bpf_maps::SYSMON_MAP_CHANGE_UNFILTERED_MAP_NAME, - pin_dir.join(crate::pinned_bpf_maps::SYSMON_MAP_CHANGE_UNFILTERED_MAP_NAME), - ); - - Ok(loader.load(obj)?) -} - -#[cfg(feature = "sysmon-ebpf")] -fn configure_sysmon_exec_comm_filter( - bpf: &mut aya::Ebpf, - target: Option<&Path>, -) -> anyhow::Result<()> { - use aya::maps::Array; - - let mut filter_bytes = [0u8; 16]; - let mut filter_len = 0usize; - if let Some(tpath) = target { - if !crate::util::is_shared_object(tpath) { - if let Some(name) = tpath.file_name().and_then(|s| s.to_str()) { - let bytes = name.as_bytes(); - // task->comm stores at most TASK_COMM_LEN - 1 visible bytes plus NUL. - // Keep the filter null-terminated so long executable basenames compare - // against the same truncation that bpf_get_current_comm() returns. - let len = bytes.len().min(filter_bytes.len() - 1); - filter_bytes[..len].copy_from_slice(&bytes[..len]); - filter_len = len; - } else { - tracing::warn!( - "Sysmon: target basename contains non-UTF8 bytes; exec comm filter disabled" - ); - } - } - } - - if let Some(map) = bpf.map_mut("target_exec_comm") { - let mut array: Array<_, [u8; 16]> = map.try_into()?; - array.set(0, filter_bytes, 0)?; - if filter_len > 0 { - match std::str::from_utf8(&filter_bytes[..filter_len]) { - Ok(name_str) => { - tracing::info!("Sysmon: exec comm filter configured for '{}'", name_str) - } - Err(_) => tracing::info!( - "Sysmon: exec comm filter configured (non-UTF8 basename, len={})", - filter_len - ), - } - } else { - tracing::info!("Sysmon: exec comm filter disabled"); - } - } else if filter_len > 0 { - tracing::warn!("Sysmon: target_exec_comm map missing; exec filtering unavailable"); - } - - Ok(()) -} - -#[cfg(feature = "sysmon-ebpf")] -fn configure_sysmon_event_filter( - bpf: &mut aya::Ebpf, - event_mask: SysmonEventMask, - map_change_unfiltered: bool, - watched_pid: Option, - watched_pid_ns: Option, - event_pid_ns: Option, -) -> anyhow::Result<()> { - use aya::maps::Array; - - if let Some(map) = bpf.map_mut("sysmon_event_mask") { - let mut array: Array<_, u32> = map.try_into()?; - array.set(0, event_mask.bits(), 0)?; - tracing::info!( - "Sysmon: event mask configured (exec={}, fork={}, exit={}, map_change={})", - event_mask.exec, - event_mask.fork, - event_mask.exit, - event_mask.map_change - ); - } else { - tracing::warn!("Sysmon: sysmon_event_mask map missing; event filtering unavailable"); - } - - if let Some(map) = bpf.map_mut("sysmon_map_change_unfiltered") { - let mut array: Array<_, u32> = map.try_into()?; - array.set(0, u32::from(map_change_unfiltered), 0)?; - tracing::info!( - "Sysmon: map-change pre-allowlist emission {}", - if map_change_unfiltered { - "enabled" - } else { - "disabled" - } - ); - } else if map_change_unfiltered { - tracing::warn!( - "Sysmon: sysmon_map_change_unfiltered map missing; pre-allowlist map-change events unavailable" - ); - } - - if let Some(map) = bpf.map_mut("sysmon_watched_pid") { - let mut array: Array<_, u32> = map.try_into()?; - array.set(0, watched_pid.unwrap_or(0), 0)?; - if let Some(pid) = watched_pid { - tracing::info!("Sysmon: watched event pid configured: {}", pid); - } else { - tracing::info!("Sysmon: watched event pid disabled"); - } - } else if watched_pid.is_some() { - tracing::warn!("Sysmon: sysmon_watched_pid map missing; PID filtering unavailable"); - } - - let watched_pid_ns = watched_pid.and(watched_pid_ns); - let ns_spec = watched_pid_ns.and_then(|pid_ns| pid_ns.helper_dev_inode()); - let (ns_dev, ns_ino) = ns_spec.unwrap_or((0, 0)); - - if let Some(map) = bpf.map_mut("sysmon_watched_pid_ns_dev") { - let mut array: Array<_, u64> = map.try_into()?; - array.set(0, ns_dev, 0)?; - } else if ns_spec.is_some() { - tracing::warn!( - "Sysmon: sysmon_watched_pid_ns_dev map missing; namespace PID filtering unavailable" - ); - } - - if let Some(map) = bpf.map_mut("sysmon_watched_pid_ns_ino") { - let mut array: Array<_, u64> = map.try_into()?; - array.set(0, ns_ino, 0)?; - } else if ns_spec.is_some() { - tracing::warn!( - "Sysmon: sysmon_watched_pid_ns_ino map missing; namespace PID filtering unavailable" - ); - } - - if let (Some(pid), Some((dev, ino))) = (watched_pid, ns_spec) { - tracing::info!( - "Sysmon: watched PID namespace configured: pid={} ns_dev={} ns_inode={}", - pid, - dev, - ino - ); - } - - let event_ns_spec = event_pid_ns.and_then(|pid_ns| pid_ns.helper_dev_inode()); - let (event_ns_dev, event_ns_ino) = event_ns_spec.unwrap_or((0, 0)); - - if let Some(map) = bpf.map_mut("sysmon_event_pid_ns_dev") { - let mut array: Array<_, u64> = map.try_into()?; - array.set(0, event_ns_dev, 0)?; - } else if event_ns_spec.is_some() { - tracing::warn!( - "Sysmon: sysmon_event_pid_ns_dev map missing; event namespace reporting unavailable" - ); - } - - if let Some(map) = bpf.map_mut("sysmon_event_pid_ns_ino") { - let mut array: Array<_, u64> = map.try_into()?; - array.set(0, event_ns_ino, 0)?; - } else if event_ns_spec.is_some() { - tracing::warn!( - "Sysmon: sysmon_event_pid_ns_ino map missing; event namespace reporting unavailable" - ); - } - - if let Some((dev, ino)) = event_ns_spec { - tracing::info!( - "Sysmon: event PID namespace configured: ns_dev={} ns_inode={}", - dev, - ino - ); - } - - Ok(()) -} - -#[cfg(feature = "sysmon-ebpf")] -fn attach_sysmon_backend(bpf: &mut aya::Ebpf, backend: SysmonAttachBackend) -> anyhow::Result<()> { - match backend { - SysmonAttachBackend::Raw => attach_raw_sysmon_tracepoints(bpf), - SysmonAttachBackend::Btf => attach_btf_sysmon_tracepoints(bpf), - SysmonAttachBackend::Classic => attach_classic_sysmon_tracepoints(bpf), - } -} - -#[cfg(feature = "sysmon-ebpf")] -fn attach_raw_sysmon_tracepoints(bpf: &mut aya::Ebpf) -> anyhow::Result<()> { - use aya::programs::RawTracePoint; - - for spec in SYSMON_TRACEPOINTS { - let prog = bpf.program_mut(spec.raw_program).ok_or_else(|| { - anyhow::anyhow!("missing program '{}' in sysmon-bpf", spec.raw_program) - })?; - let tp: &mut RawTracePoint = prog.try_into()?; - tp.load()?; - tp.attach(spec.event)?; - info!("Attached raw tracepoint: {}", spec.event); - } - Ok(()) -} - -#[cfg(feature = "sysmon-ebpf")] -fn attach_btf_sysmon_tracepoints(bpf: &mut aya::Ebpf) -> anyhow::Result<()> { - use anyhow::Context as _; - use aya::{programs::BtfTracePoint, Btf}; - - let btf = Btf::from_sys_fs().context("kernel BTF is unavailable")?; - for spec in SYSMON_TRACEPOINTS { - let prog = bpf.program_mut(spec.btf_program).ok_or_else(|| { - anyhow::anyhow!("missing program '{}' in sysmon-bpf", spec.btf_program) - })?; - let tp: &mut BtfTracePoint = prog.try_into()?; - tp.load(spec.event, &btf)?; - tp.attach()?; - info!("Attached BTF tracepoint: {}", spec.event); - } - Ok(()) -} - -#[cfg(feature = "sysmon-ebpf")] -fn attach_classic_sysmon_tracepoints(bpf: &mut aya::Ebpf) -> anyhow::Result<()> { - use aya::programs::TracePoint; - - for spec in SYSMON_TRACEPOINTS { - let prog = bpf.program_mut(spec.classic_program).ok_or_else(|| { - anyhow::anyhow!("missing program '{}' in sysmon-bpf", spec.classic_program) - })?; - let tp: &mut TracePoint = prog.try_into()?; - tp.load()?; - tp.attach(spec.category, spec.event)?; - info!( - "Attached classic tracepoint: {}:{}", - spec.category, spec.event - ); - } - Ok(()) -} - -#[cfg(feature = "sysmon-ebpf")] -fn attach_classic_map_change_tracepoints(bpf: &mut aya::Ebpf) -> anyhow::Result { - use aya::programs::TracePoint; - - let mut attached = 0usize; - for spec in SYSMON_MAP_CHANGE_TRACEPOINTS { - let Some(prog) = bpf.program_mut(spec.program) else { - tracing::warn!( - "Sysmon: missing map-change program '{}' in sysmon-bpf", - spec.program - ); - continue; - }; - let attach_result = (|| { - let tp: &mut TracePoint = prog.try_into()?; - tp.load()?; - tp.attach(spec.category, spec.event)?; - Ok::<_, anyhow::Error>(()) - })(); - match attach_result { - Ok(()) => { - attached += 1; - info!( - "Attached map-change tracepoint: {}:{}", - spec.category, spec.event - ); - } - Err(err) => { - tracing::warn!( - "Sysmon: map-change tracepoint {}:{} unavailable: {:#}", - spec.category, - spec.event, - err - ); - } - } - } - - Ok(attached) -} - -#[cfg(feature = "sysmon-ebpf")] -fn load_and_attach_sysmon_bpf( - obj: &[u8], - cfg: &SysmonConfig, - use_verbose: bool, -) -> anyhow::Result { - let mut failures = Vec::new(); - for backend in [ - SysmonAttachBackend::Raw, - SysmonAttachBackend::Btf, - SysmonAttachBackend::Classic, - ] { - tracing::info!("Sysmon: trying {} backend", backend.label()); - let result = (|| { - let mut bpf = load_sysmon_bpf(obj, use_verbose)?; - configure_sysmon_exec_comm_filter(&mut bpf, cfg.target_module.as_deref())?; - configure_sysmon_event_filter( - &mut bpf, - cfg.event_mask, - cfg.map_change_unfiltered, - cfg.watched_pid, - cfg.watched_pid_ns, - cfg.event_pid_ns, - )?; - attach_sysmon_backend(&mut bpf, backend)?; - if cfg.event_mask.map_change { - let map_attached = attach_classic_map_change_tracepoints(&mut bpf)?; - if map_attached == 0 { - if !cfg.event_mask.has_lifecycle_events() { - return Err(anyhow::anyhow!( - "map-change events requested but no syscall tracepoints attached" - )); - } - - let fallback_mask = cfg.event_mask.without_map_change(); - configure_sysmon_event_filter( - &mut bpf, - fallback_mask, - cfg.map_change_unfiltered, - cfg.watched_pid, - cfg.watched_pid_ns, - cfg.event_pid_ns, - )?; - tracing::warn!( - "Sysmon: map-change events requested but no syscall tracepoints attached; \ - continuing with exec/fork/exit lifecycle events only" - ); - } - } - Ok::<_, anyhow::Error>(bpf) - })(); - - match result { - Ok(bpf) => { - tracing::info!("Sysmon: using {} backend", backend.label()); - return Ok(bpf); - } - Err(err) => { - tracing::warn!("Sysmon: {} backend unavailable: {:#}", backend.label(), err); - failures.push(format!("{}: {err:#}", backend.label())); - } - } - } - - Err(anyhow::anyhow!( - "no sysmon tracepoint backend available ({})", - failures.join("; ") - )) -} - -#[cfg(feature = "sysmon-ebpf")] -fn run_sysmon_loop( - mgr: Arc>, - cfg: SysmonConfig, - pending: Arc>, - pending_map_refreshes: Arc>, - tx: mpsc::SyncSender, -) -> anyhow::Result<()> { - use aya::include_bytes_aligned; - use aya::maps::{ - perf::{PerfEvent, PerfEventArray}, - ring_buf::RingBuf, - MapData, - }; - use log::{log_enabled, Level as LogLevel}; - // Load eBPF object (copied to OUT_DIR at build time) - #[allow(unused_variables)] - let obj_le: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/sysmon-bpf.bpfel.o")); - #[allow(unused_variables)] - let obj_be: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/sysmon-bpf.bpfeb.o")); - let obj: &[u8] = if cfg!(target_endian = "little") { - obj_le - } else { - obj_be - }; - if obj.is_empty() { - warn!("sysmon-bpf object missing; running in stub mode (no realtime process events)"); - return Ok(()); - } - let target = cfg.target_module.clone(); - let use_verbose = - cfg!(debug_assertions) || log_enabled!(LogLevel::Trace) || log_enabled!(LogLevel::Debug); - let mut bpf = load_and_attach_sysmon_bpf(obj, &cfg, use_verbose)?; - let proc_pid_for_event = sysmon_proc_pid_resolver(cfg.watched_pid, cfg.watched_proc_pid); - - // Using allowlist-based gating in kernel; userspace decides allow on exec. - - // Initial prefill for late-start cases: compute and insert offsets for already-running PIDs. - if let Some(tpath) = &target { - let mut initial_target_pids: BTreeSet = BTreeSet::new(); - if let Ok(mut guard) = mgr.lock() { - if let Ok(prefilled) = guard.ensure_prefill_module(tpath.to_string_lossy().as_ref()) { - tracing::info!( - "Sysmon: initial prefill cached {} pid(s) for module {}", - prefilled, - tpath.display() - ); - let entries = guard.cached_offsets_for_module(tpath.to_string_lossy().as_ref()); - if !entries.is_empty() { - use crate::pinned_bpf_maps::ProcModuleOffsetsValue; - let mut by_pid: HashMap> = - HashMap::new(); - for (pid, cookie, off, base, size) in entries { - if is_current_process_pid(pid) { - continue; - } - by_pid.entry(pid).or_default().push(( - cookie, - ProcModuleOffsetsValue::new( - off.text, off.rodata, off.data, off.bss, base, size, - ), - )); - } - let mut total = 0usize; - for (pid, items) in by_pid { - initial_target_pids.insert(pid); - // Add event PID (kernel namespace) to allowlist so subsequent - // fork/exit events are filtered in-kernel. - let event_pid = resolve_event_pid_for_proc(pid); - let runtime_pids = runtime_pid_keys_for_proc_event(pid, event_pid, []); - for runtime_pid in &runtime_pids { - write_pinned_runtime_pid_alias(*runtime_pid, pid); - guard.record_runtime_pid_alias(*runtime_pid, pid); - } - if let Ok(n) = publish_offsets_for_runtime_pid_keys( - pid, - event_pid, - &runtime_pids, - &items, - "initial prefill", - ) { - total += n; - } - insert_allowed_runtime_pid_keys(&runtime_pids); - } - tracing::info!( - "Sysmon: initial inserted {} offset entries for module {}", - total, - tpath.display() - ); - } - } - } - for pid in initial_target_pids { - let event_pid = resolve_event_pid_for_proc(pid); - if let Err(e) = - prefill_full_offsets_for_pid_if_new(&mgr, event_pid, &proc_pid_for_event) - { - tracing::debug!( - "Sysmon: initial full offset prefill failed for proc pid {} (event pid {}): {}", - pid, - event_pid, - e - ); - } - } - } - tracing::info!("Sysmon: setup complete"); - // Initial prefill already ran above. Do not make the first periodic module - // refresh immediately due: for `-t executable`, the exec event is the fast - // path that inserts proc_module_offsets and allowed_pids. A fallback /proc - // scan here can delay a short-lived target past its only probe. - let mut last_module_refresh = Instant::now(); - let mut target_pid_map_signatures = HashMap::::new(); - - // Event loop: prefer ringbuf; fallback to perf - if let Some(map) = bpf.take_map("sysmon_events") { - let mut rb: RingBuf = map.try_into()?; - loop { - let mut had_event = false; - // Drain queued lifecycle events before periodic refresh. In the - // short-lived `-t executable` path, sched_process_exec must be - // handled promptly so offsets are ready before the first uprobe. - while let Some(item) = rb.next() { - had_event = true; - if item.len() == core::mem::size_of::() { - // SAFETY: The ring buffer sample length was checked to match SysEvent; - // read_unaligned handles any alignment from the byte slice. - let ev = unsafe { core::ptr::read_unaligned(item.as_ptr() as *const SysEvent) }; - let matched = dispatch_sysmon_event( - &mgr, - &target, - &pending, - &pending_map_refreshes, - &proc_pid_for_event, - &ev, - ); - if matched { - try_publish_sys_event(&tx, ev); - } - } - } - poll_pending_offsets(&mgr, &pending, &proc_pid_for_event); - poll_pending_map_refreshes( - &mgr, - target.as_deref(), - &pending_map_refreshes, - &pending, - &tx, - ); - refresh_target_module_offsets( - &mgr, - target.as_deref(), - &mut last_module_refresh, - &mut target_pid_map_signatures, - &tx, - ); - if !had_event { - std::thread::sleep(std::time::Duration::from_millis(5)); - } - } - } else if let Some(map) = bpf.take_map("sysmon_events_perf") { - let mut perf: PerfEventArray<_> = map.try_into()?; - let online = aya::util::online_cpus().map_err(|(_, e)| anyhow::anyhow!(e))?; - let mut bufs = Vec::new(); - for cpu in online { - match perf.open(cpu, cfg.perf_page_count) { - Ok(buf) => bufs.push(buf), - Err(e) => warn!("Perf open failed for CPU {}: {}", cpu, e), - } - } - if bufs.is_empty() { - return Err(anyhow::anyhow!("No perf buffers opened")); - } - loop { - std::thread::sleep(std::time::Duration::from_millis(10)); - for buf in bufs.iter_mut() { - if !buf.readable() { - continue; - } - buf.for_each(|event| match event { - PerfEvent::Sample { head, tail } => { - let mut raw = [0u8; core::mem::size_of::()]; - let mut copied = 0; - for chunk in [head, tail] { - let remaining = raw.len().saturating_sub(copied); - if remaining == 0 { - break; - } - let take = chunk.len().min(remaining); - raw[copied..copied + take].copy_from_slice(&chunk[..take]); - copied += take; - } - if copied == raw.len() { - // SAFETY: raw is exactly the size of SysEvent and read_unaligned - // handles the byte array's alignment. - let ev = unsafe { - core::ptr::read_unaligned(raw.as_ptr() as *const SysEvent) - }; - let matched = dispatch_sysmon_event( - &mgr, - &target, - &pending, - &pending_map_refreshes, - &proc_pid_for_event, - &ev, - ); - if matched { - try_publish_sys_event(&tx, ev); - } - } - } - PerfEvent::Lost { count } => { - warn!("Perf event buffer lost {} sysmon events", count); - } - }); - } - poll_pending_offsets(&mgr, &pending, &proc_pid_for_event); - poll_pending_map_refreshes( - &mgr, - target.as_deref(), - &pending_map_refreshes, - &pending, - &tx, - ); - refresh_target_module_offsets( - &mgr, - target.as_deref(), - &mut last_module_refresh, - &mut target_pid_map_signatures, - &tx, - ); - } - } else { - return Err(anyhow::anyhow!("No sysmon events map found (ringbuf/perf)")); - } -} - -/* moved to ghostscope_process::util::is_shared_object -fn looks_like_shared_object(path: &Path) -> bool { - // Determine shared object by ELF metadata: - // - ET_EXEC => executable (not shared) - // - ET_DYN + PT_INTERP present => PIE executable (not shared) - // - ET_DYN without PT_INTERP => shared library - use std::fs::File; - use std::io::{Read, Seek, SeekFrom}; - - const EI_CLASS: usize = 4; // 1=32-bit, 2=64-bit - const EI_DATA: usize = 5; // 1=little, 2=big - const ET_EXEC: u16 = 2; - const ET_DYN: u16 = 3; - const PT_INTERP: u32 = 3; - - let mut f = match File::open(path) { - Ok(f) => f, - Err(_) => return false, // conservative: treat as executable (enable filtering) - }; - let mut ehdr = [0u8; 64]; - if f.read(&mut ehdr).ok().filter(|&n| n >= 52).is_none() { - return false; - } - // ELF magic - if &ehdr[0..4] != b"\x7FELF" { - return false; - } - let class = ehdr[EI_CLASS]; - let data = ehdr[EI_DATA]; - let is_le = data == 1; - // read u16/u32/u64 helpers - let rd16 = |b: &[u8]| -> u16 { - if is_le { - u16::from_le_bytes([b[0], b[1]]) - } else { - u16::from_be_bytes([b[0], b[1]]) - } - }; - let rd32 = |b: &[u8]| -> u32 { - if is_le { - u32::from_le_bytes([b[0], b[1], b[2], b[3]]) - } else { - u32::from_be_bytes([b[0], b[1], b[2], b[3]]) - } - }; - let rd64 = |b: &[u8]| -> u64 { - if is_le { - u64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]) - } else { - u64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]) - } - }; - - // e_type at 0x10 - let e_type = rd16(&ehdr[16..18]); - if e_type == ET_EXEC { - return false; // executable - } - - // program header table offsets - let (e_phoff, e_phentsize, e_phnum) = match class { - 1 => { - // ELF32: e_phoff @0x1C (4), e_phentsize @0x2A (2), e_phnum @0x2C (2) - let phoff = rd32(&ehdr[28..32]) as u64; - let entsz = rd16(&ehdr[42..44]) as u64; - let phnum = rd16(&ehdr[44..46]) as u64; - (phoff, entsz, phnum) - } - 2 => { - // ELF64: e_phoff @0x20 (8), e_phentsize @0x36 (2), e_phnum @0x38 (2) - let phoff = rd64(&ehdr[32..40]); - let entsz = rd16(&ehdr[54..56]) as u64; - let phnum = rd16(&ehdr[56..58]) as u64; - (phoff, entsz, phnum) - } - _ => return false, - }; - - if e_type == ET_DYN { - // Scan program headers for PT_INTERP - if e_phoff == 0 || e_phentsize < 4 || e_phnum == 0 { - // malformed - // If cannot inspect, be conservative and treat as shared library (disable filtering) - return true; - } - // Seek and read each p_type - for i in 0..e_phnum { - let off = e_phoff + i * e_phentsize; - if f.seek(SeekFrom::Start(off)).is_err() { - return true; - } - let mut p = [0u8; 8]; - if f.read(&mut p[..4]).ok().filter(|&n| n == 4).is_none() { - return true; - } - let p_type = rd32(&p[..4]); - if p_type == PT_INTERP { - return false; // PIE executable - } - } - return true; // ET_DYN w/o PT_INTERP => shared library - } - - // Unknown types: default to 'not shared' (enable filtering) - false -} -*/ - -fn pid_alive(pid: u32) -> bool { - std::path::Path::new(&format!("/proc/{pid}")).exists() -} - -fn filter_entries_for_target<'a>( - entries: &'a [PidOffsetsEntry], - target: Option<&Path>, -) -> Vec<&'a PidOffsetsEntry> { - use std::fs; - use std::os::unix::fs::MetadataExt; - - if let Some(tpath) = target { - match fs::metadata(tpath) { - Ok(meta) => { - let t_dev = meta.dev(); - let t_ino = meta.ino(); - entries - .iter() - .filter(|e| { - fs::metadata(&e.module_path) - .map(|m| m.dev() == t_dev && m.ino() == t_ino) - .unwrap_or(false) - }) - .collect() - } - Err(_) => { - let tc = cookie_for_path(&tpath.to_string_lossy()); - let by_cookie: Vec<_> = entries.iter().filter(|e| e.cookie == tc).collect(); - if !by_cookie.is_empty() { - by_cookie - } else { - let tnorm = tpath.to_string_lossy().replace("/./", "/"); - entries.iter().filter(|e| e.module_path == tnorm).collect() - } - } - } - } else { - entries.iter().collect() - } -} - -fn prefill_offsets_for_pid( - mgr: &Arc>, - event_pid: u32, - target: Option<&Path>, - proc_pid_for_event: &impl Fn(u32) -> u32, -) -> anyhow::Result { - write_offsets_for_pid(mgr, event_pid, target, false, &[], proc_pid_for_event) -} - -fn refresh_offsets_for_known_proc_pid( - mgr: &Arc>, - event_pid: u32, - host_pid: u32, - proc_pid: u32, -) -> anyhow::Result { - let proc_pid_for_event = |_: u32| proc_pid; - write_offsets_for_pid(mgr, event_pid, None, true, &[host_pid], &proc_pid_for_event) -} - -fn write_offsets_for_pid( - mgr: &Arc>, - event_pid: u32, - target: Option<&Path>, - force_refresh: bool, - extra_runtime_pids: &[u32], - proc_pid_for_event: &impl Fn(u32) -> u32, -) -> anyhow::Result { - let proc_pid = proc_pid_for_event(event_pid); - let runtime_pids = - runtime_pid_keys_for_proc_event(proc_pid, event_pid, extra_runtime_pids.iter().copied()); - record_runtime_pid_aliases_for_keys(mgr, proc_pid, &runtime_pids); - let mut inserted_any = false; - if let Ok(mut guard) = mgr.lock() { - if target.is_some() && is_current_process_pid(proc_pid) { - tracing::debug!( - "Sysmon: skipping self proc pid {} for target-module offset prefill", - proc_pid - ); - return Ok(false); - } - let prefilled = match if force_refresh { - guard.refresh_prefill_pid(proc_pid) - } else { - guard.ensure_prefill_pid(proc_pid) - } { - Ok(v) => v, - Err(e) => { - // In private PID namespaces, sysmon event PID may be in the initial namespace - // and not resolvable via /proc/. Fall back to module-wide refresh. - if let Some(target_path) = target { - let module_path = target_path.to_string_lossy().to_string(); - tracing::debug!( - "Sysmon: pid prefill failed for event pid {} (proc pid {}): {}; falling back to module refresh for {}", - event_pid, - proc_pid, - e, - module_path - ); - let refreshed = guard.refresh_prefill_module(&module_path)?; - if refreshed > 0 { - tracing::info!( - "Sysmon: module refresh cached {} pid(s) for {}", - refreshed, - module_path - ); - } - let mut target_pids = BTreeSet::new(); - for (pid, _, _, _, _) in guard.cached_offsets_for_module(&module_path) { - target_pids.insert(pid); - } - drop(guard); - - for pid in target_pids { - let runtime_pid = resolve_event_pid_for_proc(pid); - match refresh_full_offsets_for_pid(mgr, pid, runtime_pid) { - Ok(true) => inserted_any = true, - Ok(false) => {} - Err(err) => tracing::warn!( - "Sysmon: module refresh failed to publish full snapshot for proc pid {}: {}", - pid, - err - ), - } - } - return Ok(inserted_any); - } - return Err(e); - } - }; - if prefilled > 0 { - info!( - "Sysmon: {} {} entries for event pid {} (proc pid {})", - if force_refresh { - "refreshed" - } else { - "prefilled" - }, - prefilled, - event_pid, - proc_pid - ); - } - let mut entries = guard - .cached_offsets_with_paths_for_pid(proc_pid) - .map(|entries| entries.to_vec()) - .unwrap_or_default(); - let mut target_match_count = filter_entries_for_target(&entries, target).len(); - - if target_match_count == 0 && target.is_some() { - let refreshed = guard.refresh_prefill_pid(proc_pid)?; - if refreshed > 0 { - tracing::debug!( - "Sysmon: refreshed {} cached entries for event pid {} (proc pid {})", - refreshed, - event_pid, - proc_pid - ); - } - entries = guard - .cached_offsets_with_paths_for_pid(proc_pid) - .map(|entries| entries.to_vec()) - .unwrap_or_default(); - target_match_count = filter_entries_for_target(&entries, target).len(); - } - - if force_refresh && target.is_none() { - let purged = purge_offsets_for_runtime_pid_keys(&runtime_pids)?; - if purged > 0 { - tracing::debug!( - "Sysmon: purged {} stale offset entries before map-change refresh for event pid {} (proc pid {}, runtime keys={:?})", - purged, - event_pid, - proc_pid, - runtime_pids - ); - } - } - - if !entries.is_empty() && (target.is_none() || target_match_count > 0) { - let items = offset_items_from_entries(entries.iter()); - match publish_offsets_for_runtime_pid_keys( - proc_pid, - event_pid, - &runtime_pids, - &items, - "prefill", - ) { - Ok(inserted) => { - if inserted == 0 { - tracing::warn!( - "Sysmon: no offsets inserted for event pid {} (proc pid {}, runtime keys={:?}) (entry count={})", - event_pid, - proc_pid, - runtime_pids, - items.len() - ); - } else { - tracing::info!( - "Sysmon: inserted {} offset entries for event pid {} (proc pid {}, runtime keys={:?})", - inserted, - event_pid, - proc_pid, - runtime_pids - ); - insert_allowed_runtime_pid_keys(&runtime_pids); - inserted_any = true; - } - } - Err(e) => { - tracing::warn!( - "Sysmon: failed to insert offsets for event pid {} (proc pid {}): {}", - event_pid, - proc_pid, - e - ); - } - } - } else if target.is_some() { - tracing::debug!( - "Sysmon: event pid {} (proc pid {}) does not map target module; skip", - event_pid, - proc_pid - ); - } - } - Ok(inserted_any) -} - -fn prefill_full_offsets_for_pid_if_new( - mgr: &Arc>, - event_pid: u32, - proc_pid_for_event: &impl Fn(u32) -> u32, -) -> anyhow::Result { - let proc_pid = proc_pid_for_event(event_pid); - let runtime_pids = runtime_pid_keys_for_proc_event(proc_pid, event_pid, []); - record_runtime_pid_aliases_for_keys(mgr, proc_pid, &runtime_pids); - - let items = { - let Ok(mut guard) = mgr.lock() else { - return Ok(false); - }; - let prefilled = guard.ensure_prefill_pid(proc_pid)?; - if prefilled == 0 { - return Ok(false); - } - let Some(entries) = guard.cached_offsets_with_paths_for_pid(proc_pid) else { - return Ok(false); - }; - offset_items_from_entries(entries.iter()) - }; - - if items.is_empty() { - return Ok(false); - } - - match publish_offsets_for_runtime_pid_keys( - proc_pid, - event_pid, - &runtime_pids, - &items, - "full prefill", - ) { - Ok(inserted) if inserted > 0 => { - tracing::info!( - "Sysmon: inserted {} full offset entries for event pid {} (proc pid {}, runtime keys={:?})", - inserted, - event_pid, - proc_pid, - runtime_pids - ); - insert_allowed_runtime_pid_keys(&runtime_pids); - Ok(true) - } - Ok(_) => Ok(false), - Err(e) => { - tracing::warn!( - "Sysmon: failed to insert full offsets for event pid {} (proc pid {}): {}", - event_pid, - proc_pid, - e - ); - Ok(false) - } - } -} - -type PidMapsSignature = Vec<(String, u64, u64, u64, u64, u64, bool)>; - -fn pid_maps_signature(pid: u32) -> anyhow::Result { - let mut signature = read_proc_maps(pid)? - .into_iter() - .filter_map(|entry| { - let path = entry.path()?; - if should_skip_mapped_module_path(path) { - return None; - } - Some(( - normalize_mapped_module_path(path).to_string(), - entry.start, - entry.end, - entry.offset, - entry.inode, - (entry.dev_major << 32) | entry.dev_minor, - entry.executable(), - )) - }) - .collect::>(); - signature.sort_unstable(); - Ok(signature) -} - -fn refresh_full_offsets_for_pid( - mgr: &Arc>, - proc_pid: u32, - event_pid: u32, -) -> anyhow::Result { - let runtime_pids = runtime_pid_keys_for_proc_event(proc_pid, event_pid, []); - record_runtime_pid_aliases_for_keys(mgr, proc_pid, &runtime_pids); - - let items = { - let Ok(mut guard) = mgr.lock() else { - return Ok(false); - }; - guard.refresh_prefill_pid(proc_pid)?; - let Some(entries) = guard.cached_offsets_with_paths_for_pid(proc_pid) else { - return Ok(false); - }; - offset_items_from_entries(entries.iter()) - }; - - if items.is_empty() { - return Ok(false); - } - - let purged = purge_offsets_for_runtime_pid_keys(&runtime_pids)?; - if purged > 0 { - tracing::debug!( - "Sysmon: purged {} stale offset entries before periodic full refresh for proc pid {} (runtime keys={:?})", - purged, - proc_pid, - runtime_pids - ); - } - let inserted = publish_offsets_for_runtime_pid_keys( - proc_pid, - event_pid, - &runtime_pids, - &items, - "periodic full refresh", - )?; - insert_allowed_runtime_pid_keys(&runtime_pids); - tracing::debug!( - "Sysmon: periodic full refresh wrote {} offset entries for proc pid {} (event pid {}, runtime keys={:?})", - inserted, - proc_pid, - event_pid, - runtime_pids - ); - Ok(inserted > 0) -} - -fn offset_items_from_entries<'a>( - entries: impl IntoIterator, -) -> Vec<(u64, crate::pinned_bpf_maps::ProcModuleOffsetsValue)> { - entries - .into_iter() - .map(|e| { - ( - e.cookie, - crate::pinned_bpf_maps::ProcModuleOffsetsValue::new( - e.offsets.text, - e.offsets.rodata, - e.offsets.data, - e.offsets.bss, - e.base, - e.size, - ), - ) - }) - .collect() -} - -fn refresh_target_module_offsets( - mgr: &Arc>, - target: Option<&Path>, - last_refresh: &mut Instant, - target_pid_map_signatures: &mut HashMap, - tx: &mpsc::SyncSender, -) { - use crate::pinned_bpf_maps::{allowed_pid_exists, ProcModuleOffsetsValue}; - - let Some(target_path) = target else { - return; - }; - let now = Instant::now(); - if now.duration_since(*last_refresh) < MODULE_REFRESH_INTERVAL { - return; - } - *last_refresh = now; - - let module_path = target_path.to_string_lossy().to_string(); - let mut by_pid: HashMap> = HashMap::new(); - let mut target_pids: BTreeSet = BTreeSet::new(); - if let Ok(mut guard) = mgr.lock() { - if let Err(e) = guard.refresh_prefill_module(&module_path) { - tracing::debug!( - "Sysmon: periodic module refresh failed for {}: {}", - module_path, - e - ); - return; - } - for (pid, cookie, off, base, size) in guard.cached_offsets_for_module(&module_path) { - if is_current_process_pid(pid) { - continue; - } - target_pids.insert(pid); - by_pid.entry(pid).or_default().push(( - cookie, - ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss, base, size), - )); - } - } - if by_pid.is_empty() { - return; - } - - let mut total = 0usize; - let mut newly_allowed_event_pids = BTreeSet::new(); - for (pid, items) in by_pid { - let event_pid = resolve_event_pid_for_proc(pid); - let runtime_pids = runtime_pid_keys_for_proc_event(pid, event_pid, []); - record_runtime_pid_aliases_for_keys(mgr, pid, &runtime_pids); - let was_allowed = match allowed_pid_exists(event_pid) { - Ok(value) => value, - Err(e) => { - tracing::debug!( - "Sysmon: allowed_pids lookup failed for event pid {} (proc pid {}): {}", - event_pid, - pid, - e - ); - false - } - }; - match publish_offsets_for_runtime_pid_keys( - pid, - event_pid, - &runtime_pids, - &items, - "periodic module refresh", - ) { - Ok(inserted) => { - if inserted > 0 { - total += inserted; - insert_allowed_runtime_pid_keys(&runtime_pids); - if !was_allowed { - tracing::debug!( - "Sysmon: event pid {} became allowed during periodic module refresh", - event_pid - ); - newly_allowed_event_pids.insert(event_pid); - } - } - } - Err(e) => tracing::debug!( - "Sysmon: periodic module refresh insert failed for pid {} ({}): {}", - pid, - module_path, - e - ), - } - } - for pid in &target_pids { - let event_pid = resolve_event_pid_for_proc(*pid); - let maps_signature = match pid_maps_signature(*pid) { - Ok(signature) => signature, - Err(e) => { - tracing::debug!( - "Sysmon: periodic maps signature failed for proc pid {} (event pid {}): {}", - *pid, - event_pid, - e - ); - continue; - } - }; - if target_pid_map_signatures.get(pid) == Some(&maps_signature) { - continue; - } - target_pid_map_signatures.insert(*pid, maps_signature); - - match refresh_full_offsets_for_pid(mgr, *pid, event_pid) { - Ok(true) => { - newly_allowed_event_pids.insert(event_pid); - } - Ok(false) => {} - Err(e) => { - tracing::debug!( - "Sysmon: periodic full offset refresh failed for proc pid {} (event pid {}): {}", - *pid, - event_pid, - e - ); - } - } - } - target_pid_map_signatures.retain(|pid, _| target_pids.contains(pid)); - for event_pid in newly_allowed_event_pids { - let ev = SysEvent { - tgid: event_pid, - host_tgid: event_pid, - kind: SysEventKind::MapChange.as_u32(), - }; - if try_publish_sys_event(tx, ev) { - tracing::debug!( - "Sysmon: published synthetic map-change for newly discovered target pid {}", - event_pid - ); - } - } - if total > 0 { - tracing::debug!( - "Sysmon: periodic module refresh inserted {} offset entries for {}", - total, - module_path - ); - } -} - -fn poll_pending_offsets( - mgr: &Arc>, - pending: &Arc>, - proc_pid_for_event: &impl Fn(u32) -> u32, -) { - let due = if let Ok(mut guard) = pending.lock() { - guard.take_due() - } else { - Vec::new() - }; - - if due.is_empty() { - return; - } - - let mut to_remove: Vec = Vec::new(); - let mut to_exhaust: Vec = Vec::new(); - - for due in due { - let event_pid = due.event_pid; - let target_path = due.target_path; - let attempts = due.attempts; - let proc_pid = proc_pid_for_event(event_pid); - if !pid_alive(proc_pid) { - tracing::debug!( - "Sysmon: event pid {} (proc pid {}) exited while waiting for offsets; removing from retry queue", - event_pid, - proc_pid - ); - to_remove.push(event_pid); - continue; - } - - if !pid_maps_target_module(proc_pid, &target_path) { - if attempts >= PENDING_MAX_ATTEMPTS { - if due.kind.keep_for_map_changes_after_retry_exhaustion() { - tracing::debug!( - "Sysmon: event pid {} (proc pid {}) still missing module {} after {} retries; waiting for map-change trigger", - event_pid, - proc_pid, - target_path.display(), - attempts - ); - to_exhaust.push(event_pid); - } else { - tracing::warn!( - "Sysmon: event pid {} (proc pid {}) still missing module {} after {} retries; giving up", - event_pid, - proc_pid, - target_path.display(), - attempts - ); - to_remove.push(event_pid); - } - } - continue; - } - - match prefill_offsets_for_pid( - mgr, - event_pid, - Some(target_path.as_path()), - proc_pid_for_event, - ) { - Ok(true) => { - tracing::info!( - "Sysmon: deferred prefill succeeded for event pid {} (proc pid {}) (module {})", - event_pid, - proc_pid, - target_path.display() - ); - to_remove.push(event_pid); - } - Ok(false) => { - if attempts >= PENDING_MAX_ATTEMPTS { - if due.kind.keep_for_map_changes_after_retry_exhaustion() { - tracing::debug!( - "Sysmon: deferred prefill produced no entries for event pid {} (proc pid {}) after {} retries; waiting for map-change trigger", - event_pid, - proc_pid, - attempts - ); - to_exhaust.push(event_pid); - } else { - tracing::warn!( - "Sysmon: deferred prefill produced no entries for event pid {} (proc pid {}) after {} retries; giving up", - event_pid, - proc_pid, - attempts - ); - to_remove.push(event_pid); - } - } - } - Err(e) => { - tracing::warn!( - "Sysmon: deferred prefill failed for event pid {} (proc pid {}) (attempt {}): {}", - event_pid, - proc_pid, - attempts, - e - ); - if attempts >= PENDING_MAX_ATTEMPTS { - if due.kind.keep_for_map_changes_after_retry_exhaustion() { - to_exhaust.push(event_pid); - } else { - to_remove.push(event_pid); - } - } - } - } - } - - if !to_remove.is_empty() || !to_exhaust.is_empty() { - if let Ok(mut guard) = pending.lock() { - for pid in to_remove { - guard.remove(pid); - } - for pid in to_exhaust { - guard.mark_retry_exhausted(pid); - } - } - } -} - -fn cached_offsets_exist_for_target_pid( - mgr: &Arc>, - target_path: &Path, - proc_pid: u32, -) -> bool { - let module_path = target_path.to_string_lossy().to_string(); - mgr.lock() - .ok() - .map(|guard| { - guard.cached_offsets_with_paths_for_pid(proc_pid).is_some() - || guard - .cached_offsets_for_module(&module_path) - .iter() - .any(|(pid, _, _, _, _)| *pid == proc_pid) - }) - .unwrap_or(false) -} - -fn forget_pid_offsets_after_target_unmap( - mgr: &Arc>, - event_pid: u32, - host_pid: u32, - proc_pid: u32, -) { - if let Ok(mut guard) = mgr.lock() { - guard.forget_pid(proc_pid); - if proc_pid != event_pid { - guard.forget_pid(event_pid); - } - if host_pid != event_pid && host_pid != proc_pid { - guard.forget_pid(host_pid); - } - } - - let purged = purge_runtime_pid_artifacts(proc_pid, event_pid, host_pid); - if purged > 0 { - tracing::info!( - "Sysmon: target unmapped for event pid {} (host pid {}, proc pid {}); purged {} offset entries", - event_pid, - host_pid, - proc_pid, - purged - ); - } -} - -fn poll_pending_map_refreshes( - mgr: &Arc>, - target: Option<&Path>, - pending_map_refreshes: &Arc>, - pending: &Arc>, - tx: &mpsc::SyncSender, -) { - let due = if let Ok(mut guard) = pending_map_refreshes.lock() { - guard.take_due() - } else { - Vec::new() - }; - - if due.is_empty() { - return; - } - - for event in due { - let event_pid = event.event_pid; - let host_pid = event.host_pid; - let proc_pid = target - .map(|target_path| { - canonicalize_cached_target_proc_pid(mgr, target_path, event.proc_pid) - }) - .unwrap_or(event.proc_pid); - if !pid_alive(proc_pid) { - tracing::trace!( - "Sysmon: event pid {} (proc pid {}) exited before map refresh", - event_pid, - proc_pid - ); - continue; - } - - if let Some(target_path) = target { - if !pid_maps_target_module(proc_pid, target_path) { - if cached_offsets_exist_for_target_pid(mgr, target_path, proc_pid) { - forget_pid_offsets_after_target_unmap(mgr, event_pid, host_pid, proc_pid); - let ev = SysEvent { - tgid: event_pid, - host_tgid: host_pid, - kind: SysEventKind::MapChange.as_u32(), - }; - try_publish_sys_event(tx, ev); - } - tracing::trace!( - "Sysmon: event pid {} (proc pid {}) map-change does not include target {}; skip", - event_pid, - proc_pid, - target_path.display() - ); - continue; - } - } - - match refresh_offsets_for_known_proc_pid(mgr, event_pid, host_pid, proc_pid) { - Ok(true) => { - tracing::debug!( - "Sysmon: refreshed offsets after map-change for event pid {} (host pid {}, proc pid {})", - event_pid, - host_pid, - proc_pid - ); - if let Ok(mut guard) = pending.lock() { - guard.remove(event_pid); - if host_pid != event_pid { - guard.remove(host_pid); - } - } - let ev = SysEvent { - tgid: event_pid, - host_tgid: host_pid, - kind: SysEventKind::MapChange.as_u32(), - }; - try_publish_sys_event(tx, ev); - } - Ok(false) => tracing::trace!( - "Sysmon: map-change refresh inserted no offsets for event pid {} (proc pid {})", - event_pid, - proc_pid - ), - Err(e) => tracing::debug!( - "Sysmon: map-change refresh failed for event pid {} (proc pid {}): {}", - event_pid, - proc_pid, - e - ), - } - } -} - -fn get_comm_from_proc(pid: u32) -> Option { - use std::io::Read; - let path = format!("/proc/{pid}/comm"); - let mut f = std::fs::File::open(path).ok()?; - let mut s = String::new(); - f.read_to_string(&mut s).ok()?; - if s.ends_with('\n') { - s.pop(); - if s.ends_with('\r') { - s.pop(); - } - } - // Kernel task->comm is at most 15 bytes; /proc returns without NUL. We compare as-is. - Some(s) -} - -fn truncate_basename_to_comm(path: &Path) -> Vec { - use std::ffi::OsStr; - let mut buf = Vec::with_capacity(16); - if let Some(name) = path.file_name().and_then(OsStr::to_str) { - let bytes = name.as_bytes(); - let n = core::cmp::min(bytes.len(), 15); - buf.extend_from_slice(&bytes[..n]); - } - buf -} - -fn pid_maps_target_module(pid: u32, target: &Path) -> bool { - let target = ModuleIdentity::from_path(target); - let mut matched = false; - - if visit_proc_maps(pid, |entry| { - if target.matches(&entry) { - matched = true; - return ControlFlow::Break(()); - } - ControlFlow::Continue(()) - }) - .is_err() - { - return false; - } - - matched -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn sysmon_target_mode_does_not_enable_map_change_events() { - let mask = SysmonEventMask::target_mode(); - assert!(mask.exec); - assert!(mask.fork); - assert!(mask.exit); - assert!(!mask.map_change); - } - - #[test] - fn sysmon_pid_module_changes_only_enable_map_change_events() { - let mask = SysmonEventMask::pid_module_changes(); - assert!(!mask.exec); - assert!(!mask.fork); - assert!(!mask.exit); - assert!(mask.map_change); - } - - #[test] - fn sysmon_event_publish_drops_when_queue_is_full() { - let (tx, rx) = mpsc::sync_channel(1); - - assert!(try_publish_sys_event( - &tx, - SysEvent { - tgid: 1, - host_tgid: 1, - kind: 1 - } - )); - assert!(!try_publish_sys_event( - &tx, - SysEvent { - tgid: 2, - host_tgid: 2, - kind: 2 - } - )); - - let queued = rx.try_recv().expect("first event should be queued"); - assert_eq!(queued.tgid, 1); - assert_eq!(queued.kind, 1); - assert!(matches!(rx.try_recv(), Err(mpsc::TryRecvError::Empty))); - } - - #[test] - fn invisible_shared_object_exec_does_not_queue_unresolved_alias() -> anyhow::Result<()> { - let dir = tempfile::tempdir()?; - let target_path = dir.path().join("libtarget.so"); - let mut elf = [0u8; 64]; - elf[0..4].copy_from_slice(b"\x7FELF"); - elf[4] = 2; - elf[5] = 1; - elf[16..18].copy_from_slice(&3u16.to_le_bytes()); - std::fs::write(&target_path, elf)?; - - let mgr = Arc::new(Mutex::new(ProcessManager::new())); - let pending = Arc::new(Mutex::new(PendingOffsets::new())); - let ev = SysEvent { - tgid: u32::MAX - 1, - host_tgid: u32::MAX - 2, - kind: SysEventKind::Exec.as_u32(), - }; - - ProcessSysmon::handle_event_with_proc_pid_resolver( - &mgr, - &Some(target_path), - &pending, - &ev, - |pid| pid, - )?; - - assert!( - pending - .lock() - .expect("pending offsets lock") - .entries - .is_empty(), - "unresolved exec events must not be deferred without target-map proof" - ); - Ok(()) - } - - #[test] - fn visible_shared_object_exec_queues_map_change_candidate() -> anyhow::Result<()> { - let dir = tempfile::tempdir()?; - let target_path = dir.path().join("libtarget.so"); - let mut elf = [0u8; 64]; - elf[0..4].copy_from_slice(b"\x7FELF"); - elf[4] = 2; - elf[5] = 1; - elf[16..18].copy_from_slice(&3u16.to_le_bytes()); - std::fs::write(&target_path, elf)?; - - let event_pid = std::process::id(); - let mgr = Arc::new(Mutex::new(ProcessManager::new())); - let pending = Arc::new(Mutex::new(PendingOffsets::new())); - let ev = SysEvent { - tgid: event_pid, - host_tgid: event_pid, - kind: SysEventKind::Exec.as_u32(), - }; - - ProcessSysmon::handle_event_with_proc_pid_resolver( - &mgr, - &Some(target_path.clone()), - &pending, - &ev, - |pid| pid, - )?; - - let guard = pending.lock().expect("pending offsets lock"); - let entry = guard - .entries - .get(&event_pid) - .expect("visible shared-object exec should be queued for retry"); - assert_eq!(entry.target_path, target_path); - assert_eq!(entry.kind, PendingOffsetsKind::MapChangeCandidate); - Ok(()) - } - - #[test] - fn invisible_shared_object_exec_queues_visible_host_pid_retry() -> anyhow::Result<()> { - let dir = tempfile::tempdir()?; - let target_path = dir.path().join("libtarget.so"); - let mut elf = [0u8; 64]; - elf[0..4].copy_from_slice(b"\x7FELF"); - elf[4] = 2; - elf[5] = 1; - elf[16..18].copy_from_slice(&3u16.to_le_bytes()); - std::fs::write(&target_path, elf)?; - - let host_pid = std::process::id(); - let mgr = Arc::new(Mutex::new(ProcessManager::new())); - let pending = Arc::new(Mutex::new(PendingOffsets::new())); - let ev = SysEvent { - tgid: u32::MAX - 1, - host_tgid: host_pid, - kind: SysEventKind::Exec.as_u32(), - }; - - ProcessSysmon::handle_event_with_proc_pid_resolver( - &mgr, - &Some(target_path.clone()), - &pending, - &ev, - |pid| pid, - )?; - - let guard = pending.lock().expect("pending offsets lock"); - let entry = guard - .entries - .get(&host_pid) - .expect("visible host PID should be queued for maps-based retry"); - assert_eq!(entry.target_path, target_path); - assert_eq!(entry.kind, PendingOffsetsKind::MapChangeCandidate); - Ok(()) - } - - #[test] - fn shared_object_candidate_survives_retry_exhaustion() -> anyhow::Result<()> { - let dir = tempfile::tempdir()?; - let target_path = dir.path().join("libtarget.so"); - std::fs::write(&target_path, b"not actually mapped")?; - - let event_pid = std::process::id(); - let pending = Arc::new(Mutex::new(PendingOffsets::new())); - { - let mut guard = pending.lock().expect("pending offsets lock"); - guard.register_map_change_candidate(event_pid, &target_path); - let entry = guard - .entries - .get_mut(&event_pid) - .expect("pending candidate"); - entry.attempts = PENDING_MAX_ATTEMPTS - 1; - entry.last_poll = Instant::now() - PENDING_POLL_INTERVAL; - } - - let mgr = Arc::new(Mutex::new(ProcessManager::new())); - poll_pending_offsets(&mgr, &pending, &|pid| pid); - - let guard = pending.lock().expect("pending offsets lock"); - let entry = guard - .entries - .get(&event_pid) - .expect("map-change candidate should remain after retry exhaustion"); - assert!(entry.retry_exhausted); - assert!(guard.contains_map_change_candidate(event_pid, &target_path)); - Ok(()) - } -} diff --git a/ghostscope-process/src/sysmon/attach.rs b/ghostscope-process/src/sysmon/attach.rs new file mode 100644 index 00000000..609dd164 --- /dev/null +++ b/ghostscope-process/src/sysmon/attach.rs @@ -0,0 +1,458 @@ +use super::*; + +#[cfg(feature = "sysmon-ebpf")] +#[derive(Debug, Clone, Copy)] +pub(super) enum SysmonAttachBackend { + Raw, + Btf, + Classic, +} + +#[cfg(feature = "sysmon-ebpf")] +impl SysmonAttachBackend { + fn label(self) -> &'static str { + match self { + SysmonAttachBackend::Raw => "raw tracepoint", + SysmonAttachBackend::Btf => "BTF tracepoint", + SysmonAttachBackend::Classic => "classic tracepoint", + } + } +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) struct SysmonTracepoint { + event: &'static str, + category: &'static str, + classic_program: &'static str, + raw_program: &'static str, + btf_program: &'static str, +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) const SYSMON_TRACEPOINTS: &[SysmonTracepoint] = &[ + SysmonTracepoint { + event: "sched_process_exec", + category: "sched", + classic_program: "sched_process_exec", + raw_program: "raw_sched_process_exec", + btf_program: "btf_sched_process_exec", + }, + SysmonTracepoint { + event: "sched_process_exit", + category: "sched", + classic_program: "sched_process_exit", + raw_program: "raw_sched_process_exit", + btf_program: "btf_sched_process_exit", + }, + SysmonTracepoint { + event: "sched_process_fork", + category: "sched", + classic_program: "sched_process_fork", + raw_program: "raw_sched_process_fork", + btf_program: "btf_sched_process_fork", + }, +]; + +#[cfg(feature = "sysmon-ebpf")] +pub(super) struct SysmonMapChangeTracepoint { + event: &'static str, + category: &'static str, + program: &'static str, +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) const SYSMON_MAP_CHANGE_TRACEPOINTS: &[SysmonMapChangeTracepoint] = &[ + SysmonMapChangeTracepoint { + event: "sys_exit_mmap", + category: "syscalls", + program: "sys_exit_mmap", + }, + SysmonMapChangeTracepoint { + event: "sys_exit_mprotect", + category: "syscalls", + program: "sys_exit_mprotect", + }, + SysmonMapChangeTracepoint { + event: "sys_exit_munmap", + category: "syscalls", + program: "sys_exit_munmap", + }, + SysmonMapChangeTracepoint { + event: "sys_exit_mremap", + category: "syscalls", + program: "sys_exit_mremap", + }, +]; + +#[cfg(feature = "sysmon-ebpf")] +pub(super) fn load_sysmon_bpf(obj: &[u8], use_verbose: bool) -> anyhow::Result { + use aya::{EbpfLoader, VerifierLogLevel}; + + let mut loader = EbpfLoader::new(); + if use_verbose { + loader.verifier_log_level(VerifierLogLevel::VERBOSE | VerifierLogLevel::STATS); + tracing::info!("Sysmon verifier logs: VERBOSE (debug build/log)"); + } else { + loader.verifier_log_level(VerifierLogLevel::DEBUG | VerifierLogLevel::STATS); + tracing::info!("Sysmon verifier logs: DEBUG (release/info)"); + } + + let pin_dir = crate::pinned_bpf_maps::proc_offsets_pin_dir()?; + loader.map_pin_path( + crate::pinned_bpf_maps::ALLOWED_PIDS_MAP_NAME, + pin_dir.join(crate::pinned_bpf_maps::ALLOWED_PIDS_MAP_NAME), + ); + loader.map_pin_path( + crate::pinned_bpf_maps::TARGET_EXEC_COMM_MAP_NAME, + pin_dir.join(crate::pinned_bpf_maps::TARGET_EXEC_COMM_MAP_NAME), + ); + loader.map_pin_path( + crate::pinned_bpf_maps::SYSMON_MAP_CHANGE_UNFILTERED_MAP_NAME, + pin_dir.join(crate::pinned_bpf_maps::SYSMON_MAP_CHANGE_UNFILTERED_MAP_NAME), + ); + + Ok(loader.load(obj)?) +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) fn configure_sysmon_exec_comm_filter( + bpf: &mut aya::Ebpf, + target: Option<&Path>, +) -> anyhow::Result<()> { + use aya::maps::Array; + + let mut filter_bytes = [0u8; 16]; + let mut filter_len = 0usize; + if let Some(tpath) = target { + if !crate::util::is_shared_object(tpath) { + if let Some(name) = tpath.file_name().and_then(|s| s.to_str()) { + let bytes = name.as_bytes(); + // task->comm stores at most TASK_COMM_LEN - 1 visible bytes plus NUL. + // Keep the filter null-terminated so long executable basenames compare + // against the same truncation that bpf_get_current_comm() returns. + let len = bytes.len().min(filter_bytes.len() - 1); + filter_bytes[..len].copy_from_slice(&bytes[..len]); + filter_len = len; + } else { + tracing::warn!( + "Sysmon: target basename contains non-UTF8 bytes; exec comm filter disabled" + ); + } + } + } + + if let Some(map) = bpf.map_mut("target_exec_comm") { + let mut array: Array<_, [u8; 16]> = map.try_into()?; + array.set(0, filter_bytes, 0)?; + if filter_len > 0 { + match std::str::from_utf8(&filter_bytes[..filter_len]) { + Ok(name_str) => { + tracing::info!("Sysmon: exec comm filter configured for '{}'", name_str) + } + Err(_) => tracing::info!( + "Sysmon: exec comm filter configured (non-UTF8 basename, len={})", + filter_len + ), + } + } else { + tracing::info!("Sysmon: exec comm filter disabled"); + } + } else if filter_len > 0 { + tracing::warn!("Sysmon: target_exec_comm map missing; exec filtering unavailable"); + } + + Ok(()) +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) fn configure_sysmon_event_filter( + bpf: &mut aya::Ebpf, + event_mask: SysmonEventMask, + map_change_unfiltered: bool, + watched_pid: Option, + watched_pid_ns: Option, + event_pid_ns: Option, +) -> anyhow::Result<()> { + use aya::maps::Array; + + if let Some(map) = bpf.map_mut("sysmon_event_mask") { + let mut array: Array<_, u32> = map.try_into()?; + array.set(0, event_mask.bits(), 0)?; + tracing::info!( + "Sysmon: event mask configured (exec={}, fork={}, exit={}, map_change={})", + event_mask.exec, + event_mask.fork, + event_mask.exit, + event_mask.map_change + ); + } else { + tracing::warn!("Sysmon: sysmon_event_mask map missing; event filtering unavailable"); + } + + if let Some(map) = bpf.map_mut("sysmon_map_change_unfiltered") { + let mut array: Array<_, u32> = map.try_into()?; + array.set(0, u32::from(map_change_unfiltered), 0)?; + tracing::info!( + "Sysmon: map-change pre-allowlist emission {}", + if map_change_unfiltered { + "enabled" + } else { + "disabled" + } + ); + } else if map_change_unfiltered { + tracing::warn!( + "Sysmon: sysmon_map_change_unfiltered map missing; pre-allowlist map-change events unavailable" + ); + } + + if let Some(map) = bpf.map_mut("sysmon_watched_pid") { + let mut array: Array<_, u32> = map.try_into()?; + array.set(0, watched_pid.unwrap_or(0), 0)?; + if let Some(pid) = watched_pid { + tracing::info!("Sysmon: watched event pid configured: {}", pid); + } else { + tracing::info!("Sysmon: watched event pid disabled"); + } + } else if watched_pid.is_some() { + tracing::warn!("Sysmon: sysmon_watched_pid map missing; PID filtering unavailable"); + } + + let watched_pid_ns = watched_pid.and(watched_pid_ns); + let ns_spec = watched_pid_ns.and_then(|pid_ns| pid_ns.helper_dev_inode()); + let (ns_dev, ns_ino) = ns_spec.unwrap_or((0, 0)); + + if let Some(map) = bpf.map_mut("sysmon_watched_pid_ns_dev") { + let mut array: Array<_, u64> = map.try_into()?; + array.set(0, ns_dev, 0)?; + } else if ns_spec.is_some() { + tracing::warn!( + "Sysmon: sysmon_watched_pid_ns_dev map missing; namespace PID filtering unavailable" + ); + } + + if let Some(map) = bpf.map_mut("sysmon_watched_pid_ns_ino") { + let mut array: Array<_, u64> = map.try_into()?; + array.set(0, ns_ino, 0)?; + } else if ns_spec.is_some() { + tracing::warn!( + "Sysmon: sysmon_watched_pid_ns_ino map missing; namespace PID filtering unavailable" + ); + } + + if let (Some(pid), Some((dev, ino))) = (watched_pid, ns_spec) { + tracing::info!( + "Sysmon: watched PID namespace configured: pid={} ns_dev={} ns_inode={}", + pid, + dev, + ino + ); + } + + let event_ns_spec = event_pid_ns.and_then(|pid_ns| pid_ns.helper_dev_inode()); + let (event_ns_dev, event_ns_ino) = event_ns_spec.unwrap_or((0, 0)); + + if let Some(map) = bpf.map_mut("sysmon_event_pid_ns_dev") { + let mut array: Array<_, u64> = map.try_into()?; + array.set(0, event_ns_dev, 0)?; + } else if event_ns_spec.is_some() { + tracing::warn!( + "Sysmon: sysmon_event_pid_ns_dev map missing; event namespace reporting unavailable" + ); + } + + if let Some(map) = bpf.map_mut("sysmon_event_pid_ns_ino") { + let mut array: Array<_, u64> = map.try_into()?; + array.set(0, event_ns_ino, 0)?; + } else if event_ns_spec.is_some() { + tracing::warn!( + "Sysmon: sysmon_event_pid_ns_ino map missing; event namespace reporting unavailable" + ); + } + + if let Some((dev, ino)) = event_ns_spec { + tracing::info!( + "Sysmon: event PID namespace configured: ns_dev={} ns_inode={}", + dev, + ino + ); + } + + Ok(()) +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) fn attach_sysmon_backend( + bpf: &mut aya::Ebpf, + backend: SysmonAttachBackend, +) -> anyhow::Result<()> { + match backend { + SysmonAttachBackend::Raw => attach_raw_sysmon_tracepoints(bpf), + SysmonAttachBackend::Btf => attach_btf_sysmon_tracepoints(bpf), + SysmonAttachBackend::Classic => attach_classic_sysmon_tracepoints(bpf), + } +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) fn attach_raw_sysmon_tracepoints(bpf: &mut aya::Ebpf) -> anyhow::Result<()> { + use aya::programs::RawTracePoint; + + for spec in SYSMON_TRACEPOINTS { + let prog = bpf.program_mut(spec.raw_program).ok_or_else(|| { + anyhow::anyhow!("missing program '{}' in sysmon-bpf", spec.raw_program) + })?; + let tp: &mut RawTracePoint = prog.try_into()?; + tp.load()?; + tp.attach(spec.event)?; + info!("Attached raw tracepoint: {}", spec.event); + } + Ok(()) +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) fn attach_btf_sysmon_tracepoints(bpf: &mut aya::Ebpf) -> anyhow::Result<()> { + use anyhow::Context as _; + use aya::{programs::BtfTracePoint, Btf}; + + let btf = Btf::from_sys_fs().context("kernel BTF is unavailable")?; + for spec in SYSMON_TRACEPOINTS { + let prog = bpf.program_mut(spec.btf_program).ok_or_else(|| { + anyhow::anyhow!("missing program '{}' in sysmon-bpf", spec.btf_program) + })?; + let tp: &mut BtfTracePoint = prog.try_into()?; + tp.load(spec.event, &btf)?; + tp.attach()?; + info!("Attached BTF tracepoint: {}", spec.event); + } + Ok(()) +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) fn attach_classic_sysmon_tracepoints(bpf: &mut aya::Ebpf) -> anyhow::Result<()> { + use aya::programs::TracePoint; + + for spec in SYSMON_TRACEPOINTS { + let prog = bpf.program_mut(spec.classic_program).ok_or_else(|| { + anyhow::anyhow!("missing program '{}' in sysmon-bpf", spec.classic_program) + })?; + let tp: &mut TracePoint = prog.try_into()?; + tp.load()?; + tp.attach(spec.category, spec.event)?; + info!( + "Attached classic tracepoint: {}:{}", + spec.category, spec.event + ); + } + Ok(()) +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) fn attach_classic_map_change_tracepoints(bpf: &mut aya::Ebpf) -> anyhow::Result { + use aya::programs::TracePoint; + + let mut attached = 0usize; + for spec in SYSMON_MAP_CHANGE_TRACEPOINTS { + let Some(prog) = bpf.program_mut(spec.program) else { + tracing::warn!( + "Sysmon: missing map-change program '{}' in sysmon-bpf", + spec.program + ); + continue; + }; + let attach_result = (|| { + let tp: &mut TracePoint = prog.try_into()?; + tp.load()?; + tp.attach(spec.category, spec.event)?; + Ok::<_, anyhow::Error>(()) + })(); + match attach_result { + Ok(()) => { + attached += 1; + info!( + "Attached map-change tracepoint: {}:{}", + spec.category, spec.event + ); + } + Err(err) => { + tracing::warn!( + "Sysmon: map-change tracepoint {}:{} unavailable: {:#}", + spec.category, + spec.event, + err + ); + } + } + } + + Ok(attached) +} + +#[cfg(feature = "sysmon-ebpf")] +pub(super) fn load_and_attach_sysmon_bpf( + obj: &[u8], + cfg: &SysmonConfig, + use_verbose: bool, +) -> anyhow::Result { + let mut failures = Vec::new(); + for backend in [ + SysmonAttachBackend::Raw, + SysmonAttachBackend::Btf, + SysmonAttachBackend::Classic, + ] { + tracing::info!("Sysmon: trying {} backend", backend.label()); + let result = (|| { + let mut bpf = load_sysmon_bpf(obj, use_verbose)?; + configure_sysmon_exec_comm_filter(&mut bpf, cfg.target_module.as_deref())?; + configure_sysmon_event_filter( + &mut bpf, + cfg.event_mask, + cfg.map_change_unfiltered, + cfg.watched_pid, + cfg.watched_pid_ns, + cfg.event_pid_ns, + )?; + attach_sysmon_backend(&mut bpf, backend)?; + if cfg.event_mask.map_change { + let map_attached = attach_classic_map_change_tracepoints(&mut bpf)?; + if map_attached == 0 { + if !cfg.event_mask.has_lifecycle_events() { + return Err(anyhow::anyhow!( + "map-change events requested but no syscall tracepoints attached" + )); + } + + let fallback_mask = cfg.event_mask.without_map_change(); + configure_sysmon_event_filter( + &mut bpf, + fallback_mask, + cfg.map_change_unfiltered, + cfg.watched_pid, + cfg.watched_pid_ns, + cfg.event_pid_ns, + )?; + tracing::warn!( + "Sysmon: map-change events requested but no syscall tracepoints attached; \ + continuing with exec/fork/exit lifecycle events only" + ); + } + } + Ok::<_, anyhow::Error>(bpf) + })(); + + match result { + Ok(bpf) => { + tracing::info!("Sysmon: using {} backend", backend.label()); + return Ok(bpf); + } + Err(err) => { + tracing::warn!("Sysmon: {} backend unavailable: {:#}", backend.label(), err); + failures.push(format!("{}: {err:#}", backend.label())); + } + } + } + + Err(anyhow::anyhow!( + "no sysmon tracepoint backend available ({})", + failures.join("; ") + )) +} diff --git a/ghostscope-process/src/sysmon/events.rs b/ghostscope-process/src/sysmon/events.rs new file mode 100644 index 00000000..94c050e8 --- /dev/null +++ b/ghostscope-process/src/sysmon/events.rs @@ -0,0 +1,316 @@ +use super::offset_refresh::*; +use super::pending::*; +use super::pid_alias::*; +use super::*; + +pub(super) fn try_publish_sys_event(tx: &mpsc::SyncSender, ev: SysEvent) -> bool { + match tx.try_send(ev) { + Ok(()) => true, + Err(mpsc::TrySendError::Full(ev)) => { + tracing::trace!( + "Sysmon event queue full; dropping lifecycle notification for pid {} kind {}", + ev.tgid, + ev.kind + ); + false + } + Err(mpsc::TrySendError::Disconnected(ev)) => { + tracing::trace!( + "Sysmon event receiver disconnected; dropping lifecycle notification for pid {} kind {}", + ev.tgid, + ev.kind + ); + false + } + } +} + +pub(super) fn dispatch_sysmon_event( + mgr: &Arc>, + target: &Option, + pending: &Arc>, + pending_map_refreshes: &Arc>, + proc_pid_for_event: &impl Fn(u32) -> u32, + ev: &SysEvent, +) -> bool { + match SysEventKind::from_u32(ev.kind) { + Some(SysEventKind::MapChange) => { + if let Some(proc_pid) = + proc_pid_for_map_change_event(mgr, target, proc_pid_for_event, ev) + { + let proc_pid = target + .as_deref() + .map(|target_path| { + canonicalize_cached_target_proc_pid(mgr, target_path, proc_pid) + }) + .unwrap_or(proc_pid); + record_runtime_pid_aliases_for_sys_event(mgr, ev, proc_pid); + if let Ok(mut guard) = pending_map_refreshes.lock() { + guard.register(ev.tgid, sys_event_host_pid(ev), proc_pid); + } + true + } else if let Some(target_path) = target.as_deref() { + let candidates = pending + .lock() + .ok() + .map(|guard| { + pending_map_change_candidates(&guard, target_path, proc_pid_for_event, ev) + }) + .unwrap_or_default(); + + if candidates.is_empty() { + tracing::trace!( + "Sysmon: map-change event pid {} (host pid {}) did not resolve to a target /proc pid; skipping per-pid refresh", + ev.tgid, + sys_event_host_pid(ev) + ); + false + } else { + if let Ok(mut guard) = pending_map_refreshes.lock() { + for candidate in &candidates { + guard.register( + candidate.event_pid, + candidate.host_pid, + candidate.proc_pid, + ); + } + } + tracing::trace!( + "Sysmon: queued map-change refresh for {} pending target candidate(s) from event pid {} (host pid {})", + candidates.len(), + ev.tgid, + sys_event_host_pid(ev) + ); + false + } + } else { + tracing::trace!( + "Sysmon: map-change event pid {} (host pid {}) did not resolve to a target /proc pid; skipping per-pid refresh", + ev.tgid, + sys_event_host_pid(ev) + ); + false + } + } + Some(_) => { + match ProcessSysmon::handle_event_with_proc_pid_resolver( + mgr, + target, + pending, + ev, + proc_pid_for_event, + ) { + Ok(()) => true, + Err(e) => { + tracing::debug!( + "Sysmon: handle_event failed for pid {} kind {}: {}", + ev.tgid, + ev.kind, + e + ); + false + } + } + } + None => { + match ProcessSysmon::handle_event_with_proc_pid_resolver( + mgr, + target, + pending, + ev, + proc_pid_for_event, + ) { + Ok(()) => true, + Err(e) => { + tracing::debug!( + "Sysmon: handle_event rejected invalid event for pid {} kind {}: {}", + ev.tgid, + ev.kind, + e + ); + false + } + } + } + } +} + +pub(super) fn proc_pid_for_map_change_event( + mgr: &Arc>, + target: &Option, + proc_pid_for_event: &impl Fn(u32) -> u32, + ev: &SysEvent, +) -> Option { + let host_pid = sys_event_host_pid(ev); + let mut candidates = Vec::with_capacity(2); + push_unique_pid(&mut candidates, proc_pid_for_event(ev.tgid)); + if host_pid != ev.tgid { + push_unique_pid(&mut candidates, proc_pid_for_event(host_pid)); + } + + for proc_pid in candidates { + if !pid_alive(proc_pid) { + continue; + } + if target.is_some() && is_current_process_pid(proc_pid) { + tracing::trace!( + "Sysmon: ignoring self map-change candidate proc pid {}", + proc_pid + ); + continue; + } + + let Some(target_path) = target.as_deref() else { + return Some(proc_pid); + }; + + if pid_maps_target_module(proc_pid, target_path) + || cached_offsets_exist_for_target_pid(mgr, target_path, proc_pid) + { + return Some(proc_pid); + } + } + + None +} + +pub(super) fn pending_map_change_candidates( + pending: &PendingOffsets, + target_path: &Path, + proc_pid_for_event: &impl Fn(u32) -> u32, + ev: &SysEvent, +) -> Vec { + let host_pid = sys_event_host_pid(ev); + let mut candidates = Vec::with_capacity(2); + push_pending_map_change_candidate( + &mut candidates, + pending, + target_path, + proc_pid_for_event, + ev.tgid, + host_pid, + ); + if host_pid != ev.tgid { + push_pending_map_change_candidate( + &mut candidates, + pending, + target_path, + proc_pid_for_event, + host_pid, + host_pid, + ); + } + candidates +} + +pub(super) fn push_pending_map_change_candidate( + candidates: &mut Vec, + pending: &PendingOffsets, + target_path: &Path, + proc_pid_for_event: &impl Fn(u32) -> u32, + event_pid: u32, + host_pid: u32, +) { + if !pending.contains_map_change_candidate(event_pid, target_path) { + return; + } + + let proc_pid = proc_pid_for_event(event_pid); + if !pid_alive(proc_pid) || is_current_process_pid(proc_pid) { + return; + } + + if candidates + .iter() + .any(|candidate| candidate.proc_pid == proc_pid) + { + return; + } + + candidates.push(PendingMapChangeCandidate { + event_pid, + host_pid, + proc_pid, + }); +} + +pub(super) fn is_current_process_pid(proc_pid: u32) -> bool { + proc_pid == std::process::id() +} + +pub(super) fn cached_single_target_proc_pid( + mgr: &Arc>, + target_path: &Path, +) -> Option { + let module_path = target_path.to_string_lossy(); + let mut target_pids = BTreeSet::new(); + let guard = mgr.lock().ok()?; + for (pid, _, _, _, _) in guard.cached_offsets_for_module(module_path.as_ref()) { + if !is_current_process_pid(pid) + && pid_alive(pid) + && pid_maps_target_module(pid, target_path) + { + target_pids.insert(pid); + } + } + + if target_pids.len() == 1 { + target_pids.iter().next().copied() + } else { + None + } +} + +pub(super) fn canonicalize_cached_target_proc_pid( + mgr: &Arc>, + target_path: &Path, + fallback_proc_pid: u32, +) -> u32 { + if pid_alive(fallback_proc_pid) && pid_maps_target_module(fallback_proc_pid, target_path) { + return fallback_proc_pid; + } + + let Some(proc_pid) = cached_single_target_proc_pid(mgr, target_path) else { + return fallback_proc_pid; + }; + + if proc_pid != fallback_proc_pid { + tracing::debug!( + "Sysmon: canonicalized map-change proc pid {} -> {} for target {}", + fallback_proc_pid, + proc_pid, + target_path.display() + ); + } + + proc_pid +} + +pub(super) fn push_unique_pid(pids: &mut Vec, pid: u32) { + if !pids.contains(&pid) { + pids.push(pid); + } +} + +pub(super) fn sysmon_proc_pid_resolver( + watched_event_pid: Option, + watched_proc_pid: Option, +) -> impl Fn(u32) -> u32 { + move |event_pid| { + if watched_event_pid == Some(event_pid) { + if let Some(proc_pid) = watched_proc_pid { + return proc_pid; + } + } + + resolve_proc_pid_for_event(event_pid) + } +} + +pub(super) fn sys_event_host_pid(ev: &SysEvent) -> u32 { + if ev.host_tgid != 0 { + ev.host_tgid + } else { + ev.tgid + } +} diff --git a/ghostscope-process/src/sysmon/mod.rs b/ghostscope-process/src/sysmon/mod.rs new file mode 100644 index 00000000..4a2a5899 --- /dev/null +++ b/ghostscope-process/src/sysmon/mod.rs @@ -0,0 +1,664 @@ +use crate::{ + module_probe::cookie_for_path, + offsets::{PidOffsetsEntry, ProcessManager}, + pid::{ + resolve_event_pid_for_proc, resolve_proc_pid_for_event, runtime_pid_candidates_for_proc, + PidNamespaceId, + }, + pinned_bpf_maps, + proc_maps::{ + normalize_mapped_module_path, read_proc_maps, should_skip_mapped_module_path, + visit_proc_maps, ModuleIdentity, + }, +}; +use std::collections::{BTreeSet, HashMap}; +use std::ops::ControlFlow; +use std::path::{Path, PathBuf}; +use std::sync::{mpsc, Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; +use tracing::{error, info, warn}; + +/// Kind of process lifecycle event +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SysEventKind { + Exec, + Fork, + Exit, + MapChange, +} + +impl SysEventKind { + fn from_u32(v: u32) -> Option { + match v { + 1 => Some(SysEventKind::Exec), + 2 => Some(SysEventKind::Fork), + 3 => Some(SysEventKind::Exit), + 4 => Some(SysEventKind::MapChange), + _ => None, + } + } + + fn as_u32(self) -> u32 { + match self { + SysEventKind::Exec => 1, + SysEventKind::Fork => 2, + SysEventKind::Exit => 3, + SysEventKind::MapChange => 4, + } + } +} + +/// Internal sysmon event selector. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SysmonEventMask { + pub exec: bool, + pub fork: bool, + pub exit: bool, + pub map_change: bool, +} + +impl SysmonEventMask { + pub fn target_mode() -> Self { + Self { + exec: true, + fork: true, + exit: true, + map_change: false, + } + } + + pub fn target_mode_with_map_changes() -> Self { + Self { + exec: true, + fork: true, + exit: true, + map_change: true, + } + } + + pub fn pid_module_changes() -> Self { + Self { + exec: false, + fork: false, + exit: false, + map_change: true, + } + } + + fn without_map_change(self) -> Self { + Self { + map_change: false, + ..self + } + } + + fn has_lifecycle_events(self) -> bool { + self.exec || self.fork || self.exit + } + + #[cfg(feature = "sysmon-ebpf")] + fn bits(self) -> u32 { + let mut bits = 0u32; + if self.exec { + bits |= SYSMON_EVENT_MASK_EXEC; + } + if self.fork { + bits |= SYSMON_EVENT_MASK_FORK; + } + if self.exit { + bits |= SYSMON_EVENT_MASK_EXIT; + } + if self.map_change { + bits |= SYSMON_EVENT_MASK_MAP_CHANGE; + } + bits + } +} + +impl Default for SysmonEventMask { + fn default() -> Self { + Self::target_mode() + } +} + +/// Raw SysEvent ABI — must match eBPF side exactly +/// ABI note: This layout is mirrored in eBPF at +/// `ghostscope-process/ebpf/sysmon-bpf/src/lib.rs`. We intentionally keep +/// two copies for now to avoid entangling the BPF build with the workspace. +/// Keep repr(C), field order and sizes identical on both sides. Current +/// layout (12 bytes): { tgid: u32, host_tgid: u32, kind: u32 }. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct SysEvent { + /// Runtime TGID in the configured sysmon event namespace when available. + pub tgid: u32, + /// Host/initial-namespace TGID from bpf_get_current_pid_tgid(). + pub host_tgid: u32, + pub kind: u32, // 1=exec,2=fork,3=exit,4=map-change +} + +impl SysEvent { + pub fn event_kind(self) -> Option { + SysEventKind::from_u32(self.kind) + } +} + +const PENDING_POLL_INTERVAL: Duration = Duration::from_millis(150); +const PENDING_MAX_ATTEMPTS: u32 = 20; +const MAP_CHANGE_DEBOUNCE_INTERVAL: Duration = Duration::from_millis(75); +const MODULE_REFRESH_INTERVAL: Duration = Duration::from_millis(250); +const SYSMON_EVENT_QUEUE_CAPACITY: usize = 1024; + +#[cfg(feature = "sysmon-ebpf")] +const SYSMON_EVENT_MASK_EXEC: u32 = 1 << 0; +#[cfg(feature = "sysmon-ebpf")] +const SYSMON_EVENT_MASK_FORK: u32 = 1 << 1; +#[cfg(feature = "sysmon-ebpf")] +const SYSMON_EVENT_MASK_EXIT: u32 = 1 << 2; +#[cfg(feature = "sysmon-ebpf")] +const SYSMON_EVENT_MASK_MAP_CHANGE: u32 = 1 << 3; + +#[cfg(feature = "sysmon-ebpf")] +mod attach; +mod events; +mod offset_refresh; +mod pending; +mod pid_alias; +#[cfg(feature = "sysmon-ebpf")] +mod runtime_loop; + +use events::sys_event_host_pid; +use offset_refresh::{ + get_comm_from_proc, pid_alive, pid_maps_target_module, prefill_offsets_for_pid, + truncate_basename_to_comm, +}; +use pending::{PendingMapRefreshes, PendingOffsets}; +use pid_alias::{purge_runtime_pid_artifacts, record_runtime_pid_aliases_for_sys_event}; +#[cfg(feature = "sysmon-ebpf")] +use runtime_loop::run_sysmon_loop; + +#[cfg(test)] +use events::try_publish_sys_event; +#[cfg(test)] +use offset_refresh::poll_pending_offsets; +#[cfg(test)] +use pending::PendingOffsetsKind; + +/// Configuration for sysmon +#[derive(Debug, Clone)] +pub struct SysmonConfig { + /// If set, only attempt offsets prefill for events whose binary/module path matches this target. + pub target_module: Option, + /// Maximum number of entries for the pinned proc offsets map (used when ensuring existence). + pub proc_offsets_max_entries: u32, + /// PerfEventArray per-CPU buffer pages (used when ringbuf is unavailable). + pub perf_page_count: Option, + /// Internal event selector for the sysmon eBPF side. + pub event_mask: SysmonEventMask, + /// Whether map-change events should be emitted before the PID allowlist is populated. + pub map_change_unfiltered: bool, + /// Optional event PID to watch. `None` means system-wide. + pub watched_pid: Option, + /// Optional PID namespace for interpreting `watched_pid`. + pub watched_pid_ns: Option, + /// Optional PID namespace for reporting target-mode event TGIDs. + pub event_pid_ns: Option, + /// Optional `/proc` PID corresponding to `watched_pid`. + pub watched_proc_pid: Option, +} + +impl SysmonConfig { + pub fn new() -> Self { + Self { + target_module: None, + proc_offsets_max_entries: 4096, + perf_page_count: None, + event_mask: SysmonEventMask::target_mode(), + map_change_unfiltered: false, + watched_pid: None, + watched_pid_ns: None, + event_pid_ns: None, + watched_proc_pid: None, + } + } +} + +impl Default for SysmonConfig { + fn default() -> Self { + Self::new() + } +} + +/// Process sysmon — userspace controller that listens for process lifecycle events and +/// performs incremental prefill/cleanup of offsets. +/// +/// Note: The low-level event source (tracepoints via eBPF or kernel proc connector) is pluggable. +/// This initial implementation provides the public API and a background loop stub; the event source +/// integration will be wired subsequently. +pub struct ProcessSysmon { + cfg: SysmonConfig, + mgr: Arc>, // shared manager to compute/prefill offsets + tx: mpsc::SyncSender, + rx: mpsc::Receiver, + pending_offsets: Arc>, + pending_map_refreshes: Arc>, + handle: Option>, +} + +impl core::fmt::Debug for ProcessSysmon { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("ProcessSysmon{..}") + } +} + +impl ProcessSysmon { + /// Create a new sysmon instance with shared ProcessManager and config. + pub fn new(mgr: Arc>, cfg: SysmonConfig) -> Self { + let (tx, rx) = mpsc::sync_channel(SYSMON_EVENT_QUEUE_CAPACITY); + Self { + cfg, + mgr, + tx, + rx, + pending_offsets: Arc::new(Mutex::new(PendingOffsets::new())), + pending_map_refreshes: Arc::new(Mutex::new(PendingMapRefreshes::new())), + handle: None, + } + } + + /// Start background monitoring thread and return immediately. + /// + /// In the next iteration we will attach eBPF tracepoints (sched_process_exec/exit/fork) + /// and stream events into this channel. For now, we ensure the pinned offsets map exists + /// and keep a placeholder loop that can be extended to consume a real source. + pub fn start(&mut self) { + let _ = + pinned_bpf_maps::ensure_pinned_proc_offsets_exists(self.cfg.proc_offsets_max_entries); + let _ = + pinned_bpf_maps::ensure_pinned_pid_aliases_exists(self.cfg.proc_offsets_max_entries); + let _ = pinned_bpf_maps::ensure_pinned_proc_module_ranges_exist( + self.cfg.proc_offsets_max_entries, + ); + let _ = pinned_bpf_maps::ensure_pinned_allowed_pids_exists(16_384); + + let tx = self.tx.clone(); + let mgr = Arc::clone(&self.mgr); + let pending = Arc::clone(&self.pending_offsets); + let pending_map_refreshes = Arc::clone(&self.pending_map_refreshes); + let cfg = self.cfg.clone(); + + let handle = thread::Builder::new() + .name("gs-sysmon".to_string()) + .spawn(move || { + info!("ProcessSysmon thread started"); + #[cfg(feature = "sysmon-ebpf")] + { + if let Err(e) = run_sysmon_loop(mgr, cfg, pending, pending_map_refreshes, tx) { + error!("Sysmon loop error: {}", e); + } + } + #[cfg(not(feature = "sysmon-ebpf"))] + { + let _ = pending; + let _ = pending_map_refreshes; + let _ = cfg; + warn!("sysmon-ebpf feature is disabled; sysmon is in stub mode"); + loop { + std::thread::sleep(std::time::Duration::from_millis(5000)); + } + } + info!("ProcessSysmon thread exiting"); + }); + match handle { + Ok(h) => self.handle = Some(h), + Err(e) => { + error!("Failed to spawn ProcessSysmon thread: {}", e); + self.handle = None; + } + } + } + + /// Blocking poll (with timeout) for the next system event. + pub fn recv_timeout(&self, timeout: std::time::Duration) -> Option { + match self.rx.recv_timeout(timeout) { + Ok(ev) => Some(ev), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => None, + } + } + + /// Handle one system event: prefill on Exec/Fork, cleanup on Exit. + fn handle_event_with_proc_pid_resolver( + mgr: &Arc>, + target: &Option, + pending: &Arc>, + ev: &SysEvent, + proc_pid_for_event: impl Fn(u32) -> u32, + ) -> anyhow::Result<()> { + let kind = match SysEventKind::from_u32(ev.kind) { + Some(k) => k, + None => { + tracing::warn!( + "Sysmon: invalid event kind {} for pid {}; ignoring", + ev.kind, + ev.tgid + ); + return Ok(()); + } + }; + tracing::trace!("Sysmon event: kind={:?} event_pid={}", kind, ev.tgid); + match kind { + SysEventKind::Exec | SysEventKind::Fork => { + let proc_pid = proc_pid_for_event(ev.tgid); + record_runtime_pid_aliases_for_sys_event(mgr, ev, proc_pid); + if let Some(tpath) = target { + let path = tpath.as_path(); + if crate::util::is_shared_object(path) { + if kind == SysEventKind::Exec && !pid_maps_target_module(proc_pid, path) { + if pid_alive(proc_pid) { + tracing::debug!( + "Sysmon: event pid {} (proc pid {}) does not map target module yet; scheduling retry", + ev.tgid, + proc_pid + ); + if let Ok(mut guard) = pending.lock() { + guard.register_map_change_candidate(ev.tgid, path); + } + } else { + let host_pid = sys_event_host_pid(ev); + if host_pid != ev.tgid && pid_alive(host_pid) { + tracing::debug!( + "Sysmon: event pid {} is not visible in current /proc namespace; scheduling host pid {} retry", + ev.tgid, + host_pid + ); + if let Ok(mut guard) = pending.lock() { + guard.register_map_change_candidate(host_pid, path); + } + } else { + tracing::debug!( + "Sysmon: event pid {} (host pid {}) is not visible in current /proc namespace; skip exec-based target retry", + ev.tgid, + host_pid + ); + } + } + return Ok(()); + } else if let Ok(mut guard) = pending.lock() { + guard.remove(ev.tgid); + } + } else if kind == SysEventKind::Exec { + if let Some(actual) = get_comm_from_proc(proc_pid) { + let expected = truncate_basename_to_comm(path); + if actual.as_bytes() != expected.as_slice() { + tracing::warn!( + "Sysmon: comm mismatch for event pid {} (proc pid {}) (actual='{}', expected='{}'); skip prefill/insert", + ev.tgid, + proc_pid, + actual, + core::str::from_utf8(&expected).unwrap_or("") + ); + return Ok(()); + } + } + } + } + let inserted = + prefill_offsets_for_pid(mgr, ev.tgid, target.as_deref(), &proc_pid_for_event)?; + if inserted { + let host_pid = sys_event_host_pid(ev); + if host_pid != ev.tgid { + let _ = crate::pinned_bpf_maps::insert_allowed_pid(host_pid); + } + } + if kind == SysEventKind::Exec { + if let Some(tpath) = target { + if inserted { + if let Ok(mut guard) = pending.lock() { + guard.remove(ev.tgid); + } + } else if let Ok(mut guard) = pending.lock() { + tracing::debug!( + "Sysmon: event pid {} (proc pid {}) prefill inserted no matching offsets; scheduling retry", + ev.tgid, + proc_pid + ); + guard.register(ev.tgid, tpath.as_path()); + } + } + } + } + SysEventKind::Exit => { + let proc_pid = proc_pid_for_event(ev.tgid); + let host_pid = sys_event_host_pid(ev); + if let Ok(mut guard) = pending.lock() { + guard.remove(ev.tgid); + if host_pid != ev.tgid { + guard.remove(host_pid); + } + } + if let Ok(mut guard) = mgr.lock() { + guard.forget_pid(proc_pid); + if proc_pid != ev.tgid { + guard.forget_pid(ev.tgid); + } + if host_pid != ev.tgid && host_pid != proc_pid { + guard.forget_pid(host_pid); + } + } + let purged = purge_runtime_pid_artifacts(proc_pid, ev.tgid, host_pid); + info!( + "Sysmon: observed exit for event pid {} (host pid {}, proc pid {}) (purged {} entries)", + ev.tgid, + host_pid, + proc_pid, + purged + ); + } + SysEventKind::MapChange => { + tracing::trace!( + "Sysmon: map-change event for pid {} is handled by the debounce queue", + ev.tgid + ); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sysmon_target_mode_does_not_enable_map_change_events() { + let mask = SysmonEventMask::target_mode(); + assert!(mask.exec); + assert!(mask.fork); + assert!(mask.exit); + assert!(!mask.map_change); + } + + #[test] + fn sysmon_pid_module_changes_only_enable_map_change_events() { + let mask = SysmonEventMask::pid_module_changes(); + assert!(!mask.exec); + assert!(!mask.fork); + assert!(!mask.exit); + assert!(mask.map_change); + } + + #[test] + fn sysmon_event_publish_drops_when_queue_is_full() { + let (tx, rx) = mpsc::sync_channel(1); + + assert!(try_publish_sys_event( + &tx, + SysEvent { + tgid: 1, + host_tgid: 1, + kind: 1 + } + )); + assert!(!try_publish_sys_event( + &tx, + SysEvent { + tgid: 2, + host_tgid: 2, + kind: 2 + } + )); + + let queued = rx.try_recv().expect("first event should be queued"); + assert_eq!(queued.tgid, 1); + assert_eq!(queued.kind, 1); + assert!(matches!(rx.try_recv(), Err(mpsc::TryRecvError::Empty))); + } + + #[test] + fn invisible_shared_object_exec_does_not_queue_unresolved_alias() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let target_path = dir.path().join("libtarget.so"); + let mut elf = [0u8; 64]; + elf[0..4].copy_from_slice(b"\x7FELF"); + elf[4] = 2; + elf[5] = 1; + elf[16..18].copy_from_slice(&3u16.to_le_bytes()); + std::fs::write(&target_path, elf)?; + + let mgr = Arc::new(Mutex::new(ProcessManager::new())); + let pending = Arc::new(Mutex::new(PendingOffsets::new())); + let ev = SysEvent { + tgid: u32::MAX - 1, + host_tgid: u32::MAX - 2, + kind: SysEventKind::Exec.as_u32(), + }; + + ProcessSysmon::handle_event_with_proc_pid_resolver( + &mgr, + &Some(target_path), + &pending, + &ev, + |pid| pid, + )?; + + assert!( + pending + .lock() + .expect("pending offsets lock") + .entries + .is_empty(), + "unresolved exec events must not be deferred without target-map proof" + ); + Ok(()) + } + + #[test] + fn visible_shared_object_exec_queues_map_change_candidate() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let target_path = dir.path().join("libtarget.so"); + let mut elf = [0u8; 64]; + elf[0..4].copy_from_slice(b"\x7FELF"); + elf[4] = 2; + elf[5] = 1; + elf[16..18].copy_from_slice(&3u16.to_le_bytes()); + std::fs::write(&target_path, elf)?; + + let event_pid = std::process::id(); + let mgr = Arc::new(Mutex::new(ProcessManager::new())); + let pending = Arc::new(Mutex::new(PendingOffsets::new())); + let ev = SysEvent { + tgid: event_pid, + host_tgid: event_pid, + kind: SysEventKind::Exec.as_u32(), + }; + + ProcessSysmon::handle_event_with_proc_pid_resolver( + &mgr, + &Some(target_path.clone()), + &pending, + &ev, + |pid| pid, + )?; + + let guard = pending.lock().expect("pending offsets lock"); + let entry = guard + .entries + .get(&event_pid) + .expect("visible shared-object exec should be queued for retry"); + assert_eq!(entry.target_path, target_path); + assert_eq!(entry.kind, PendingOffsetsKind::MapChangeCandidate); + Ok(()) + } + + #[test] + fn invisible_shared_object_exec_queues_visible_host_pid_retry() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let target_path = dir.path().join("libtarget.so"); + let mut elf = [0u8; 64]; + elf[0..4].copy_from_slice(b"\x7FELF"); + elf[4] = 2; + elf[5] = 1; + elf[16..18].copy_from_slice(&3u16.to_le_bytes()); + std::fs::write(&target_path, elf)?; + + let host_pid = std::process::id(); + let mgr = Arc::new(Mutex::new(ProcessManager::new())); + let pending = Arc::new(Mutex::new(PendingOffsets::new())); + let ev = SysEvent { + tgid: u32::MAX - 1, + host_tgid: host_pid, + kind: SysEventKind::Exec.as_u32(), + }; + + ProcessSysmon::handle_event_with_proc_pid_resolver( + &mgr, + &Some(target_path.clone()), + &pending, + &ev, + |pid| pid, + )?; + + let guard = pending.lock().expect("pending offsets lock"); + let entry = guard + .entries + .get(&host_pid) + .expect("visible host PID should be queued for maps-based retry"); + assert_eq!(entry.target_path, target_path); + assert_eq!(entry.kind, PendingOffsetsKind::MapChangeCandidate); + Ok(()) + } + + #[test] + fn shared_object_candidate_survives_retry_exhaustion() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let target_path = dir.path().join("libtarget.so"); + std::fs::write(&target_path, b"not actually mapped")?; + + let event_pid = std::process::id(); + let pending = Arc::new(Mutex::new(PendingOffsets::new())); + { + let mut guard = pending.lock().expect("pending offsets lock"); + guard.register_map_change_candidate(event_pid, &target_path); + let entry = guard + .entries + .get_mut(&event_pid) + .expect("pending candidate"); + entry.attempts = PENDING_MAX_ATTEMPTS - 1; + entry.last_poll = Instant::now() - PENDING_POLL_INTERVAL; + } + + let mgr = Arc::new(Mutex::new(ProcessManager::new())); + poll_pending_offsets(&mgr, &pending, &|pid| pid); + + let guard = pending.lock().expect("pending offsets lock"); + let entry = guard + .entries + .get(&event_pid) + .expect("map-change candidate should remain after retry exhaustion"); + assert!(entry.retry_exhausted); + assert!(guard.contains_map_change_candidate(event_pid, &target_path)); + Ok(()) + } +} diff --git a/ghostscope-process/src/sysmon/offset_refresh.rs b/ghostscope-process/src/sysmon/offset_refresh.rs new file mode 100644 index 00000000..39ba5cd0 --- /dev/null +++ b/ghostscope-process/src/sysmon/offset_refresh.rs @@ -0,0 +1,951 @@ +use super::events::*; +use super::pending::*; +use super::pid_alias::*; +use super::*; + +/* moved to ghostscope_process::util::is_shared_object +pub(super) fn looks_like_shared_object(path: &Path) -> bool { + // Determine shared object by ELF metadata: + // - ET_EXEC => executable (not shared) + // - ET_DYN + PT_INTERP present => PIE executable (not shared) + // - ET_DYN without PT_INTERP => shared library + use std::fs::File; + use std::io::{Read, Seek, SeekFrom}; + + const EI_CLASS: usize = 4; // 1=32-bit, 2=64-bit + const EI_DATA: usize = 5; // 1=little, 2=big + const ET_EXEC: u16 = 2; + const ET_DYN: u16 = 3; + const PT_INTERP: u32 = 3; + + let mut f = match File::open(path) { + Ok(f) => f, + Err(_) => return false, // conservative: treat as executable (enable filtering) + }; + let mut ehdr = [0u8; 64]; + if f.read(&mut ehdr).ok().filter(|&n| n >= 52).is_none() { + return false; + } + // ELF magic + if &ehdr[0..4] != b"\x7FELF" { + return false; + } + let class = ehdr[EI_CLASS]; + let data = ehdr[EI_DATA]; + let is_le = data == 1; + // read u16/u32/u64 helpers + let rd16 = |b: &[u8]| -> u16 { + if is_le { + u16::from_le_bytes([b[0], b[1]]) + } else { + u16::from_be_bytes([b[0], b[1]]) + } + }; + let rd32 = |b: &[u8]| -> u32 { + if is_le { + u32::from_le_bytes([b[0], b[1], b[2], b[3]]) + } else { + u32::from_be_bytes([b[0], b[1], b[2], b[3]]) + } + }; + let rd64 = |b: &[u8]| -> u64 { + if is_le { + u64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]) + } else { + u64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]) + } + }; + + // e_type at 0x10 + let e_type = rd16(&ehdr[16..18]); + if e_type == ET_EXEC { + return false; // executable + } + + // program header table offsets + let (e_phoff, e_phentsize, e_phnum) = match class { + 1 => { + // ELF32: e_phoff @0x1C (4), e_phentsize @0x2A (2), e_phnum @0x2C (2) + let phoff = rd32(&ehdr[28..32]) as u64; + let entsz = rd16(&ehdr[42..44]) as u64; + let phnum = rd16(&ehdr[44..46]) as u64; + (phoff, entsz, phnum) + } + 2 => { + // ELF64: e_phoff @0x20 (8), e_phentsize @0x36 (2), e_phnum @0x38 (2) + let phoff = rd64(&ehdr[32..40]); + let entsz = rd16(&ehdr[54..56]) as u64; + let phnum = rd16(&ehdr[56..58]) as u64; + (phoff, entsz, phnum) + } + _ => return false, + }; + + if e_type == ET_DYN { + // Scan program headers for PT_INTERP + if e_phoff == 0 || e_phentsize < 4 || e_phnum == 0 { + // malformed + // If cannot inspect, be conservative and treat as shared library (disable filtering) + return true; + } + // Seek and read each p_type + for i in 0..e_phnum { + let off = e_phoff + i * e_phentsize; + if f.seek(SeekFrom::Start(off)).is_err() { + return true; + } + let mut p = [0u8; 8]; + if f.read(&mut p[..4]).ok().filter(|&n| n == 4).is_none() { + return true; + } + let p_type = rd32(&p[..4]); + if p_type == PT_INTERP { + return false; // PIE executable + } + } + return true; // ET_DYN w/o PT_INTERP => shared library + } + + // Unknown types: default to 'not shared' (enable filtering) + false +} +*/ + +pub(super) fn pid_alive(pid: u32) -> bool { + std::path::Path::new(&format!("/proc/{pid}")).exists() +} + +pub(super) fn filter_entries_for_target<'a>( + entries: &'a [PidOffsetsEntry], + target: Option<&Path>, +) -> Vec<&'a PidOffsetsEntry> { + use std::fs; + use std::os::unix::fs::MetadataExt; + + if let Some(tpath) = target { + match fs::metadata(tpath) { + Ok(meta) => { + let t_dev = meta.dev(); + let t_ino = meta.ino(); + entries + .iter() + .filter(|e| { + fs::metadata(&e.module_path) + .map(|m| m.dev() == t_dev && m.ino() == t_ino) + .unwrap_or(false) + }) + .collect() + } + Err(_) => { + let tc = cookie_for_path(&tpath.to_string_lossy()); + let by_cookie: Vec<_> = entries.iter().filter(|e| e.cookie == tc).collect(); + if !by_cookie.is_empty() { + by_cookie + } else { + let tnorm = tpath.to_string_lossy().replace("/./", "/"); + entries.iter().filter(|e| e.module_path == tnorm).collect() + } + } + } + } else { + entries.iter().collect() + } +} + +pub(super) fn prefill_offsets_for_pid( + mgr: &Arc>, + event_pid: u32, + target: Option<&Path>, + proc_pid_for_event: &impl Fn(u32) -> u32, +) -> anyhow::Result { + write_offsets_for_pid(mgr, event_pid, target, false, &[], proc_pid_for_event) +} + +pub(super) fn refresh_offsets_for_known_proc_pid( + mgr: &Arc>, + event_pid: u32, + host_pid: u32, + proc_pid: u32, +) -> anyhow::Result { + let proc_pid_for_event = |_: u32| proc_pid; + write_offsets_for_pid(mgr, event_pid, None, true, &[host_pid], &proc_pid_for_event) +} + +pub(super) fn write_offsets_for_pid( + mgr: &Arc>, + event_pid: u32, + target: Option<&Path>, + force_refresh: bool, + extra_runtime_pids: &[u32], + proc_pid_for_event: &impl Fn(u32) -> u32, +) -> anyhow::Result { + let proc_pid = proc_pid_for_event(event_pid); + let runtime_pids = + runtime_pid_keys_for_proc_event(proc_pid, event_pid, extra_runtime_pids.iter().copied()); + record_runtime_pid_aliases_for_keys(mgr, proc_pid, &runtime_pids); + let mut inserted_any = false; + if let Ok(mut guard) = mgr.lock() { + if target.is_some() && is_current_process_pid(proc_pid) { + tracing::debug!( + "Sysmon: skipping self proc pid {} for target-module offset prefill", + proc_pid + ); + return Ok(false); + } + let prefilled = match if force_refresh { + guard.refresh_prefill_pid(proc_pid) + } else { + guard.ensure_prefill_pid(proc_pid) + } { + Ok(v) => v, + Err(e) => { + // In private PID namespaces, sysmon event PID may be in the initial namespace + // and not resolvable via /proc/. Fall back to module-wide refresh. + if let Some(target_path) = target { + let module_path = target_path.to_string_lossy().to_string(); + tracing::debug!( + "Sysmon: pid prefill failed for event pid {} (proc pid {}): {}; falling back to module refresh for {}", + event_pid, + proc_pid, + e, + module_path + ); + let refreshed = guard.refresh_prefill_module(&module_path)?; + if refreshed > 0 { + tracing::info!( + "Sysmon: module refresh cached {} pid(s) for {}", + refreshed, + module_path + ); + } + let mut target_pids = BTreeSet::new(); + for (pid, _, _, _, _) in guard.cached_offsets_for_module(&module_path) { + target_pids.insert(pid); + } + drop(guard); + + for pid in target_pids { + let runtime_pid = resolve_event_pid_for_proc(pid); + match refresh_full_offsets_for_pid(mgr, pid, runtime_pid) { + Ok(true) => inserted_any = true, + Ok(false) => {} + Err(err) => tracing::warn!( + "Sysmon: module refresh failed to publish full snapshot for proc pid {}: {}", + pid, + err + ), + } + } + return Ok(inserted_any); + } + return Err(e); + } + }; + if prefilled > 0 { + info!( + "Sysmon: {} {} entries for event pid {} (proc pid {})", + if force_refresh { + "refreshed" + } else { + "prefilled" + }, + prefilled, + event_pid, + proc_pid + ); + } + let mut entries = guard + .cached_offsets_with_paths_for_pid(proc_pid) + .map(|entries| entries.to_vec()) + .unwrap_or_default(); + let mut target_match_count = filter_entries_for_target(&entries, target).len(); + + if target_match_count == 0 && target.is_some() { + let refreshed = guard.refresh_prefill_pid(proc_pid)?; + if refreshed > 0 { + tracing::debug!( + "Sysmon: refreshed {} cached entries for event pid {} (proc pid {})", + refreshed, + event_pid, + proc_pid + ); + } + entries = guard + .cached_offsets_with_paths_for_pid(proc_pid) + .map(|entries| entries.to_vec()) + .unwrap_or_default(); + target_match_count = filter_entries_for_target(&entries, target).len(); + } + + if force_refresh && target.is_none() { + let purged = purge_offsets_for_runtime_pid_keys(&runtime_pids)?; + if purged > 0 { + tracing::debug!( + "Sysmon: purged {} stale offset entries before map-change refresh for event pid {} (proc pid {}, runtime keys={:?})", + purged, + event_pid, + proc_pid, + runtime_pids + ); + } + } + + if !entries.is_empty() && (target.is_none() || target_match_count > 0) { + let items = offset_items_from_entries(entries.iter()); + match publish_offsets_for_runtime_pid_keys( + proc_pid, + event_pid, + &runtime_pids, + &items, + "prefill", + ) { + Ok(inserted) => { + if inserted == 0 { + tracing::warn!( + "Sysmon: no offsets inserted for event pid {} (proc pid {}, runtime keys={:?}) (entry count={})", + event_pid, + proc_pid, + runtime_pids, + items.len() + ); + } else { + tracing::info!( + "Sysmon: inserted {} offset entries for event pid {} (proc pid {}, runtime keys={:?})", + inserted, + event_pid, + proc_pid, + runtime_pids + ); + insert_allowed_runtime_pid_keys(&runtime_pids); + inserted_any = true; + } + } + Err(e) => { + tracing::warn!( + "Sysmon: failed to insert offsets for event pid {} (proc pid {}): {}", + event_pid, + proc_pid, + e + ); + } + } + } else if target.is_some() { + tracing::debug!( + "Sysmon: event pid {} (proc pid {}) does not map target module; skip", + event_pid, + proc_pid + ); + } + } + Ok(inserted_any) +} + +pub(super) fn prefill_full_offsets_for_pid_if_new( + mgr: &Arc>, + event_pid: u32, + proc_pid_for_event: &impl Fn(u32) -> u32, +) -> anyhow::Result { + let proc_pid = proc_pid_for_event(event_pid); + let runtime_pids = runtime_pid_keys_for_proc_event(proc_pid, event_pid, []); + record_runtime_pid_aliases_for_keys(mgr, proc_pid, &runtime_pids); + + let items = { + let Ok(mut guard) = mgr.lock() else { + return Ok(false); + }; + let prefilled = guard.ensure_prefill_pid(proc_pid)?; + if prefilled == 0 { + return Ok(false); + } + let Some(entries) = guard.cached_offsets_with_paths_for_pid(proc_pid) else { + return Ok(false); + }; + offset_items_from_entries(entries.iter()) + }; + + if items.is_empty() { + return Ok(false); + } + + match publish_offsets_for_runtime_pid_keys( + proc_pid, + event_pid, + &runtime_pids, + &items, + "full prefill", + ) { + Ok(inserted) if inserted > 0 => { + tracing::info!( + "Sysmon: inserted {} full offset entries for event pid {} (proc pid {}, runtime keys={:?})", + inserted, + event_pid, + proc_pid, + runtime_pids + ); + insert_allowed_runtime_pid_keys(&runtime_pids); + Ok(true) + } + Ok(_) => Ok(false), + Err(e) => { + tracing::warn!( + "Sysmon: failed to insert full offsets for event pid {} (proc pid {}): {}", + event_pid, + proc_pid, + e + ); + Ok(false) + } + } +} + +pub(super) type PidMapsSignature = Vec<(String, u64, u64, u64, u64, u64, bool)>; + +pub(super) fn pid_maps_signature(pid: u32) -> anyhow::Result { + let mut signature = read_proc_maps(pid)? + .into_iter() + .filter_map(|entry| { + let path = entry.path()?; + if should_skip_mapped_module_path(path) { + return None; + } + Some(( + normalize_mapped_module_path(path).to_string(), + entry.start, + entry.end, + entry.offset, + entry.inode, + (entry.dev_major << 32) | entry.dev_minor, + entry.executable(), + )) + }) + .collect::>(); + signature.sort_unstable(); + Ok(signature) +} + +pub(super) fn refresh_full_offsets_for_pid( + mgr: &Arc>, + proc_pid: u32, + event_pid: u32, +) -> anyhow::Result { + let runtime_pids = runtime_pid_keys_for_proc_event(proc_pid, event_pid, []); + record_runtime_pid_aliases_for_keys(mgr, proc_pid, &runtime_pids); + + let items = { + let Ok(mut guard) = mgr.lock() else { + return Ok(false); + }; + guard.refresh_prefill_pid(proc_pid)?; + let Some(entries) = guard.cached_offsets_with_paths_for_pid(proc_pid) else { + return Ok(false); + }; + offset_items_from_entries(entries.iter()) + }; + + if items.is_empty() { + return Ok(false); + } + + let purged = purge_offsets_for_runtime_pid_keys(&runtime_pids)?; + if purged > 0 { + tracing::debug!( + "Sysmon: purged {} stale offset entries before periodic full refresh for proc pid {} (runtime keys={:?})", + purged, + proc_pid, + runtime_pids + ); + } + let inserted = publish_offsets_for_runtime_pid_keys( + proc_pid, + event_pid, + &runtime_pids, + &items, + "periodic full refresh", + )?; + insert_allowed_runtime_pid_keys(&runtime_pids); + tracing::debug!( + "Sysmon: periodic full refresh wrote {} offset entries for proc pid {} (event pid {}, runtime keys={:?})", + inserted, + proc_pid, + event_pid, + runtime_pids + ); + Ok(inserted > 0) +} + +pub(super) fn offset_items_from_entries<'a>( + entries: impl IntoIterator, +) -> Vec<(u64, crate::pinned_bpf_maps::ProcModuleOffsetsValue)> { + entries + .into_iter() + .map(|e| { + ( + e.cookie, + crate::pinned_bpf_maps::ProcModuleOffsetsValue::new( + e.offsets.text, + e.offsets.rodata, + e.offsets.data, + e.offsets.bss, + e.base, + e.size, + ), + ) + }) + .collect() +} + +pub(super) fn refresh_target_module_offsets( + mgr: &Arc>, + target: Option<&Path>, + last_refresh: &mut Instant, + target_pid_map_signatures: &mut HashMap, + tx: &mpsc::SyncSender, +) { + use crate::pinned_bpf_maps::{allowed_pid_exists, ProcModuleOffsetsValue}; + + let Some(target_path) = target else { + return; + }; + let now = Instant::now(); + if now.duration_since(*last_refresh) < MODULE_REFRESH_INTERVAL { + return; + } + *last_refresh = now; + + let module_path = target_path.to_string_lossy().to_string(); + let mut by_pid: HashMap> = HashMap::new(); + let mut target_pids: BTreeSet = BTreeSet::new(); + if let Ok(mut guard) = mgr.lock() { + if let Err(e) = guard.refresh_prefill_module(&module_path) { + tracing::debug!( + "Sysmon: periodic module refresh failed for {}: {}", + module_path, + e + ); + return; + } + for (pid, cookie, off, base, size) in guard.cached_offsets_for_module(&module_path) { + if is_current_process_pid(pid) { + continue; + } + target_pids.insert(pid); + by_pid.entry(pid).or_default().push(( + cookie, + ProcModuleOffsetsValue::new(off.text, off.rodata, off.data, off.bss, base, size), + )); + } + } + if by_pid.is_empty() { + return; + } + + let mut total = 0usize; + let mut newly_allowed_event_pids = BTreeSet::new(); + for (pid, items) in by_pid { + let event_pid = resolve_event_pid_for_proc(pid); + let runtime_pids = runtime_pid_keys_for_proc_event(pid, event_pid, []); + record_runtime_pid_aliases_for_keys(mgr, pid, &runtime_pids); + let was_allowed = match allowed_pid_exists(event_pid) { + Ok(value) => value, + Err(e) => { + tracing::debug!( + "Sysmon: allowed_pids lookup failed for event pid {} (proc pid {}): {}", + event_pid, + pid, + e + ); + false + } + }; + match publish_offsets_for_runtime_pid_keys( + pid, + event_pid, + &runtime_pids, + &items, + "periodic module refresh", + ) { + Ok(inserted) => { + if inserted > 0 { + total += inserted; + insert_allowed_runtime_pid_keys(&runtime_pids); + if !was_allowed { + tracing::debug!( + "Sysmon: event pid {} became allowed during periodic module refresh", + event_pid + ); + newly_allowed_event_pids.insert(event_pid); + } + } + } + Err(e) => tracing::debug!( + "Sysmon: periodic module refresh insert failed for pid {} ({}): {}", + pid, + module_path, + e + ), + } + } + for pid in &target_pids { + let event_pid = resolve_event_pid_for_proc(*pid); + let maps_signature = match pid_maps_signature(*pid) { + Ok(signature) => signature, + Err(e) => { + tracing::debug!( + "Sysmon: periodic maps signature failed for proc pid {} (event pid {}): {}", + *pid, + event_pid, + e + ); + continue; + } + }; + if target_pid_map_signatures.get(pid) == Some(&maps_signature) { + continue; + } + target_pid_map_signatures.insert(*pid, maps_signature); + + match refresh_full_offsets_for_pid(mgr, *pid, event_pid) { + Ok(true) => { + newly_allowed_event_pids.insert(event_pid); + } + Ok(false) => {} + Err(e) => { + tracing::debug!( + "Sysmon: periodic full offset refresh failed for proc pid {} (event pid {}): {}", + *pid, + event_pid, + e + ); + } + } + } + target_pid_map_signatures.retain(|pid, _| target_pids.contains(pid)); + for event_pid in newly_allowed_event_pids { + let ev = SysEvent { + tgid: event_pid, + host_tgid: event_pid, + kind: SysEventKind::MapChange.as_u32(), + }; + if try_publish_sys_event(tx, ev) { + tracing::debug!( + "Sysmon: published synthetic map-change for newly discovered target pid {}", + event_pid + ); + } + } + if total > 0 { + tracing::debug!( + "Sysmon: periodic module refresh inserted {} offset entries for {}", + total, + module_path + ); + } +} + +pub(super) fn poll_pending_offsets( + mgr: &Arc>, + pending: &Arc>, + proc_pid_for_event: &impl Fn(u32) -> u32, +) { + let due = if let Ok(mut guard) = pending.lock() { + guard.take_due() + } else { + Vec::new() + }; + + if due.is_empty() { + return; + } + + let mut to_remove: Vec = Vec::new(); + let mut to_exhaust: Vec = Vec::new(); + + for due in due { + let event_pid = due.event_pid; + let target_path = due.target_path; + let attempts = due.attempts; + let proc_pid = proc_pid_for_event(event_pid); + if !pid_alive(proc_pid) { + tracing::debug!( + "Sysmon: event pid {} (proc pid {}) exited while waiting for offsets; removing from retry queue", + event_pid, + proc_pid + ); + to_remove.push(event_pid); + continue; + } + + if !pid_maps_target_module(proc_pid, &target_path) { + if attempts >= PENDING_MAX_ATTEMPTS { + if due.kind.keep_for_map_changes_after_retry_exhaustion() { + tracing::debug!( + "Sysmon: event pid {} (proc pid {}) still missing module {} after {} retries; waiting for map-change trigger", + event_pid, + proc_pid, + target_path.display(), + attempts + ); + to_exhaust.push(event_pid); + } else { + tracing::warn!( + "Sysmon: event pid {} (proc pid {}) still missing module {} after {} retries; giving up", + event_pid, + proc_pid, + target_path.display(), + attempts + ); + to_remove.push(event_pid); + } + } + continue; + } + + match prefill_offsets_for_pid( + mgr, + event_pid, + Some(target_path.as_path()), + proc_pid_for_event, + ) { + Ok(true) => { + tracing::info!( + "Sysmon: deferred prefill succeeded for event pid {} (proc pid {}) (module {})", + event_pid, + proc_pid, + target_path.display() + ); + to_remove.push(event_pid); + } + Ok(false) => { + if attempts >= PENDING_MAX_ATTEMPTS { + if due.kind.keep_for_map_changes_after_retry_exhaustion() { + tracing::debug!( + "Sysmon: deferred prefill produced no entries for event pid {} (proc pid {}) after {} retries; waiting for map-change trigger", + event_pid, + proc_pid, + attempts + ); + to_exhaust.push(event_pid); + } else { + tracing::warn!( + "Sysmon: deferred prefill produced no entries for event pid {} (proc pid {}) after {} retries; giving up", + event_pid, + proc_pid, + attempts + ); + to_remove.push(event_pid); + } + } + } + Err(e) => { + tracing::warn!( + "Sysmon: deferred prefill failed for event pid {} (proc pid {}) (attempt {}): {}", + event_pid, + proc_pid, + attempts, + e + ); + if attempts >= PENDING_MAX_ATTEMPTS { + if due.kind.keep_for_map_changes_after_retry_exhaustion() { + to_exhaust.push(event_pid); + } else { + to_remove.push(event_pid); + } + } + } + } + } + + if !to_remove.is_empty() || !to_exhaust.is_empty() { + if let Ok(mut guard) = pending.lock() { + for pid in to_remove { + guard.remove(pid); + } + for pid in to_exhaust { + guard.mark_retry_exhausted(pid); + } + } + } +} + +pub(super) fn cached_offsets_exist_for_target_pid( + mgr: &Arc>, + target_path: &Path, + proc_pid: u32, +) -> bool { + let module_path = target_path.to_string_lossy().to_string(); + mgr.lock() + .ok() + .map(|guard| { + guard.cached_offsets_with_paths_for_pid(proc_pid).is_some() + || guard + .cached_offsets_for_module(&module_path) + .iter() + .any(|(pid, _, _, _, _)| *pid == proc_pid) + }) + .unwrap_or(false) +} + +pub(super) fn forget_pid_offsets_after_target_unmap( + mgr: &Arc>, + event_pid: u32, + host_pid: u32, + proc_pid: u32, +) { + if let Ok(mut guard) = mgr.lock() { + guard.forget_pid(proc_pid); + if proc_pid != event_pid { + guard.forget_pid(event_pid); + } + if host_pid != event_pid && host_pid != proc_pid { + guard.forget_pid(host_pid); + } + } + + let purged = purge_runtime_pid_artifacts(proc_pid, event_pid, host_pid); + if purged > 0 { + tracing::info!( + "Sysmon: target unmapped for event pid {} (host pid {}, proc pid {}); purged {} offset entries", + event_pid, + host_pid, + proc_pid, + purged + ); + } +} + +pub(super) fn poll_pending_map_refreshes( + mgr: &Arc>, + target: Option<&Path>, + pending_map_refreshes: &Arc>, + pending: &Arc>, + tx: &mpsc::SyncSender, +) { + let due = if let Ok(mut guard) = pending_map_refreshes.lock() { + guard.take_due() + } else { + Vec::new() + }; + + if due.is_empty() { + return; + } + + for event in due { + let event_pid = event.event_pid; + let host_pid = event.host_pid; + let proc_pid = target + .map(|target_path| { + canonicalize_cached_target_proc_pid(mgr, target_path, event.proc_pid) + }) + .unwrap_or(event.proc_pid); + if !pid_alive(proc_pid) { + tracing::trace!( + "Sysmon: event pid {} (proc pid {}) exited before map refresh", + event_pid, + proc_pid + ); + continue; + } + + if let Some(target_path) = target { + if !pid_maps_target_module(proc_pid, target_path) { + if cached_offsets_exist_for_target_pid(mgr, target_path, proc_pid) { + forget_pid_offsets_after_target_unmap(mgr, event_pid, host_pid, proc_pid); + let ev = SysEvent { + tgid: event_pid, + host_tgid: host_pid, + kind: SysEventKind::MapChange.as_u32(), + }; + try_publish_sys_event(tx, ev); + } + tracing::trace!( + "Sysmon: event pid {} (proc pid {}) map-change does not include target {}; skip", + event_pid, + proc_pid, + target_path.display() + ); + continue; + } + } + + match refresh_offsets_for_known_proc_pid(mgr, event_pid, host_pid, proc_pid) { + Ok(true) => { + tracing::debug!( + "Sysmon: refreshed offsets after map-change for event pid {} (host pid {}, proc pid {})", + event_pid, + host_pid, + proc_pid + ); + if let Ok(mut guard) = pending.lock() { + guard.remove(event_pid); + if host_pid != event_pid { + guard.remove(host_pid); + } + } + let ev = SysEvent { + tgid: event_pid, + host_tgid: host_pid, + kind: SysEventKind::MapChange.as_u32(), + }; + try_publish_sys_event(tx, ev); + } + Ok(false) => tracing::trace!( + "Sysmon: map-change refresh inserted no offsets for event pid {} (proc pid {})", + event_pid, + proc_pid + ), + Err(e) => tracing::debug!( + "Sysmon: map-change refresh failed for event pid {} (proc pid {}): {}", + event_pid, + proc_pid, + e + ), + } + } +} + +pub(super) fn get_comm_from_proc(pid: u32) -> Option { + use std::io::Read; + let path = format!("/proc/{pid}/comm"); + let mut f = std::fs::File::open(path).ok()?; + let mut s = String::new(); + f.read_to_string(&mut s).ok()?; + if s.ends_with('\n') { + s.pop(); + if s.ends_with('\r') { + s.pop(); + } + } + // Kernel task->comm is at most 15 bytes; /proc returns without NUL. We compare as-is. + Some(s) +} + +pub(super) fn truncate_basename_to_comm(path: &Path) -> Vec { + use std::ffi::OsStr; + let mut buf = Vec::with_capacity(16); + if let Some(name) = path.file_name().and_then(OsStr::to_str) { + let bytes = name.as_bytes(); + let n = core::cmp::min(bytes.len(), 15); + buf.extend_from_slice(&bytes[..n]); + } + buf +} + +pub(super) fn pid_maps_target_module(pid: u32, target: &Path) -> bool { + let target = ModuleIdentity::from_path(target); + let mut matched = false; + + if visit_proc_maps(pid, |entry| { + if target.matches(&entry) { + matched = true; + return ControlFlow::Break(()); + } + ControlFlow::Continue(()) + }) + .is_err() + { + return false; + } + + matched +} diff --git a/ghostscope-process/src/sysmon/pending.rs b/ghostscope-process/src/sysmon/pending.rs new file mode 100644 index 00000000..93132eff --- /dev/null +++ b/ghostscope-process/src/sysmon/pending.rs @@ -0,0 +1,178 @@ +use super::*; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PendingOffsetsKind { + Retry, + MapChangeCandidate, +} + +impl PendingOffsetsKind { + pub(super) fn keep_for_map_changes_after_retry_exhaustion(self) -> bool { + matches!(self, Self::MapChangeCandidate) + } +} + +#[derive(Debug, Clone)] +pub(super) struct PendingOffsetsEntry { + pub(super) target_path: PathBuf, + pub(super) attempts: u32, + pub(super) kind: PendingOffsetsKind, + pub(super) retry_exhausted: bool, + pub(super) last_poll: Instant, +} + +#[derive(Debug, Clone)] +pub(super) struct PendingOffsetsDue { + pub(super) event_pid: u32, + pub(super) target_path: PathBuf, + pub(super) attempts: u32, + pub(super) kind: PendingOffsetsKind, +} + +#[derive(Debug, Default)] +pub(super) struct PendingOffsets { + pub(super) entries: HashMap, +} + +impl PendingOffsets { + pub(super) fn new() -> Self { + Self { + entries: HashMap::new(), + } + } + + pub(super) fn register(&mut self, pid: u32, target: &Path) { + self.register_with_kind(pid, target, PendingOffsetsKind::Retry); + } + + pub(super) fn register_map_change_candidate(&mut self, pid: u32, target: &Path) { + self.register_with_kind(pid, target, PendingOffsetsKind::MapChangeCandidate); + } + + pub(super) fn register_with_kind(&mut self, pid: u32, target: &Path, kind: PendingOffsetsKind) { + let now = Instant::now(); + let last_poll = now.checked_sub(PENDING_POLL_INTERVAL).unwrap_or(now); + self.entries + .entry(pid) + .and_modify(|entry| { + entry.target_path = target.to_path_buf(); + entry.attempts = 0; + entry.kind = kind; + entry.retry_exhausted = false; + entry.last_poll = last_poll; + }) + .or_insert(PendingOffsetsEntry { + target_path: target.to_path_buf(), + attempts: 0, + kind, + retry_exhausted: false, + last_poll, + }); + } + + pub(super) fn remove(&mut self, pid: u32) { + self.entries.remove(&pid); + } + + pub(super) fn contains_map_change_candidate(&self, pid: u32, target: &Path) -> bool { + self.entries + .get(&pid) + .map(|entry| { + entry.kind == PendingOffsetsKind::MapChangeCandidate && entry.target_path == target + }) + .unwrap_or(false) + } + + pub(super) fn mark_retry_exhausted(&mut self, pid: u32) { + if let Some(entry) = self.entries.get_mut(&pid) { + entry.retry_exhausted = true; + } + } + + pub(super) fn take_due(&mut self) -> Vec { + let mut due = Vec::new(); + let now = Instant::now(); + for (&pid, entry) in self.entries.iter_mut() { + if entry.retry_exhausted { + continue; + } + if now.duration_since(entry.last_poll) >= PENDING_POLL_INTERVAL { + entry.last_poll = now; + entry.attempts = entry.attempts.saturating_add(1); + due.push(PendingOffsetsDue { + event_pid: pid, + target_path: entry.target_path.clone(), + attempts: entry.attempts, + kind: entry.kind, + }); + } + } + due + } +} + +#[derive(Debug, Clone)] +pub(super) struct PendingMapRefreshEntry { + pub(super) last_seen: Instant, + pub(super) event_pid: u32, + pub(super) host_pid: u32, +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct PendingMapRefreshDue { + pub(super) event_pid: u32, + pub(super) host_pid: u32, + pub(super) proc_pid: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct PendingMapChangeCandidate { + pub(super) event_pid: u32, + pub(super) host_pid: u32, + pub(super) proc_pid: u32, +} + +#[derive(Debug, Default)] +pub(super) struct PendingMapRefreshes { + pub(super) entries: HashMap, +} + +impl PendingMapRefreshes { + pub(super) fn new() -> Self { + Self { + entries: HashMap::new(), + } + } + + pub(super) fn register(&mut self, event_pid: u32, host_pid: u32, proc_pid: u32) { + self.entries.insert( + proc_pid, + PendingMapRefreshEntry { + last_seen: Instant::now(), + event_pid, + host_pid, + }, + ); + } + + pub(super) fn take_due(&mut self) -> Vec { + let now = Instant::now(); + let due: Vec = self + .entries + .iter() + .filter_map(|(&proc_pid, entry)| { + (now.duration_since(entry.last_seen) >= MAP_CHANGE_DEBOUNCE_INTERVAL).then_some( + PendingMapRefreshDue { + event_pid: entry.event_pid, + host_pid: entry.host_pid, + proc_pid, + }, + ) + }) + .collect(); + for entry in &due { + self.entries.remove(&entry.proc_pid); + } + due + } +} diff --git a/ghostscope-process/src/sysmon/pid_alias.rs b/ghostscope-process/src/sysmon/pid_alias.rs new file mode 100644 index 00000000..029d2725 --- /dev/null +++ b/ghostscope-process/src/sysmon/pid_alias.rs @@ -0,0 +1,187 @@ +use super::events::sys_event_host_pid; +use super::*; + +pub(super) fn write_pinned_runtime_pid_alias(runtime_pid: u32, proc_pid: u32) { + if runtime_pid == proc_pid { + return; + } + match crate::pinned_bpf_maps::insert_pid_alias(runtime_pid, proc_pid) { + Ok(()) => tracing::trace!( + "Sysmon: inserted PID alias runtime pid {} -> proc pid {}", + runtime_pid, + proc_pid + ), + Err(e) => { + tracing::debug!( + "Sysmon: failed to insert PID alias runtime pid {} -> proc pid {}: {}", + runtime_pid, + proc_pid, + e + ); + } + } +} + +pub(super) fn record_runtime_pid_alias_for_event( + mgr: &Arc>, + runtime_pid: u32, + proc_pid: u32, +) { + write_pinned_runtime_pid_alias(runtime_pid, proc_pid); + if let Ok(mut guard) = mgr.lock() { + guard.record_runtime_pid_alias(runtime_pid, proc_pid); + } +} + +pub(super) fn record_runtime_pid_aliases_for_proc_pid_locked( + guard: &mut ProcessManager, + proc_pid: u32, +) { + for runtime_pid in runtime_pid_candidates_for_proc(proc_pid) { + write_pinned_runtime_pid_alias(runtime_pid, proc_pid); + guard.record_runtime_pid_alias(runtime_pid, proc_pid); + } +} + +pub(super) fn record_runtime_pid_aliases_for_proc_pid( + mgr: &Arc>, + proc_pid: u32, +) { + if let Ok(mut guard) = mgr.lock() { + record_runtime_pid_aliases_for_proc_pid_locked(&mut guard, proc_pid); + } else { + for runtime_pid in runtime_pid_candidates_for_proc(proc_pid) { + write_pinned_runtime_pid_alias(runtime_pid, proc_pid); + } + } +} + +pub(super) fn runtime_pid_keys_for_proc_event( + proc_pid: u32, + event_pid: u32, + extra_runtime_pids: impl IntoIterator, +) -> Vec { + let mut keys = BTreeSet::new(); + keys.insert(proc_pid); + keys.insert(event_pid); + for runtime_pid in runtime_pid_candidates_for_proc(proc_pid) { + keys.insert(runtime_pid); + } + for runtime_pid in extra_runtime_pids { + if runtime_pid != 0 { + keys.insert(runtime_pid); + } + } + keys.into_iter().collect() +} + +pub(super) fn record_runtime_pid_aliases_for_keys( + mgr: &Arc>, + proc_pid: u32, + runtime_pids: &[u32], +) { + for runtime_pid in runtime_pids { + write_pinned_runtime_pid_alias(*runtime_pid, proc_pid); + } + if let Ok(mut guard) = mgr.lock() { + for runtime_pid in runtime_pids { + guard.record_runtime_pid_alias(*runtime_pid, proc_pid); + } + } +} + +pub(super) fn insert_allowed_runtime_pid_keys(runtime_pids: &[u32]) { + for runtime_pid in runtime_pids { + let _ = crate::pinned_bpf_maps::insert_allowed_pid(*runtime_pid); + } +} + +pub(super) fn publish_offsets_for_runtime_pid_keys( + proc_pid: u32, + event_pid: u32, + runtime_pids: &[u32], + items: &[(u64, crate::pinned_bpf_maps::ProcModuleOffsetsValue)], + log_context: &str, +) -> anyhow::Result { + use crate::pinned_bpf_maps::{insert_offsets_for_pid, replace_ranges_for_pid}; + + let mut total_inserted = 0usize; + for runtime_pid in runtime_pids { + match insert_offsets_for_pid(*runtime_pid, items) { + Ok(inserted) => { + if inserted == 0 { + tracing::warn!( + "Sysmon: no offsets inserted for {} runtime pid {} (event pid {}, proc pid {}) (entry count={})", + log_context, + runtime_pid, + event_pid, + proc_pid, + items.len() + ); + continue; + } + total_inserted += inserted; + if let Err(e) = replace_ranges_for_pid(*runtime_pid, items) { + tracing::warn!( + "Sysmon: failed to replace module ranges for {} runtime pid {} (event pid {}, proc pid {}): {}", + log_context, + runtime_pid, + event_pid, + proc_pid, + e + ); + } + } + Err(e) => { + tracing::warn!( + "Sysmon: failed to insert offsets for {} runtime pid {} (event pid {}, proc pid {}): {}", + log_context, + runtime_pid, + event_pid, + proc_pid, + e + ); + } + } + } + + Ok(total_inserted) +} + +pub(super) fn purge_offsets_for_runtime_pid_keys(runtime_pids: &[u32]) -> anyhow::Result { + let mut purged = 0usize; + for runtime_pid in runtime_pids { + purged += crate::pinned_bpf_maps::purge_offsets_for_pid(*runtime_pid)?; + let _ = crate::pinned_bpf_maps::purge_ranges_for_pid(*runtime_pid); + } + Ok(purged) +} + +pub(super) fn purge_runtime_pid_artifacts(proc_pid: u32, event_pid: u32, host_pid: u32) -> usize { + let runtime_pids = runtime_pid_keys_for_proc_event(proc_pid, event_pid, [host_pid]); + let mut purged_offsets = 0usize; + for runtime_pid in runtime_pids { + if let Ok(purged) = crate::pinned_bpf_maps::purge_offsets_for_pid(runtime_pid) { + purged_offsets += purged; + } + let _ = crate::pinned_bpf_maps::purge_ranges_for_pid(runtime_pid); + let _ = crate::pinned_bpf_maps::remove_allowed_pid(runtime_pid); + if runtime_pid != proc_pid { + let _ = crate::pinned_bpf_maps::remove_pid_alias(runtime_pid); + } + } + purged_offsets +} + +pub(super) fn record_runtime_pid_aliases_for_sys_event( + mgr: &Arc>, + ev: &SysEvent, + proc_pid: u32, +) { + record_runtime_pid_alias_for_event(mgr, ev.tgid, proc_pid); + let host_pid = sys_event_host_pid(ev); + if host_pid != ev.tgid { + record_runtime_pid_alias_for_event(mgr, host_pid, proc_pid); + } + record_runtime_pid_aliases_for_proc_pid(mgr, proc_pid); +} diff --git a/ghostscope-process/src/sysmon/runtime_loop.rs b/ghostscope-process/src/sysmon/runtime_loop.rs new file mode 100644 index 00000000..1390f092 --- /dev/null +++ b/ghostscope-process/src/sysmon/runtime_loop.rs @@ -0,0 +1,243 @@ +use super::attach::*; +use super::events::*; +use super::offset_refresh::*; +use super::pid_alias::*; +use super::*; + +#[cfg(feature = "sysmon-ebpf")] +pub(super) fn run_sysmon_loop( + mgr: Arc>, + cfg: SysmonConfig, + pending: Arc>, + pending_map_refreshes: Arc>, + tx: mpsc::SyncSender, +) -> anyhow::Result<()> { + use aya::include_bytes_aligned; + use aya::maps::{ + perf::{PerfEvent, PerfEventArray}, + ring_buf::RingBuf, + MapData, + }; + use log::{log_enabled, Level as LogLevel}; + // Load eBPF object (copied to OUT_DIR at build time) + #[allow(unused_variables)] + let obj_le: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/sysmon-bpf.bpfel.o")); + #[allow(unused_variables)] + let obj_be: &[u8] = include_bytes_aligned!(concat!(env!("OUT_DIR"), "/sysmon-bpf.bpfeb.o")); + let obj: &[u8] = if cfg!(target_endian = "little") { + obj_le + } else { + obj_be + }; + if obj.is_empty() { + warn!("sysmon-bpf object missing; running in stub mode (no realtime process events)"); + return Ok(()); + } + let target = cfg.target_module.clone(); + let use_verbose = + cfg!(debug_assertions) || log_enabled!(LogLevel::Trace) || log_enabled!(LogLevel::Debug); + let mut bpf = load_and_attach_sysmon_bpf(obj, &cfg, use_verbose)?; + let proc_pid_for_event = sysmon_proc_pid_resolver(cfg.watched_pid, cfg.watched_proc_pid); + + // Using allowlist-based gating in kernel; userspace decides allow on exec. + + // Initial prefill for late-start cases: compute and insert offsets for already-running PIDs. + if let Some(tpath) = &target { + let mut initial_target_pids: BTreeSet = BTreeSet::new(); + if let Ok(mut guard) = mgr.lock() { + if let Ok(prefilled) = guard.ensure_prefill_module(tpath.to_string_lossy().as_ref()) { + tracing::info!( + "Sysmon: initial prefill cached {} pid(s) for module {}", + prefilled, + tpath.display() + ); + let entries = guard.cached_offsets_for_module(tpath.to_string_lossy().as_ref()); + if !entries.is_empty() { + use crate::pinned_bpf_maps::ProcModuleOffsetsValue; + let mut by_pid: HashMap> = + HashMap::new(); + for (pid, cookie, off, base, size) in entries { + if is_current_process_pid(pid) { + continue; + } + by_pid.entry(pid).or_default().push(( + cookie, + ProcModuleOffsetsValue::new( + off.text, off.rodata, off.data, off.bss, base, size, + ), + )); + } + let mut total = 0usize; + for (pid, items) in by_pid { + initial_target_pids.insert(pid); + // Add event PID (kernel namespace) to allowlist so subsequent + // fork/exit events are filtered in-kernel. + let event_pid = resolve_event_pid_for_proc(pid); + let runtime_pids = runtime_pid_keys_for_proc_event(pid, event_pid, []); + for runtime_pid in &runtime_pids { + write_pinned_runtime_pid_alias(*runtime_pid, pid); + guard.record_runtime_pid_alias(*runtime_pid, pid); + } + if let Ok(n) = publish_offsets_for_runtime_pid_keys( + pid, + event_pid, + &runtime_pids, + &items, + "initial prefill", + ) { + total += n; + } + insert_allowed_runtime_pid_keys(&runtime_pids); + } + tracing::info!( + "Sysmon: initial inserted {} offset entries for module {}", + total, + tpath.display() + ); + } + } + } + for pid in initial_target_pids { + let event_pid = resolve_event_pid_for_proc(pid); + if let Err(e) = + prefill_full_offsets_for_pid_if_new(&mgr, event_pid, &proc_pid_for_event) + { + tracing::debug!( + "Sysmon: initial full offset prefill failed for proc pid {} (event pid {}): {}", + pid, + event_pid, + e + ); + } + } + } + tracing::info!("Sysmon: setup complete"); + // Initial prefill already ran above. Do not make the first periodic module + // refresh immediately due: for `-t executable`, the exec event is the fast + // path that inserts proc_module_offsets and allowed_pids. A fallback /proc + // scan here can delay a short-lived target past its only probe. + let mut last_module_refresh = Instant::now(); + let mut target_pid_map_signatures = HashMap::::new(); + + // Event loop: prefer ringbuf; fallback to perf + if let Some(map) = bpf.take_map("sysmon_events") { + let mut rb: RingBuf = map.try_into()?; + loop { + let mut had_event = false; + // Drain queued lifecycle events before periodic refresh. In the + // short-lived `-t executable` path, sched_process_exec must be + // handled promptly so offsets are ready before the first uprobe. + while let Some(item) = rb.next() { + had_event = true; + if item.len() == core::mem::size_of::() { + // SAFETY: The ring buffer sample length was checked to match SysEvent; + // read_unaligned handles any alignment from the byte slice. + let ev = unsafe { core::ptr::read_unaligned(item.as_ptr() as *const SysEvent) }; + let matched = dispatch_sysmon_event( + &mgr, + &target, + &pending, + &pending_map_refreshes, + &proc_pid_for_event, + &ev, + ); + if matched { + try_publish_sys_event(&tx, ev); + } + } + } + poll_pending_offsets(&mgr, &pending, &proc_pid_for_event); + poll_pending_map_refreshes( + &mgr, + target.as_deref(), + &pending_map_refreshes, + &pending, + &tx, + ); + refresh_target_module_offsets( + &mgr, + target.as_deref(), + &mut last_module_refresh, + &mut target_pid_map_signatures, + &tx, + ); + if !had_event { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + } + } else if let Some(map) = bpf.take_map("sysmon_events_perf") { + let mut perf: PerfEventArray<_> = map.try_into()?; + let online = aya::util::online_cpus().map_err(|(_, e)| anyhow::anyhow!(e))?; + let mut bufs = Vec::new(); + for cpu in online { + match perf.open(cpu, cfg.perf_page_count) { + Ok(buf) => bufs.push(buf), + Err(e) => warn!("Perf open failed for CPU {}: {}", cpu, e), + } + } + if bufs.is_empty() { + return Err(anyhow::anyhow!("No perf buffers opened")); + } + loop { + std::thread::sleep(std::time::Duration::from_millis(10)); + for buf in bufs.iter_mut() { + if !buf.readable() { + continue; + } + buf.for_each(|event| match event { + PerfEvent::Sample { head, tail } => { + let mut raw = [0u8; core::mem::size_of::()]; + let mut copied = 0; + for chunk in [head, tail] { + let remaining = raw.len().saturating_sub(copied); + if remaining == 0 { + break; + } + let take = chunk.len().min(remaining); + raw[copied..copied + take].copy_from_slice(&chunk[..take]); + copied += take; + } + if copied == raw.len() { + // SAFETY: raw is exactly the size of SysEvent and read_unaligned + // handles the byte array's alignment. + let ev = unsafe { + core::ptr::read_unaligned(raw.as_ptr() as *const SysEvent) + }; + let matched = dispatch_sysmon_event( + &mgr, + &target, + &pending, + &pending_map_refreshes, + &proc_pid_for_event, + &ev, + ); + if matched { + try_publish_sys_event(&tx, ev); + } + } + } + PerfEvent::Lost { count } => { + warn!("Perf event buffer lost {} sysmon events", count); + } + }); + } + poll_pending_offsets(&mgr, &pending, &proc_pid_for_event); + poll_pending_map_refreshes( + &mgr, + target.as_deref(), + &pending_map_refreshes, + &pending, + &tx, + ); + refresh_target_module_offsets( + &mgr, + target.as_deref(), + &mut last_module_refresh, + &mut target_pid_map_signatures, + &tx, + ); + } + } else { + return Err(anyhow::anyhow!("No sysmon events map found (ringbuf/perf)")); + } +}