From d33370f3a56db56f3275dc45ccd599a37e1f97c6 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Mon, 22 Jun 2026 14:14:49 +0200 Subject: [PATCH 01/23] feat: encoder v1 to v04 + refacto --- .../src/msgpack_encoder/v04/mod.rs | 132 +++ .../src/msgpack_encoder/v04/span.rs | 56 - .../src/msgpack_encoder/v04/span_v1_to_v04.rs | 1037 +++++++++++++++++ .../src/msgpack_encoder/v1/mod.rs | 4 +- .../v1/{span_v04.rs => span_v04_to_v1.rs} | 0 ..._and_v1_to_v04_encoders_snapshot_test.json | 38 + libdd-trace-utils/tests/test_send_data.rs | 99 +- 7 files changed, 1301 insertions(+), 65 deletions(-) create mode 100644 libdd-trace-utils/src/msgpack_encoder/v04/span_v1_to_v04.rs rename libdd-trace-utils/src/msgpack_encoder/v1/{span_v04.rs => span_v04_to_v1.rs} (100%) create mode 100644 libdd-trace-utils/tests/snapshots/compare_v04_native_and_v1_to_v04_encoders_snapshot_test.json diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs index 8d89567f0a..2b896b5f51 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs @@ -2,11 +2,71 @@ // SPDX-License-Identifier: Apache-2.0 use crate::span::v04::Span; +use crate::span::v1::TracerPayload; use crate::span::TraceData; use libdd_common::ResultInfallibleExt; use rmp::encode::{write_array_len, ByteBuf, RmpWrite, ValueWriteError}; +const fn msp_string_encoding_len(s: &str) -> usize { + let length_marker_len = if s.len() < 32 { + 1 + } else if s.len() < 256 { + 2 + } else if s.len() <= (u16::MAX as usize) { + 3 + } else { + 5 + }; + length_marker_len + s.len() +} + +// Compute the encoding of a string to messagepack in a const manner +const fn msp_const_string_encoding(s: &str) -> [u8; ENCODING_LEN] { + // copy_to_slice is not const yet, so we make a helper + const fn copy_to_slice(dest: &mut [u8], src: &[u8], n: usize) { + let mut i = 0; + while i < n { + dest[i] = src[i]; + i += 1; + } + } + + let mut storage = [0; ENCODING_LEN]; + let len = s.len() as u64; + let len_bytes = if len < 32 { + storage[0] = 0xa0 | (len as u8 & 0x1f); + 0 + } else if len < 256 { + storage[0] = 0xd9; + 1 + } else if len <= (u16::MAX as u64) { + storage[0] = 0xda; + 2 + } else { + storage[0] = 0xdb; + 4 + }; + copy_to_slice(storage.split_at_mut(1).1, &len.to_be_bytes(), len_bytes); + copy_to_slice(storage.split_at_mut(1 + len_bytes).1, s.as_bytes(), s.len()); + storage +} + +// `super::` lets the macro body resolve the const helpers from any submodule of `v04/` +// (`span.rs`, `span_v1_to_v04.rs`, …) without each one importing them explicitly. +macro_rules! write_const_msg_pack_str { + ($writer:expr, $str:expr) => {{ + use rmp::encode::ValueWriteError; + const STRING_ENCODING_LEN: usize = super::msp_string_encoding_len($str); + const STRING_ENCODING: [u8; STRING_ENCODING_LEN] = super::msp_const_string_encoding($str); + + $writer + .write_bytes(&STRING_ENCODING) + .map_err(ValueWriteError::InvalidDataWrite) + }}; +} + mod span; +mod span_v1_to_v04; #[inline(always)] fn to_writer]>>( @@ -168,3 +228,75 @@ pub fn to_encoded_byte_len]>>(traces: &[S]) -> u let _ = to_writer(&mut counter, traces); counter.0 } + +/// Encodes a [`TracerPayload`] in the v0.4 wire format (downgrade path used when the agent +/// does not advertise `/v1.0/traces`). The output is a msgpack array of traces, where each +/// trace is itself a msgpack array of v0.4-shaped spans — matching the existing v0.4 wire +/// format produced by [`to_vec`]. +fn encode_payload_v1_to_v04( + writer: &mut W, + payload: &TracerPayload, +) -> Result<(), ValueWriteError> { + use span_v1_to_v04::{encode_span_v1_to_v04, ChunkContext}; + + write_array_len(writer, payload.chunks.len() as u32)?; + for chunk in &payload.chunks { + let ctx = ChunkContext { + trace_id: &chunk.trace_id, + priority: chunk.priority, + origin: &chunk.origin, + sampling_mechanism: chunk.sampling_mechanism, + attributes: &chunk.attributes, + }; + write_array_len(writer, chunk.spans.len() as u32)?; + for span in &chunk.spans { + encode_span_v1_to_v04(writer, span, &ctx)?; + } + } + Ok(()) +} + +/// Serializes a [`TracerPayload`] (V1 data model) as a v0.4 msgpack payload. +/// +/// Used by the trace exporter when the agent has not advertised `/v1.0/traces` via `/info`. +/// The output is byte-compatible with [`to_vec`] for equivalent data — chunk-level fields are +/// propagated to every span and typed attributes are bucketed into the v0.4 `meta` / +/// `metrics` / `meta_struct` maps per [`span_v1_to_v04`]'s mapping table. +pub fn to_vec_v1_to_v04(payload: &TracerPayload) -> Vec { + to_vec_v1_to_v04_with_capacity(payload, 0) +} + +/// Serializes a [`TracerPayload`] as a v0.4 msgpack payload with a caller-supplied initial +/// capacity. Use this when you can size the buffer up front (e.g. from +/// [`to_encoded_byte_len_v1_to_v04`]) to avoid reallocations. +pub fn to_vec_v1_to_v04_with_capacity( + payload: &TracerPayload, + capacity: u32, +) -> Vec { + let mut buf = ByteBuf::with_capacity(capacity as usize); + encode_payload_v1_to_v04(&mut buf, payload) + .map_err(super::flatten_value_write_infallible) + .unwrap_infallible(); + buf.into_vec() +} + +/// Encodes a [`TracerPayload`] as v0.4 msgpack into the provided slice. Useful for callers +/// that own a pre-sized buffer (e.g. for FFI / zero-copy paths). +/// +/// # Errors +/// +/// Returns any [`ValueWriteError`] from the underlying writer (typically buffer-too-small). +pub fn write_v1_to_v04_to_slice( + slice: &mut &mut [u8], + payload: &TracerPayload, +) -> Result<(), ValueWriteError> { + encode_payload_v1_to_v04(slice, payload) +} + +/// Returns the exact number of bytes [`to_vec_v1_to_v04`] would write for `payload`. Walks +/// the payload through a counting writer without allocating an output buffer. +pub fn to_encoded_byte_len_v1_to_v04(payload: &TracerPayload) -> u32 { + let mut counter = super::CountLength(0); + let _ = encode_payload_v1_to_v04(&mut counter, payload); + counter.0 +} diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span.rs index b4e8f7301c..cb9c60b944 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span.rs @@ -9,62 +9,6 @@ use rmp::encode::{ }; use std::borrow::Borrow; -const fn msp_string_encoding_len(s: &str) -> usize { - let length_marker_len = if s.len() < 32 { - 1 - } else if s.len() < 256 { - 2 - } else if s.len() <= (u16::MAX as usize) { - 3 - } else { - 5 - }; - length_marker_len + s.len() -} - -// Compute the encoding of a string to messagepack in a const manner -const fn msp_const_string_encoding(s: &str) -> [u8; ENCODING_LEN] { - // copy_to_slice is not const yet, so we make a helper - const fn copy_to_slice(dest: &mut [u8], src: &[u8], n: usize) { - let mut i = 0; - while i < n { - dest[i] = src[i]; - i += 1; - } - } - - let mut storage = [0; ENCODING_LEN]; - let len = s.len() as u64; - let len_bytes = if len < 32 { - storage[0] = 0xa0 | (len as u8 & 0x1f); - 0 - } else if len < 256 { - storage[0] = 0xd9; - 1 - } else if len <= (u16::MAX as u64) { - storage[0] = 0xda; - 2 - } else { - storage[0] = 0xdb; - 4 - }; - copy_to_slice(storage.split_at_mut(1).1, &len.to_be_bytes(), len_bytes); - copy_to_slice(storage.split_at_mut(1 + len_bytes).1, s.as_bytes(), s.len()); - storage -} - -macro_rules! write_const_msg_pack_str { - ($writer:expr, $str:expr) => {{ - use rmp::encode::ValueWriteError; - const STRING_ENCODING_LEN: usize = msp_string_encoding_len($str); - const STRING_ENCODING: [u8; STRING_ENCODING_LEN] = msp_const_string_encoding($str); - - $writer - .write_bytes(&STRING_ENCODING) - .map_err(ValueWriteError::InvalidDataWrite) - }}; -} - /// Encodes a `SpanLink` object into a slice of bytes. /// /// # Arguments diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1_to_v04.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1_to_v04.rs new file mode 100644 index 0000000000..62ee9611ad --- /dev/null +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1_to_v04.rs @@ -0,0 +1,1037 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Downgrade encoder: writes a [`crate::span::v1::Span`] in the v0.4 msgpack wire format. +//! +//! Used when the receiving agent does not advertise the `/v1.0/traces` endpoint and the tracer +//! must fall back to v0.4. The mapping is: +//! +//! | v1::Span field / attribute | v0.4 field | +//! |---------------------------------------|---------------------------------------------| +//! | `env` / `version` / `component` | `meta["env"]` / `meta["version"]` / ... | +//! | `span_kind` | `meta["span.kind"]` (lowercase string) | +//! | `AttributeValue::String` / `Bool` | `meta[k]` (`"true"` / `"false"` for bool) | +//! | `AttributeValue::Float` / `Int` | `metrics[k]` (Int cast to `f64`) | +//! | `AttributeValue::Bytes` / `KeyValue` | `meta_struct[k]` (re-encoded as msgpack) | +//! | `AttributeValue::List` | `meta_struct[k]` (re-encoded as msgpack) | +//! | `error: bool` | `error: i32` (`true → 1`, `false → 0`) | +//! | Chunk `trace_id: [u8; 16]` | `trace_id: u64` (low 64) + `meta["_dd.p.tid"]` (hex of high 64, when non-zero) | +//! | Chunk `origin` | `meta["_dd.origin"]` | +//! | Chunk `priority` | `metrics["_sampling_priority_v1"]` | +//! | Chunk `sampling_mechanism` | `meta["_dd.p.dm"]` (`"-{mechanism}"`) | +//! | Chunk `attributes` | Applied to every span in the chunk | + +use crate::span::v1::{AttributeValue, Span, SpanEvent, SpanKind, SpanLink}; +use crate::span::vec_map::VecMap; +use crate::span::TraceData; +use rmp::encode::{ + write_array_len, write_bin, write_bool, write_f64, write_i64, write_map_len, write_sint, + write_str, write_u32, write_u64, write_u8, RmpWrite, ValueWriteError, +}; +use std::borrow::Borrow; + +/// Chunk-level context propagated into every span when downgrading to v0.4. Built once per +/// chunk by the top-level encoder and passed by reference to `encode_span_v1_to_v04`. +pub(super) struct ChunkContext<'a, T: TraceData> { + pub trace_id: &'a [u8; 16], + pub priority: Option, + pub origin: &'a T::Text, + pub sampling_mechanism: Option, + pub attributes: &'a VecMap>, +} + +/// Maps a `SpanKind` to its v0.4 `span.kind` meta string. Returns `None` for `Internal` so +/// callers can skip emitting the default value. +fn span_kind_to_meta(kind: SpanKind) -> Option<&'static str> { + match kind { + SpanKind::Internal => None, + SpanKind::Server => Some("server"), + SpanKind::Client => Some("client"), + SpanKind::Producer => Some("producer"), + SpanKind::Consumer => Some("consumer"), + } +} + +/// Splits a 128-bit big-endian trace_id into `(low_64, high_64)`. The low half maps to v0.4's +/// `trace_id` field; the high half goes to `meta["_dd.p.tid"]` as hex when non-zero. +#[inline] +fn split_trace_id(trace_id: &[u8; 16]) -> (u64, u64) { + let mut high_bytes = [0u8; 8]; + let mut low_bytes = [0u8; 8]; + high_bytes.copy_from_slice(&trace_id[..8]); + low_bytes.copy_from_slice(&trace_id[8..]); + ( + u64::from_be_bytes(low_bytes), + u64::from_be_bytes(high_bytes), + ) +} + +/// Per-bucket counts for the v0.4 `meta`, `metrics`, and `meta_struct` maps. +#[derive(Default)] +struct BucketCounts { + meta: u32, + metrics: u32, + meta_struct: u32, +} + +impl BucketCounts { + /// Routes a typed attribute into its bucket per the mapping table in the module docs. + fn count_attr(&mut self, v: &AttributeValue) { + match v { + AttributeValue::String(_) | AttributeValue::Bool(_) => self.meta += 1, + AttributeValue::Float(_) | AttributeValue::Int(_) => self.metrics += 1, + AttributeValue::Bytes(_) | AttributeValue::KeyValue(_) | AttributeValue::List(_) => { + self.meta_struct += 1 + } + } + } +} + +/// Encodes a `Span` in the v0.4 wire format, injecting chunk-level context into the span's +/// `meta` / `metrics` / `meta_struct` maps. +/// +/// # Arguments +/// +/// * `writer` - A RmpWriter compatible with rmp writing functions. +/// * `span` - The v1::Span to encode. +/// * `chunk` - Chunk-level context (`trace_id`, `origin`, `priority`, `sampling_mechanism`, chunk +/// attributes) propagated into the span on the v0.4 wire. +/// +/// # Returns +/// +/// * `Ok(())` - Nothing if successful. +/// * `Err(ValueWriteError)` - An error if the writing fails. +/// +/// # Errors +/// +/// This function will return any error emitted by the writer. +pub(super) fn encode_span_v1_to_v04( + writer: &mut W, + span: &Span, + chunk: &ChunkContext<'_, T>, +) -> Result<(), ValueWriteError> { + let span_attrs_dd = span.attributes.defensive_dedup(); + let chunk_attrs_dd = chunk.attributes.defensive_dedup(); + + let (trace_id_low, trace_id_high) = split_trace_id(chunk.trace_id); + let kind_meta = span_kind_to_meta(span.span_kind); + + // First pass: count bucket sizes so each msgpack map header carries the exact length. + let mut counts = BucketCounts::default(); + counts.meta += !span.env.borrow().is_empty() as u32; + counts.meta += !span.version.borrow().is_empty() as u32; + counts.meta += !span.component.borrow().is_empty() as u32; + counts.meta += kind_meta.is_some() as u32; + counts.meta += (trace_id_high != 0) as u32; + counts.meta += !chunk.origin.borrow().is_empty() as u32; + counts.meta += chunk.sampling_mechanism.is_some() as u32; + counts.metrics += chunk.priority.is_some() as u32; + for (_, v) in span_attrs_dd.iter() { + counts.count_attr(v); + } + // Chunk attributes are propagated to every span. Duplicate keys between chunk and span are + // tolerated here — msgpack map decode keeps the last value (the span's), which matches the + // "span overrides chunk" precedence. + for (_, v) in chunk_attrs_dd.iter() { + counts.count_attr(v); + } + + let span_len = 7 // service, name, resource, trace_id, span_id, start, duration (always) + + (!span.r#type.borrow().is_empty()) as u32 + + (span.parent_id != 0) as u32 + + span.error as u32 + + (counts.meta > 0) as u32 + + (counts.metrics > 0) as u32 + + (counts.meta_struct > 0) as u32 + + (!span.span_links.is_empty()) as u32 + + (!span.span_events.is_empty()) as u32; + + write_map_len(writer, span_len)?; + + write_const_msg_pack_str!(writer, "service")?; + write_str(writer, span.service.borrow())?; + + write_const_msg_pack_str!(writer, "name")?; + write_str(writer, span.name.borrow())?; + + write_const_msg_pack_str!(writer, "resource")?; + write_str(writer, span.resource.borrow())?; + + write_const_msg_pack_str!(writer, "trace_id")?; + write_u64(writer, trace_id_low)?; + + write_const_msg_pack_str!(writer, "span_id")?; + write_u64(writer, span.span_id)?; + + if span.parent_id != 0 { + write_const_msg_pack_str!(writer, "parent_id")?; + write_u64(writer, span.parent_id)?; + } + + write_const_msg_pack_str!(writer, "start")?; + write_i64(writer, span.start)?; + + write_const_msg_pack_str!(writer, "duration")?; + write_sint(writer, span.duration)?; + + if span.error { + write_const_msg_pack_str!(writer, "error")?; + write_sint(writer, 1)?; + } + + if counts.meta > 0 { + write_const_msg_pack_str!(writer, "meta")?; + write_map_len(writer, counts.meta)?; + + if !span.env.borrow().is_empty() { + write_const_msg_pack_str!(writer, "env")?; + write_str(writer, span.env.borrow())?; + } + if !span.version.borrow().is_empty() { + write_const_msg_pack_str!(writer, "version")?; + write_str(writer, span.version.borrow())?; + } + if !span.component.borrow().is_empty() { + write_const_msg_pack_str!(writer, "component")?; + write_str(writer, span.component.borrow())?; + } + if let Some(kind_str) = kind_meta { + write_const_msg_pack_str!(writer, "span.kind")?; + write_str(writer, kind_str)?; + } + if trace_id_high != 0 { + // Lower-case hex without `0x` prefix — the agent expects this format. + write_const_msg_pack_str!(writer, "_dd.p.tid")?; + write_str(writer, &format!("{trace_id_high:016x}"))?; + } + if !chunk.origin.borrow().is_empty() { + write_const_msg_pack_str!(writer, "_dd.origin")?; + write_str(writer, chunk.origin.borrow())?; + } + if let Some(mechanism) = chunk.sampling_mechanism { + write_const_msg_pack_str!(writer, "_dd.p.dm")?; + write_str(writer, &format!("-{mechanism}"))?; + } + // Span attrs first, then chunk attrs (duplicate keys resolve to the span value via + // msgpack last-write-wins on the decoder side). + for (k, v) in span_attrs_dd.iter() { + match v { + AttributeValue::String(s) => { + write_str(writer, k.borrow())?; + write_str(writer, s.borrow())?; + } + AttributeValue::Bool(b) => { + write_str(writer, k.borrow())?; + write_str(writer, if *b { "true" } else { "false" })?; + } + _ => {} + } + } + for (k, v) in chunk_attrs_dd.iter() { + match v { + AttributeValue::String(s) => { + write_str(writer, k.borrow())?; + write_str(writer, s.borrow())?; + } + AttributeValue::Bool(b) => { + write_str(writer, k.borrow())?; + write_str(writer, if *b { "true" } else { "false" })?; + } + _ => {} + } + } + } + + if counts.metrics > 0 { + write_const_msg_pack_str!(writer, "metrics")?; + write_map_len(writer, counts.metrics)?; + + if let Some(priority) = chunk.priority { + write_const_msg_pack_str!(writer, "_sampling_priority_v1")?; + write_f64(writer, priority as f64)?; + } + for (k, v) in span_attrs_dd.iter() { + match v { + AttributeValue::Float(f) => { + write_str(writer, k.borrow())?; + write_f64(writer, *f)?; + } + AttributeValue::Int(i) => { + write_str(writer, k.borrow())?; + write_f64(writer, *i as f64)?; + } + _ => {} + } + } + for (k, v) in chunk_attrs_dd.iter() { + match v { + AttributeValue::Float(f) => { + write_str(writer, k.borrow())?; + write_f64(writer, *f)?; + } + AttributeValue::Int(i) => { + write_str(writer, k.borrow())?; + write_f64(writer, *i as f64)?; + } + _ => {} + } + } + } + + if !span.r#type.borrow().is_empty() { + write_const_msg_pack_str!(writer, "type")?; + write_str(writer, span.r#type.borrow())?; + } + + if counts.meta_struct > 0 { + write_const_msg_pack_str!(writer, "meta_struct")?; + write_map_len(writer, counts.meta_struct)?; + + for (k, v) in span_attrs_dd.iter() { + write_meta_struct_entry(writer, k, v)?; + } + for (k, v) in chunk_attrs_dd.iter() { + write_meta_struct_entry(writer, k, v)?; + } + } + + if !span.span_links.is_empty() { + encode_span_links_v1_to_v04(writer, &span.span_links)?; + } + if !span.span_events.is_empty() { + encode_span_events_v1_to_v04(writer, &span.span_events)?; + } + + Ok(()) +} + +/// Writes a single `meta_struct` entry. `Bytes` goes through as msgpack `bin`; structural +/// `KeyValue` / `List` would need a recursive msgpack re-encoding — stubbed as empty `bin` for +/// now (TODO: implement structural serialization). +fn write_meta_struct_entry( + writer: &mut W, + k: &T::Text, + v: &AttributeValue, +) -> Result<(), ValueWriteError> { + match v { + AttributeValue::Bytes(b) => { + write_str(writer, k.borrow())?; + write_bin(writer, b.borrow())?; + } + AttributeValue::KeyValue(_) | AttributeValue::List(_) => { + write_str(writer, k.borrow())?; + write_bin(writer, &[])?; + } + _ => {} + } + Ok(()) +} + +/// Converts v1 `SpanLink`s to the v0.4 wire format. The 128-bit `trace_id` is split into +/// `(trace_id, trace_id_high)` u64s. Typed link attributes are downgraded to strings (`String` +/// passes through, `Bool` becomes `"true"` / `"false"`); non-string-coercible variants are +/// dropped because v0.4 link attributes are `String → String` only. +fn encode_span_links_v1_to_v04( + writer: &mut W, + span_links: &[SpanLink], +) -> Result<(), ValueWriteError> { + write_const_msg_pack_str!(writer, "span_links")?; + write_array_len(writer, span_links.len() as u32)?; + + for link in span_links { + let (trace_id_low, trace_id_high) = split_trace_id(&link.trace_id); + let attrs_dd = link.attributes.defensive_dedup(); + let attr_count = attrs_dd + .iter() + .filter(|(_, v)| matches!(v, AttributeValue::String(_) | AttributeValue::Bool(_))) + .count() as u32; + + let link_len = 3 // trace_id, trace_id_high, span_id (always) + + (attr_count > 0) as u32 + + (!link.tracestate.borrow().is_empty()) as u32 + + (link.flags != 0) as u32; + + write_map_len(writer, link_len)?; + + write_const_msg_pack_str!(writer, "trace_id")?; + write_u64(writer, trace_id_low)?; + + write_const_msg_pack_str!(writer, "trace_id_high")?; + write_u64(writer, trace_id_high)?; + + write_const_msg_pack_str!(writer, "span_id")?; + write_u64(writer, link.span_id)?; + + if attr_count > 0 { + write_const_msg_pack_str!(writer, "attributes")?; + write_map_len(writer, attr_count)?; + for (k, v) in attrs_dd.iter() { + match v { + AttributeValue::String(s) => { + write_str(writer, k.borrow())?; + write_str(writer, s.borrow())?; + } + AttributeValue::Bool(b) => { + write_str(writer, k.borrow())?; + write_str(writer, if *b { "true" } else { "false" })?; + } + _ => {} + } + } + } + + if !link.tracestate.borrow().is_empty() { + write_const_msg_pack_str!(writer, "tracestate")?; + write_str(writer, link.tracestate.borrow())?; + } + + if link.flags != 0 { + write_const_msg_pack_str!(writer, "flags")?; + write_u32(writer, link.flags)?; + } + } + + Ok(()) +} + +/// Converts v1 `SpanEvent`s to the v0.4 wire format. Typed attributes are downgraded to the +/// v0.4 `{"type": , "_value": ...}` shape — see `write_event_attr_value`. `Bytes` +/// and `KeyValue` have no v0.4 event-attribute equivalent and are dropped. +fn encode_span_events_v1_to_v04( + writer: &mut W, + span_events: &[SpanEvent], +) -> Result<(), ValueWriteError> { + write_const_msg_pack_str!(writer, "span_events")?; + write_array_len(writer, span_events.len() as u32)?; + + for event in span_events { + let attrs_dd = event.attributes.defensive_dedup(); + let attr_count = attrs_dd + .iter() + .filter(|(_, v)| is_supported_event_attr(v)) + .count() as u32; + + let event_len = 2 // time_unix_nano, name (always) + + (attr_count > 0) as u32; + + write_map_len(writer, event_len)?; + + write_const_msg_pack_str!(writer, "time_unix_nano")?; + write_u64(writer, event.time_unix_nano)?; + + write_const_msg_pack_str!(writer, "name")?; + write_str(writer, event.name.borrow())?; + + if attr_count > 0 { + write_const_msg_pack_str!(writer, "attributes")?; + write_map_len(writer, attr_count)?; + for (k, v) in attrs_dd.iter() { + if !is_supported_event_attr(v) { + continue; + } + write_str(writer, k.borrow())?; + write_event_attr_value(writer, v)?; + } + } + } + + Ok(()) +} + +/// Returns `true` when `v` can be downgraded to a v0.4 event-attribute (scalar or scalar list). +fn is_supported_event_attr(v: &AttributeValue) -> bool { + matches!( + v, + AttributeValue::String(_) + | AttributeValue::Bool(_) + | AttributeValue::Int(_) + | AttributeValue::Float(_) + | AttributeValue::List(_) + ) +} + +/// Writes a v0.4 event-attribute value as `{"type": , "..._value": ...}`. Scalars produce a +/// 2-entry map; `List` produces `{"type": 4, "array_value": {"values": [...]}}` with each +/// element written via `write_event_array_element`. +fn write_event_attr_value( + writer: &mut W, + v: &AttributeValue, +) -> Result<(), ValueWriteError> { + match v { + AttributeValue::String(s) => { + write_map_len(writer, 2)?; + write_const_msg_pack_str!(writer, "type")?; + write_u8(writer, 0)?; + write_const_msg_pack_str!(writer, "string_value")?; + write_str(writer, s.borrow())?; + } + AttributeValue::Bool(b) => { + write_map_len(writer, 2)?; + write_const_msg_pack_str!(writer, "type")?; + write_u8(writer, 1)?; + write_const_msg_pack_str!(writer, "bool_value")?; + write_bool(writer, *b).map_err(ValueWriteError::InvalidDataWrite)?; + } + AttributeValue::Int(i) => { + write_map_len(writer, 2)?; + write_const_msg_pack_str!(writer, "type")?; + write_u8(writer, 2)?; + write_const_msg_pack_str!(writer, "int_value")?; + write_sint(writer, *i)?; + } + AttributeValue::Float(f) => { + write_map_len(writer, 2)?; + write_const_msg_pack_str!(writer, "type")?; + write_u8(writer, 3)?; + write_const_msg_pack_str!(writer, "double_value")?; + write_f64(writer, *f)?; + } + AttributeValue::List(arr) => { + // Only scalar elements survive the downgrade; nested structural entries are + // skipped because v0.4 array elements must themselves be scalar. + let scalar_elems = arr.iter().filter(|e| is_scalar_array_elem(e)); + let elem_count = scalar_elems.clone().count() as u32; + write_map_len(writer, 2)?; + write_const_msg_pack_str!(writer, "type")?; + write_u8(writer, 4)?; + write_const_msg_pack_str!(writer, "array_value")?; + write_map_len(writer, 1)?; + write_const_msg_pack_str!(writer, "values")?; + write_array_len(writer, elem_count)?; + for elem in scalar_elems { + write_event_array_element(writer, elem)?; + } + } + AttributeValue::Bytes(_) | AttributeValue::KeyValue(_) => { + // Filtered upstream by `is_supported_event_attr`; reachable only on a bug. + debug_assert!(false, "unsupported event attribute variant reached writer"); + } + } + Ok(()) +} + +/// Returns `true` when `v` is a scalar that fits in a v0.4 `AttributeArrayValue` (no nesting). +fn is_scalar_array_elem(v: &AttributeValue) -> bool { + matches!( + v, + AttributeValue::String(_) + | AttributeValue::Bool(_) + | AttributeValue::Int(_) + | AttributeValue::Float(_) + ) +} + +/// Writes a v0.4 `AttributeArrayValue` (scalar). Same `{"type", "..._value"}` shape as +/// `write_event_attr_value`, minus the `Array` variant — v0.4 array elements are scalar only. +fn write_event_array_element( + writer: &mut W, + v: &AttributeValue, +) -> Result<(), ValueWriteError> { + match v { + AttributeValue::String(s) => { + write_map_len(writer, 2)?; + write_const_msg_pack_str!(writer, "type")?; + write_u8(writer, 0)?; + write_const_msg_pack_str!(writer, "string_value")?; + write_str(writer, s.borrow())?; + } + AttributeValue::Bool(b) => { + write_map_len(writer, 2)?; + write_const_msg_pack_str!(writer, "type")?; + write_u8(writer, 1)?; + write_const_msg_pack_str!(writer, "bool_value")?; + write_bool(writer, *b).map_err(ValueWriteError::InvalidDataWrite)?; + } + AttributeValue::Int(i) => { + write_map_len(writer, 2)?; + write_const_msg_pack_str!(writer, "type")?; + write_u8(writer, 2)?; + write_const_msg_pack_str!(writer, "int_value")?; + write_sint(writer, *i)?; + } + AttributeValue::Float(f) => { + write_map_len(writer, 2)?; + write_const_msg_pack_str!(writer, "type")?; + write_u8(writer, 3)?; + write_const_msg_pack_str!(writer, "double_value")?; + write_f64(writer, *f)?; + } + _ => { + // Filtered upstream by `is_scalar_array_elem`; reachable only on a bug. + debug_assert!(false, "non-scalar array element reached writer"); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + //! Unit tests for the v1::Span → v0.4 downgrade encoder. Each test encodes a small + //! `TracerPayload` via [`super::super::to_vec_v1_to_v04`] and decodes the bytes with + //! `rmpv` to assert on the resulting v0.4 shape — this implicitly checks that the output + //! is also valid msgpack consumable by any standard v0.4 decoder (test-agent, agent, etc.). + use crate::span::v1::{ + AttributeValue, AttributeValueBytes, SpanBytes, SpanEventBytes, SpanKind, SpanLinkBytes, + TraceChunkBytes, TracerPayloadBytes, + }; + use crate::span::vec_map::VecMap; + use libdd_tinybytes::{Bytes, BytesString}; + use rmpv::Value; + use thin_vec::ThinVec; + + fn bs(s: &str) -> BytesString { + BytesString::from_slice(s.as_bytes()).expect("test string must fit in BytesString") + } + + /// Encodes `payload` and decodes back into `rmpv::Value`. The top level of v0.4 is an + /// array of traces; this helper returns it as a `Vec` so tests can index in. + fn encode_and_decode(payload: &TracerPayloadBytes) -> Vec { + let bytes = super::super::to_vec_v1_to_v04(payload); + let value = rmpv::decode::read_value(&mut &bytes[..]).expect("decode failed"); + match value { + Value::Array(traces) => traces, + other => panic!("expected top-level array, got {other:?}"), + } + } + + /// Looks up `key` in a msgpack `Value::Map`. Returns `None` when absent so callers can + /// distinguish "field missing" from "field empty". + fn map_get<'a>(map: &'a Value, key: &str) -> Option<&'a Value> { + let entries = match map { + Value::Map(m) => m, + other => panic!("expected map, got {other:?}"), + }; + entries + .iter() + .find(|(k, _)| k.as_str() == Some(key)) + .map(|(_, v)| v) + } + + /// Convenience: build a minimal single-chunk single-span payload with the v0.4-equivalent + /// of the canonical "svc/op/res" example. Tests override fields as needed. + fn minimal_payload(trace_id: [u8; 16], span: SpanBytes) -> TracerPayloadBytes { + TracerPayloadBytes { + chunks: vec![TraceChunkBytes { + trace_id, + spans: vec![span], + ..Default::default() + }], + ..Default::default() + } + } + + fn minimal_span() -> SpanBytes { + SpanBytes { + service: bs("svc"), + name: bs("op"), + resource: bs("res"), + span_id: 1, + start: 1_000, + duration: 500, + ..Default::default() + } + } + + #[test] + fn basic_span_writes_required_v04_fields() { + let payload = minimal_payload([0u8; 16], minimal_span()); + let traces = encode_and_decode(&payload); + + assert_eq!(traces.len(), 1); + let trace = traces[0].as_array().expect("trace must be array"); + assert_eq!(trace.len(), 1); + let span = &trace[0]; + + assert_eq!(map_get(span, "service").unwrap().as_str(), Some("svc")); + assert_eq!(map_get(span, "name").unwrap().as_str(), Some("op")); + assert_eq!(map_get(span, "resource").unwrap().as_str(), Some("res")); + assert_eq!(map_get(span, "span_id").unwrap().as_u64(), Some(1)); + assert_eq!(map_get(span, "trace_id").unwrap().as_u64(), Some(0)); + assert_eq!(map_get(span, "start").unwrap().as_i64(), Some(1_000)); + assert_eq!(map_get(span, "duration").unwrap().as_i64(), Some(500)); + // Optional fields must be absent when their underlying value is zero/empty. + assert!(map_get(span, "parent_id").is_none()); + assert!(map_get(span, "error").is_none()); + assert!(map_get(span, "type").is_none()); + assert!(map_get(span, "meta").is_none()); + assert!(map_get(span, "metrics").is_none()); + assert!(map_get(span, "meta_struct").is_none()); + } + + #[test] + fn promoted_fields_are_copied_into_meta() { + let span = SpanBytes { + env: bs("prod"), + version: bs("1.2.3"), + component: bs("http"), + span_kind: SpanKind::Server, + ..minimal_span() + }; + let payload = minimal_payload([0u8; 16], span); + let traces = encode_and_decode(&payload); + let meta = map_get(&traces[0][0], "meta").expect("meta must be present"); + + assert_eq!(map_get(meta, "env").unwrap().as_str(), Some("prod")); + assert_eq!(map_get(meta, "version").unwrap().as_str(), Some("1.2.3")); + assert_eq!(map_get(meta, "component").unwrap().as_str(), Some("http")); + assert_eq!(map_get(meta, "span.kind").unwrap().as_str(), Some("server")); + } + + #[test] + fn span_kind_internal_is_not_emitted() { + // Internal is the default and is implied by the absence of `meta["span.kind"]`. + let payload = minimal_payload([0u8; 16], minimal_span()); + let traces = encode_and_decode(&payload); + // meta is None overall since no other field forces it. + assert!(map_get(&traces[0][0], "meta").is_none()); + } + + #[test] + fn trace_id_128_bit_splits_into_low_field_and_high_meta() { + // trace_id = 0x_DEADBEEF__CAFEBABE_DEADBEEF__CAFEBABE (high | low) + let mut tid = [0u8; 16]; + tid[..8].copy_from_slice(&0xDEAD_BEEF_CAFE_BABE_u64.to_be_bytes()); + tid[8..].copy_from_slice(&0x0123_4567_89AB_CDEF_u64.to_be_bytes()); + let payload = minimal_payload(tid, minimal_span()); + let traces = encode_and_decode(&payload); + let span = &traces[0][0]; + + assert_eq!( + map_get(span, "trace_id").unwrap().as_u64(), + Some(0x0123_4567_89AB_CDEF) + ); + let meta = map_get(span, "meta").expect("meta must be present (carries _dd.p.tid)"); + assert_eq!( + map_get(meta, "_dd.p.tid").unwrap().as_str(), + Some("deadbeefcafebabe"), + "high 64 bits must be encoded as lower-case hex without the 0x prefix" + ); + } + + #[test] + fn trace_id_high_zero_omits_dd_p_tid() { + // When the upper 64 bits are zero, `_dd.p.tid` must be absent so v0.4 consumers don't + // see a redundant `"0x0"` entry. + let mut tid = [0u8; 16]; + tid[8..].copy_from_slice(&42u64.to_be_bytes()); + let payload = minimal_payload(tid, minimal_span()); + let traces = encode_and_decode(&payload); + assert!(map_get(&traces[0][0], "meta").is_none()); + } + + #[test] + fn error_true_emits_one_false_omits_field() { + let payload_err = minimal_payload( + [0u8; 16], + SpanBytes { + error: true, + ..minimal_span() + }, + ); + let traces_err = encode_and_decode(&payload_err); + assert_eq!( + map_get(&traces_err[0][0], "error").unwrap().as_i64(), + Some(1) + ); + + let payload_ok = minimal_payload([0u8; 16], minimal_span()); + let traces_ok = encode_and_decode(&payload_ok); + assert!(map_get(&traces_ok[0][0], "error").is_none()); + } + + #[test] + fn string_attribute_is_routed_to_meta() { + let mut attrs: VecMap = VecMap::new(); + attrs.insert(bs("http.method"), AttributeValue::String(bs("GET"))); + let payload = minimal_payload( + [0u8; 16], + SpanBytes { + attributes: attrs, + ..minimal_span() + }, + ); + let traces = encode_and_decode(&payload); + let meta = map_get(&traces[0][0], "meta").expect("meta present"); + assert_eq!(map_get(meta, "http.method").unwrap().as_str(), Some("GET")); + } + + #[test] + fn bool_attribute_is_stringified_in_meta() { + let mut attrs: VecMap = VecMap::new(); + attrs.insert(bs("retry"), AttributeValue::Bool(true)); + attrs.insert(bs("cached"), AttributeValue::Bool(false)); + let payload = minimal_payload( + [0u8; 16], + SpanBytes { + attributes: attrs, + ..minimal_span() + }, + ); + let traces = encode_and_decode(&payload); + let meta = map_get(&traces[0][0], "meta").expect("meta present"); + assert_eq!(map_get(meta, "retry").unwrap().as_str(), Some("true")); + assert_eq!(map_get(meta, "cached").unwrap().as_str(), Some("false")); + } + + #[test] + fn float_and_int_attributes_route_to_metrics_as_f64() { + let mut attrs: VecMap = VecMap::new(); + attrs.insert(bs("duration_ms"), AttributeValue::Float(12.5)); + attrs.insert(bs("status"), AttributeValue::Int(200)); + let payload = minimal_payload( + [0u8; 16], + SpanBytes { + attributes: attrs, + ..minimal_span() + }, + ); + let traces = encode_and_decode(&payload); + let metrics = map_get(&traces[0][0], "metrics").expect("metrics present"); + assert_eq!( + map_get(metrics, "duration_ms").unwrap().as_f64(), + Some(12.5) + ); + // Int is cast to f64 in the v0.4 metrics map per the mapping table. + assert_eq!(map_get(metrics, "status").unwrap().as_f64(), Some(200.0)); + } + + #[test] + fn bytes_attribute_routes_to_meta_struct_as_msgpack_bin() { + let mut attrs: VecMap = VecMap::new(); + attrs.insert( + bs("blob"), + AttributeValue::Bytes(Bytes::copy_from_slice(b"\xde\xad\xbe\xef")), + ); + let payload = minimal_payload( + [0u8; 16], + SpanBytes { + attributes: attrs, + ..minimal_span() + }, + ); + let traces = encode_and_decode(&payload); + let ms = map_get(&traces[0][0], "meta_struct").expect("meta_struct present"); + assert_eq!( + map_get(ms, "blob").and_then(|v| match v { + Value::Binary(b) => Some(b.as_slice()), + _ => None, + }), + Some(b"\xde\xad\xbe\xef".as_slice()) + ); + } + + #[test] + fn chunk_origin_priority_and_sampling_mechanism_propagate_to_span() { + let chunk_attrs: VecMap = VecMap::new(); + let payload = TracerPayloadBytes { + chunks: vec![TraceChunkBytes { + trace_id: [0u8; 16], + priority: Some(1), + origin: bs("synthetics"), + sampling_mechanism: Some(4), + attributes: chunk_attrs, + spans: vec![minimal_span()], + ..Default::default() + }], + ..Default::default() + }; + let traces = encode_and_decode(&payload); + let span = &traces[0][0]; + + let meta = map_get(span, "meta").expect("meta carries origin + sampling_mechanism"); + assert_eq!( + map_get(meta, "_dd.origin").unwrap().as_str(), + Some("synthetics") + ); + assert_eq!( + map_get(meta, "_dd.p.dm").unwrap().as_str(), + Some("-4"), + "sampling_mechanism is encoded as `-{{n}}` per the agent's convention" + ); + + let metrics = map_get(span, "metrics").expect("metrics carries sampling_priority_v1"); + assert_eq!( + map_get(metrics, "_sampling_priority_v1").unwrap().as_f64(), + Some(1.0) + ); + } + + #[test] + fn chunk_attributes_are_propagated_to_every_span_in_chunk() { + let mut chunk_attrs: VecMap = VecMap::new(); + chunk_attrs.insert(bs("region"), AttributeValue::String(bs("us-east-1"))); + let payload = TracerPayloadBytes { + chunks: vec![TraceChunkBytes { + trace_id: [0u8; 16], + attributes: chunk_attrs, + spans: vec![ + minimal_span(), + SpanBytes { + span_id: 2, + ..minimal_span() + }, + ], + ..Default::default() + }], + ..Default::default() + }; + let traces = encode_and_decode(&payload); + let trace = traces[0].as_array().expect("trace is array of spans"); + assert_eq!(trace.len(), 2); + + for span in trace { + let meta = map_get(span, "meta").expect("each span inherits chunk attrs"); + assert_eq!(map_get(meta, "region").unwrap().as_str(), Some("us-east-1")); + } + } + + #[test] + fn empty_payload_encodes_as_empty_top_level_array() { + let payload = TracerPayloadBytes::default(); + let traces = encode_and_decode(&payload); + assert!(traces.is_empty()); + } + + #[test] + fn multiple_chunks_become_multiple_traces() { + let payload = TracerPayloadBytes { + chunks: vec![ + TraceChunkBytes { + trace_id: [0u8; 16], + spans: vec![minimal_span()], + ..Default::default() + }, + TraceChunkBytes { + trace_id: [0u8; 16], + spans: vec![SpanBytes { + span_id: 99, + ..minimal_span() + }], + ..Default::default() + }, + ], + ..Default::default() + }; + let traces = encode_and_decode(&payload); + assert_eq!(traces.len(), 2); + assert_eq!(map_get(&traces[0][0], "span_id").unwrap().as_u64(), Some(1)); + assert_eq!( + map_get(&traces[1][0], "span_id").unwrap().as_u64(), + Some(99) + ); + } + + #[test] + fn span_link_splits_trace_id_into_low_and_high_fields() { + let mut link_tid = [0u8; 16]; + link_tid[..8].copy_from_slice(&0xAAAA_BBBB_CCCC_DDDD_u64.to_be_bytes()); + link_tid[8..].copy_from_slice(&0x1111_2222_3333_4444_u64.to_be_bytes()); + + let mut link_attrs: VecMap = VecMap::new(); + link_attrs.insert(bs("link.name"), AttributeValue::String(bs("job-42"))); + link_attrs.insert(bs("link.retry"), AttributeValue::Bool(true)); + // Non-string/bool typed attrs must be dropped (v0.4 SpanLink is String→String only). + link_attrs.insert(bs("link.count"), AttributeValue::Int(5)); + + let payload = minimal_payload( + [0u8; 16], + SpanBytes { + span_links: ThinVec::from_iter([SpanLinkBytes { + trace_id: link_tid, + span_id: 7, + attributes: link_attrs, + tracestate: bs("dd=t.dm:-1"), + flags: 3, + }]), + ..minimal_span() + }, + ); + + let traces = encode_and_decode(&payload); + let links = map_get(&traces[0][0], "span_links").expect("span_links present"); + let links_arr = links.as_array().expect("span_links is array"); + assert_eq!(links_arr.len(), 1); + let link = &links_arr[0]; + + assert_eq!( + map_get(link, "trace_id").unwrap().as_u64(), + Some(0x1111_2222_3333_4444) + ); + assert_eq!( + map_get(link, "trace_id_high").unwrap().as_u64(), + Some(0xAAAA_BBBB_CCCC_DDDD) + ); + assert_eq!(map_get(link, "span_id").unwrap().as_u64(), Some(7)); + assert_eq!( + map_get(link, "tracestate").unwrap().as_str(), + Some("dd=t.dm:-1") + ); + assert_eq!(map_get(link, "flags").unwrap().as_u64(), Some(3)); + + let attrs = map_get(link, "attributes").expect("string attrs preserved"); + assert_eq!( + map_get(attrs, "link.name").unwrap().as_str(), + Some("job-42") + ); + assert_eq!(map_get(attrs, "link.retry").unwrap().as_str(), Some("true")); + // Int attr was dropped — v0.4 SpanLink schema cannot carry it. + assert!(map_get(attrs, "link.count").is_none()); + } + + #[test] + fn span_event_attributes_are_downgraded_to_v04_anyvalue_shape() { + let mut event_attrs: VecMap = VecMap::new(); + event_attrs.insert(bs("kind"), AttributeValue::String(bs("exception"))); + event_attrs.insert(bs("escaped"), AttributeValue::Bool(true)); + event_attrs.insert(bs("count"), AttributeValue::Int(3)); + event_attrs.insert(bs("ratio"), AttributeValue::Float(0.75)); + + let payload = minimal_payload( + [0u8; 16], + SpanBytes { + span_events: ThinVec::from_iter([SpanEventBytes { + time_unix_nano: 1_700_000_000_000_000_000, + name: bs("oops"), + attributes: event_attrs, + }]), + ..minimal_span() + }, + ); + + let traces = encode_and_decode(&payload); + let events = map_get(&traces[0][0], "span_events").expect("span_events present"); + let events_arr = events.as_array().expect("span_events is array"); + assert_eq!(events_arr.len(), 1); + let event = &events_arr[0]; + + assert_eq!(map_get(event, "name").unwrap().as_str(), Some("oops")); + assert_eq!( + map_get(event, "time_unix_nano").unwrap().as_u64(), + Some(1_700_000_000_000_000_000) + ); + + // Each typed attribute decodes to a `{"type": , "_value": value}` map. + let attrs = map_get(event, "attributes").expect("event attributes present"); + let kind = map_get(attrs, "kind").unwrap(); + assert_eq!(map_get(kind, "type").unwrap().as_u64(), Some(0)); + assert_eq!( + map_get(kind, "string_value").unwrap().as_str(), + Some("exception") + ); + + let escaped = map_get(attrs, "escaped").unwrap(); + assert_eq!(map_get(escaped, "type").unwrap().as_u64(), Some(1)); + assert_eq!( + map_get(escaped, "bool_value").unwrap().as_bool(), + Some(true) + ); + + let count = map_get(attrs, "count").unwrap(); + assert_eq!(map_get(count, "type").unwrap().as_u64(), Some(2)); + assert_eq!(map_get(count, "int_value").unwrap().as_i64(), Some(3)); + + let ratio = map_get(attrs, "ratio").unwrap(); + assert_eq!(map_get(ratio, "type").unwrap().as_u64(), Some(3)); + assert_eq!(map_get(ratio, "double_value").unwrap().as_f64(), Some(0.75)); + } +} diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs index 9ae268c588..c861cbb53e 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs @@ -1,7 +1,7 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -mod span_v04; +mod span_v04_to_v1; mod span_v1; use crate::span::v04::Span; @@ -431,7 +431,7 @@ fn encode_chunk( write_uint8(writer, chunk_key::SPANS)?; write_array_len(writer, spans.len() as u32)?; for span in spans { - span_v04::encode_span(writer, span, table)?; + span_v04_to_v1::encode_span(writer, span, table)?; } if let Some(origin) = attrs.origin { diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs b/libdd-trace-utils/src/msgpack_encoder/v1/span_v04_to_v1.rs similarity index 100% rename from libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs rename to libdd-trace-utils/src/msgpack_encoder/v1/span_v04_to_v1.rs diff --git a/libdd-trace-utils/tests/snapshots/compare_v04_native_and_v1_to_v04_encoders_snapshot_test.json b/libdd-trace-utils/tests/snapshots/compare_v04_native_and_v1_to_v04_encoders_snapshot_test.json new file mode 100644 index 0000000000..bcb9423a20 --- /dev/null +++ b/libdd-trace-utils/tests/snapshots/compare_v04_native_and_v1_to_v04_encoders_snapshot_test.json @@ -0,0 +1,38 @@ +[[ + { + "name": "op", + "service": "svc", + "resource": "res", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "meta": { + "env": "test-env", + "http.method": "GET", + "span.kind": "server" + }, + "metrics": { + "http.duration_ms": 12.5 + }, + "duration": 5000, + "start": 1000000 + }], +[ + { + "name": "op", + "service": "svc", + "resource": "res", + "trace_id": 1, + "span_id": 1, + "parent_id": 0, + "meta": { + "env": "test-env", + "http.method": "GET", + "span.kind": "server" + }, + "metrics": { + "http.duration_ms": 12.5 + }, + "duration": 5000, + "start": 1000000 + }]] diff --git a/libdd-trace-utils/tests/test_send_data.rs b/libdd-trace-utils/tests/test_send_data.rs index 39a10b58ff..30a58847c7 100644 --- a/libdd-trace-utils/tests/test_send_data.rs +++ b/libdd-trace-utils/tests/test_send_data.rs @@ -413,10 +413,11 @@ mod tracing_integration_tests { out } - /// POSTs a raw V1 msgpack payload to the test-agent's `/v1.0/traces` and asserts the agent - /// returns 2xx. Headers are the minimum the agent needs to attach the payload to a snapshot - /// session (`X-Datadog-Test-Session-Token` query param + `Datadog-Meta-Lang*` for routing). - async fn post_v1_payload(uri: hyper::Uri, body: Vec) { + /// POSTs a raw msgpack trace payload to the test-agent (the endpoint — `/v0.4/traces` or + /// `/v1.0/traces` — is baked into `uri` by the caller). Asserts the agent returns 2xx. + /// Headers are the minimum the agent needs to attach the payload to a snapshot session + /// (`X-Datadog-Test-Session-Token` query param + `Datadog-Meta-Lang*` for routing). + async fn post_msgpack_traces(uri: hyper::Uri, body: Vec) { use libdd_capabilities_impl::HttpClientCapability; let client = NativeCapabilities::new_client(); let req = http::Request::builder() @@ -543,7 +544,7 @@ mod tracing_integration_tests { let payload = make_v1_payload("test_send_data_v1_native_snapshot"); let encoded = to_vec_from_payload_v1(&payload); - post_v1_payload(uri, encoded).await; + post_msgpack_traces(uri, encoded).await; test_agent.assert_snapshot(snapshot_name).await; } @@ -620,11 +621,95 @@ mod tracing_integration_tests { // ── POST both into the same session ──────────────────────────────────────── let bytes_v04 = to_vec(&v04_traces, &metadata); let bytes_v1 = to_vec_from_payload_v1(&v1_payload); - post_v1_payload(uri.clone(), bytes_v04).await; - post_v1_payload(uri, bytes_v1).await; + post_msgpack_traces(uri.clone(), bytes_v04).await; + post_msgpack_traces(uri, bytes_v1).await; // Both POSTs share trace_id=1, so the test-agent merges them into a single trace of // 2 decoded spans. The checked-in snapshot is the canonical equivalent decoded form. test_agent.assert_snapshot(snapshot_name).await; } + + /// Round-trip: the v1::Span → v0.4 downgrade encoder + /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec_v1_to_v04`]) must produce a v0.4 + /// payload that decodes to the same canonical form as the native v0.4 encoder + /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec`]) when fed equivalent input. + /// + /// Both encodings are POSTed to `/v0.4/traces` in the same session with distinct + /// `trace_id`s, and the checked-in snapshot records the two traces side by side. Any + /// drift in either encoder makes its trace diverge from the canonical decoded form. + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn compare_v04_native_and_v1_to_v04_encoders_snapshot_test() { + use libdd_trace_utils::msgpack_encoder::v04::to_vec as to_vec_v04_native; + use libdd_trace_utils::msgpack_encoder::v04::to_vec_v1_to_v04; + use libdd_trace_utils::span::v04::SpanBytes as V04SpanBytes; + use libdd_trace_utils::span::v1::{ + AttributeValue, SpanBytes as V1SpanBytes, SpanKind, TraceChunkBytes, TracerPayloadBytes, + }; + + let relative_snapshot_path = "libdd-trace-utils/tests/snapshots/"; + let snapshot_name = "compare_v04_native_and_v1_to_v04_encoders_snapshot_test"; + let test_agent = DatadogTestAgent::new(Some(relative_snapshot_path), None, &[]).await; + let uri = test_agent + .get_uri_for_endpoint("v0.4/traces", Some(snapshot_name)) + .await; + + test_agent.start_session(snapshot_name, None).await; + + // ── v0.4 native input — trace_id = 1 ─────────────────────────────────────── + // Note: v0.4's `meta["span.kind"]` is the downgrade target for the v1::SpanKind enum, + // so we set it explicitly here for parity with the v1::Span path below. + let mut meta_v04 = VecMap::new(); + meta_v04.insert(bs_v1("env"), bs_v1("test-env")); + meta_v04.insert(bs_v1("http.method"), bs_v1("GET")); + meta_v04.insert(bs_v1("span.kind"), bs_v1("server")); + let mut metrics_v04 = VecMap::new(); + metrics_v04.insert(bs_v1("http.duration_ms"), 12.5_f64); + let v04_traces: Vec> = vec![vec![V04SpanBytes { + service: bs_v1("svc"), + name: bs_v1("op"), + resource: bs_v1("res"), + trace_id: 1, + span_id: 1, + start: 1_000_000, + duration: 5_000, + meta: meta_v04, + metrics: metrics_v04, + ..Default::default() + }]]; + + // ── v1::Span input — trace_id = 2, semantically equivalent otherwise ─────── + // The v1 model carries `env` / `span_kind` as typed fields rather than meta strings; + // the downgrade encoder must place them back under `meta["env"]` / `meta["span.kind"]`. + let mut attrs_v1 = VecMap::new(); + attrs_v1.insert(bs_v1("http.method"), AttributeValue::String(bs_v1("GET"))); + attrs_v1.insert(bs_v1("http.duration_ms"), AttributeValue::Float(12.5)); + let v1_payload = TracerPayloadBytes { + chunks: vec![TraceChunkBytes { + trace_id: tid_bytes(0, 2), + spans: vec![V1SpanBytes { + service: bs_v1("svc"), + name: bs_v1("op"), + resource: bs_v1("res"), + span_id: 1, + start: 1_000_000, + duration: 5_000, + span_kind: SpanKind::Server, + env: bs_v1("test-env"), + attributes: attrs_v1, + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }; + + // ── Encode each via its respective encoder and POST both ─────────────────── + let bytes_v04 = to_vec_v04_native(&v04_traces); + let bytes_v1_to_v04 = to_vec_v1_to_v04(&v1_payload); + post_msgpack_traces(uri.clone(), bytes_v04).await; + post_msgpack_traces(uri, bytes_v1_to_v04).await; + + test_agent.assert_snapshot(snapshot_name).await; + } } From 02ab60b60df3cfce77461e8593df6749e133ecc7 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Tue, 23 Jun 2026 14:39:29 +0200 Subject: [PATCH 02/23] refacto: renaming --- libdd-trace-utils/src/msgpack_encoder/v04/mod.rs | 8 ++++---- .../src/msgpack_encoder/v04/{span.rs => span_v04.rs} | 4 ++++ .../v04/{span_v1_to_v04.rs => span_v1.rs} | 12 ++++++++---- libdd-trace-utils/src/msgpack_encoder/v1/mod.rs | 4 ++-- .../v1/{span_v04_to_v1.rs => span_v04.rs} | 4 ++++ libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs | 3 +++ 6 files changed, 25 insertions(+), 10 deletions(-) rename libdd-trace-utils/src/msgpack_encoder/v04/{span.rs => span_v04.rs} (97%) rename libdd-trace-utils/src/msgpack_encoder/v04/{span_v1_to_v04.rs => span_v1.rs} (98%) rename libdd-trace-utils/src/msgpack_encoder/v1/{span_v04_to_v1.rs => span_v04.rs} (98%) diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs index 2b896b5f51..8d0243c21c 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs @@ -65,8 +65,8 @@ macro_rules! write_const_msg_pack_str { }}; } -mod span; -mod span_v1_to_v04; +mod span_v04; +mod span_v1; #[inline(always)] fn to_writer]>>( @@ -77,7 +77,7 @@ fn to_writer]>>( for trace in traces { write_array_len(writer, trace.as_ref().len() as u32)?; for span in trace.as_ref() { - span::encode_span(writer, span)?; + span_v04::encode_span(writer, span)?; } } @@ -237,7 +237,7 @@ fn encode_payload_v1_to_v04( writer: &mut W, payload: &TracerPayload, ) -> Result<(), ValueWriteError> { - use span_v1_to_v04::{encode_span_v1_to_v04, ChunkContext}; + use span_v1::{encode_span_v1_to_v04, ChunkContext}; write_array_len(writer, payload.chunks.len() as u32)?; for chunk in &payload.chunks { diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs similarity index 97% rename from libdd-trace-utils/src/msgpack_encoder/v04/span.rs rename to libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs index cb9c60b944..4ccf319f30 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs @@ -1,6 +1,10 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +// Naming convention: parent module (`v04`) = output wire format, file suffix (`_v04`) = input +// span type. This file encodes a [`crate::span::v04::Span`] into the v0.4 msgpack wire format +// (native). + use crate::span::v04::{AttributeAnyValue, AttributeArrayValue, Span, SpanEvent, SpanLink}; use crate::span::TraceData; use rmp::encode::{ diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1_to_v04.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs similarity index 98% rename from libdd-trace-utils/src/msgpack_encoder/v04/span_v1_to_v04.rs rename to libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs index 62ee9611ad..f03a11741a 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1_to_v04.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs @@ -1,6 +1,10 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +// Naming convention: parent module (`v04`) = output wire format, file suffix (`_v1`) = input +// span type. This file encodes a [`crate::span::v1::Span`] into the v0.4 msgpack wire format +// (cross-format downgrade). + //! Downgrade encoder: writes a [`crate::span::v1::Span`] in the v0.4 msgpack wire format. //! //! Used when the receiving agent does not advertise the `/v1.0/traces` endpoint and the tracer @@ -296,10 +300,10 @@ pub(super) fn encode_span_v1_to_v04( } if !span.span_links.is_empty() { - encode_span_links_v1_to_v04(writer, &span.span_links)?; + encode_span_links(writer, &span.span_links)?; } if !span.span_events.is_empty() { - encode_span_events_v1_to_v04(writer, &span.span_events)?; + encode_span_events(writer, &span.span_events)?; } Ok(()) @@ -331,7 +335,7 @@ fn write_meta_struct_entry( /// `(trace_id, trace_id_high)` u64s. Typed link attributes are downgraded to strings (`String` /// passes through, `Bool` becomes `"true"` / `"false"`); non-string-coercible variants are /// dropped because v0.4 link attributes are `String → String` only. -fn encode_span_links_v1_to_v04( +fn encode_span_links( writer: &mut W, span_links: &[SpanLink], ) -> Result<(), ValueWriteError> { @@ -397,7 +401,7 @@ fn encode_span_links_v1_to_v04( /// Converts v1 `SpanEvent`s to the v0.4 wire format. Typed attributes are downgraded to the /// v0.4 `{"type": , "_value": ...}` shape — see `write_event_attr_value`. `Bytes` /// and `KeyValue` have no v0.4 event-attribute equivalent and are dropped. -fn encode_span_events_v1_to_v04( +fn encode_span_events( writer: &mut W, span_events: &[SpanEvent], ) -> Result<(), ValueWriteError> { diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs index c861cbb53e..9ae268c588 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs @@ -1,7 +1,7 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -mod span_v04_to_v1; +mod span_v04; mod span_v1; use crate::span::v04::Span; @@ -431,7 +431,7 @@ fn encode_chunk( write_uint8(writer, chunk_key::SPANS)?; write_array_len(writer, spans.len() as u32)?; for span in spans { - span_v04_to_v1::encode_span(writer, span, table)?; + span_v04::encode_span(writer, span, table)?; } if let Some(origin) = attrs.origin { diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/span_v04_to_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs similarity index 98% rename from libdd-trace-utils/src/msgpack_encoder/v1/span_v04_to_v1.rs rename to libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs index c1755623f6..56c2051133 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v1/span_v04_to_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs @@ -1,6 +1,10 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +// Naming convention: parent module (`v1`) = output wire format, file suffix (`_v04`) = input +// span type. This file encodes a [`crate::span::v04::Span`] into the V1 msgpack wire format +// (cross-format upgrade). + use crate::span::v04::{AttributeAnyValue, AttributeArrayValue, Span, SpanEvent, SpanLink}; use crate::span::TraceData; use rmp::encode::{ diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs index 0fcd9a98e4..aa3e17be20 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs @@ -1,6 +1,9 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +// Naming convention: parent module (`v1`) = output wire format, file suffix (`_v1`) = input span +// type. This file encodes a [`crate::span::v1::Span`] into the V1 msgpack wire format (native). + use crate::span::v1::{AttributeValue, Span, SpanEvent, SpanLink}; use crate::span::vec_map::VecMap; use crate::span::TraceData; From 020b6cc9f686f1b052bb83b78453032f5daa7fa4 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Thu, 25 Jun 2026 09:50:06 +0200 Subject: [PATCH 03/23] refacto name --- datadog-sidecar-ffi/src/lib.rs | 4 +- libdd-data-pipeline/src/trace_exporter/mod.rs | 26 +++--- .../src/trace_exporter/trace_serializer.rs | 4 +- libdd-trace-utils/benches/serialization.rs | 2 +- .../src/msgpack_encoder/v04/mod.rs | 87 +++++++++---------- .../src/msgpack_encoder/v04/span_v04.rs | 9 +- .../src/msgpack_encoder/v04/span_v1.rs | 22 +++-- .../src/msgpack_encoder/v1/mod.rs | 73 ++++++++-------- .../src/msgpack_encoder/v1/span_v04.rs | 16 ++-- .../src/msgpack_encoder/v1/span_v1.rs | 9 +- libdd-trace-utils/src/send_data/mod.rs | 4 +- libdd-trace-utils/tests/test_send_data.rs | 12 +-- 12 files changed, 137 insertions(+), 131 deletions(-) diff --git a/datadog-sidecar-ffi/src/lib.rs b/datadog-sidecar-ffi/src/lib.rs index 25fa4550bc..fc10d4485f 100644 --- a/datadog-sidecar-ffi/src/lib.rs +++ b/datadog-sidecar-ffi/src/lib.rs @@ -1694,7 +1694,7 @@ pub unsafe extern "C" fn ddog_send_traces_to_sidecar( // Write traces to the shared memory let mut shm_slice = mapped_shm.as_slice_mut(); let shm_slice_len = shm_slice.len(); - let written = match msgpack_encoder::v04::write_to_slice(&mut shm_slice, traces) { + let written = match msgpack_encoder::v04::write_to_slice_v04(&mut shm_slice, traces) { Ok(()) => shm_slice_len - shm_slice.len(), Err(_) => { tracing::error!("Failed serializing the traces"); @@ -1724,7 +1724,7 @@ pub unsafe extern "C" fn ddog_send_traces_to_sidecar( match blocking::send_trace_v04_bytes( &mut parameters.transport, ¶meters.instance_id, - msgpack_encoder::v04::to_vec_with_capacity(traces, written as u32), + msgpack_encoder::v04::to_vec_with_capacity_v04(traces, written as u32), check!( (¶meters.tracer_headers_tags).try_into(), "Failed to convert tracer headers tags" diff --git a/libdd-data-pipeline/src/trace_exporter/mod.rs b/libdd-data-pipeline/src/trace_exporter/mod.rs index 789db0d193..e4f239e2e7 100644 --- a/libdd-data-pipeline/src/trace_exporter/mod.rs +++ b/libdd-data-pipeline/src/trace_exporter/mod.rs @@ -1229,7 +1229,7 @@ mod tests { ..Default::default() }], ]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_v04(&traces); let _result = exporter.send(data.as_ref()).expect("failed to send trace"); @@ -1329,7 +1329,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_v04(&traces); let result = exporter.send(data.as_ref()); assert!(result.is_err()); @@ -1437,7 +1437,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_v04(&traces); let result = exporter.send(data.as_ref()); assert!(result.is_err()); @@ -1541,7 +1541,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_v04(&traces); let _result = exporter.send(data.as_ref()).expect("failed to send trace"); @@ -1601,7 +1601,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_v04(&traces); let result = exporter.send(data.as_ref()).unwrap(); assert_eq!( @@ -1643,7 +1643,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_v04(&traces); let code = match exporter.send(data.as_ref()).unwrap_err() { TraceExporterError::Request(e) => Some(e.status()), _ => None, @@ -1678,7 +1678,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_v04(&traces); let err = exporter.send(data.as_ref()); assert!(err.is_err()); @@ -2023,7 +2023,7 @@ mod tests { ..Default::default() }]; - let data = msgpack_encoder::v04::to_vec(&[trace_chunk]); + let data = msgpack_encoder::v04::to_vec_v04(&[trace_chunk]); // Wait for the info fetcher to get the config while mock_info.calls() == 0 { @@ -2091,7 +2091,7 @@ mod tests { error: 0, ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_v04(&traces); let result = exporter.send(data.as_ref()); assert!( @@ -2167,7 +2167,7 @@ mod single_threaded_tests { ..Default::default() }]; - let data = msgpack_encoder::v04::to_vec(&[trace_chunk]); + let data = msgpack_encoder::v04::to_vec_v04(&[trace_chunk]); // Wait for the info fetcher to get the config while agent_info::get_agent_info().is_none() { @@ -2268,7 +2268,7 @@ mod single_threaded_tests { ..Default::default() }]; - let data = msgpack_encoder::v04::to_vec(&[trace_chunk]); + let data = msgpack_encoder::v04::to_vec_v04(&[trace_chunk]); // Wait for agent_info to be present so that sending a trace will trigger the stats worker // to start @@ -2367,7 +2367,7 @@ mod single_threaded_tests { duration: 10, ..Default::default() }]; - let data = msgpack_encoder::v04::to_vec(&[trace_chunk]); + let data = msgpack_encoder::v04::to_vec_v04(&[trace_chunk]); let _ = exporter.send(data.as_ref()); let start = std::time::Instant::now(); @@ -2475,7 +2475,7 @@ mod single_threaded_tests { duration: 10, ..Default::default() }]; - let data = msgpack_encoder::v04::to_vec(&[trace_chunk]); + let data = msgpack_encoder::v04::to_vec_v04(&[trace_chunk]); // 1st send: /info has promoted v1_active=true, so this hits /v1.0/traces and 404s. let result1 = exporter.send(&data); diff --git a/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs b/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs index 7cfd62a5a0..8d053f69a0 100644 --- a/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs +++ b/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs @@ -120,7 +120,7 @@ impl TraceSerializer { .max(MIN_BUFFER_CAPACITY); let buff = match payload { tracer_payload::TraceChunks::V04(p) => { - msgpack_encoder::v04::to_vec_with_capacity(p, capacity as u32) + msgpack_encoder::v04::to_vec_with_capacity_v04(p, capacity as u32) } tracer_payload::TraceChunks::V05(p) => { let mut buff = Vec::with_capacity(capacity); @@ -129,7 +129,7 @@ impl TraceSerializer { buff } tracer_payload::TraceChunks::V1(p) => { - msgpack_encoder::v1::to_vec_with_capacity(p, capacity as u32, metadata) + msgpack_encoder::v1::to_vec_with_capacity_v04(p, capacity as u32, metadata) } }; self.previous_serialised_len diff --git a/libdd-trace-utils/benches/serialization.rs b/libdd-trace-utils/benches/serialization.rs index 3179b3fa73..931b96928b 100644 --- a/libdd-trace-utils/benches/serialization.rs +++ b/libdd-trace-utils/benches/serialization.rs @@ -65,7 +65,7 @@ pub fn serialize_internal_to_msgpack(c: &mut Criterion) { b.iter_batched( || vec![0u8; 12_000_000], |mut vec| { - let _ = black_box(msgpack_encoder::v04::write_to_slice( + let _ = black_box(msgpack_encoder::v04::write_to_slice_v04( &mut vec.as_mut_slice(), black_box(&data), )); diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs index 8d0243c21c..7d4b01dbf0 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs @@ -7,6 +7,21 @@ use crate::span::TraceData; use libdd_common::ResultInfallibleExt; use rmp::encode::{write_array_len, ByteBuf, RmpWrite, ValueWriteError}; +macro_rules! write_const_msg_pack_str { + ($writer:expr, $str:expr) => {{ + use rmp::encode::ValueWriteError; + const STRING_ENCODING_LEN: usize = super::msp_string_encoding_len($str); + const STRING_ENCODING: [u8; STRING_ENCODING_LEN] = super::msp_const_string_encoding($str); + + $writer + .write_bytes(&STRING_ENCODING) + .map_err(ValueWriteError::InvalidDataWrite) + }}; +} + +mod span_v04; +mod span_v1; + const fn msp_string_encoding_len(s: &str) -> usize { let length_marker_len = if s.len() < 32 { 1 @@ -51,23 +66,6 @@ const fn msp_const_string_encoding(s: &str) -> [u8; E storage } -// `super::` lets the macro body resolve the const helpers from any submodule of `v04/` -// (`span.rs`, `span_v1_to_v04.rs`, …) without each one importing them explicitly. -macro_rules! write_const_msg_pack_str { - ($writer:expr, $str:expr) => {{ - use rmp::encode::ValueWriteError; - const STRING_ENCODING_LEN: usize = super::msp_string_encoding_len($str); - const STRING_ENCODING: [u8; STRING_ENCODING_LEN] = super::msp_const_string_encoding($str); - - $writer - .write_bytes(&STRING_ENCODING) - .map_err(ValueWriteError::InvalidDataWrite) - }}; -} - -mod span_v04; -mod span_v1; - #[inline(always)] fn to_writer]>>( writer: &mut W, @@ -105,7 +103,7 @@ fn to_writer]>>( /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v04::write_to_slice; +/// use libdd_trace_utils::msgpack_encoder::v04::write_to_slice_v04; /// use libdd_trace_utils::span::v04::SpanSlice; /// /// let mut buffer = vec![0u8; 1024]; @@ -115,9 +113,9 @@ fn to_writer]>>( /// }; /// let traces = vec![vec![span]]; /// -/// write_to_slice(&mut &mut buffer[..], &traces).expect("Encoding failed"); +/// write_to_slice_v04(&mut &mut buffer[..], &traces).expect("Encoding failed"); /// ``` -pub fn write_to_slice]>>( +pub fn write_to_slice_v04]>>( slice: &mut &mut [u8], traces: &[S], ) -> Result<(), ValueWriteError> { @@ -137,7 +135,7 @@ pub fn write_to_slice]>>( /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v04::to_vec; +/// use libdd_trace_utils::msgpack_encoder::v04::to_vec_v04; /// use libdd_trace_utils::span::v04::SpanSlice; /// /// let span = SpanSlice { @@ -145,12 +143,12 @@ pub fn write_to_slice]>>( /// ..Default::default() /// }; /// let traces = vec![vec![span]]; -/// let encoded = to_vec(&traces); +/// let encoded = to_vec_v04(&traces); /// /// assert!(!encoded.is_empty()); /// ``` -pub fn to_vec]>>(traces: &[S]) -> Vec { - to_vec_with_capacity(traces, 0) +pub fn to_vec_v04]>>(traces: &[S]) -> Vec { + to_vec_with_capacity_v04(traces, 0) } /// Serializes traces into a vector of bytes with specified capacity. @@ -167,7 +165,7 @@ pub fn to_vec]>>(traces: &[S]) -> Vec { /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v04::to_vec_with_capacity; +/// use libdd_trace_utils::msgpack_encoder::v04::to_vec_with_capacity_v04; /// use libdd_trace_utils::span::v04::SpanSlice; /// /// let span = SpanSlice { @@ -175,11 +173,11 @@ pub fn to_vec]>>(traces: &[S]) -> Vec { /// ..Default::default() /// }; /// let traces = vec![vec![span]]; -/// let encoded = to_vec_with_capacity(&traces, 1024); +/// let encoded = to_vec_with_capacity_v04(&traces, 1024); /// /// assert!(encoded.capacity() >= 1024); /// ``` -pub fn to_vec_with_capacity]>>( +pub fn to_vec_with_capacity_v04]>>( traces: &[S], capacity: u32, ) -> Vec { @@ -206,7 +204,7 @@ pub fn to_vec_with_capacity]>>( /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v04::to_encoded_byte_len; +/// use libdd_trace_utils::msgpack_encoder::v04::to_encoded_byte_len_v04; /// use libdd_trace_utils::span::v04::SpanSlice; /// /// let span = SpanSlice { @@ -214,11 +212,11 @@ pub fn to_vec_with_capacity]>>( /// ..Default::default() /// }; /// let traces = vec![vec![span]]; -/// let encoded_len = to_encoded_byte_len(&traces); +/// let encoded_len = to_encoded_byte_len_v04(&traces); /// /// assert!(encoded_len > 0); /// ``` -pub fn to_encoded_byte_len]>>(traces: &[S]) -> u32 { +pub fn to_encoded_byte_len_v04]>>(traces: &[S]) -> u32 { let mut counter = super::CountLength(0); // `CountLength` impls `std::io::Write` (whose error type is `std::io::Error`, not // `Infallible`), so we can't statically prove infallibility via `unwrap_infallible` @@ -233,11 +231,11 @@ pub fn to_encoded_byte_len]>>(traces: &[S]) -> u /// does not advertise `/v1.0/traces`). The output is a msgpack array of traces, where each /// trace is itself a msgpack array of v0.4-shaped spans — matching the existing v0.4 wire /// format produced by [`to_vec`]. -fn encode_payload_v1_to_v04( +fn encode_payload_v1( writer: &mut W, payload: &TracerPayload, ) -> Result<(), ValueWriteError> { - use span_v1::{encode_span_v1_to_v04, ChunkContext}; + use span_v1::{encode_span, ChunkContext}; write_array_len(writer, payload.chunks.len() as u32)?; for chunk in &payload.chunks { @@ -250,7 +248,7 @@ fn encode_payload_v1_to_v04( }; write_array_len(writer, chunk.spans.len() as u32)?; for span in &chunk.spans { - encode_span_v1_to_v04(writer, span, &ctx)?; + encode_span(writer, span, &ctx)?; } } Ok(()) @@ -262,19 +260,16 @@ fn encode_payload_v1_to_v04( /// The output is byte-compatible with [`to_vec`] for equivalent data — chunk-level fields are /// propagated to every span and typed attributes are bucketed into the v0.4 `meta` / /// `metrics` / `meta_struct` maps per [`span_v1_to_v04`]'s mapping table. -pub fn to_vec_v1_to_v04(payload: &TracerPayload) -> Vec { - to_vec_v1_to_v04_with_capacity(payload, 0) +pub fn to_vec_v1(payload: &TracerPayload) -> Vec { + to_vec_with_capacity_v1(payload, 0) } /// Serializes a [`TracerPayload`] as a v0.4 msgpack payload with a caller-supplied initial /// capacity. Use this when you can size the buffer up front (e.g. from -/// [`to_encoded_byte_len_v1_to_v04`]) to avoid reallocations. -pub fn to_vec_v1_to_v04_with_capacity( - payload: &TracerPayload, - capacity: u32, -) -> Vec { +/// [`to_encoded_byte_len_v1`]) to avoid reallocations. +pub fn to_vec_with_capacity_v1(payload: &TracerPayload, capacity: u32) -> Vec { let mut buf = ByteBuf::with_capacity(capacity as usize); - encode_payload_v1_to_v04(&mut buf, payload) + encode_payload_v1(&mut buf, payload) .map_err(super::flatten_value_write_infallible) .unwrap_infallible(); buf.into_vec() @@ -286,17 +281,17 @@ pub fn to_vec_v1_to_v04_with_capacity( /// # Errors /// /// Returns any [`ValueWriteError`] from the underlying writer (typically buffer-too-small). -pub fn write_v1_to_v04_to_slice( +pub fn write_to_slice_v1( slice: &mut &mut [u8], payload: &TracerPayload, ) -> Result<(), ValueWriteError> { - encode_payload_v1_to_v04(slice, payload) + encode_payload_v1(slice, payload) } -/// Returns the exact number of bytes [`to_vec_v1_to_v04`] would write for `payload`. Walks +/// Returns the exact number of bytes [`to_vec_v1`] would write for `payload`. Walks /// the payload through a counting writer without allocating an output buffer. -pub fn to_encoded_byte_len_v1_to_v04(payload: &TracerPayload) -> u32 { +pub fn to_encoded_byte_len_v1(payload: &TracerPayload) -> u32 { let mut counter = super::CountLength(0); - let _ = encode_payload_v1_to_v04(&mut counter, payload); + let _ = encode_payload_v1(&mut counter, payload); counter.0 } diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs index 4ccf319f30..1a0fd19bf0 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs @@ -13,7 +13,8 @@ use rmp::encode::{ }; use std::borrow::Borrow; -/// Encodes a `SpanLink` object into a slice of bytes. +/// Encodes a [`v04::SpanLink`](crate::span::v04::SpanLink) into the v0.4 msgpack wire format +/// (native encoding: input span shape and output wire format are both v0.4). /// /// # Arguments /// @@ -74,7 +75,8 @@ pub fn encode_span_links( Ok(()) } -/// Encodes a `SpanEvent` object into a slice of bytes. +/// Encodes a [`v04::SpanEvent`](crate::span::v04::SpanEvent) into the v0.4 msgpack wire format +/// (native encoding: v0.4 input → v0.4 output). /// /// # Arguments /// @@ -172,7 +174,8 @@ pub fn encode_span_events( Ok(()) } -/// Encodes a `Span` object into a slice of bytes. +/// Encodes a [`v04::Span`](crate::span::v04::Span) into the v0.4 msgpack wire format +/// (native encoding: v0.4 input → v0.4 output). /// /// # Arguments /// diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs index f03a11741a..d580cfd9b9 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs @@ -91,8 +91,10 @@ impl BucketCounts { } } -/// Encodes a `Span` in the v0.4 wire format, injecting chunk-level context into the span's -/// `meta` / `metrics` / `meta_struct` maps. +/// Encodes a [`v1::Span`](crate::span::v1::Span) into the v0.4 msgpack wire format +/// (downgrade: v1 input → v0.4 output). Chunk-level context (`trace_id`, `origin`, `priority`, +/// `sampling_mechanism`, chunk attributes) is injected into the span's `meta` / `metrics` / +/// `meta_struct` maps since v0.4 has no chunk concept. /// /// # Arguments /// @@ -109,7 +111,7 @@ impl BucketCounts { /// # Errors /// /// This function will return any error emitted by the writer. -pub(super) fn encode_span_v1_to_v04( +pub(super) fn encode_span( writer: &mut W, span: &Span, chunk: &ChunkContext<'_, T>, @@ -331,7 +333,8 @@ fn write_meta_struct_entry( Ok(()) } -/// Converts v1 `SpanLink`s to the v0.4 wire format. The 128-bit `trace_id` is split into +/// Encodes [`v1::SpanLink`](crate::span::v1::SpanLink)s into the v0.4 msgpack wire format +/// (downgrade: v1 input → v0.4 output). The 128-bit `trace_id` is split into /// `(trace_id, trace_id_high)` u64s. Typed link attributes are downgraded to strings (`String` /// passes through, `Bool` becomes `"true"` / `"false"`); non-string-coercible variants are /// dropped because v0.4 link attributes are `String → String` only. @@ -398,9 +401,10 @@ fn encode_span_links( Ok(()) } -/// Converts v1 `SpanEvent`s to the v0.4 wire format. Typed attributes are downgraded to the -/// v0.4 `{"type": , "_value": ...}` shape — see `write_event_attr_value`. `Bytes` -/// and `KeyValue` have no v0.4 event-attribute equivalent and are dropped. +/// Encodes [`v1::SpanEvent`](crate::span::v1::SpanEvent)s into the v0.4 msgpack wire format +/// (downgrade: v1 input → v0.4 output). Typed attributes are downgraded to the v0.4 +/// `{"type": , "_value": ...}` shape — see `write_event_attr_value`. `Bytes` and +/// `KeyValue` have no v0.4 event-attribute equivalent and are dropped. fn encode_span_events( writer: &mut W, span_events: &[SpanEvent], @@ -571,7 +575,7 @@ fn write_event_array_element( #[cfg(test)] mod tests { //! Unit tests for the v1::Span → v0.4 downgrade encoder. Each test encodes a small - //! `TracerPayload` via [`super::super::to_vec_v1_to_v04`] and decodes the bytes with + //! `TracerPayload` via [`super::super::to_vec_v1`] and decodes the bytes with //! `rmpv` to assert on the resulting v0.4 shape — this implicitly checks that the output //! is also valid msgpack consumable by any standard v0.4 decoder (test-agent, agent, etc.). use crate::span::v1::{ @@ -590,7 +594,7 @@ mod tests { /// Encodes `payload` and decodes back into `rmpv::Value`. The top level of v0.4 is an /// array of traces; this helper returns it as a `Vec` so tests can index in. fn encode_and_decode(payload: &TracerPayloadBytes) -> Vec { - let bytes = super::super::to_vec_v1_to_v04(payload); + let bytes = super::super::to_vec_v1(payload); let value = rmpv::decode::read_value(&mut &bytes[..]).expect("decode failed"); match value { Value::Array(traces) => traces, diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs index 9ae268c588..001c4f60df 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs @@ -109,8 +109,9 @@ pub(super) const FLAT_ATTR_STRIDE: u32 = 3; /// incrementing integer ID. On subsequent occurrences only the ID is emitted as a msgpack `uint`. /// ID 0 is reserved for the empty string (pre-inserted in the constructor). /// -/// The string table is scoped per payload: each `to_vec` / `write_to_slice` call starts with a -/// fresh table so deduplication is payload-local. +/// The string table is scoped per payload: each `to_vec_v04` / `to_vec_from_payload_v1` (and +/// their `write_to_slice_v04` / `write_to_slice_v1` counterparts) call starts with a fresh +/// table so deduplication is payload-local. pub(crate) struct StringTable { seen: HashMap, } @@ -315,7 +316,7 @@ where /// trace_key::CHUNKS (11) → Array[Chunk, ...] /// } /// ``` -fn encode_payload]>>( +fn encode_payload_v04]>>( writer: &mut W, traces: &[S], metadata: &TracerMetadata, @@ -342,7 +343,7 @@ fn encode_payload]>>( write_uint8(writer, trace_key::CHUNKS)?; write_array_len(writer, traces.len() as u32)?; for trace in traces { - encode_chunk(writer, trace.as_ref(), &mut table)?; + encode_chunk_v04(writer, trace.as_ref(), &mut table)?; } if !metadata.language.is_empty() { @@ -411,7 +412,7 @@ fn encode_payload]>>( /// chunk_key::SPANS (4) → Array[Span, ...] /// } /// ``` -fn encode_chunk( +fn encode_chunk_v04( writer: &mut W, spans: &[Span], table: &mut StringTable, @@ -456,31 +457,31 @@ fn encode_chunk( /// /// # Errors /// Returns a `ValueWriteError` if the underlying writer fails. -pub fn write_to_slice]>>( +pub fn write_to_slice_v04]>>( // &mut &mut [u8] lets the caller see the slice shrink as bytes are written. slice: &mut &mut [u8], traces: &[S], metadata: &TracerMetadata, ) -> Result<(), ValueWriteError> { - encode_payload(slice, traces, metadata) + encode_payload_v04(slice, traces, metadata) } /// Serializes traces into a `Vec` using the V1 msgpack format. -pub fn to_vec]>>( +pub fn to_vec_v04]>>( traces: &[S], metadata: &TracerMetadata, ) -> Vec { - to_vec_with_capacity(traces, 0, metadata) + to_vec_with_capacity_v04(traces, 0, metadata) } /// Serializes traces into a `Vec` with a pre-allocated capacity. -pub fn to_vec_with_capacity]>>( +pub fn to_vec_with_capacity_v04]>>( traces: &[S], capacity: u32, metadata: &TracerMetadata, ) -> Vec { let mut buf = ByteBuf::with_capacity(capacity as usize); - encode_payload(&mut buf, traces, metadata) + encode_payload_v04(&mut buf, traces, metadata) .map_err(super::flatten_value_write_infallible) .unwrap_infallible(); buf.into_vec() @@ -497,7 +498,7 @@ pub fn to_encoded_byte_len]>>( // the way we do for `ByteBuf`. In practice `CountLength::write*` only ever return // `Ok`, so the error path here is unreachable today; should `CountLength` ever grow // a fallible code path, fuzz tests on the msgpack encoded length would catch it. - let _ = encode_payload(&mut counter, traces, metadata); + let _ = encode_payload_v04(&mut counter, traces, metadata); counter.0 } @@ -652,7 +653,7 @@ fn encode_chunk_v1( /// assert!(!encoded.is_empty()); /// ``` pub fn to_vec_from_payload_v1(payload: &TracerPayload) -> Vec { - to_vec_from_payload_with_capacity_v1(payload, 0) + to_vec_with_capacity_from_payload_v1(payload, 0) } /// Serializes a `TracerPayload` into a vector of bytes with specified capacity. @@ -669,18 +670,18 @@ pub fn to_vec_from_payload_v1(payload: &TracerPayload) -> Vec= 1024); /// ``` -pub fn to_vec_from_payload_with_capacity_v1( +pub fn to_vec_with_capacity_from_payload_v1( payload: &TracerPayload, capacity: u32, ) -> Vec { @@ -710,7 +711,7 @@ pub fn to_vec_from_payload_with_capacity_v1( /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v1::write_payload_to_slice_v1; +/// use libdd_trace_utils::msgpack_encoder::v1::write_to_slice_v1; /// use libdd_trace_utils::span::v1::TracerPayloadSlice; /// /// let mut buffer = vec![0u8; 1024]; @@ -719,9 +720,9 @@ pub fn to_vec_from_payload_with_capacity_v1( /// ..Default::default() /// }; /// -/// write_payload_to_slice_v1(&mut &mut buffer[..], &payload).expect("Encoding failed"); +/// write_to_slice_v1(&mut &mut buffer[..], &payload).expect("Encoding failed"); /// ``` -pub fn write_payload_to_slice_v1( +pub fn write_to_slice_v1( slice: &mut &mut [u8], payload: &TracerPayload, ) -> Result<(), ValueWriteError> { @@ -791,14 +792,14 @@ mod tests { fn test_to_vec_non_empty() { let spans = vec![make_span("svc", "op", 42, 1, 0)]; let traces = vec![spans]; - let encoded = to_vec(&traces, &TracerMetadata::default()); + let encoded = to_vec_v04(&traces, &TracerMetadata::default()); assert!(!encoded.is_empty()); } #[test] fn test_to_vec_empty_traces() { let traces: Vec> = vec![]; - let encoded = to_vec(&traces, &TracerMetadata::default()); + let encoded = to_vec_v04(&traces, &TracerMetadata::default()); // Must still produce a valid msgpack map with an empty chunks array. assert!(!encoded.is_empty()); } @@ -814,8 +815,8 @@ mod tests { let s_single = make_span("my-service", "op1", 1, 1, 0); let traces_single = vec![vec![s_single]]; - let encoded_two = to_vec(&traces_two, &TracerMetadata::default()); - let encoded_single = to_vec(&traces_single, &TracerMetadata::default()); + let encoded_two = to_vec_v04(&traces_two, &TracerMetadata::default()); + let encoded_single = to_vec_v04(&traces_single, &TracerMetadata::default()); // The two-trace payload should be less than 2× the single-trace payload // if interning is working (the second "my-service" is encoded as an integer). @@ -850,7 +851,7 @@ mod tests { ..Default::default() }; - let encoded = to_vec(&[vec![root]], &TracerMetadata::default()); + let encoded = to_vec_v04(&[vec![root]], &TracerMetadata::default()); assert!(!encoded.is_empty()); // The payload must contain "lambda" somewhere (the origin string). let lambda_bytes = b"lambda"; @@ -870,7 +871,7 @@ mod tests { ]; let traces = vec![spans]; let meta = TracerMetadata::default(); - let encoded = to_vec(&traces, &meta); + let encoded = to_vec_v04(&traces, &meta); let len = to_encoded_byte_len(&traces, &meta); assert_eq!(encoded.len() as u32, len); } @@ -897,7 +898,7 @@ mod tests { ..Default::default() }; - let encoded = to_vec(&[vec![root]], &TracerMetadata::default()); + let encoded = to_vec_v04(&[vec![root]], &TracerMetadata::default()); assert!(!encoded.is_empty()); } @@ -932,7 +933,7 @@ mod tests { ..Default::default() }; - let encoded = to_vec(&[vec![span]], &TracerMetadata::default()); + let encoded = to_vec_v04(&[vec![span]], &TracerMetadata::default()); let prod_bytes = b"prod"; assert!( encoded.windows(prod_bytes.len()).any(|w| w == prod_bytes), @@ -972,7 +973,7 @@ mod tests { ..Default::default() }; - let encoded = to_vec(&[vec![span]], &TracerMetadata::default()); + let encoded = to_vec_v04(&[vec![span]], &TracerMetadata::default()); // Both attribute strings must appear in the payload bytes. let ssi_bytes = b"ssi"; @@ -1002,7 +1003,7 @@ mod tests { fn test_payload_attributes_absent_when_no_relevant_tags() { // A span with no _dd.apm_mode or _dd.git.commit.sha must not produce key 10. let span = make_span("svc", "op", 1, 1, 0); - let encoded = to_vec(&[vec![span]], &TracerMetadata::default()); + let encoded = to_vec_v04(&[vec![span]], &TracerMetadata::default()); let apm_key = b"_dd.apm_mode"; assert!( !encoded.windows(apm_key.len()).any(|w| w == apm_key), @@ -1020,7 +1021,7 @@ mod tests { runtime_id: "abc-123-uuid".to_string(), ..Default::default() }; - let encoded = to_vec(&[vec![span]], &metadata); + let encoded = to_vec_v04(&[vec![span]], &metadata); for s in &[b"python" as &[u8], b"3.11", b"2.0.0", b"abc-123-uuid"] { assert!( @@ -1034,14 +1035,14 @@ mod tests { #[test] fn test_payload_metadata_absent_when_empty() { let span = make_span("svc", "op", 1, 1, 0); - let encoded_with = to_vec( + let encoded_with = to_vec_v04( &[vec![span.clone()]], &TracerMetadata { language: "go".to_string(), ..Default::default() }, ); - let encoded_without = to_vec(&[vec![span]], &TracerMetadata::default()); + let encoded_without = to_vec_v04(&[vec![span]], &TracerMetadata::default()); // Payload with metadata must be larger (it carries extra fields). assert!(encoded_with.len() > encoded_without.len()); } @@ -1065,7 +1066,7 @@ mod tests { meta, ..Default::default() }; - let encoded = to_vec(&[vec![span]], &TracerMetadata::default()); + let encoded = to_vec_v04(&[vec![span]], &TracerMetadata::default()); // Expected 16-byte BE: high = 0x640cfd5400000000, low = 0x0123456789abcdef let expected = [ @@ -1088,7 +1089,7 @@ mod tests { fn test_128bit_trace_id_without_dd_p_tid() { // Absent _dd.p.tid → high 64 bits zero. let span = make_span("svc", "op", 0x0123456789abcdef, 1, 0); - let encoded = to_vec(&[vec![span]], &TracerMetadata::default()); + let encoded = to_vec_v04(&[vec![span]], &TracerMetadata::default()); let expected = [ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, @@ -1120,7 +1121,7 @@ mod tests { meta, ..Default::default() }; - let encoded = to_vec(&[vec![root]], &TracerMetadata::default()); + let encoded = to_vec_v04(&[vec![root]], &TracerMetadata::default()); // The chunk-level sampling_mechanism (key 7) must be encoded as uint 4. // The byte sequence is `chunk_key::SAMPLING_MECHANISM (0x07)` followed by the @@ -1184,7 +1185,7 @@ mod tests { meta: meta3, ..Default::default() }; - let encoded = to_vec(&[vec![s1, s2, s3]], &TracerMetadata::default()); + let encoded = to_vec_v04(&[vec![s1, s2, s3]], &TracerMetadata::default()); // Each attribute must be present at chunk level — collected from a different // non-root span. diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs b/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs index 56c2051133..6b33f925e1 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs @@ -30,10 +30,9 @@ fn span_kind_from_str(s: &str) -> u32 { } } -/// Encodes span links into the V1 format. -/// -/// Uses integer keys and string interning for string values. Each span link's -/// trace ID is encoded as a 16-byte big-endian binary. +/// Encodes [`v04::SpanLink`](crate::span::v04::SpanLink)s into the V1 msgpack wire format +/// (upgrade: v0.4 input → V1 output). Uses integer keys and string interning for string +/// values. Each span link's trace ID is encoded as a 16-byte big-endian binary. pub fn encode_span_links( writer: &mut W, span_links: &[SpanLink], @@ -84,9 +83,9 @@ pub fn encode_span_links( Ok(()) } -/// Encodes span events into the V1 format. -/// -/// Uses integer keys and string interning. Attribute values are type-tagged. +/// Encodes [`v04::SpanEvent`](crate::span::v04::SpanEvent)s into the V1 msgpack wire format +/// (upgrade: v0.4 input → V1 output). Uses integer keys and string interning. Attribute values +/// are type-tagged. pub fn encode_span_events( writer: &mut W, span_events: &[SpanEvent], @@ -175,7 +174,8 @@ fn encode_attribute_any_value( Ok(()) } -/// Encodes a v0.4 span into the V1 msgpack format. +/// Encodes a [`v04::Span`](crate::span::v04::Span) into the V1 msgpack wire format +/// (upgrade: v0.4 input → V1 output). /// /// Key differences from v0.4: /// - Uses integer keys for all fields. diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs index aa3e17be20..9b1b6e8d52 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs @@ -111,7 +111,8 @@ pub(super) fn encode_attributes_map( Ok(()) } -/// Encodes a `SpanLink` object into a slice of bytes. +/// Encodes a [`v1::SpanLink`](crate::span::v1::SpanLink) into the V1 msgpack wire format +/// (native encoding: V1 input → V1 output). /// /// # Arguments /// @@ -171,7 +172,8 @@ pub(super) fn encode_span_links( Ok(()) } -/// Encodes a `SpanEvent` object into a slice of bytes. +/// Encodes a [`v1::SpanEvent`](crate::span::v1::SpanEvent) into the V1 msgpack wire format +/// (native encoding: V1 input → V1 output). /// /// # Arguments /// @@ -216,7 +218,8 @@ pub(super) fn encode_span_events( Ok(()) } -/// Encodes a `Span` object into a slice of bytes. +/// Encodes a [`v1::Span`](crate::span::v1::Span) into the V1 msgpack wire format +/// (native encoding: V1 input → V1 output). /// /// # Arguments /// diff --git a/libdd-trace-utils/src/send_data/mod.rs b/libdd-trace-utils/src/send_data/mod.rs index 53d14d8ecd..371c4909d5 100644 --- a/libdd-trace-utils/src/send_data/mod.rs +++ b/libdd-trace-utils/src/send_data/mod.rs @@ -384,7 +384,7 @@ impl SendData { headers.insert(DATADOG_TRACE_COUNT, chunks.into()); headers.insert(CONTENT_TYPE, APPLICATION_MSGPACK); - let payload = msgpack_encoder::v04::to_vec(payload); + let payload = msgpack_encoder::v04::to_vec_v04(payload); futures.push(self.send_payload( capabilities, @@ -534,7 +534,7 @@ mod tests { total } TracerPayloadCollection::V04(payloads) => { - msgpack_encoder::v04::to_encoded_byte_len(payloads) as usize + msgpack_encoder::v04::to_encoded_byte_len_v04(payloads) as usize } TracerPayloadCollection::V05(payloads) => rmp_serde::to_vec(payloads).unwrap().len(), } diff --git a/libdd-trace-utils/tests/test_send_data.rs b/libdd-trace-utils/tests/test_send_data.rs index 30a58847c7..7adcafd615 100644 --- a/libdd-trace-utils/tests/test_send_data.rs +++ b/libdd-trace-utils/tests/test_send_data.rs @@ -556,7 +556,7 @@ mod tracing_integration_tests { #[cfg_attr(miri, ignore)] #[tokio::test] async fn compare_v04_and_v1_encoders_snapshot_test() { - use libdd_trace_utils::msgpack_encoder::v1::{to_vec, to_vec_from_payload_v1}; + use libdd_trace_utils::msgpack_encoder::v1::{to_vec_from_payload_v1, to_vec_v04}; use libdd_trace_utils::span::v04::SpanBytes as V04SpanBytes; use libdd_trace_utils::span::v1::{ AttributeValue, SpanBytes as V1SpanBytes, SpanKind, TraceChunkBytes, TracerPayloadBytes, @@ -619,7 +619,7 @@ mod tracing_integration_tests { }; // ── POST both into the same session ──────────────────────────────────────── - let bytes_v04 = to_vec(&v04_traces, &metadata); + let bytes_v04 = to_vec_v04(&v04_traces, &metadata); let bytes_v1 = to_vec_from_payload_v1(&v1_payload); post_msgpack_traces(uri.clone(), bytes_v04).await; post_msgpack_traces(uri, bytes_v1).await; @@ -630,7 +630,7 @@ mod tracing_integration_tests { } /// Round-trip: the v1::Span → v0.4 downgrade encoder - /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec_v1_to_v04`]) must produce a v0.4 + /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec_v1`]) must produce a v0.4 /// payload that decodes to the same canonical form as the native v0.4 encoder /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec`]) when fed equivalent input. /// @@ -640,8 +640,8 @@ mod tracing_integration_tests { #[cfg_attr(miri, ignore)] #[tokio::test] async fn compare_v04_native_and_v1_to_v04_encoders_snapshot_test() { - use libdd_trace_utils::msgpack_encoder::v04::to_vec as to_vec_v04_native; - use libdd_trace_utils::msgpack_encoder::v04::to_vec_v1_to_v04; + use libdd_trace_utils::msgpack_encoder::v04::to_vec_v04 as to_vec_v04_native; + use libdd_trace_utils::msgpack_encoder::v04::to_vec_v1; use libdd_trace_utils::span::v04::SpanBytes as V04SpanBytes; use libdd_trace_utils::span::v1::{ AttributeValue, SpanBytes as V1SpanBytes, SpanKind, TraceChunkBytes, TracerPayloadBytes, @@ -706,7 +706,7 @@ mod tracing_integration_tests { // ── Encode each via its respective encoder and POST both ─────────────────── let bytes_v04 = to_vec_v04_native(&v04_traces); - let bytes_v1_to_v04 = to_vec_v1_to_v04(&v1_payload); + let bytes_v1_to_v04 = to_vec_v1(&v1_payload); post_msgpack_traces(uri.clone(), bytes_v04).await; post_msgpack_traces(uri, bytes_v1_to_v04).await; From 22c104454dd099a582c621d3a40a6efc0d87add0 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Thu, 25 Jun 2026 14:52:48 +0200 Subject: [PATCH 04/23] fix: apply comments --- .../src/msgpack_encoder/v04/span_v1.rs | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs index d580cfd9b9..200d128cbe 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs @@ -34,6 +34,15 @@ use rmp::encode::{ }; use std::borrow::Borrow; +/// Writes a `bool` as the v0.4 string representation (`"true"` / `"false"`). Used wherever a +/// typed V1 `Bool` attribute is downgraded into v0.4 `meta` (which is `String → String` only). +fn write_bool_as_str( + writer: &mut W, + b: bool, +) -> Result<(), ValueWriteError> { + write_str(writer, if b { "true" } else { "false" }) +} + /// Chunk-level context propagated into every span when downgrading to v0.4. Built once per /// chunk by the top-level encoder and passed by reference to `encode_span_v1_to_v04`. pub(super) struct ChunkContext<'a, T: TraceData> { @@ -56,8 +65,8 @@ fn span_kind_to_meta(kind: SpanKind) -> Option<&'static str> { } } -/// Splits a 128-bit big-endian trace_id into `(low_64, high_64)`. The low half maps to v0.4's -/// `trace_id` field; the high half goes to `meta["_dd.p.tid"]` as hex when non-zero. +/// Splits a 128-bit big-endian trace_id into big-endian `(low_64, high_64)`. The low half maps to +/// v0.4's `trace_id` field; the high half goes to `meta["_dd.p.tid"]` as hex when non-zero. #[inline] fn split_trace_id(trace_id: &[u8; 16]) -> (u64, u64) { let mut high_bytes = [0u8; 8]; @@ -228,7 +237,7 @@ pub(super) fn encode_span( } AttributeValue::Bool(b) => { write_str(writer, k.borrow())?; - write_str(writer, if *b { "true" } else { "false" })?; + write_bool_as_str(writer, *b)?; } _ => {} } @@ -241,7 +250,7 @@ pub(super) fn encode_span( } AttributeValue::Bool(b) => { write_str(writer, k.borrow())?; - write_str(writer, if *b { "true" } else { "false" })?; + write_bool_as_str(writer, *b)?; } _ => {} } @@ -326,7 +335,7 @@ fn write_meta_struct_entry( } AttributeValue::KeyValue(_) | AttributeValue::List(_) => { write_str(writer, k.borrow())?; - write_bin(writer, &[])?; + write_bin(writer, todo!())?; } _ => {} } @@ -335,9 +344,9 @@ fn write_meta_struct_entry( /// Encodes [`v1::SpanLink`](crate::span::v1::SpanLink)s into the v0.4 msgpack wire format /// (downgrade: v1 input → v0.4 output). The 128-bit `trace_id` is split into -/// `(trace_id, trace_id_high)` u64s. Typed link attributes are downgraded to strings (`String` -/// passes through, `Bool` becomes `"true"` / `"false"`); non-string-coercible variants are -/// dropped because v0.4 link attributes are `String → String` only. +/// `(trace_id, trace_id_high)` u64s. Typed link attributes are downgraded to strings; +/// non-string-coercible variants are dropped because v0.4 link attributes are `String → String` +/// only. fn encode_span_links( writer: &mut W, span_links: &[SpanLink], @@ -380,7 +389,7 @@ fn encode_span_links( } AttributeValue::Bool(b) => { write_str(writer, k.borrow())?; - write_str(writer, if *b { "true" } else { "false" })?; + write_bool_as_str(writer, *b)?; } _ => {} } From 8936d5bb09ebebf132acbb31b25292bb6efa07ad Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Mon, 29 Jun 2026 15:58:58 +0200 Subject: [PATCH 05/23] feat: add v1 decoder --- .../src/trace_exporter/trace_serializer.rs | 61 +- libdd-trace-utils/src/msgpack_decoder/mod.rs | 1 + .../src/msgpack_decoder/v1/mod.rs | 658 ++++++++++++++++++ .../src/msgpack_decoder/v1/span.rs | 330 +++++++++ libdd-trace-utils/src/send_data/mod.rs | 21 + libdd-trace-utils/src/trace_utils.rs | 62 +- libdd-trace-utils/src/tracer_payload.rs | 90 ++- ...compare_v1_full_payload_snapshot_test.json | 60 ++ libdd-trace-utils/tests/test_send_data.rs | 125 ++++ 9 files changed, 1344 insertions(+), 64 deletions(-) create mode 100644 libdd-trace-utils/src/msgpack_decoder/v1/mod.rs create mode 100644 libdd-trace-utils/src/msgpack_decoder/v1/span.rs create mode 100644 libdd-trace-utils/tests/snapshots/compare_v1_full_payload_snapshot_test.json diff --git a/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs b/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs index 7cfd62a5a0..37eb1ea7cd 100644 --- a/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs +++ b/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs @@ -17,7 +17,7 @@ use libdd_trace_utils::msgpack_encoder; use libdd_trace_utils::span::{v04::Span, TraceData}; use libdd_trace_utils::trace_utils::{self, TracerHeaderTags}; use libdd_trace_utils::tracer_metadata::TracerMetadata; -use libdd_trace_utils::tracer_payload::{self, TraceEncoding}; +use libdd_trace_utils::tracer_payload::{self}; /// Minimal capacity of fresh buffers allocated to encode traces, in bytes. const MIN_BUFFER_CAPACITY: usize = 1024; @@ -59,7 +59,7 @@ impl TraceSerializer { let chunks = payload.size(); let headers = self.build_traces_headers(header_tags, chunks, agent_payload_response_version); - let mp_payload = self.serialize_payload(&payload, metadata)?; + let mp_payload = self.serialize_payload(&payload, metadata, output_format)?; Ok(PreparedTracesPayload { data: mp_payload, @@ -78,12 +78,19 @@ impl TraceSerializer { TraceExporterError::Deserialization(DecodeError::InvalidFormat(e.to_string())) }; match output_format { - TraceExporterOutputFormat::V1 => Ok(tracer_payload::TraceChunks::V1(traces)), - TraceExporterOutputFormat::V04 => { - trace_utils::collect_trace_chunks(traces, TraceEncoding::V04).map_err(map_err) + // v0.4 input spans are kept as-is in `TraceChunks::V04`. Whether they go out as v0.4 + // or are cross-encoded into V1 on the wire is decided in `serialize_payload`. + // + // APMSP-2812 - TODO: when the data-pipeline gains a V1-native input model (its own + // `v1::Span`-shaped builder), route `OutputFormat::V1` to + // `TraceChunks::V1(v1::TracerPayload)` instead and serialize via + // `to_vec_from_payload_v1`. A `StatSpan` impl on `v1::Span` will also be needed + // if client-side stats are enabled on the V1-native path. + TraceExporterOutputFormat::V04 | TraceExporterOutputFormat::V1 => { + Ok(tracer_payload::TraceChunks::V04(traces)) } TraceExporterOutputFormat::V05 => { - trace_utils::collect_trace_chunks(traces, TraceEncoding::V05).map_err(map_err) + trace_utils::convert_trace_chunks_v04_to_v05(traces).map_err(map_err) } } } @@ -113,23 +120,43 @@ impl TraceSerializer { &self, payload: &tracer_payload::TraceChunks, metadata: &TracerMetadata, + output_format: TraceExporterOutputFormat, ) -> Result, TraceExporterError> { let capacity = self .previous_serialised_len .load(Ordering::Relaxed) .max(MIN_BUFFER_CAPACITY); - let buff = match payload { - tracer_payload::TraceChunks::V04(p) => { + let buff = match (payload, output_format) { + (tracer_payload::TraceChunks::V04(p), TraceExporterOutputFormat::V04) => { msgpack_encoder::v04::to_vec_with_capacity(p, capacity as u32) } - tracer_payload::TraceChunks::V05(p) => { + // v0.4 spans cross-encoded as V1 on the wire — used when the agent advertises + // /v1.0/traces. Same in-memory shape as the v0.4 native path, different encoder. + (tracer_payload::TraceChunks::V04(p), TraceExporterOutputFormat::V1) => { + msgpack_encoder::v1::to_vec_with_capacity(p, capacity as u32, metadata) + } + (tracer_payload::TraceChunks::V05(p), TraceExporterOutputFormat::V05) => { let mut buff = Vec::with_capacity(capacity); rmp_serde::encode::write(&mut buff, p) .map_err(TraceExporterError::Serialization)?; buff } - tracer_payload::TraceChunks::V1(p) => { - msgpack_encoder::v1::to_vec_with_capacity(p, capacity as u32, metadata) + // APMSP-2812 - TODO: native V1 input — call + // `msgpack_encoder::v1::to_vec_from_payload_v1` on the carried + // `v1::TracerPayload`. Not yet reachable: `collect_and_process_traces` + // never produces `TraceChunks::V1` in the current data-pipeline path. + (tracer_payload::TraceChunks::V1(_), TraceExporterOutputFormat::V1) => { + todo!("Native V1 input serialization not yet implemented (APMSP-2812)") + } + // `collect_and_process_traces` only produces (V04, V04|V1), (V05, V05), + // or (V1, V1) — any other combination here is a programming error. + _ => { + return Err(TraceExporterError::Deserialization( + DecodeError::InvalidFormat( + "Unsupported (TraceChunks, OutputFormat) combination for serialization" + .to_owned(), + ), + )); } }; self.previous_serialised_len @@ -277,7 +304,11 @@ mod tests { .collect_and_process_traces(original_traces.clone(), TraceExporterOutputFormat::V04) .unwrap(); - let result = serializer.serialize_payload(&payload, &TracerMetadata::default()); + let result = serializer.serialize_payload( + &payload, + &TracerMetadata::default(), + TraceExporterOutputFormat::V04, + ); assert!(result.is_ok()); let serialized = result.unwrap(); @@ -312,7 +343,11 @@ mod tests { .collect_and_process_traces(original_traces.clone(), TraceExporterOutputFormat::V05) .unwrap(); - let result = serializer.serialize_payload(&payload, &TracerMetadata::default()); + let result = serializer.serialize_payload( + &payload, + &TracerMetadata::default(), + TraceExporterOutputFormat::V05, + ); assert!(result.is_ok()); let serialized = result.unwrap(); diff --git a/libdd-trace-utils/src/msgpack_decoder/mod.rs b/libdd-trace-utils/src/msgpack_decoder/mod.rs index 6a07361389..d22f2fd788 100644 --- a/libdd-trace-utils/src/msgpack_decoder/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/mod.rs @@ -4,3 +4,4 @@ pub mod decode; pub mod v04; pub mod v05; +pub mod v1; diff --git a/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs new file mode 100644 index 0000000000..a91568bd08 --- /dev/null +++ b/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs @@ -0,0 +1,658 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +pub(super) mod span; + +use crate::msgpack_decoder::decode::buffer::Buffer; +use crate::msgpack_decoder::decode::error::DecodeError; +use crate::span::v1::{TraceChunk, TracerPayload, TracerPayloadBytes, TracerPayloadSlice}; +use crate::span::DeserializableTraceData; +use rmp::decode; +use std::borrow::Borrow; + +// Integer keys used by the V1 wire format. Kept in sync with the encoder side +// (`msgpack_encoder::v1::{trace_key, chunk_key, SpanKey, SpanLinkKey, SpanEventKey, AnyValueKey}`). + +pub(super) mod trace_key { + pub const LANGUAGE_NAME: u8 = 3; + pub const LANGUAGE_VERSION: u8 = 4; + pub const TRACER_VERSION: u8 = 5; + pub const RUNTIME_ID: u8 = 6; + pub const ENV_REF: u8 = 7; + pub const HOSTNAME_REF: u8 = 8; + pub const APP_VERSION_REF: u8 = 9; + pub const ATTRIBUTES: u8 = 10; + pub const CHUNKS: u8 = 11; +} + +pub(super) mod chunk_key { + pub const PRIORITY: u8 = 1; + pub const ORIGIN: u8 = 2; + pub const ATTRIBUTES: u8 = 3; + pub const SPANS: u8 = 4; + pub const DROPPED_TRACE: u8 = 5; + pub const TRACE_ID: u8 = 6; + pub const SAMPLING_MECHANISM: u8 = 7; +} + +pub(super) mod span_key { + pub const SERVICE: u8 = 1; + pub const NAME: u8 = 2; + pub const RESOURCE: u8 = 3; + pub const SPAN_ID: u8 = 4; + pub const PARENT_ID: u8 = 5; + pub const START: u8 = 6; + pub const DURATION: u8 = 7; + pub const ERROR: u8 = 8; + pub const ATTRIBUTES: u8 = 9; + pub const TYPE: u8 = 10; + pub const SPAN_LINKS: u8 = 11; + pub const SPAN_EVENTS: u8 = 12; + pub const ENV: u8 = 13; + pub const VERSION: u8 = 14; + pub const COMPONENT: u8 = 15; + pub const KIND: u8 = 16; +} + +pub(super) mod span_link_key { + pub const TRACE_ID: u8 = 1; + pub const SPAN_ID: u8 = 2; + pub const ATTRIBUTES: u8 = 3; + pub const TRACE_STATE: u8 = 4; + pub const FLAGS: u8 = 5; +} + +pub(super) mod span_event_key { + pub const TIME: u8 = 1; + pub const NAME: u8 = 2; + pub const ATTRIBUTES: u8 = 3; +} + +pub(super) const ANY_VALUE_KEY_STRING: u8 = 1; +pub(super) const ANY_VALUE_KEY_BOOL: u8 = 2; +pub(super) const ANY_VALUE_KEY_DOUBLE: u8 = 3; +pub(super) const ANY_VALUE_KEY_INT64: u8 = 4; +pub(super) const ANY_VALUE_KEY_BYTES: u8 = 5; +pub(super) const ANY_VALUE_KEY_ARRAY: u8 = 6; +pub(super) const ANY_VALUE_KEY_KEY_VALUE_LIST: u8 = 7; + +/// Number of msgpack items consumed per `[type, value]` pair in a typed `Array`. +pub(super) const TYPED_VALUE_STRIDE: u32 = 2; + +/// Number of msgpack items consumed per `[key, type, value]` triplet in a typed attributes map. +pub(super) const FLAT_ATTR_STRIDE: u32 = 3; + +/// Streaming string intern table built up as the payload is decoded. +/// +/// V1 strings are encoded inline the first time they appear (as msgpack `str`), and as a +/// msgpack `uint` reference on every subsequent occurrence. ID 0 is reserved for the empty +/// string and is pre-inserted on construction. +pub(super) struct StringTable +where + T::Text: Clone, +{ + seen: Vec, +} + +impl StringTable +where + T::Text: Clone, +{ + pub(super) fn new() -> Self { + Self { + seen: vec![T::Text::default()], + } + } + + /// Resolves a string reference by ID (encoded inline as msgpack `uint`). + fn resolve(&self, id: u64) -> Result { + usize::try_from(id) + .ok() + .and_then(|i| self.seen.get(i).cloned()) + .ok_or_else(|| { + DecodeError::InvalidFormat(format!( + "V1 string table reference out of range: id={id}, table_len={}", + self.seen.len() + )) + }) + } + + /// Records a freshly-read inline string and returns it (cloned for reuse). + fn record(&mut self, s: T::Text) -> T::Text { + self.seen.push(s.clone()); + s + } +} + +/// Reads a string-or-reference value at the current buffer position. +/// +/// Decides based on the next msgpack marker: +/// - `str`/`fixstr` → read and intern, return the value +/// - any unsigned int marker → resolve the table reference +pub(super) fn read_interned_string( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result +where + T::Text: Clone, +{ + let slice: &[u8] = buf.as_mut_slice(); + let marker_byte = *slice.first().ok_or_else(|| { + DecodeError::InvalidFormat( + "Unexpected end of V1 buffer when reading interned string".to_owned(), + ) + })?; + + // msgpack markers: + // fixstr : 0xa0..=0xbf + // str8/str16/str32 : 0xd9, 0xda, 0xdb + // fixint (positive): 0x00..=0x7f + // uint8/16/32/64 : 0xcc, 0xcd, 0xce, 0xcf + let is_string = matches!(marker_byte, 0xa0..=0xbf | 0xd9 | 0xda | 0xdb); + let is_uint = matches!(marker_byte, 0x00..=0x7f | 0xcc | 0xcd | 0xce | 0xcf); + + if is_string { + let s = buf.read_string()?; + Ok(table.record(s)) + } else if is_uint { + let id: u64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 interned string reference uint read failure".to_owned()) + })?; + table.resolve(id) + } else { + Err(DecodeError::InvalidFormat(format!( + "Unexpected msgpack marker 0x{marker_byte:02x} for V1 interned string" + ))) + } +} + +/// Decodes a V1 msgpack payload from owned bytes into a [`TracerPayloadBytes`]. +/// +/// # Returns +/// +/// * `Ok((payload, payload_size))` — the decoded payload and the number of bytes consumed from the +/// buffer. +/// * `Err(DecodeError)` — if the payload is malformed. +/// +/// # Errors +/// +/// Returns an error for any malformed map / array length, unknown map key, missing required +/// field, or any embedded msgpack read failure. +pub fn from_bytes( + data: libdd_tinybytes::Bytes, +) -> Result<(TracerPayloadBytes, usize), DecodeError> { + from_buffer(&mut Buffer::new(data)) +} + +/// Decodes a V1 msgpack payload from a borrowed slice into a [`TracerPayloadSlice`]. +/// The resulting payload borrows from the input buffer (same lifetime). +pub fn from_slice(data: &[u8]) -> Result<(TracerPayloadSlice<'_>, usize), DecodeError> { + from_buffer(&mut Buffer::new(data)) +} + +/// Generic over the deserialization mode (owned `BytesData` or borrowed `SliceData`). +#[allow(clippy::type_complexity)] +pub fn from_buffer( + data: &mut Buffer, +) -> Result<(TracerPayload, usize), DecodeError> +where + T::Text: Clone, +{ + let start_len = data.len(); + let mut table = StringTable::::new(); + let payload = decode_payload(data, &mut table)?; + let consumed = start_len - data.len(); + Ok((payload, consumed)) +} + +/// Decodes the top-level V1 payload map: tracer metadata fields + chunks array. +fn decode_payload( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result, DecodeError> +where + T::Text: Clone, +{ + let map_len = decode::read_map_len(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("Unable to read V1 payload map len".to_owned()))?; + + let mut payload = TracerPayload::::default(); + let mut saw_chunks = false; + + for _ in 0..map_len { + let key = decode::read_int::(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 payload key (u8) read failure".to_owned()) + })?; + match key { + trace_key::CHUNKS => { + payload.chunks = decode_chunks(buf, table)?; + saw_chunks = true; + } + trace_key::LANGUAGE_NAME => payload.language_name = read_interned_string(buf, table)?, + trace_key::LANGUAGE_VERSION => { + payload.language_version = read_interned_string(buf, table)? + } + trace_key::TRACER_VERSION => payload.tracer_version = read_interned_string(buf, table)?, + trace_key::RUNTIME_ID => payload.runtime_id = read_interned_string(buf, table)?, + trace_key::ENV_REF => payload.env = read_interned_string(buf, table)?, + trace_key::HOSTNAME_REF => payload.hostname = read_interned_string(buf, table)?, + trace_key::APP_VERSION_REF => payload.app_version = read_interned_string(buf, table)?, + trace_key::ATTRIBUTES => { + payload.attributes = span::read_attributes_map(buf, table)?; + } + unknown => { + return Err(DecodeError::InvalidFormat(format!( + "Unknown V1 payload key: {unknown}" + ))); + } + } + } + + if !saw_chunks { + return Err(DecodeError::InvalidFormat( + "V1 payload is missing the chunks field".to_owned(), + )); + } + + Ok(payload) +} + +fn decode_chunks( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result>, DecodeError> +where + T::Text: Clone, +{ + let count = decode::read_array_len(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("V1 chunks array len read failure".to_owned()))?; + let mut chunks = Vec::with_capacity(count as usize); + for _ in 0..count { + chunks.push(decode_chunk(buf, table)?); + } + Ok(chunks) +} + +fn decode_chunk( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result, DecodeError> +where + T::Text: Clone, +{ + let map_len = decode::read_map_len(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("V1 chunk map len read failure".to_owned()))?; + let mut chunk = TraceChunk::::default(); + let mut saw_trace_id = false; + let mut saw_spans = false; + + for _ in 0..map_len { + let key = decode::read_int::(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("V1 chunk key (u8) read failure".to_owned()))?; + match key { + chunk_key::TRACE_ID => { + let len = decode::read_bin_len(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 chunk trace_id bin len read failure".to_owned()) + })?; + if len != 16 { + return Err(DecodeError::InvalidFormat(format!( + "V1 chunk trace_id must be 16 bytes, got {len}" + ))); + } + let bytes = buf.try_slice_and_advance(16).ok_or_else(|| { + DecodeError::InvalidFormat("V1 chunk trace_id payload truncated".to_owned()) + })?; + let slice: &[u8] = bytes.borrow(); + chunk.trace_id.copy_from_slice(slice); + saw_trace_id = true; + } + chunk_key::SPANS => { + let count = decode::read_array_len(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 chunk spans array len read failure".to_owned()) + })?; + let mut spans = Vec::with_capacity(count as usize); + for _ in 0..count { + spans.push(span::decode_span(buf, table)?); + } + chunk.spans = spans; + saw_spans = true; + } + chunk_key::ORIGIN => chunk.origin = read_interned_string(buf, table)?, + chunk_key::PRIORITY => { + let v: i64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 chunk priority read failure".to_owned()) + })?; + chunk.priority = Some(v as i32); + } + chunk_key::SAMPLING_MECHANISM => { + let v: u64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat( + "V1 chunk sampling_mechanism read failure".to_owned(), + ) + })?; + chunk.sampling_mechanism = Some(v as u32); + } + chunk_key::ATTRIBUTES => { + chunk.attributes = span::read_attributes_map(buf, table)?; + } + chunk_key::DROPPED_TRACE => { + chunk.dropped_trace = decode::read_bool(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat( + "V1 chunk dropped_trace bool read failure".to_owned(), + ) + })?; + } + unknown => { + return Err(DecodeError::InvalidFormat(format!( + "Unknown V1 chunk key: {unknown}" + ))); + } + } + } + + if !saw_trace_id { + return Err(DecodeError::InvalidFormat( + "V1 chunk is missing trace_id".to_owned(), + )); + } + if !saw_spans { + return Err(DecodeError::InvalidFormat( + "V1 chunk is missing spans array".to_owned(), + )); + } + + Ok(chunk) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::msgpack_encoder::v1::to_vec_from_payload_v1; + use crate::span::v1::{ + AttributeValue, Span as V1Span, SpanBytes as V1SpanBytes, SpanKind, TraceChunkBytes, + TracerPayloadBytes, + }; + use crate::span::vec_map::VecMap; + use bolero::check; + use libdd_tinybytes::{Bytes, BytesString}; + + fn bs(s: &str) -> BytesString { + BytesString::from_slice(s.as_bytes()).expect("test string must fit in BytesString") + } + + fn sample_payload() -> TracerPayloadBytes { + let mut attrs = VecMap::>::new(); + attrs.insert(bs("http.method"), AttributeValue::String(bs("GET"))); + attrs.insert(bs("http.status"), AttributeValue::Int(200)); + attrs.insert(bs("is_root"), AttributeValue::Bool(true)); + attrs.insert(bs("ratio"), AttributeValue::Float(0.75)); + attrs.insert( + bs("ids"), + AttributeValue::List(vec![AttributeValue::Int(1), AttributeValue::Int(2)]), + ); + + let span = V1Span { + service: bs("svc"), + name: bs("GET /users"), + resource: bs("/users"), + r#type: bs("web"), + span_id: 42, + parent_id: 7, + start: 1_700_000_000_000, + duration: 1_500, + error: true, + span_kind: SpanKind::Server, + env: bs("prod"), + version: bs("1.2.3"), + component: bs("net/http"), + attributes: attrs, + ..Default::default() + }; + + let mut chunk_attrs = VecMap::>::new(); + chunk_attrs.insert(bs("_dd.p.dm"), AttributeValue::String(bs("-1"))); + + let chunk = TraceChunkBytes { + trace_id: [1u8; 16], + priority: Some(1), + origin: bs("synthetic"), + sampling_mechanism: Some(2), + dropped_trace: false, + attributes: chunk_attrs, + spans: vec![span], + }; + + TracerPayloadBytes { + language_name: bs("rust"), + language_version: bs("1.87"), + tracer_version: bs("9.9.9"), + runtime_id: bs("abcd-1234"), + env: bs("prod"), + hostname: bs("host-1"), + app_version: bs("1.2.3"), + chunks: vec![chunk], + ..Default::default() + } + } + + #[test] + fn roundtrip_full_payload() { + let original = sample_payload(); + let bytes = to_vec_from_payload_v1(&original); + let payload_len = bytes.len(); + let (decoded, consumed) = + from_bytes(Bytes::from(bytes)).expect("decoder should succeed on encoder output"); + + assert_eq!(consumed, payload_len, "decoder should consume all bytes"); + + // Tracer-level metadata + assert_eq!(decoded.language_name.as_str(), "rust"); + assert_eq!(decoded.language_version.as_str(), "1.87"); + assert_eq!(decoded.tracer_version.as_str(), "9.9.9"); + assert_eq!(decoded.runtime_id.as_str(), "abcd-1234"); + assert_eq!(decoded.env.as_str(), "prod"); + assert_eq!(decoded.hostname.as_str(), "host-1"); + assert_eq!(decoded.app_version.as_str(), "1.2.3"); + + // Chunk + assert_eq!(decoded.chunks.len(), 1); + let chunk = &decoded.chunks[0]; + assert_eq!(chunk.trace_id, [1u8; 16]); + assert_eq!(chunk.priority, Some(1)); + assert_eq!(chunk.sampling_mechanism, Some(2)); + assert_eq!(chunk.origin.as_str(), "synthetic"); + assert_eq!(chunk.attributes.len(), 1); + + // Span + assert_eq!(chunk.spans.len(), 1); + let span = &chunk.spans[0]; + assert_eq!(span.service.as_str(), "svc"); + assert_eq!(span.name.as_str(), "GET /users"); + assert_eq!(span.resource.as_str(), "/users"); + assert_eq!(span.r#type.as_str(), "web"); + assert_eq!(span.span_id, 42); + assert_eq!(span.parent_id, 7); + assert_eq!(span.start, 1_700_000_000_000); + assert_eq!(span.duration, 1_500); + assert!(span.error); + assert_eq!(span.span_kind, SpanKind::Server); + assert_eq!(span.env.as_str(), "prod"); + assert_eq!(span.version.as_str(), "1.2.3"); + assert_eq!(span.component.as_str(), "net/http"); + assert_eq!(span.attributes.len(), 5); + } + + #[test] + fn empty_payload_roundtrip() { + let original = TracerPayloadBytes::default(); + let bytes = to_vec_from_payload_v1(&original); + let (decoded, _) = + from_bytes(Bytes::from(bytes)).expect("decoder should succeed on empty payload"); + assert!(decoded.chunks.is_empty()); + assert!(decoded.language_name.as_str().is_empty()); + } + + #[test] + fn missing_chunks_field_is_rejected() { + // Manually encode a payload map with only one entry (env), no chunks field. + // `0x81` = fixmap len 1, key 0x07 (ENV_REF), value = inline str "x" (`0xa1 0x78`). + let bytes = vec![0x81, 0x07, 0xa1, 0x78]; + let err = from_bytes(Bytes::from(bytes)).expect_err("missing chunks must error"); + assert!(matches!(err, DecodeError::InvalidFormat(_))); + } + + #[test] + fn string_interning_resolves_across_chunks() { + // Two chunks sharing the same service name. The decoded service strings must both + // be "shared" — verifying the streaming string table is preserved across chunks. + let span_a = V1Span { + service: bs("shared"), + name: bs("a"), + span_id: 1, + start: 1, + ..Default::default() + }; + let span_b = V1Span { + service: bs("shared"), + name: bs("b"), + span_id: 2, + start: 1, + ..Default::default() + }; + let payload = TracerPayloadBytes { + chunks: vec![ + TraceChunkBytes { + trace_id: [1u8; 16], + spans: vec![span_a], + ..Default::default() + }, + TraceChunkBytes { + trace_id: [2u8; 16], + spans: vec![span_b], + ..Default::default() + }, + ], + ..Default::default() + }; + let bytes = to_vec_from_payload_v1(&payload); + let (decoded, _) = + from_bytes(Bytes::from(bytes)).expect("decoder should resolve interned strings"); + assert_eq!(decoded.chunks[0].spans[0].service.as_str(), "shared"); + assert_eq!(decoded.chunks[1].spans[0].service.as_str(), "shared"); + } + + #[test] + fn nested_keyvalue_attribute_roundtrip() { + let mut inner = VecMap::>::new(); + inner.insert(bs("k"), AttributeValue::String(bs("v"))); + let mut attrs = VecMap::>::new(); + attrs.insert(bs("nested"), AttributeValue::KeyValue(inner)); + + let span = V1Span { + service: bs("svc"), + name: bs("op"), + span_id: 1, + start: 1, + attributes: attrs, + ..Default::default() + }; + let payload = TracerPayloadBytes { + chunks: vec![TraceChunkBytes { + trace_id: [0u8; 16], + spans: vec![span], + ..Default::default() + }], + ..Default::default() + }; + let bytes = to_vec_from_payload_v1(&payload); + let (decoded, _) = from_bytes(Bytes::from(bytes)).expect("nested KeyValue roundtrip"); + + let decoded_attrs = &decoded.chunks[0].spans[0].attributes; + match decoded_attrs.get(&bs("nested")) { + Some(AttributeValue::KeyValue(map)) => { + assert_eq!(map.len(), 1); + match map.get(&bs("k")) { + Some(AttributeValue::String(v)) => assert_eq!(v.as_str(), "v"), + _ => panic!("inner value should be String"), + } + } + _ => panic!("attribute should decode as KeyValue"), + } + } + + /// Fuzz test: bolero generates random strings + numbers for the V1 payload, the encoder + /// serialises it, and the decoder must accept its own output (no panic, no error). Mirrors + /// the v04 `fuzz_from_bytes` pattern. Bolero caps tuples at 12 fields — extra metadata is + /// either omitted or filled with deterministic defaults. + #[test] + #[cfg_attr(miri, ignore)] + fn fuzz_from_bytes() { + check!() + .with_type::<( + String, // language_name + String, // env (payload-level) + String, // service + String, // name + String, // resource + String, // span env + String, // attr_key + String, // attr_value + u64, // span_id + u64, // parent_id + u64, // start + bool, // error + )>() + .cloned() + .for_each( + |( + lang, + payload_env, + service, + name, + resource, + span_env, + attr_key, + attr_value, + span_id, + parent_id, + start, + error, + )| { + let bs = |s: &str| BytesString::from_slice(s.as_ref()).unwrap(); + let mut attrs = VecMap::>::new(); + attrs.insert(bs(&attr_key), AttributeValue::String(bs(&attr_value))); + + let span = V1SpanBytes { + service: bs(&service), + name: bs(&name), + resource: bs(&resource), + span_id, + parent_id, + start: start as i64, + error, + env: bs(&span_env), + attributes: attrs, + ..Default::default() + }; + + let payload = TracerPayloadBytes { + language_name: bs(&lang), + env: bs(&payload_env), + chunks: vec![TraceChunkBytes { + trace_id: [0xab; 16], + spans: vec![span], + ..Default::default() + }], + ..Default::default() + }; + + let encoded = to_vec_from_payload_v1(&payload); + let result = from_bytes(Bytes::from(encoded)); + assert!( + result.is_ok(), + "decoder rejected its own encoded output: {result:?}" + ); + }, + ); + } +} diff --git a/libdd-trace-utils/src/msgpack_decoder/v1/span.rs b/libdd-trace-utils/src/msgpack_decoder/v1/span.rs new file mode 100644 index 0000000000..16822f959e --- /dev/null +++ b/libdd-trace-utils/src/msgpack_decoder/v1/span.rs @@ -0,0 +1,330 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +// Naming convention: parent module (`v1`) = input wire format being decoded. This file decodes +// a V1 msgpack span into a [`crate::span::v1::Span`]. + +use super::{ + read_interned_string, span_event_key, span_key, span_link_key, StringTable, + ANY_VALUE_KEY_ARRAY, ANY_VALUE_KEY_BOOL, ANY_VALUE_KEY_BYTES, ANY_VALUE_KEY_DOUBLE, + ANY_VALUE_KEY_INT64, ANY_VALUE_KEY_KEY_VALUE_LIST, ANY_VALUE_KEY_STRING, FLAT_ATTR_STRIDE, + TYPED_VALUE_STRIDE, +}; +use crate::msgpack_decoder::decode::buffer::Buffer; +use crate::msgpack_decoder::decode::error::DecodeError; +use crate::span::v1::{AttributeValue, Span, SpanEvent, SpanKind, SpanLink, ThinVec}; +use crate::span::vec_map::VecMap; +use crate::span::DeserializableTraceData; +use rmp::decode; +use std::borrow::Borrow; + +/// Decodes a V1 span (msgpack map with integer keys) into a [`Span`]. +/// +/// The streaming `StringTable` is shared across the whole payload, so interned references in +/// this span can resolve to strings that appeared in an earlier chunk or payload header. +pub(super) fn decode_span( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result, DecodeError> +where + T::Text: Clone, +{ + let map_len = decode::read_map_len(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("Unable to read V1 span map len".to_owned()))?; + + let mut span = Span::::default(); + + for _ in 0..map_len { + let key = decode::read_int::(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("V1 span key (u8) read failure".to_owned()))?; + + match key { + span_key::SERVICE => span.service = read_interned_string(buf, table)?, + span_key::NAME => span.name = read_interned_string(buf, table)?, + span_key::RESOURCE => span.resource = read_interned_string(buf, table)?, + span_key::SPAN_ID => { + span.span_id = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 span_id u64 read failure".to_owned()) + })? + } + span_key::PARENT_ID => { + span.parent_id = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 parent_id u64 read failure".to_owned()) + })? + } + span_key::START => { + span.start = decode::read_int::(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 span start u64 read failure".to_owned()) + })? as i64; + } + span_key::DURATION => { + span.duration = decode::read_int::(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 span duration u64 read failure".to_owned()) + })? as i64; + } + span_key::ERROR => { + span.error = decode::read_bool(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 span error bool read failure".to_owned()) + })?; + } + span_key::TYPE => span.r#type = read_interned_string(buf, table)?, + span_key::ATTRIBUTES => span.attributes = read_attributes_map(buf, table)?, + span_key::SPAN_LINKS => span.span_links = read_span_links(buf, table)?, + span_key::SPAN_EVENTS => span.span_events = read_span_events(buf, table)?, + span_key::ENV => span.env = read_interned_string(buf, table)?, + span_key::VERSION => span.version = read_interned_string(buf, table)?, + span_key::COMPONENT => span.component = read_interned_string(buf, table)?, + span_key::KIND => { + let kind: u32 = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 span_kind u32 read failure".to_owned()) + })?; + // OTEL spec: unset / unknown → Internal. + span.span_kind = match kind { + 2 => SpanKind::Server, + 3 => SpanKind::Client, + 4 => SpanKind::Producer, + 5 => SpanKind::Consumer, + _ => SpanKind::Internal, + }; + } + unknown => { + return Err(DecodeError::InvalidFormat(format!( + "Unknown V1 span key: {unknown}" + ))); + } + } + } + + span.attributes.mark_deduped(); + + Ok(span) +} + +/// Reads a V1 attributes map encoded as a flat array of `[key, type_uint8, value, ...]` triplets. +pub(super) fn read_attributes_map( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result>, DecodeError> +where + T::Text: Clone, +{ + let flat_len = decode::read_array_len(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 attributes flat array len read failure".to_owned()) + })?; + + if flat_len % FLAT_ATTR_STRIDE != 0 { + return Err(DecodeError::InvalidFormat(format!( + "V1 attributes flat array length {flat_len} is not a multiple of {FLAT_ATTR_STRIDE}" + ))); + } + + let entries = (flat_len / FLAT_ATTR_STRIDE) as usize; + let mut map = VecMap::with_capacity(entries); + + for _ in 0..entries { + let key = read_interned_string(buf, table)?; + let value = read_typed_attribute_value(buf, table)?; + map.insert(key, value); + } + + map.mark_deduped(); + + Ok(map) +} + +/// Reads `[type_uint8, value]` and dispatches by type discriminant. Recurses into `Array` and +/// `KeyValueList`. +pub(super) fn read_typed_attribute_value( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result, DecodeError> +where + T::Text: Clone, +{ + let ty = decode::read_int::(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 attribute type discriminant read failure".to_owned()) + })?; + + match ty { + ANY_VALUE_KEY_STRING => Ok(AttributeValue::String(read_interned_string(buf, table)?)), + ANY_VALUE_KEY_BOOL => { + let b = decode::read_bool(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 attribute Bool read failure".to_owned()) + })?; + Ok(AttributeValue::Bool(b)) + } + ANY_VALUE_KEY_DOUBLE => { + let f = decode::read_f64(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 attribute Double read failure".to_owned()) + })?; + Ok(AttributeValue::Float(f)) + } + ANY_VALUE_KEY_INT64 => { + // Encoder writes signed via `write_sint`, which can emit any int marker. `read_int` + // accepts any integer marker that fits in i64. + let i: i64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 attribute Int64 read failure".to_owned()) + })?; + Ok(AttributeValue::Int(i)) + } + ANY_VALUE_KEY_BYTES => Ok(AttributeValue::Bytes(read_bin(buf)?)), + ANY_VALUE_KEY_ARRAY => { + let array_len_with_stride = + decode::read_array_len(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 attribute Array len read failure".to_owned()) + })?; + // Array is a flat sequence of [type, value] pairs. + if array_len_with_stride % TYPED_VALUE_STRIDE != 0 { + return Err(DecodeError::InvalidFormat(format!( + "V1 attribute Array length {array_len_with_stride} is not a multiple of {TYPED_VALUE_STRIDE}" + ))); + } + let n = (array_len_with_stride / TYPED_VALUE_STRIDE) as usize; + let mut items = Vec::with_capacity(n); + for _ in 0..n { + items.push(read_typed_attribute_value(buf, table)?); + } + Ok(AttributeValue::List(items)) + } + ANY_VALUE_KEY_KEY_VALUE_LIST => { + Ok(AttributeValue::KeyValue(read_attributes_map(buf, table)?)) + } + unknown => Err(DecodeError::InvalidFormat(format!( + "Unknown V1 AnyValue type discriminant: {unknown}" + ))), + } +} + +/// Reads a msgpack `bin` and slices the matching range out of the buffer. +fn read_bin(buf: &mut Buffer) -> Result { + let len = decode::read_bin_len(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("V1 bin len read failure".to_owned()))?; + buf.try_slice_and_advance(len as usize) + .ok_or_else(|| DecodeError::InvalidFormat("V1 bin payload truncated".to_owned())) +} + +/// Reads the span_links array. The `SpanLinks` map key has already been consumed by the caller. +pub(super) fn read_span_links( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result>, DecodeError> +where + T::Text: Clone, +{ + let count = decode::read_array_len(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("V1 span_links len read failure".to_owned()))?; + let mut links = ThinVec::with_capacity(count as usize); + for _ in 0..count { + links.push(decode_span_link(buf, table)?); + } + Ok(links) +} + +fn decode_span_link( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result, DecodeError> +where + T::Text: Clone, +{ + let map_len = decode::read_map_len(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("V1 span_link map len read failure".to_owned()))?; + let mut link = SpanLink::::default(); + + for _ in 0..map_len { + let key = decode::read_int::(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 span_link key (u8) read failure".to_owned()) + })?; + match key { + span_link_key::TRACE_ID => { + let bytes = read_bin(buf)?; + let slice: &[u8] = bytes.borrow(); + if slice.len() != 16 { + return Err(DecodeError::InvalidFormat(format!( + "V1 span_link trace_id expected 16 bytes, got {}", + slice.len() + ))); + } + let mut arr = [0u8; 16]; + arr.copy_from_slice(slice); + link.trace_id = arr; + } + span_link_key::SPAN_ID => { + link.span_id = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 span_link span_id read failure".to_owned()) + })?; + } + span_link_key::ATTRIBUTES => { + link.attributes = read_attributes_map(buf, table)?; + } + span_link_key::TRACE_STATE => { + link.tracestate = read_interned_string(buf, table)?; + } + span_link_key::FLAGS => { + let v: u64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 span_link flags read failure".to_owned()) + })?; + link.flags = v as u32; + } + unknown => { + return Err(DecodeError::InvalidFormat(format!( + "Unknown V1 span_link key: {unknown}" + ))); + } + } + } + Ok(link) +} + +pub(super) fn read_span_events( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result>, DecodeError> +where + T::Text: Clone, +{ + let count = decode::read_array_len(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("V1 span_events len read failure".to_owned()))?; + let mut events = ThinVec::with_capacity(count as usize); + for _ in 0..count { + events.push(decode_span_event(buf, table)?); + } + Ok(events) +} + +fn decode_span_event( + buf: &mut Buffer, + table: &mut StringTable, +) -> Result, DecodeError> +where + T::Text: Clone, +{ + let map_len = decode::read_map_len(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("V1 span_event map len read failure".to_owned()))?; + let mut event = SpanEvent::::default(); + + for _ in 0..map_len { + let key = decode::read_int::(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 span_event key (u8) read failure".to_owned()) + })?; + match key { + span_event_key::TIME => { + event.time_unix_nano = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat("V1 span_event time read failure".to_owned()) + })?; + } + span_event_key::NAME => { + event.name = read_interned_string(buf, table)?; + } + span_event_key::ATTRIBUTES => { + event.attributes = read_attributes_map(buf, table)?; + } + unknown => { + return Err(DecodeError::InvalidFormat(format!( + "Unknown V1 span_event key: {unknown}" + ))); + } + } + } + Ok(event) +} diff --git a/libdd-trace-utils/src/send_data/mod.rs b/libdd-trace-utils/src/send_data/mod.rs index 53d14d8ecd..cb6e7fe2f0 100644 --- a/libdd-trace-utils/src/send_data/mod.rs +++ b/libdd-trace-utils/src/send_data/mod.rs @@ -407,6 +407,24 @@ impl SendData { Err(e) => return result.error(anyhow!(e)), }; + futures.push(self.send_payload( + capabilities, + chunks, + payload, + headers, + endpoint.as_ref(), + )); + } + TracerPayloadCollection::V1(payload) => { + #[allow(clippy::unwrap_used)] + let chunks = u64::try_from(self.tracer_payloads.size()).unwrap(); + let mut headers = self.headers.clone(); + headers.reserve(2); + headers.insert(DATADOG_TRACE_COUNT, chunks.into()); + headers.insert(CONTENT_TYPE, APPLICATION_MSGPACK); + + let payload = msgpack_encoder::v1::to_vec_from_payload_v1(payload); + futures.push(self.send_payload( capabilities, chunks, @@ -537,6 +555,9 @@ mod tests { msgpack_encoder::v04::to_encoded_byte_len(payloads) as usize } TracerPayloadCollection::V05(payloads) => rmp_serde::to_vec(payloads).unwrap().len(), + TracerPayloadCollection::V1(payload) => { + msgpack_encoder::v1::to_encoded_byte_len_from_payload_v1(payload) as usize + } } } diff --git a/libdd-trace-utils/src/trace_utils.rs b/libdd-trace-utils/src/trace_utils.rs index 851ac54beb..f0398235f6 100644 --- a/libdd-trace-utils/src/trace_utils.rs +++ b/libdd-trace-utils/src/trace_utils.rs @@ -7,7 +7,7 @@ use crate::span::v05::dict::SharedDict; use crate::span::{v05, TraceData}; pub use crate::tracer_header_tags::TracerHeaderTags; use crate::tracer_payload::TracerPayloadCollection; -use crate::tracer_payload::{self, TraceChunks, TraceEncoding}; +use crate::tracer_payload::{self, TraceChunks}; use anyhow::anyhow; use bytes::buf::Reader; use bytes::Buf; @@ -583,25 +583,28 @@ pub fn enrich_span_with_azure_function_metadata(span: &mut pb::Span) { } } -pub fn collect_trace_chunks( +/// Converts v0.4-shaped span chunks into the v0.5 wire representation. +/// +/// v0.5 deduplicates every string field across the whole payload through a shared dictionary +/// and replaces them with `u32` indices. This walks each span via [`v05::from_v04_span`], +/// interning strings into the [`SharedDict`] as it goes, and returns the resulting +/// `(dict, traces)` pair wrapped in [`TraceChunks::V05`]. +/// +/// Returns `Err` if any span fails to convert (e.g. unsupported field value); the partial +/// dictionary built so far is discarded. +pub fn convert_trace_chunks_v04_to_v05( traces: Vec>>, - format: TraceEncoding, ) -> anyhow::Result> { - match format { - TraceEncoding::V05 => { - let mut shared_dict = SharedDict::default(); - let mut v05_traces: Vec> = Vec::with_capacity(traces.len()); - for trace in traces { - let v05_trace = trace - .into_iter() - .map(|span| v05::from_v04_span(span, &mut shared_dict)) - .collect::>>()?; - v05_traces.push(v05_trace); - } - Ok(TraceChunks::V05((shared_dict, v05_traces))) - } - TraceEncoding::V04 => Ok(TraceChunks::V04(traces)), + let mut shared_dict = SharedDict::default(); + let mut v05_traces: Vec> = Vec::with_capacity(traces.len()); + for trace in traces { + let v05_trace = trace + .into_iter() + .map(|span| v05::from_v04_span(span, &mut shared_dict)) + .collect::>>()?; + v05_traces.push(v05_trace); } + Ok(TraceChunks::V05((shared_dict, v05_traces))) } pub fn collect_pb_trace_chunks( @@ -1118,10 +1121,10 @@ mod tests { } #[test] - fn test_collect_trace_chunks_v05() { + fn test_convert_trace_chunks_v04_to_v05() { let chunk = vec![create_test_no_alloc_span(123, 456, 789, 1, true)]; - let collection = collect_trace_chunks(vec![chunk], TraceEncoding::V05).unwrap(); + let collection = convert_trace_chunks_v04_to_v05(vec![chunk]).unwrap(); let (dict, traces) = match collection { TraceChunks::V05(payload) => payload, @@ -1192,27 +1195,6 @@ mod tests { ); } - #[test] - fn test_collect_trace_chunks_v04() { - let chunk = vec![create_test_no_alloc_span(123, 456, 789, 1, true)]; - - let collection = collect_trace_chunks(vec![chunk], TraceEncoding::V04).unwrap(); - - let traces = match collection { - TraceChunks::V04(traces) => traces, - _ => panic!("Unexpected type"), - }; - - assert_eq!(traces.len(), 1); - assert_eq!(traces[0].len(), 1); - let span = &traces[0][0]; - assert_eq!(span.trace_id, 123); - assert_eq!(span.span_id, 456); - assert_eq!(span.parent_id, 789); - assert_eq!(span.start, 1); - assert_eq!(span.error, 0); - } - #[test] fn test_rmp_serde_deserialize_meta_with_null_values() { // Create a JSON representation with null value in meta diff --git a/libdd-trace-utils/src/tracer_payload.rs b/libdd-trace-utils/src/tracer_payload.rs index 30752cabdf..b6cedbc70a 100644 --- a/libdd-trace-utils/src/tracer_payload.rs +++ b/libdd-trace-utils/src/tracer_payload.rs @@ -2,12 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 use crate::span::v05::dict::SharedDict; -use crate::span::{v04, v05, BytesData, SharedDictBytes, TraceData}; -use crate::trace_utils::collect_trace_chunks; +use crate::span::{v04, v05, v1, BytesData, SharedDictBytes, TraceData}; +use crate::trace_utils::convert_trace_chunks_v04_to_v05; use crate::{msgpack_decoder, trace_utils::cmp_send_data_payloads}; +use anyhow::Ok; use libdd_trace_protobuf::pb; use std::cmp::Ordering; use std::iter::Iterator; +use tracing::warn; pub type TracerPayloadV04 = Vec; pub type TracerPayloadV05 = Vec; @@ -19,6 +21,8 @@ pub enum TraceEncoding { V04, /// v0.5 encoding (TracerPayloadV05). V05, + /// v1 encoding (TracerPayloadV1). + V1, } #[derive(Debug)] @@ -28,7 +32,7 @@ pub enum TraceChunks { /// Collection of TraceChunkSpan with de-duplicated strings. V05((SharedDict, Vec>)), /// Collection of v0.4 spans to be serialized as a V1 msgpack payload. - V1(Vec>>), + V1(v1::TracerPayload), } impl TraceChunks { @@ -36,8 +40,7 @@ impl TraceChunks { match self { TraceChunks::V04(traces) => TracerPayloadCollection::V04(traces), TraceChunks::V05(traces) => TracerPayloadCollection::V05(traces), - // V1 uses the same underlying span structure as V04. - TraceChunks::V1(traces) => TracerPayloadCollection::V04(traces), + TraceChunks::V1(traces) => TracerPayloadCollection::V1(traces), } } } @@ -48,7 +51,7 @@ impl TraceChunks { match self { TraceChunks::V04(traces) => traces.len(), TraceChunks::V05((_, traces)) => traces.len(), - TraceChunks::V1(traces) => traces.len(), + TraceChunks::V1(trace) => trace.chunks.len(), } } } @@ -62,6 +65,8 @@ pub enum TracerPayloadCollection { V04(Vec>), /// Collection of TraceChunkSpan with de-duplicated strings. V05((SharedDictBytes, Vec>)), + // /// V0.4-shaped spans that must be serialized as a V1 msgpack payload on send. + V1(v1::TracerPayload), } impl TracerPayloadCollection { @@ -92,6 +97,18 @@ impl TracerPayloadCollection { dest.append(src) } } + TracerPayloadCollection::V1(dest) => { + if let TracerPayloadCollection::V1(src) = other { + // Same-target SendData entries are coalesced by + // trace_utils::coalesce_send_data, so both V1 payloads + // typically share tracer-level metadata. If all metadata + // fields match we append `src`'s chunks into `dest`; if any diverge we no-op + // (logging a warning) rather than silently dropping `src`'s metadata. + if metadata_matches_v1(dest, src) { + dest.chunks.append(&mut src.chunks); + } + } + } // TODO: Properly handle non-OK states to prevent possible panics (APMSP-18190). #[allow(clippy::unimplemented)] TracerPayloadCollection::V05(_) => unimplemented!("Append for V05 not implemented"), @@ -144,6 +161,7 @@ impl TracerPayloadCollection { } TracerPayloadCollection::V04(collection) => collection.len(), TracerPayloadCollection::V05((_, collection)) => collection.len(), + TracerPayloadCollection::V1(collection) => collection.chunks.len(), } } } @@ -228,13 +246,63 @@ pub fn decode_to_trace_chunks( data: libdd_tinybytes::Bytes, encoding_type: TraceEncoding, ) -> Result<(TraceChunks, usize), anyhow::Error> { - let (data, size) = match encoding_type { - TraceEncoding::V04 => msgpack_decoder::v04::from_bytes(data), - TraceEncoding::V05 => msgpack_decoder::v05::from_bytes(data), + match encoding_type { + TraceEncoding::V04 => { + let (data, size) = msgpack_decoder::v04::from_bytes(data).map_err(|e| { + anyhow::format_err!("Error deserializing trace from request body: {e}") + })?; + Ok((TraceChunks::V04(data), size)) + } + TraceEncoding::V05 => { + let (data, size) = msgpack_decoder::v05::from_bytes(data).map_err(|e| { + anyhow::format_err!("Error deserializing trace from request body: {e}") + })?; + Ok((convert_trace_chunks_v04_to_v05(data)?, size)) + } + TraceEncoding::V1 => { + let (data, size) = msgpack_decoder::v1::from_bytes(data).map_err(|e| { + anyhow::format_err!("Error deserializing trace from request body: {e}") + })?; + Ok((TraceChunks::V1(data), size)) + } } - .map_err(|e| anyhow::format_err!("Error deserializing trace from request body: {e}"))?; +} - Ok((collect_trace_chunks(data, encoding_type)?, size)) +/// Returns `true` iff every tracer-level metadata string field of `src` matches `dest`. +/// +/// V1 payloads carry tracer metadata (env, hostname, language, …) inside the payload itself, so +/// merging two payloads whose metadata diverges would silently drop one set of values. Callers +/// use this to gate the merge: on a `false` return, append is skipped (no-op) and the two +/// payloads stay separate. A warning is logged listing the diverging fields so the situation +/// is observable rather than silent. +fn metadata_matches_v1( + dest: &v1::TracerPayload, + src: &v1::TracerPayload, +) -> bool { + let differing: Vec<&'static str> = [ + ("language_name", dest.language_name == src.language_name), + ( + "language_version", + dest.language_version == src.language_version, + ), + ("tracer_version", dest.tracer_version == src.tracer_version), + ("runtime_id", dest.runtime_id == src.runtime_id), + ("env", dest.env == src.env), + ("hostname", dest.hostname == src.hostname), + ("app_version", dest.app_version == src.app_version), + ] + .into_iter() + .filter_map(|(label, eq)| (!eq).then_some(label)) + .collect(); + + if !differing.is_empty() { + warn!( + "Skipping V1 TracerPayload append: diverging metadata fields {:?}", + differing + ); + return false; + } + true } #[cfg(test)] diff --git a/libdd-trace-utils/tests/snapshots/compare_v1_full_payload_snapshot_test.json b/libdd-trace-utils/tests/snapshots/compare_v1_full_payload_snapshot_test.json new file mode 100644 index 0000000000..265c35f962 --- /dev/null +++ b/libdd-trace-utils/tests/snapshots/compare_v1_full_payload_snapshot_test.json @@ -0,0 +1,60 @@ +[[ + { + "name": "test_send_data_v1_full_root", + "service": "test-service", + "resource": "/api/users", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "web", + "meta": { + "_dd.origin": "synthetics", + "_dd.p.dm": "-4", + "_dd.p.tid": "0123456789abcdef", + "component": "http", + "env": "test-env", + "http.method": "POST", + "span.kind": "server", + "version": "1.2.3" + }, + "metrics": { + "_sampling_priority_v1": 1, + "http.duration_ms": 42.75, + "http.status_code": 201, + "http.success": 1 + }, + "span_links": [ + { + "trace_id": 18364758544493064720, + "trace_id_high": 81985529216486895, + "span_id": 11574427654092267680, + "attributes": { + "link.reason": "retry" + }, + "tracestate": "dd=t.tid:abc", + "flags": 1 + } + ], + "span_events": [ + { + "time_unix_nano": 1727211691770715042, + "name": "exception", + "attributes": { + "exception.code": { + "type": 2, + "int_value": 500 + }, + "exception.handled": { + "type": 1, + "bool_value": false + }, + "exception.type": { + "type": 0, + "string_value": "RuntimeError" + } + } + } + ], + "duration": 5000, + "start": 1000000 + }]] diff --git a/libdd-trace-utils/tests/test_send_data.rs b/libdd-trace-utils/tests/test_send_data.rs index 39a10b58ff..a3f3070ff2 100644 --- a/libdd-trace-utils/tests/test_send_data.rs +++ b/libdd-trace-utils/tests/test_send_data.rs @@ -627,4 +627,129 @@ mod tracing_integration_tests { // 2 decoded spans. The checked-in snapshot is the canonical equivalent decoded form. test_agent.assert_snapshot(snapshot_name).await; } + + /// Builds a V1 payload exercising the full attribute surface that the current test-agent + /// (v1.56.0) is able to decode: `AttributeValue::String/Bool/Int/Float`, span links with + /// non-empty attributes, span events with non-empty attributes, plus chunk-level attributes. + /// + /// APMSP-3479 - TODO: `AttributeValue::Bytes`, `AttributeValue::List` and + /// `AttributeValue::KeyValue` are deliberately omitted — `ddapm-test-agent` v1.56.0 rejects + /// Bytes with `400 "Bytes values are not supported yet."` and does not handle List / KeyValue. + /// Add them here once test-agent V1 support catches up. + fn make_v1_full_payload(name_prefix: &str) -> libdd_trace_utils::span::v1::TracerPayloadBytes { + use libdd_trace_utils::span::v1::{ + AttributeValue, AttributeValueBytes, SpanBytes as V1SpanBytes, SpanEventBytes, + SpanKind, SpanLinkBytes, TraceChunkBytes, TracerPayloadBytes, + }; + + // Root span attributes — primitives supported by the test-agent today. + let mut root_attrs: VecMap = VecMap::new(); + root_attrs.insert(bs_v1("http.method"), AttributeValue::String(bs_v1("POST"))); + root_attrs.insert(bs_v1("http.status_code"), AttributeValue::Int(201)); + root_attrs.insert(bs_v1("http.success"), AttributeValue::Bool(true)); + root_attrs.insert(bs_v1("http.duration_ms"), AttributeValue::Float(42.75)); + + // Span link with non-empty attributes (String-only — primitives are enough to exercise + // the encoder/decoder paths for link attribute maps). + let mut link_attrs: VecMap = VecMap::new(); + link_attrs.insert(bs_v1("link.reason"), AttributeValue::String(bs_v1("retry"))); + let span_link = SpanLinkBytes { + trace_id: tid_bytes(0x0123_4567_89ab_cdef, 0xfedc_ba98_7654_3210), + span_id: 0xa0a0_a0a0_a0a0_a0a0, + tracestate: bs_v1("dd=t.tid:abc"), + flags: 1, + attributes: link_attrs, + }; + + // Span event with attributes (String + Int + Bool to exercise typed values). + let mut event_attrs: VecMap = VecMap::new(); + event_attrs.insert( + bs_v1("exception.type"), + AttributeValue::String(bs_v1("RuntimeError")), + ); + event_attrs.insert(bs_v1("exception.code"), AttributeValue::Int(500)); + event_attrs.insert(bs_v1("exception.handled"), AttributeValue::Bool(false)); + let span_event = SpanEventBytes { + time_unix_nano: 1_727_211_691_770_715_042, + name: bs_v1("exception"), + attributes: event_attrs, + }; + + let root_span = V1SpanBytes { + service: bs_v1("test-service"), + name: bs_v1(&format!("{name_prefix}_root")), + resource: bs_v1("/api/users"), + r#type: bs_v1("web"), + span_id: 1, + parent_id: 0, + start: 1_000_000, + duration: 5_000, + span_kind: SpanKind::Server, + env: bs_v1("test-env"), + version: bs_v1("1.2.3"), + component: bs_v1("http"), + attributes: root_attrs, + span_links: thin_vec::thin_vec![span_link], + span_events: thin_vec::thin_vec![span_event], + ..Default::default() + }; + + let mut chunk_attrs = VecMap::new(); + chunk_attrs.insert(bs_v1("_dd.p.dm"), AttributeValue::String(bs_v1("-4"))); + chunk_attrs.insert( + bs_v1("_dd.p.tid"), + AttributeValue::String(bs_v1("0123456789abcdef")), + ); + + let chunk = TraceChunkBytes { + trace_id: tid_bytes(0, 0xfeedface), + priority: Some(1), + origin: bs_v1("synthetics"), + sampling_mechanism: Some(4), + attributes: chunk_attrs, + dropped_trace: false, + spans: vec![root_span], + }; + + TracerPayloadBytes { + language_name: bs_v1("test-lang"), + language_version: bs_v1("2.0"), + tracer_version: bs_v1("1.0"), + runtime_id: bs_v1("test-runtime-id-full"), + env: bs_v1("test-env"), + hostname: bs_v1("test-host"), + app_version: bs_v1("1.2.3"), + attributes: VecMap::new(), + chunks: vec![chunk], + } + } + + /// Big V1 integration test: encodes a payload covering every primitive attribute variant + /// the test-agent supports today (String/Bool/Int/Float) plus non-empty span_link and + /// span_event attribute maps, then POSTs to `/v1.0/traces` and asserts the snapshot. + /// + /// APMSP-3479 - TODO: extend this with `AttributeValue::Bytes`, `AttributeValue::List` + /// and `AttributeValue::KeyValue` once `ddapm-test-agent` v1 support catches up. See + /// [`make_v1_full_payload`] for the payload builder and the omitted variants. + #[cfg_attr(miri, ignore)] + #[tokio::test] + async fn compare_v1_full_payload_snapshot_test() { + use libdd_trace_utils::msgpack_encoder::v1::to_vec_from_payload_v1; + + let relative_snapshot_path = "libdd-trace-utils/tests/snapshots/"; + let snapshot_name = "compare_v1_full_payload_snapshot_test"; + let test_agent = DatadogTestAgent::new(Some(relative_snapshot_path), None, &[]).await; + let uri = test_agent + .get_uri_for_endpoint("v1.0/traces", Some(snapshot_name)) + .await; + + test_agent.start_session(snapshot_name, None).await; + + let payload = make_v1_full_payload("test_send_data_v1_full"); + let encoded = to_vec_from_payload_v1(&payload); + + post_v1_payload(uri, encoded).await; + + test_agent.assert_snapshot(snapshot_name).await; + } } From dd161cd64561296baa1c0d0f1f7a5e5385df3d07 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Mon, 29 Jun 2026 19:24:02 +0200 Subject: [PATCH 06/23] fix: comments --- .../src/msgpack_encoder/v04/span_v1.rs | 222 ++++++++++++++---- 1 file changed, 176 insertions(+), 46 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs index 200d128cbe..134514701f 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs @@ -28,11 +28,13 @@ use crate::span::v1::{AttributeValue, Span, SpanEvent, SpanKind, SpanLink}; use crate::span::vec_map::VecMap; use crate::span::TraceData; +use libdd_common::ResultInfallibleExt; use rmp::encode::{ write_array_len, write_bin, write_bool, write_f64, write_i64, write_map_len, write_sint, - write_str, write_u32, write_u64, write_u8, RmpWrite, ValueWriteError, + write_str, write_u32, write_u64, write_u8, ByteBuf, RmpWrite, ValueWriteError, }; use std::borrow::Borrow; +use std::collections::HashSet; /// Writes a `bool` as the v0.4 string representation (`"true"` / `"false"`). Used wherever a /// typed V1 `Bool` attribute is downgraded into v0.4 `meta` (which is `String → String` only). @@ -128,6 +130,19 @@ pub(super) fn encode_span( let span_attrs_dd = span.attributes.defensive_dedup(); let chunk_attrs_dd = chunk.attributes.defensive_dedup(); + // Merge span + chunk attributes upfront with explicit "span overrides chunk" precedence. + // We don't rely on msgpack map last-write-wins decoding here: the v0.4 / msgpack specs do + // not formalize behavior for duplicate map keys, so we emit each key exactly once. + let span_keys: HashSet<&T::Text> = span_attrs_dd.iter().map(|(k, _)| k).collect(); + let merged_attrs: Vec<(&T::Text, &AttributeValue)> = span_attrs_dd + .iter() + .chain( + chunk_attrs_dd + .iter() + .filter(|(k, _)| !span_keys.contains(k)), + ) + .collect(); + let (trace_id_low, trace_id_high) = split_trace_id(chunk.trace_id); let kind_meta = span_kind_to_meta(span.span_kind); @@ -141,13 +156,7 @@ pub(super) fn encode_span( counts.meta += !chunk.origin.borrow().is_empty() as u32; counts.meta += chunk.sampling_mechanism.is_some() as u32; counts.metrics += chunk.priority.is_some() as u32; - for (_, v) in span_attrs_dd.iter() { - counts.count_attr(v); - } - // Chunk attributes are propagated to every span. Duplicate keys between chunk and span are - // tolerated here — msgpack map decode keeps the last value (the span's), which matches the - // "span overrides chunk" precedence. - for (_, v) in chunk_attrs_dd.iter() { + for (_, v) in &merged_attrs { counts.count_attr(v); } @@ -227,22 +236,7 @@ pub(super) fn encode_span( write_const_msg_pack_str!(writer, "_dd.p.dm")?; write_str(writer, &format!("-{mechanism}"))?; } - // Span attrs first, then chunk attrs (duplicate keys resolve to the span value via - // msgpack last-write-wins on the decoder side). - for (k, v) in span_attrs_dd.iter() { - match v { - AttributeValue::String(s) => { - write_str(writer, k.borrow())?; - write_str(writer, s.borrow())?; - } - AttributeValue::Bool(b) => { - write_str(writer, k.borrow())?; - write_bool_as_str(writer, *b)?; - } - _ => {} - } - } - for (k, v) in chunk_attrs_dd.iter() { + for &(k, v) in &merged_attrs { match v { AttributeValue::String(s) => { write_str(writer, k.borrow())?; @@ -265,20 +259,7 @@ pub(super) fn encode_span( write_const_msg_pack_str!(writer, "_sampling_priority_v1")?; write_f64(writer, priority as f64)?; } - for (k, v) in span_attrs_dd.iter() { - match v { - AttributeValue::Float(f) => { - write_str(writer, k.borrow())?; - write_f64(writer, *f)?; - } - AttributeValue::Int(i) => { - write_str(writer, k.borrow())?; - write_f64(writer, *i as f64)?; - } - _ => {} - } - } - for (k, v) in chunk_attrs_dd.iter() { + for &(k, v) in &merged_attrs { match v { AttributeValue::Float(f) => { write_str(writer, k.borrow())?; @@ -302,10 +283,7 @@ pub(super) fn encode_span( write_const_msg_pack_str!(writer, "meta_struct")?; write_map_len(writer, counts.meta_struct)?; - for (k, v) in span_attrs_dd.iter() { - write_meta_struct_entry(writer, k, v)?; - } - for (k, v) in chunk_attrs_dd.iter() { + for &(k, v) in &merged_attrs { write_meta_struct_entry(writer, k, v)?; } } @@ -320,9 +298,10 @@ pub(super) fn encode_span( Ok(()) } -/// Writes a single `meta_struct` entry. `Bytes` goes through as msgpack `bin`; structural -/// `KeyValue` / `List` would need a recursive msgpack re-encoding — stubbed as empty `bin` for -/// now (TODO: implement structural serialization). +/// Writes a single `meta_struct` entry. `Bytes` passes through as a msgpack `bin`. `KeyValue` +/// and `List` are recursively re-encoded as standalone msgpack (no V1 type tags, no string +/// interning) into a scratch buffer, then emitted as the `bin` payload — matching the v0.4 +/// `meta_struct` contract of `String → msgpack-encoded bytes`. fn write_meta_struct_entry( writer: &mut W, k: &T::Text, @@ -335,13 +314,58 @@ fn write_meta_struct_entry( } AttributeValue::KeyValue(_) | AttributeValue::List(_) => { write_str(writer, k.borrow())?; - write_bin(writer, todo!())?; + let mut buf = ByteBuf::with_capacity(0); + encode_attribute_value(&mut buf, v) + .map_err(super::super::flatten_value_write_infallible) + .unwrap_infallible(); + write_bin(writer, buf.as_ref())?; } _ => {} } Ok(()) } +/// Recursively encodes a [`v1::AttributeValue`] as a self-contained msgpack value using natural +/// types — `str`/`bool`/`int`/`f64`/`bin`/`array`/`map` — without the V1 `[type, value]` framing +/// or its streaming string table. Used to build the body of a v0.4 `meta_struct` `bin` payload. +fn encode_attribute_value( + writer: &mut W, + v: &AttributeValue, +) -> Result<(), ValueWriteError> { + match v { + AttributeValue::String(s) => { + write_str(writer, s.borrow())?; + } + AttributeValue::Bool(b) => { + write_bool(writer, *b).map_err(ValueWriteError::InvalidDataWrite)?; + } + AttributeValue::Int(i) => { + write_sint(writer, *i)?; + } + AttributeValue::Float(f) => { + write_f64(writer, *f)?; + } + AttributeValue::Bytes(b) => { + write_bin(writer, b.borrow())?; + } + AttributeValue::List(items) => { + write_array_len(writer, items.len() as u32)?; + for item in items { + encode_attribute_value(writer, item)?; + } + } + AttributeValue::KeyValue(map) => { + let deduped = map.defensive_dedup(); + write_map_len(writer, deduped.len() as u32)?; + for (k, v) in deduped.iter() { + write_str(writer, k.borrow())?; + encode_attribute_value(writer, v)?; + } + } + } + Ok(()) +} + /// Encodes [`v1::SpanLink`](crate::span::v1::SpanLink)s into the v0.4 msgpack wire format /// (downgrade: v1 input → v0.4 output). The 128-bit `trace_id` is split into /// `(trace_id, trace_id_high)` u64s. Typed link attributes are downgraded to strings; @@ -837,6 +861,112 @@ mod tests { ); } + /// Extracts the standalone msgpack value packed inside a `meta_struct` `bin` payload. + fn meta_struct_inner(meta_struct: &Value, key: &str) -> Value { + let bin = match map_get(meta_struct, key) { + Some(Value::Binary(b)) => b, + other => panic!("expected meta_struct[{key}] to be a Binary, got {other:?}"), + }; + rmpv::decode::read_value(&mut bin.as_slice()).expect("inner msgpack decode failed") + } + + #[test] + fn list_attribute_routes_to_meta_struct_as_nested_msgpack_array() { + let mut attrs: VecMap = VecMap::new(); + attrs.insert( + bs("ids"), + AttributeValue::List(vec![ + AttributeValue::Int(1), + AttributeValue::Int(2), + AttributeValue::String(bs("three")), + ]), + ); + let payload = minimal_payload( + [0u8; 16], + SpanBytes { + attributes: attrs, + ..minimal_span() + }, + ); + let traces = encode_and_decode(&payload); + let ms = map_get(&traces[0][0], "meta_struct").expect("meta_struct present"); + + let inner = meta_struct_inner(ms, "ids"); + let items = match inner { + Value::Array(a) => a, + other => panic!("expected msgpack array inside bin, got {other:?}"), + }; + assert_eq!(items.len(), 3); + assert_eq!(items[0].as_i64(), Some(1)); + assert_eq!(items[1].as_i64(), Some(2)); + assert_eq!(items[2].as_str(), Some("three")); + } + + #[test] + fn keyvalue_attribute_routes_to_meta_struct_as_nested_msgpack_map() { + let mut inner_kv: VecMap = VecMap::new(); + inner_kv.insert(bs("user_id"), AttributeValue::Int(42)); + inner_kv.insert(bs("name"), AttributeValue::String(bs("alice"))); + inner_kv.insert(bs("active"), AttributeValue::Bool(true)); + + let mut attrs: VecMap = VecMap::new(); + attrs.insert(bs("user"), AttributeValue::KeyValue(inner_kv)); + + let payload = minimal_payload( + [0u8; 16], + SpanBytes { + attributes: attrs, + ..minimal_span() + }, + ); + let traces = encode_and_decode(&payload); + let ms = map_get(&traces[0][0], "meta_struct").expect("meta_struct present"); + + let inner = meta_struct_inner(ms, "user"); + assert_eq!(inner.as_map().expect("inner is map").len(), 3); + assert_eq!(map_get(&inner, "user_id").unwrap().as_i64(), Some(42)); + assert_eq!(map_get(&inner, "name").unwrap().as_str(), Some("alice")); + assert_eq!(map_get(&inner, "active").unwrap().as_bool(), Some(true)); + } + + #[test] + fn nested_keyvalue_and_list_recurse_through_meta_struct() { + // Build: {"outer": KeyValue { "items": List [String "a", KeyValue {"k": Int 1}] }} + let mut nested_kv: VecMap = VecMap::new(); + nested_kv.insert(bs("k"), AttributeValue::Int(1)); + + let mut middle_kv: VecMap = VecMap::new(); + middle_kv.insert( + bs("items"), + AttributeValue::List(vec![ + AttributeValue::String(bs("a")), + AttributeValue::KeyValue(nested_kv), + ]), + ); + + let mut attrs: VecMap = VecMap::new(); + attrs.insert(bs("outer"), AttributeValue::KeyValue(middle_kv)); + + let payload = minimal_payload( + [0u8; 16], + SpanBytes { + attributes: attrs, + ..minimal_span() + }, + ); + let traces = encode_and_decode(&payload); + let ms = map_get(&traces[0][0], "meta_struct").expect("meta_struct present"); + + let outer = meta_struct_inner(ms, "outer"); + let items = map_get(&outer, "items").expect("items present"); + let items_arr = items.as_array().expect("items is array"); + assert_eq!(items_arr.len(), 2); + assert_eq!(items_arr[0].as_str(), Some("a")); + let nested_map = items_arr[1].as_map().expect("nested KeyValue is map"); + assert_eq!(nested_map.len(), 1); + assert_eq!(map_get(&items_arr[1], "k").unwrap().as_i64(), Some(1)); + } + #[test] fn chunk_origin_priority_and_sampling_mechanism_propagate_to_span() { let chunk_attrs: VecMap = VecMap::new(); From 8c5d507990004df045dc7551e4513f6faa1bd284 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Mon, 29 Jun 2026 19:30:21 +0200 Subject: [PATCH 07/23] fix: clippy --- libdd-data-pipeline/src/trace_exporter/trace_serializer.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs b/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs index 37eb1ea7cd..e60eef5f10 100644 --- a/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs +++ b/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs @@ -130,8 +130,8 @@ impl TraceSerializer { (tracer_payload::TraceChunks::V04(p), TraceExporterOutputFormat::V04) => { msgpack_encoder::v04::to_vec_with_capacity(p, capacity as u32) } - // v0.4 spans cross-encoded as V1 on the wire — used when the agent advertises - // /v1.0/traces. Same in-memory shape as the v0.4 native path, different encoder. + // v0.4 spans cross-encoded as V1 on the wire (used when the agent advertises + // /v1.0/traces). (tracer_payload::TraceChunks::V04(p), TraceExporterOutputFormat::V1) => { msgpack_encoder::v1::to_vec_with_capacity(p, capacity as u32, metadata) } @@ -145,8 +145,9 @@ impl TraceSerializer { // `msgpack_encoder::v1::to_vec_from_payload_v1` on the carried // `v1::TracerPayload`. Not yet reachable: `collect_and_process_traces` // never produces `TraceChunks::V1` in the current data-pipeline path. + #[allow(clippy::unimplemented)] (tracer_payload::TraceChunks::V1(_), TraceExporterOutputFormat::V1) => { - todo!("Native V1 input serialization not yet implemented (APMSP-2812)") + unimplemented!("Native V1 input serialization not yet implemented (APMSP-2812)") } // `collect_and_process_traces` only produces (V04, V04|V1), (V05, V05), // or (V1, V1) — any other combination here is a programming error. From 7cf69fd461fa62ea7f9dc2ba095487d771854dc4 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Mon, 29 Jun 2026 19:51:15 +0200 Subject: [PATCH 08/23] fix: clippy --- libdd-data-pipeline/src/trace_exporter/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libdd-data-pipeline/src/trace_exporter/mod.rs b/libdd-data-pipeline/src/trace_exporter/mod.rs index 371d456bc2..bae2d96675 100644 --- a/libdd-data-pipeline/src/trace_exporter/mod.rs +++ b/libdd-data-pipeline/src/trace_exporter/mod.rs @@ -1335,7 +1335,7 @@ mod tests { span_id: 2, ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_v04(&traces); let resp = exporter.send(data.as_ref()).unwrap(); assert!(matches!(resp, AgentResponse::Unchanged)); @@ -1376,7 +1376,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_v04(&traces); // `send` is synchronous and, in log mode, returns after writing through the // capability without initiating any HTTP; combined with the structural assert // above this is deterministic (no background worker can race the mock). From 8d3a26b9ba54fc4266245e79c84f2840efa4c5f4 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Wed, 1 Jul 2026 14:30:25 +0200 Subject: [PATCH 09/23] fix: comments --- datadog-sidecar-ffi/src/lib.rs | 4 +- .../src/trace_exporter/builder.rs | 1 + libdd-data-pipeline/src/trace_exporter/mod.rs | 30 ++-- .../src/trace_exporter/trace_serializer.rs | 4 +- libdd-trace-utils/benches/serialization.rs | 2 +- libdd-trace-utils/src/msgpack_encoder/mod.rs | 26 +++ .../src/msgpack_encoder/v04/mod.rs | 88 ++++++----- .../src/msgpack_encoder/v04/span_v04.rs | 69 ++++---- .../src/msgpack_encoder/v04/span_v1.rs | 113 +++++++------ .../src/msgpack_encoder/v1/mod.rs | 148 ++++++++++-------- .../src/msgpack_encoder/v1/span_v04.rs | 5 +- .../src/msgpack_encoder/v1/span_v1.rs | 4 +- libdd-trace-utils/src/send_data/mod.rs | 4 +- libdd-trace-utils/src/span/v1/mod.rs | 1 + libdd-trace-utils/src/tracer_metadata.rs | 2 + libdd-trace-utils/tests/test_send_data.rs | 23 +-- 16 files changed, 284 insertions(+), 240 deletions(-) diff --git a/datadog-sidecar-ffi/src/lib.rs b/datadog-sidecar-ffi/src/lib.rs index 8d6278b732..daced522f6 100644 --- a/datadog-sidecar-ffi/src/lib.rs +++ b/datadog-sidecar-ffi/src/lib.rs @@ -1731,7 +1731,7 @@ pub unsafe extern "C" fn ddog_send_traces_to_sidecar( // Write traces to the shared memory let mut shm_slice = mapped_shm.as_slice_mut(); let shm_slice_len = shm_slice.len(); - let written = match msgpack_encoder::v04::write_to_slice_v04(&mut shm_slice, traces) { + let written = match msgpack_encoder::v04::write_to_slice_from_v04(&mut shm_slice, traces) { Ok(()) => shm_slice_len - shm_slice.len(), Err(_) => { tracing::error!("Failed serializing the traces"); @@ -1761,7 +1761,7 @@ pub unsafe extern "C" fn ddog_send_traces_to_sidecar( match blocking::send_trace_v04_bytes( &mut parameters.transport, ¶meters.instance_id, - msgpack_encoder::v04::to_vec_with_capacity_v04(traces, written as u32), + msgpack_encoder::v04::to_vec_with_capacity_from_v04(traces, written as u32), check!( (¶meters.tracer_headers_tags).try_into(), "Failed to convert tracer headers tags" diff --git a/libdd-data-pipeline/src/trace_exporter/builder.rs b/libdd-data-pipeline/src/trace_exporter/builder.rs index 1fab5f43e3..a20e020e5d 100644 --- a/libdd-data-pipeline/src/trace_exporter/builder.rs +++ b/libdd-data-pipeline/src/trace_exporter/builder.rs @@ -725,6 +725,7 @@ impl TraceExporterBuilder { app_version: self.app_version, runtime_id, service: self.service, + container_id: String::new(), }, input_format: self.input_format, output_format: self.output_format, diff --git a/libdd-data-pipeline/src/trace_exporter/mod.rs b/libdd-data-pipeline/src/trace_exporter/mod.rs index bae2d96675..353cfbc129 100644 --- a/libdd-data-pipeline/src/trace_exporter/mod.rs +++ b/libdd-data-pipeline/src/trace_exporter/mod.rs @@ -1335,7 +1335,7 @@ mod tests { span_id: 2, ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec_v04(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); let resp = exporter.send(data.as_ref()).unwrap(); assert!(matches!(resp, AgentResponse::Unchanged)); @@ -1376,7 +1376,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec_v04(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); // `send` is synchronous and, in log mode, returns after writing through the // capability without initiating any HTTP; combined with the structural assert // above this is deterministic (no background worker can race the mock). @@ -1417,7 +1417,7 @@ mod tests { ..Default::default() }], ]; - let data = msgpack_encoder::v04::to_vec_v04(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); let _result = exporter.send(data.as_ref()).expect("failed to send trace"); @@ -1517,7 +1517,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec_v04(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); let result = exporter.send(data.as_ref()); assert!(result.is_err()); @@ -1625,7 +1625,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec_v04(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); let result = exporter.send(data.as_ref()); assert!(result.is_err()); @@ -1729,7 +1729,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec_v04(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); let _result = exporter.send(data.as_ref()).expect("failed to send trace"); @@ -1789,7 +1789,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec_v04(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); let result = exporter.send(data.as_ref()).unwrap(); assert_eq!( @@ -1831,7 +1831,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec_v04(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); let code = match exporter.send(data.as_ref()).unwrap_err() { TraceExporterError::Request(e) => Some(e.status()), _ => None, @@ -1866,7 +1866,7 @@ mod tests { name: BytesString::from_slice(b"test").unwrap(), ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec_v04(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); let err = exporter.send(data.as_ref()); assert!(err.is_err()); @@ -2211,7 +2211,7 @@ mod tests { ..Default::default() }]; - let data = msgpack_encoder::v04::to_vec_v04(&[trace_chunk]); + let data = msgpack_encoder::v04::to_vec_from_v04(&[trace_chunk]); // Wait for the info fetcher to get the config while mock_info.calls() == 0 { @@ -2279,7 +2279,7 @@ mod tests { error: 0, ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec_v04(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); let result = exporter.send(data.as_ref()); assert!( @@ -2356,7 +2356,7 @@ mod single_threaded_tests { ..Default::default() }]; - let data = msgpack_encoder::v04::to_vec_v04(&[trace_chunk]); + let data = msgpack_encoder::v04::to_vec_from_v04(&[trace_chunk]); // Wait for the info fetcher to get the config while agent_info::get_agent_info().is_none() { @@ -2457,7 +2457,7 @@ mod single_threaded_tests { ..Default::default() }]; - let data = msgpack_encoder::v04::to_vec_v04(&[trace_chunk]); + let data = msgpack_encoder::v04::to_vec_from_v04(&[trace_chunk]); // Wait for agent_info to be present so that sending a trace will trigger the stats worker // to start @@ -2556,7 +2556,7 @@ mod single_threaded_tests { duration: 10, ..Default::default() }]; - let data = msgpack_encoder::v04::to_vec_v04(&[trace_chunk]); + let data = msgpack_encoder::v04::to_vec_from_v04(&[trace_chunk]); let _ = exporter.send(data.as_ref()); let start = std::time::Instant::now(); @@ -2664,7 +2664,7 @@ mod single_threaded_tests { duration: 10, ..Default::default() }]; - let data = msgpack_encoder::v04::to_vec_v04(&[trace_chunk]); + let data = msgpack_encoder::v04::to_vec_from_v04(&[trace_chunk]); // 1st send: /info has promoted v1_active=true, so this hits /v1.0/traces and 404s. let result1 = exporter.send(&data); diff --git a/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs b/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs index 8d053f69a0..98a0c980f2 100644 --- a/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs +++ b/libdd-data-pipeline/src/trace_exporter/trace_serializer.rs @@ -120,7 +120,7 @@ impl TraceSerializer { .max(MIN_BUFFER_CAPACITY); let buff = match payload { tracer_payload::TraceChunks::V04(p) => { - msgpack_encoder::v04::to_vec_with_capacity_v04(p, capacity as u32) + msgpack_encoder::v04::to_vec_with_capacity_from_v04(p, capacity as u32) } tracer_payload::TraceChunks::V05(p) => { let mut buff = Vec::with_capacity(capacity); @@ -129,7 +129,7 @@ impl TraceSerializer { buff } tracer_payload::TraceChunks::V1(p) => { - msgpack_encoder::v1::to_vec_with_capacity_v04(p, capacity as u32, metadata) + msgpack_encoder::v1::to_vec_with_capacity_from_v04(p, capacity as u32, metadata) } }; self.previous_serialised_len diff --git a/libdd-trace-utils/benches/serialization.rs b/libdd-trace-utils/benches/serialization.rs index 931b96928b..8d41e18b70 100644 --- a/libdd-trace-utils/benches/serialization.rs +++ b/libdd-trace-utils/benches/serialization.rs @@ -65,7 +65,7 @@ pub fn serialize_internal_to_msgpack(c: &mut Criterion) { b.iter_batched( || vec![0u8; 12_000_000], |mut vec| { - let _ = black_box(msgpack_encoder::v04::write_to_slice_v04( + let _ = black_box(msgpack_encoder::v04::write_to_slice_from_v04( &mut vec.as_mut_slice(), black_box(&data), )); diff --git a/libdd-trace-utils/src/msgpack_encoder/mod.rs b/libdd-trace-utils/src/msgpack_encoder/mod.rs index 2d54a03349..2370de9071 100644 --- a/libdd-trace-utils/src/msgpack_encoder/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/mod.rs @@ -1,6 +1,32 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +//! # Encoder layout & naming convention +//! +//! ```text +//! msgpack_encoder/ +//! ├── v04/ +//! │ ├── mod.rs // public API + payload-level helpers +//! │ ├── span_v04.rs // v0.4 in-memory Span → v0.4 wire (native) +//! │ └── span_v1.rs // v1 in-memory Span → v0.4 wire (downgrade) +//! └── v1/ +//! ├── mod.rs +//! ├── span_v04.rs // v0.4 in-memory Span → V1 wire (upgrade) +//! └── span_v1.rs // v1 in-memory Span → V1 wire (native) +//! ``` +//! +//! - **Module (`v04`/`v1`) = output wire format.** +//! - **File suffix (`_v04`/`_v1`) = input span type.** +//! - **Public functions carry a `_from_` suffix**, so a caller reads the *output* from the +//! module path and the *input* from the function name: +//! +//! | Module | Function | Input → Output | +//! |--------|----------|----------------| +//! | `v04::` | `to_vec_from_v04`, `write_to_slice_from_v04`, `to_encoded_byte_len_from_v04` | v04 → v0.4 (native) | +//! | `v04::` | `to_vec_from_v1`, `write_to_slice_from_v1`, `to_encoded_byte_len_from_v1` | v1 → v0.4 (downgrade) | +//! | `v1::` | `to_vec_from_v04`, `write_to_slice_from_v04`, `to_encoded_byte_len_from_v04` | v04 → V1 (upgrade) | +//! | `v1::` | `to_vec_from_v1`, `write_to_slice_from_v1`, `to_encoded_byte_len_from_v1` | v1 → V1 (native) | + pub mod v04; pub mod v1; diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs index 7d4b01dbf0..22e6cd1473 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs @@ -7,22 +7,7 @@ use crate::span::TraceData; use libdd_common::ResultInfallibleExt; use rmp::encode::{write_array_len, ByteBuf, RmpWrite, ValueWriteError}; -macro_rules! write_const_msg_pack_str { - ($writer:expr, $str:expr) => {{ - use rmp::encode::ValueWriteError; - const STRING_ENCODING_LEN: usize = super::msp_string_encoding_len($str); - const STRING_ENCODING: [u8; STRING_ENCODING_LEN] = super::msp_const_string_encoding($str); - - $writer - .write_bytes(&STRING_ENCODING) - .map_err(ValueWriteError::InvalidDataWrite) - }}; -} - -mod span_v04; -mod span_v1; - -const fn msp_string_encoding_len(s: &str) -> usize { +const fn msgpack_string_encoding_len(s: &str) -> usize { let length_marker_len = if s.len() < 32 { 1 } else if s.len() < 256 { @@ -35,8 +20,8 @@ const fn msp_string_encoding_len(s: &str) -> usize { length_marker_len + s.len() } -// Compute the encoding of a string to messagepack in a const manner -const fn msp_const_string_encoding(s: &str) -> [u8; ENCODING_LEN] { +// Compute the encoding of a string to msgpack in a const manner +const fn msgpack_const_string_encoding(s: &str) -> [u8; ENCODING_LEN] { // copy_to_slice is not const yet, so we make a helper const fn copy_to_slice(dest: &mut [u8], src: &[u8], n: usize) { let mut i = 0; @@ -66,6 +51,22 @@ const fn msp_const_string_encoding(s: &str) -> [u8; E storage } +macro_rules! write_const_msgpack_str { + ($writer:expr, $str:expr) => {{ + use rmp::encode::ValueWriteError; + const STRING_ENCODING_LEN: usize = super::msgpack_string_encoding_len($str); + const STRING_ENCODING: [u8; STRING_ENCODING_LEN] = + super::msgpack_const_string_encoding($str); + + $writer + .write_bytes(&STRING_ENCODING) + .map_err(ValueWriteError::InvalidDataWrite) + }}; +} + +mod span_v04; +mod span_v1; + #[inline(always)] fn to_writer]>>( writer: &mut W, @@ -103,7 +104,7 @@ fn to_writer]>>( /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v04::write_to_slice_v04; +/// use libdd_trace_utils::msgpack_encoder::v04::write_to_slice_from_v04; /// use libdd_trace_utils::span::v04::SpanSlice; /// /// let mut buffer = vec![0u8; 1024]; @@ -113,9 +114,9 @@ fn to_writer]>>( /// }; /// let traces = vec![vec![span]]; /// -/// write_to_slice_v04(&mut &mut buffer[..], &traces).expect("Encoding failed"); +/// write_to_slice_from_v04(&mut &mut buffer[..], &traces).expect("Encoding failed"); /// ``` -pub fn write_to_slice_v04]>>( +pub fn write_to_slice_from_v04]>>( slice: &mut &mut [u8], traces: &[S], ) -> Result<(), ValueWriteError> { @@ -135,7 +136,7 @@ pub fn write_to_slice_v04]>>( /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v04::to_vec_v04; +/// use libdd_trace_utils::msgpack_encoder::v04::to_vec_from_v04; /// use libdd_trace_utils::span::v04::SpanSlice; /// /// let span = SpanSlice { @@ -143,12 +144,12 @@ pub fn write_to_slice_v04]>>( /// ..Default::default() /// }; /// let traces = vec![vec![span]]; -/// let encoded = to_vec_v04(&traces); +/// let encoded = to_vec_from_v04(&traces); /// /// assert!(!encoded.is_empty()); /// ``` -pub fn to_vec_v04]>>(traces: &[S]) -> Vec { - to_vec_with_capacity_v04(traces, 0) +pub fn to_vec_from_v04]>>(traces: &[S]) -> Vec { + to_vec_with_capacity_from_v04(traces, 0) } /// Serializes traces into a vector of bytes with specified capacity. @@ -165,7 +166,7 @@ pub fn to_vec_v04]>>(traces: &[S]) -> Vec { /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v04::to_vec_with_capacity_v04; +/// use libdd_trace_utils::msgpack_encoder::v04::to_vec_with_capacity_from_v04; /// use libdd_trace_utils::span::v04::SpanSlice; /// /// let span = SpanSlice { @@ -173,11 +174,11 @@ pub fn to_vec_v04]>>(traces: &[S]) -> Vec { /// ..Default::default() /// }; /// let traces = vec![vec![span]]; -/// let encoded = to_vec_with_capacity_v04(&traces, 1024); +/// let encoded = to_vec_with_capacity_from_v04(&traces, 1024); /// /// assert!(encoded.capacity() >= 1024); /// ``` -pub fn to_vec_with_capacity_v04]>>( +pub fn to_vec_with_capacity_from_v04]>>( traces: &[S], capacity: u32, ) -> Vec { @@ -204,7 +205,7 @@ pub fn to_vec_with_capacity_v04]>>( /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v04::to_encoded_byte_len_v04; +/// use libdd_trace_utils::msgpack_encoder::v04::to_encoded_byte_len_from_v04; /// use libdd_trace_utils::span::v04::SpanSlice; /// /// let span = SpanSlice { @@ -212,11 +213,11 @@ pub fn to_vec_with_capacity_v04]>>( /// ..Default::default() /// }; /// let traces = vec![vec![span]]; -/// let encoded_len = to_encoded_byte_len_v04(&traces); +/// let encoded_len = to_encoded_byte_len_from_v04(&traces); /// /// assert!(encoded_len > 0); /// ``` -pub fn to_encoded_byte_len_v04]>>(traces: &[S]) -> u32 { +pub fn to_encoded_byte_len_from_v04]>>(traces: &[S]) -> u32 { let mut counter = super::CountLength(0); // `CountLength` impls `std::io::Write` (whose error type is `std::io::Error`, not // `Infallible`), so we can't statically prove infallibility via `unwrap_infallible` @@ -231,7 +232,7 @@ pub fn to_encoded_byte_len_v04]>>(traces: &[S]) /// does not advertise `/v1.0/traces`). The output is a msgpack array of traces, where each /// trace is itself a msgpack array of v0.4-shaped spans — matching the existing v0.4 wire /// format produced by [`to_vec`]. -fn encode_payload_v1( +fn encode_payload_from_v1( writer: &mut W, payload: &TracerPayload, ) -> Result<(), ValueWriteError> { @@ -260,16 +261,19 @@ fn encode_payload_v1( /// The output is byte-compatible with [`to_vec`] for equivalent data — chunk-level fields are /// propagated to every span and typed attributes are bucketed into the v0.4 `meta` / /// `metrics` / `meta_struct` maps per [`span_v1_to_v04`]'s mapping table. -pub fn to_vec_v1(payload: &TracerPayload) -> Vec { - to_vec_with_capacity_v1(payload, 0) +pub fn to_vec_from_v1(payload: &TracerPayload) -> Vec { + to_vec_with_capacity_from_v1(payload, 0) } /// Serializes a [`TracerPayload`] as a v0.4 msgpack payload with a caller-supplied initial /// capacity. Use this when you can size the buffer up front (e.g. from -/// [`to_encoded_byte_len_v1`]) to avoid reallocations. -pub fn to_vec_with_capacity_v1(payload: &TracerPayload, capacity: u32) -> Vec { +/// [`to_encoded_byte_len_from_v1`]) to avoid reallocations. +pub fn to_vec_with_capacity_from_v1( + payload: &TracerPayload, + capacity: u32, +) -> Vec { let mut buf = ByteBuf::with_capacity(capacity as usize); - encode_payload_v1(&mut buf, payload) + encode_payload_from_v1(&mut buf, payload) .map_err(super::flatten_value_write_infallible) .unwrap_infallible(); buf.into_vec() @@ -281,17 +285,17 @@ pub fn to_vec_with_capacity_v1(payload: &TracerPayload, capacit /// # Errors /// /// Returns any [`ValueWriteError`] from the underlying writer (typically buffer-too-small). -pub fn write_to_slice_v1( +pub fn write_to_slice_from_v1( slice: &mut &mut [u8], payload: &TracerPayload, ) -> Result<(), ValueWriteError> { - encode_payload_v1(slice, payload) + encode_payload_from_v1(slice, payload) } -/// Returns the exact number of bytes [`to_vec_v1`] would write for `payload`. Walks +/// Returns the exact number of bytes [`to_vec_from_v1`] would write for `payload`. Walks /// the payload through a counting writer without allocating an output buffer. -pub fn to_encoded_byte_len_v1(payload: &TracerPayload) -> u32 { +pub fn to_encoded_byte_len_from_v1(payload: &TracerPayload) -> u32 { let mut counter = super::CountLength(0); - let _ = encode_payload_v1(&mut counter, payload); + let _ = encode_payload_from_v1(&mut counter, payload); counter.0 } diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs index 1a0fd19bf0..9d079f3649 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v04.rs @@ -1,9 +1,8 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -// Naming convention: parent module (`v04`) = output wire format, file suffix (`_v04`) = input -// span type. This file encodes a [`crate::span::v04::Span`] into the v0.4 msgpack wire format -// (native). +//! Native v0.4 span encoder: `crate::span::v04::Span` → v0.4 msgpack wire. +//! (Convention documented in [`crate::msgpack_encoder`].) use crate::span::v04::{AttributeAnyValue, AttributeArrayValue, Span, SpanEvent, SpanLink}; use crate::span::TraceData; @@ -32,7 +31,7 @@ pub fn encode_span_links( writer: &mut W, span_links: &[SpanLink], ) -> Result<(), ValueWriteError> { - write_const_msg_pack_str!(writer, "span_links")?; + write_const_msgpack_str!(writer, "span_links")?; rmp::encode::write_array_len(writer, span_links.len() as u32)?; for link in span_links.iter() { @@ -43,17 +42,17 @@ pub fn encode_span_links( rmp::encode::write_map_len(writer, link_len)?; - write_const_msg_pack_str!(writer, "trace_id")?; + write_const_msgpack_str!(writer, "trace_id")?; write_u64(writer, link.trace_id)?; - write_const_msg_pack_str!(writer, "trace_id_high")?; + write_const_msgpack_str!(writer, "trace_id_high")?; write_u64(writer, link.trace_id_high)?; - write_const_msg_pack_str!(writer, "span_id")?; + write_const_msgpack_str!(writer, "span_id")?; write_u64(writer, link.span_id)?; if !link.attributes.is_empty() { - write_const_msg_pack_str!(writer, "attributes")?; + write_const_msgpack_str!(writer, "attributes")?; rmp::encode::write_map_len(writer, link.attributes.len() as u32)?; for (k, v) in link.attributes.iter() { write_str(writer, k.borrow())?; @@ -62,12 +61,12 @@ pub fn encode_span_links( } if !link.tracestate.borrow().is_empty() { - write_const_msg_pack_str!(writer, "tracestate")?; + write_const_msgpack_str!(writer, "tracestate")?; write_str(writer, link.tracestate.borrow())?; } if link.flags != 0 { - write_const_msg_pack_str!(writer, "flags")?; + write_const_msgpack_str!(writer, "flags")?; write_u32(writer, link.flags)?; } } @@ -94,7 +93,7 @@ pub fn encode_span_events( writer: &mut W, span_events: &[SpanEvent], ) -> Result<(), ValueWriteError> { - write_const_msg_pack_str!(writer, "span_events")?; + write_const_msgpack_str!(writer, "span_events")?; rmp::encode::write_array_len(writer, span_events.len() as u32)?; for event in span_events.iter() { let event_len = 2 /* minimal span event: time_unix_nano, name */ @@ -102,14 +101,14 @@ pub fn encode_span_events( rmp::encode::write_map_len(writer, event_len)?; - write_const_msg_pack_str!(writer, "time_unix_nano")?; + write_const_msgpack_str!(writer, "time_unix_nano")?; write_u64(writer, event.time_unix_nano)?; - write_const_msg_pack_str!(writer, "name")?; + write_const_msgpack_str!(writer, "name")?; write_str(writer, event.name.borrow())?; if !event.attributes.is_empty() { - write_const_msg_pack_str!(writer, "attributes")?; + write_const_msgpack_str!(writer, "attributes")?; rmp::encode::write_map_len(writer, event.attributes.len() as u32)?; for (k, attribute) in event.attributes.iter() { write_str(writer, k.borrow())?; @@ -120,26 +119,26 @@ pub fn encode_span_events( ) -> Result<(), ValueWriteError> { rmp::encode::write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; match value { AttributeArrayValue::String(s) => { write_u8(writer, 0)?; - write_const_msg_pack_str!(writer, "string_value")?; + write_const_msgpack_str!(writer, "string_value")?; write_str(writer, s.borrow())?; } AttributeArrayValue::Boolean(bool) => { write_u8(writer, 1)?; - write_const_msg_pack_str!(writer, "bool_value")?; + write_const_msgpack_str!(writer, "bool_value")?; write_bool(writer, *bool).map_err(ValueWriteError::InvalidDataWrite)?; } AttributeArrayValue::Integer(int) => { write_u8(writer, 2)?; - write_const_msg_pack_str!(writer, "int_value")?; + write_const_msgpack_str!(writer, "int_value")?; write_sint(writer, *int)?; } AttributeArrayValue::Double(double) => { write_u8(writer, 3)?; - write_const_msg_pack_str!(writer, "double_value")?; + write_const_msgpack_str!(writer, "double_value")?; write_f64(writer, *double)?; } }; @@ -154,13 +153,13 @@ pub fn encode_span_events( AttributeAnyValue::Array(array) => { rmp::encode::write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_u8(writer, 4)?; - write_const_msg_pack_str!(writer, "array_value")?; + write_const_msgpack_str!(writer, "array_value")?; rmp::encode::write_map_len(writer, 1)?; - write_const_msg_pack_str!(writer, "values")?; + write_const_msgpack_str!(writer, "values")?; rmp::encode::write_array_len(writer, array.len() as u32)?; for v in array.iter() { write_array_value(writer, v)?; @@ -206,39 +205,39 @@ pub fn encode_span( rmp::encode::write_map_len(writer, span_len)?; - write_const_msg_pack_str!(writer, "service")?; + write_const_msgpack_str!(writer, "service")?; write_str(writer, span.service.borrow())?; - write_const_msg_pack_str!(writer, "name")?; + write_const_msgpack_str!(writer, "name")?; write_str(writer, span.name.borrow())?; - write_const_msg_pack_str!(writer, "resource")?; + write_const_msgpack_str!(writer, "resource")?; write_str(writer, span.resource.borrow())?; - write_const_msg_pack_str!(writer, "trace_id")?; + write_const_msgpack_str!(writer, "trace_id")?; write_u64(writer, span.trace_id as u64)?; - write_const_msg_pack_str!(writer, "span_id")?; + write_const_msgpack_str!(writer, "span_id")?; write_u64(writer, span.span_id)?; if span.parent_id != 0 { - write_const_msg_pack_str!(writer, "parent_id")?; + write_const_msgpack_str!(writer, "parent_id")?; write_u64(writer, span.parent_id)?; } - write_const_msg_pack_str!(writer, "start")?; + write_const_msgpack_str!(writer, "start")?; write_i64(writer, span.start)?; - write_const_msg_pack_str!(writer, "duration")?; + write_const_msgpack_str!(writer, "duration")?; write_sint(writer, span.duration)?; if span.error != 0 { - write_const_msg_pack_str!(writer, "error")?; + write_const_msgpack_str!(writer, "error")?; write_sint(writer, span.error as i64)?; } if !span.meta.is_empty() { - write_const_msg_pack_str!(writer, "meta")?; + write_const_msgpack_str!(writer, "meta")?; let meta_dd = span.meta.defensive_dedup(); rmp::encode::write_map_len(writer, meta_dd.len() as u32)?; for (k, v) in meta_dd.iter() { @@ -248,7 +247,7 @@ pub fn encode_span( } if !span.metrics.is_empty() { - write_const_msg_pack_str!(writer, "metrics")?; + write_const_msgpack_str!(writer, "metrics")?; let metrics_dd = span.metrics.defensive_dedup(); rmp::encode::write_map_len(writer, metrics_dd.len() as u32)?; for (k, v) in metrics_dd.iter() { @@ -258,12 +257,12 @@ pub fn encode_span( } if !span.r#type.borrow().is_empty() { - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_str(writer, span.r#type.borrow())?; } if !span.meta_struct.is_empty() { - write_const_msg_pack_str!(writer, "meta_struct")?; + write_const_msgpack_str!(writer, "meta_struct")?; let meta_struct_dd = span.meta_struct.defensive_dedup(); rmp::encode::write_map_len(writer, meta_struct_dd.len() as u32)?; for (k, v) in meta_struct_dd.iter() { diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs index 134514701f..2b16399328 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs @@ -1,11 +1,8 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -// Naming convention: parent module (`v04`) = output wire format, file suffix (`_v1`) = input -// span type. This file encodes a [`crate::span::v1::Span`] into the v0.4 msgpack wire format -// (cross-format downgrade). - -//! Downgrade encoder: writes a [`crate::span::v1::Span`] in the v0.4 msgpack wire format. +//! Downgrade encoder: `crate::span::v1::Span` → v0.4 msgpack wire. +//! (Convention documented in [`crate::msgpack_encoder`].) //! //! Used when the receiving agent does not advertise the `/v1.0/traces` endpoint and the tracer //! must fall back to v0.4. The mapping is: @@ -172,68 +169,68 @@ pub(super) fn encode_span( write_map_len(writer, span_len)?; - write_const_msg_pack_str!(writer, "service")?; + write_const_msgpack_str!(writer, "service")?; write_str(writer, span.service.borrow())?; - write_const_msg_pack_str!(writer, "name")?; + write_const_msgpack_str!(writer, "name")?; write_str(writer, span.name.borrow())?; - write_const_msg_pack_str!(writer, "resource")?; + write_const_msgpack_str!(writer, "resource")?; write_str(writer, span.resource.borrow())?; - write_const_msg_pack_str!(writer, "trace_id")?; + write_const_msgpack_str!(writer, "trace_id")?; write_u64(writer, trace_id_low)?; - write_const_msg_pack_str!(writer, "span_id")?; + write_const_msgpack_str!(writer, "span_id")?; write_u64(writer, span.span_id)?; if span.parent_id != 0 { - write_const_msg_pack_str!(writer, "parent_id")?; + write_const_msgpack_str!(writer, "parent_id")?; write_u64(writer, span.parent_id)?; } - write_const_msg_pack_str!(writer, "start")?; + write_const_msgpack_str!(writer, "start")?; write_i64(writer, span.start)?; - write_const_msg_pack_str!(writer, "duration")?; + write_const_msgpack_str!(writer, "duration")?; write_sint(writer, span.duration)?; if span.error { - write_const_msg_pack_str!(writer, "error")?; + write_const_msgpack_str!(writer, "error")?; write_sint(writer, 1)?; } if counts.meta > 0 { - write_const_msg_pack_str!(writer, "meta")?; + write_const_msgpack_str!(writer, "meta")?; write_map_len(writer, counts.meta)?; if !span.env.borrow().is_empty() { - write_const_msg_pack_str!(writer, "env")?; + write_const_msgpack_str!(writer, "env")?; write_str(writer, span.env.borrow())?; } if !span.version.borrow().is_empty() { - write_const_msg_pack_str!(writer, "version")?; + write_const_msgpack_str!(writer, "version")?; write_str(writer, span.version.borrow())?; } if !span.component.borrow().is_empty() { - write_const_msg_pack_str!(writer, "component")?; + write_const_msgpack_str!(writer, "component")?; write_str(writer, span.component.borrow())?; } if let Some(kind_str) = kind_meta { - write_const_msg_pack_str!(writer, "span.kind")?; + write_const_msgpack_str!(writer, "span.kind")?; write_str(writer, kind_str)?; } if trace_id_high != 0 { // Lower-case hex without `0x` prefix — the agent expects this format. - write_const_msg_pack_str!(writer, "_dd.p.tid")?; + write_const_msgpack_str!(writer, "_dd.p.tid")?; write_str(writer, &format!("{trace_id_high:016x}"))?; } if !chunk.origin.borrow().is_empty() { - write_const_msg_pack_str!(writer, "_dd.origin")?; + write_const_msgpack_str!(writer, "_dd.origin")?; write_str(writer, chunk.origin.borrow())?; } if let Some(mechanism) = chunk.sampling_mechanism { - write_const_msg_pack_str!(writer, "_dd.p.dm")?; + write_const_msgpack_str!(writer, "_dd.p.dm")?; write_str(writer, &format!("-{mechanism}"))?; } for &(k, v) in &merged_attrs { @@ -252,11 +249,11 @@ pub(super) fn encode_span( } if counts.metrics > 0 { - write_const_msg_pack_str!(writer, "metrics")?; + write_const_msgpack_str!(writer, "metrics")?; write_map_len(writer, counts.metrics)?; if let Some(priority) = chunk.priority { - write_const_msg_pack_str!(writer, "_sampling_priority_v1")?; + write_const_msgpack_str!(writer, "_sampling_priority_v1")?; write_f64(writer, priority as f64)?; } for &(k, v) in &merged_attrs { @@ -275,12 +272,12 @@ pub(super) fn encode_span( } if !span.r#type.borrow().is_empty() { - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_str(writer, span.r#type.borrow())?; } if counts.meta_struct > 0 { - write_const_msg_pack_str!(writer, "meta_struct")?; + write_const_msgpack_str!(writer, "meta_struct")?; write_map_len(writer, counts.meta_struct)?; for &(k, v) in &merged_attrs { @@ -375,7 +372,7 @@ fn encode_span_links( writer: &mut W, span_links: &[SpanLink], ) -> Result<(), ValueWriteError> { - write_const_msg_pack_str!(writer, "span_links")?; + write_const_msgpack_str!(writer, "span_links")?; write_array_len(writer, span_links.len() as u32)?; for link in span_links { @@ -393,17 +390,17 @@ fn encode_span_links( write_map_len(writer, link_len)?; - write_const_msg_pack_str!(writer, "trace_id")?; + write_const_msgpack_str!(writer, "trace_id")?; write_u64(writer, trace_id_low)?; - write_const_msg_pack_str!(writer, "trace_id_high")?; + write_const_msgpack_str!(writer, "trace_id_high")?; write_u64(writer, trace_id_high)?; - write_const_msg_pack_str!(writer, "span_id")?; + write_const_msgpack_str!(writer, "span_id")?; write_u64(writer, link.span_id)?; if attr_count > 0 { - write_const_msg_pack_str!(writer, "attributes")?; + write_const_msgpack_str!(writer, "attributes")?; write_map_len(writer, attr_count)?; for (k, v) in attrs_dd.iter() { match v { @@ -421,12 +418,12 @@ fn encode_span_links( } if !link.tracestate.borrow().is_empty() { - write_const_msg_pack_str!(writer, "tracestate")?; + write_const_msgpack_str!(writer, "tracestate")?; write_str(writer, link.tracestate.borrow())?; } if link.flags != 0 { - write_const_msg_pack_str!(writer, "flags")?; + write_const_msgpack_str!(writer, "flags")?; write_u32(writer, link.flags)?; } } @@ -442,7 +439,7 @@ fn encode_span_events( writer: &mut W, span_events: &[SpanEvent], ) -> Result<(), ValueWriteError> { - write_const_msg_pack_str!(writer, "span_events")?; + write_const_msgpack_str!(writer, "span_events")?; write_array_len(writer, span_events.len() as u32)?; for event in span_events { @@ -457,14 +454,14 @@ fn encode_span_events( write_map_len(writer, event_len)?; - write_const_msg_pack_str!(writer, "time_unix_nano")?; + write_const_msgpack_str!(writer, "time_unix_nano")?; write_u64(writer, event.time_unix_nano)?; - write_const_msg_pack_str!(writer, "name")?; + write_const_msgpack_str!(writer, "name")?; write_str(writer, event.name.borrow())?; if attr_count > 0 { - write_const_msg_pack_str!(writer, "attributes")?; + write_const_msgpack_str!(writer, "attributes")?; write_map_len(writer, attr_count)?; for (k, v) in attrs_dd.iter() { if !is_supported_event_attr(v) { @@ -501,30 +498,30 @@ fn write_event_attr_value( match v { AttributeValue::String(s) => { write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_u8(writer, 0)?; - write_const_msg_pack_str!(writer, "string_value")?; + write_const_msgpack_str!(writer, "string_value")?; write_str(writer, s.borrow())?; } AttributeValue::Bool(b) => { write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_u8(writer, 1)?; - write_const_msg_pack_str!(writer, "bool_value")?; + write_const_msgpack_str!(writer, "bool_value")?; write_bool(writer, *b).map_err(ValueWriteError::InvalidDataWrite)?; } AttributeValue::Int(i) => { write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_u8(writer, 2)?; - write_const_msg_pack_str!(writer, "int_value")?; + write_const_msgpack_str!(writer, "int_value")?; write_sint(writer, *i)?; } AttributeValue::Float(f) => { write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_u8(writer, 3)?; - write_const_msg_pack_str!(writer, "double_value")?; + write_const_msgpack_str!(writer, "double_value")?; write_f64(writer, *f)?; } AttributeValue::List(arr) => { @@ -533,11 +530,11 @@ fn write_event_attr_value( let scalar_elems = arr.iter().filter(|e| is_scalar_array_elem(e)); let elem_count = scalar_elems.clone().count() as u32; write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_u8(writer, 4)?; - write_const_msg_pack_str!(writer, "array_value")?; + write_const_msgpack_str!(writer, "array_value")?; write_map_len(writer, 1)?; - write_const_msg_pack_str!(writer, "values")?; + write_const_msgpack_str!(writer, "values")?; write_array_len(writer, elem_count)?; for elem in scalar_elems { write_event_array_element(writer, elem)?; @@ -571,30 +568,30 @@ fn write_event_array_element( match v { AttributeValue::String(s) => { write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_u8(writer, 0)?; - write_const_msg_pack_str!(writer, "string_value")?; + write_const_msgpack_str!(writer, "string_value")?; write_str(writer, s.borrow())?; } AttributeValue::Bool(b) => { write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_u8(writer, 1)?; - write_const_msg_pack_str!(writer, "bool_value")?; + write_const_msgpack_str!(writer, "bool_value")?; write_bool(writer, *b).map_err(ValueWriteError::InvalidDataWrite)?; } AttributeValue::Int(i) => { write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_u8(writer, 2)?; - write_const_msg_pack_str!(writer, "int_value")?; + write_const_msgpack_str!(writer, "int_value")?; write_sint(writer, *i)?; } AttributeValue::Float(f) => { write_map_len(writer, 2)?; - write_const_msg_pack_str!(writer, "type")?; + write_const_msgpack_str!(writer, "type")?; write_u8(writer, 3)?; - write_const_msg_pack_str!(writer, "double_value")?; + write_const_msgpack_str!(writer, "double_value")?; write_f64(writer, *f)?; } _ => { @@ -608,7 +605,7 @@ fn write_event_array_element( #[cfg(test)] mod tests { //! Unit tests for the v1::Span → v0.4 downgrade encoder. Each test encodes a small - //! `TracerPayload` via [`super::super::to_vec_v1`] and decodes the bytes with + //! `TracerPayload` via [`super::super::to_vec_from_v1`] and decodes the bytes with //! `rmpv` to assert on the resulting v0.4 shape — this implicitly checks that the output //! is also valid msgpack consumable by any standard v0.4 decoder (test-agent, agent, etc.). use crate::span::v1::{ @@ -627,7 +624,7 @@ mod tests { /// Encodes `payload` and decodes back into `rmpv::Value`. The top level of v0.4 is an /// array of traces; this helper returns it as a `Vec` so tests can index in. fn encode_and_decode(payload: &TracerPayloadBytes) -> Vec { - let bytes = super::super::to_vec_v1(payload); + let bytes = super::super::to_vec_from_v1(payload); let value = rmpv::decode::read_value(&mut &bytes[..]).expect("decode failed"); match value { Value::Array(traces) => traces, diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs index 001c4f60df..d64f80e28d 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v1/mod.rs @@ -18,6 +18,7 @@ use std::collections::HashMap; /// Integer keys for the top-level V1 trace payload map. mod trace_key { + pub const CONTAINER_ID: u8 = 2; pub const LANGUAGE_NAME: u8 = 3; pub const LANGUAGE_VERSION: u8 = 4; pub const TRACER_VERSION: u8 = 5; @@ -109,9 +110,9 @@ pub(super) const FLAT_ATTR_STRIDE: u32 = 3; /// incrementing integer ID. On subsequent occurrences only the ID is emitted as a msgpack `uint`. /// ID 0 is reserved for the empty string (pre-inserted in the constructor). /// -/// The string table is scoped per payload: each `to_vec_v04` / `to_vec_from_payload_v1` (and -/// their `write_to_slice_v04` / `write_to_slice_v1` counterparts) call starts with a fresh -/// table so deduplication is payload-local. +/// The string table is scoped per payload: each `to_vec_from_v04` / `to_vec_from_v1` (and +/// their `write_to_slice_from_v04` / `write_to_slice_from_v1` counterparts) call starts with a +/// fresh table so deduplication is payload-local. pub(crate) struct StringTable { seen: HashMap, } @@ -309,6 +310,7 @@ where /// Top-level format: /// ```text /// Map { +/// trace_key::CONTAINER_ID (2) → str|uint // optional, interned /// trace_key::ENV_REF (7) → str|uint // optional, interned /// trace_key::HOSTNAME_REF (8) → str|uint // optional, interned /// trace_key::APP_VERSION (9) → str|uint // optional, interned @@ -316,7 +318,7 @@ where /// trace_key::CHUNKS (11) → Array[Chunk, ...] /// } /// ``` -fn encode_payload_v04]>>( +fn encode_payload_from_v04]>>( writer: &mut W, traces: &[S], metadata: &TracerMetadata, @@ -333,6 +335,7 @@ fn encode_payload_v04]>>( + (!metadata.language_version.is_empty()) as u32 + (!metadata.tracer_version.is_empty()) as u32 + (!metadata.runtime_id.is_empty()) as u32 + + (!metadata.container_id.is_empty()) as u32 + payload_attrs.env.is_some() as u32 + payload_attrs.hostname.is_some() as u32 + payload_attrs.app_version.is_some() as u32 @@ -343,7 +346,7 @@ fn encode_payload_v04]>>( write_uint8(writer, trace_key::CHUNKS)?; write_array_len(writer, traces.len() as u32)?; for trace in traces { - encode_chunk_v04(writer, trace.as_ref(), &mut table)?; + encode_chunk_from_v04(writer, trace.as_ref(), &mut table)?; } if !metadata.language.is_empty() { @@ -366,6 +369,11 @@ fn encode_payload_v04]>>( table.write_interned(writer, &metadata.runtime_id)?; } + if !metadata.container_id.is_empty() { + write_uint8(writer, trace_key::CONTAINER_ID)?; + table.write_interned(writer, &metadata.container_id)?; + } + if let Some(env) = payload_attrs.env { write_uint8(writer, trace_key::ENV_REF)?; table.write_interned(writer, env)?; @@ -412,7 +420,7 @@ fn encode_payload_v04]>>( /// chunk_key::SPANS (4) → Array[Span, ...] /// } /// ``` -fn encode_chunk_v04( +fn encode_chunk_from_v04( writer: &mut W, spans: &[Span], table: &mut StringTable, @@ -457,38 +465,38 @@ fn encode_chunk_v04( /// /// # Errors /// Returns a `ValueWriteError` if the underlying writer fails. -pub fn write_to_slice_v04]>>( +pub fn write_to_slice_from_v04]>>( // &mut &mut [u8] lets the caller see the slice shrink as bytes are written. slice: &mut &mut [u8], traces: &[S], metadata: &TracerMetadata, ) -> Result<(), ValueWriteError> { - encode_payload_v04(slice, traces, metadata) + encode_payload_from_v04(slice, traces, metadata) } /// Serializes traces into a `Vec` using the V1 msgpack format. -pub fn to_vec_v04]>>( +pub fn to_vec_from_v04]>>( traces: &[S], metadata: &TracerMetadata, ) -> Vec { - to_vec_with_capacity_v04(traces, 0, metadata) + to_vec_with_capacity_from_v04(traces, 0, metadata) } /// Serializes traces into a `Vec` with a pre-allocated capacity. -pub fn to_vec_with_capacity_v04]>>( +pub fn to_vec_with_capacity_from_v04]>>( traces: &[S], capacity: u32, metadata: &TracerMetadata, ) -> Vec { let mut buf = ByteBuf::with_capacity(capacity as usize); - encode_payload_v04(&mut buf, traces, metadata) + encode_payload_from_v04(&mut buf, traces, metadata) .map_err(super::flatten_value_write_infallible) .unwrap_infallible(); buf.into_vec() } /// Returns the number of bytes the V1 payload for `traces` would occupy. -pub fn to_encoded_byte_len]>>( +pub fn to_encoded_byte_len_from_v04]>>( traces: &[S], metadata: &TracerMetadata, ) -> u32 { @@ -498,12 +506,12 @@ pub fn to_encoded_byte_len]>>( // the way we do for `ByteBuf`. In practice `CountLength::write*` only ever return // `Ok`, so the error path here is unreachable today; should `CountLength` ever grow // a fallible code path, fuzz tests on the msgpack encoded length would catch it. - let _ = encode_payload_v04(&mut counter, traces, metadata); + let _ = encode_payload_from_v04(&mut counter, traces, metadata); counter.0 } /// Encodes a [`TracerPayload`] (V1 data model) as a V1 msgpack payload. -fn encode_payload_v1( +fn encode_payload_from_v1( writer: &mut W, payload: &TracerPayload, ) -> Result<(), ValueWriteError> { @@ -512,6 +520,7 @@ fn encode_payload_v1( let has_attributes = !payload.attributes.is_empty(); let map_len = 1u32 // chunks always present + + (!payload.container_id.borrow().is_empty()) as u32 + (!payload.language_name.borrow().is_empty()) as u32 + (!payload.language_version.borrow().is_empty()) as u32 + (!payload.tracer_version.borrow().is_empty()) as u32 @@ -526,7 +535,12 @@ fn encode_payload_v1( write_uint8(writer, trace_key::CHUNKS)?; write_array_len(writer, payload.chunks.len() as u32)?; for chunk in &payload.chunks { - encode_chunk_v1(writer, chunk, &mut table)?; + encode_chunk_from_v1(writer, chunk, &mut table)?; + } + + if !payload.container_id.borrow().is_empty() { + write_uint8(writer, trace_key::CONTAINER_ID)?; + table.write_interned(writer, payload.container_id.borrow())?; } if !payload.language_name.borrow().is_empty() { @@ -573,7 +587,7 @@ fn encode_payload_v1( } /// Encodes one V1 chunk (a group of spans sharing a trace ID). -fn encode_chunk_v1( +fn encode_chunk_from_v1( writer: &mut W, chunk: &crate::span::v1::TraceChunk, table: &mut StringTable, @@ -641,19 +655,19 @@ fn encode_chunk_v1( /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v1::to_vec_from_payload_v1; +/// use libdd_trace_utils::msgpack_encoder::v1::to_vec_from_v1; /// use libdd_trace_utils::span::v1::TracerPayloadSlice; /// /// let payload = TracerPayloadSlice { /// language_name: "rust".into(), /// ..Default::default() /// }; -/// let encoded = to_vec_from_payload_v1(&payload); +/// let encoded = to_vec_from_v1(&payload); /// /// assert!(!encoded.is_empty()); /// ``` -pub fn to_vec_from_payload_v1(payload: &TracerPayload) -> Vec { - to_vec_with_capacity_from_payload_v1(payload, 0) +pub fn to_vec_from_v1(payload: &TracerPayload) -> Vec { + to_vec_with_capacity_from_v1(payload, 0) } /// Serializes a `TracerPayload` into a vector of bytes with specified capacity. @@ -670,23 +684,23 @@ pub fn to_vec_from_payload_v1(payload: &TracerPayload) -> Vec= 1024); /// ``` -pub fn to_vec_with_capacity_from_payload_v1( +pub fn to_vec_with_capacity_from_v1( payload: &TracerPayload, capacity: u32, ) -> Vec { let mut buf = ByteBuf::with_capacity(capacity as usize); - encode_payload_v1(&mut buf, payload) + encode_payload_from_v1(&mut buf, payload) .map_err(super::flatten_value_write_infallible) .unwrap_infallible(); buf.into_vec() @@ -711,7 +725,7 @@ pub fn to_vec_with_capacity_from_payload_v1( /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v1::write_to_slice_v1; +/// use libdd_trace_utils::msgpack_encoder::v1::write_to_slice_from_v1; /// use libdd_trace_utils::span::v1::TracerPayloadSlice; /// /// let mut buffer = vec![0u8; 1024]; @@ -720,13 +734,13 @@ pub fn to_vec_with_capacity_from_payload_v1( /// ..Default::default() /// }; /// -/// write_to_slice_v1(&mut &mut buffer[..], &payload).expect("Encoding failed"); +/// write_to_slice_from_v1(&mut &mut buffer[..], &payload).expect("Encoding failed"); /// ``` -pub fn write_to_slice_v1( +pub fn write_to_slice_from_v1( slice: &mut &mut [u8], payload: &TracerPayload, ) -> Result<(), ValueWriteError> { - encode_payload_v1(slice, payload) + encode_payload_from_v1(slice, payload) } /// Computes the number of bytes required to encode the given `TracerPayload`. @@ -745,20 +759,20 @@ pub fn write_to_slice_v1( /// # Examples /// /// ``` -/// use libdd_trace_utils::msgpack_encoder::v1::to_encoded_byte_len_from_payload_v1; +/// use libdd_trace_utils::msgpack_encoder::v1::to_encoded_byte_len_from_v1; /// use libdd_trace_utils::span::v1::TracerPayloadSlice; /// /// let payload = TracerPayloadSlice { /// language_name: "rust".into(), /// ..Default::default() /// }; -/// let encoded_len = to_encoded_byte_len_from_payload_v1(&payload); +/// let encoded_len = to_encoded_byte_len_from_v1(&payload); /// /// assert!(encoded_len > 0); /// ``` -pub fn to_encoded_byte_len_from_payload_v1(payload: &TracerPayload) -> u32 { +pub fn to_encoded_byte_len_from_v1(payload: &TracerPayload) -> u32 { let mut counter = super::CountLength(0); - let _ = encode_payload_v1(&mut counter, payload); + let _ = encode_payload_from_v1(&mut counter, payload); counter.0 } @@ -792,14 +806,14 @@ mod tests { fn test_to_vec_non_empty() { let spans = vec![make_span("svc", "op", 42, 1, 0)]; let traces = vec![spans]; - let encoded = to_vec_v04(&traces, &TracerMetadata::default()); + let encoded = to_vec_from_v04(&traces, &TracerMetadata::default()); assert!(!encoded.is_empty()); } #[test] fn test_to_vec_empty_traces() { let traces: Vec> = vec![]; - let encoded = to_vec_v04(&traces, &TracerMetadata::default()); + let encoded = to_vec_from_v04(&traces, &TracerMetadata::default()); // Must still produce a valid msgpack map with an empty chunks array. assert!(!encoded.is_empty()); } @@ -815,8 +829,8 @@ mod tests { let s_single = make_span("my-service", "op1", 1, 1, 0); let traces_single = vec![vec![s_single]]; - let encoded_two = to_vec_v04(&traces_two, &TracerMetadata::default()); - let encoded_single = to_vec_v04(&traces_single, &TracerMetadata::default()); + let encoded_two = to_vec_from_v04(&traces_two, &TracerMetadata::default()); + let encoded_single = to_vec_from_v04(&traces_single, &TracerMetadata::default()); // The two-trace payload should be less than 2× the single-trace payload // if interning is working (the second "my-service" is encoded as an integer). @@ -851,7 +865,7 @@ mod tests { ..Default::default() }; - let encoded = to_vec_v04(&[vec![root]], &TracerMetadata::default()); + let encoded = to_vec_from_v04(&[vec![root]], &TracerMetadata::default()); assert!(!encoded.is_empty()); // The payload must contain "lambda" somewhere (the origin string). let lambda_bytes = b"lambda"; @@ -871,8 +885,8 @@ mod tests { ]; let traces = vec![spans]; let meta = TracerMetadata::default(); - let encoded = to_vec_v04(&traces, &meta); - let len = to_encoded_byte_len(&traces, &meta); + let encoded = to_vec_from_v04(&traces, &meta); + let len = to_encoded_byte_len_from_v04(&traces, &meta); assert_eq!(encoded.len() as u32, len); } @@ -898,7 +912,7 @@ mod tests { ..Default::default() }; - let encoded = to_vec_v04(&[vec![root]], &TracerMetadata::default()); + let encoded = to_vec_from_v04(&[vec![root]], &TracerMetadata::default()); assert!(!encoded.is_empty()); } @@ -933,7 +947,7 @@ mod tests { ..Default::default() }; - let encoded = to_vec_v04(&[vec![span]], &TracerMetadata::default()); + let encoded = to_vec_from_v04(&[vec![span]], &TracerMetadata::default()); let prod_bytes = b"prod"; assert!( encoded.windows(prod_bytes.len()).any(|w| w == prod_bytes), @@ -973,7 +987,7 @@ mod tests { ..Default::default() }; - let encoded = to_vec_v04(&[vec![span]], &TracerMetadata::default()); + let encoded = to_vec_from_v04(&[vec![span]], &TracerMetadata::default()); // Both attribute strings must appear in the payload bytes. let ssi_bytes = b"ssi"; @@ -1003,7 +1017,7 @@ mod tests { fn test_payload_attributes_absent_when_no_relevant_tags() { // A span with no _dd.apm_mode or _dd.git.commit.sha must not produce key 10. let span = make_span("svc", "op", 1, 1, 0); - let encoded = to_vec_v04(&[vec![span]], &TracerMetadata::default()); + let encoded = to_vec_from_v04(&[vec![span]], &TracerMetadata::default()); let apm_key = b"_dd.apm_mode"; assert!( !encoded.windows(apm_key.len()).any(|w| w == apm_key), @@ -1021,7 +1035,7 @@ mod tests { runtime_id: "abc-123-uuid".to_string(), ..Default::default() }; - let encoded = to_vec_v04(&[vec![span]], &metadata); + let encoded = to_vec_from_v04(&[vec![span]], &metadata); for s in &[b"python" as &[u8], b"3.11", b"2.0.0", b"abc-123-uuid"] { assert!( @@ -1035,14 +1049,14 @@ mod tests { #[test] fn test_payload_metadata_absent_when_empty() { let span = make_span("svc", "op", 1, 1, 0); - let encoded_with = to_vec_v04( + let encoded_with = to_vec_from_v04( &[vec![span.clone()]], &TracerMetadata { language: "go".to_string(), ..Default::default() }, ); - let encoded_without = to_vec_v04(&[vec![span]], &TracerMetadata::default()); + let encoded_without = to_vec_from_v04(&[vec![span]], &TracerMetadata::default()); // Payload with metadata must be larger (it carries extra fields). assert!(encoded_with.len() > encoded_without.len()); } @@ -1066,7 +1080,7 @@ mod tests { meta, ..Default::default() }; - let encoded = to_vec_v04(&[vec![span]], &TracerMetadata::default()); + let encoded = to_vec_from_v04(&[vec![span]], &TracerMetadata::default()); // Expected 16-byte BE: high = 0x640cfd5400000000, low = 0x0123456789abcdef let expected = [ @@ -1089,7 +1103,7 @@ mod tests { fn test_128bit_trace_id_without_dd_p_tid() { // Absent _dd.p.tid → high 64 bits zero. let span = make_span("svc", "op", 0x0123456789abcdef, 1, 0); - let encoded = to_vec_v04(&[vec![span]], &TracerMetadata::default()); + let encoded = to_vec_from_v04(&[vec![span]], &TracerMetadata::default()); let expected = [ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, @@ -1121,7 +1135,7 @@ mod tests { meta, ..Default::default() }; - let encoded = to_vec_v04(&[vec![root]], &TracerMetadata::default()); + let encoded = to_vec_from_v04(&[vec![root]], &TracerMetadata::default()); // The chunk-level sampling_mechanism (key 7) must be encoded as uint 4. // The byte sequence is `chunk_key::SAMPLING_MECHANISM (0x07)` followed by the @@ -1185,7 +1199,7 @@ mod tests { meta: meta3, ..Default::default() }; - let encoded = to_vec_v04(&[vec![s1, s2, s3]], &TracerMetadata::default()); + let encoded = to_vec_from_v04(&[vec![s1, s2, s3]], &TracerMetadata::default()); // Each attribute must be present at chunk level — collected from a different // non-root span. @@ -1211,7 +1225,7 @@ mod tests { #[cfg(test)] mod v1_payload_tests { - //! Unit tests for the v1::Span encoder (`encode_payload_v1`). + //! Unit tests for the v1::Span encoder (`encode_payload_from_v1`). //! //! Verifies the encoder produces a valid V1 payload from the canonical //! [`crate::span::v1::TracerPayload`] data model and that core invariants (interning, byte @@ -1252,7 +1266,7 @@ mod v1_payload_tests { #[test] fn empty_payload_is_valid_msgpack_map() { let payload = TracerPayloadBytes::default(); - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); // Map with a single entry (chunks), then an empty array. `0x81` = fixmap of length 1, // followed by chunk key (0x0b), then `0x90` (fixarray length 0). assert_eq!(encoded, vec![0x81, 0x0b, 0x90]); @@ -1265,8 +1279,8 @@ mod v1_payload_tests { chunks: vec![chunk], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); - let len = to_encoded_byte_len_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); + let len = to_encoded_byte_len_from_v1(&payload); assert_eq!(encoded.len() as u32, len); } @@ -1279,7 +1293,7 @@ mod v1_payload_tests { chunks: vec![chunk], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); let pat = [0x10u8, 0x01u8]; assert!( encoded.windows(2).any(|w| w == pat), @@ -1306,7 +1320,7 @@ mod v1_payload_tests { chunks: vec![chunk], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); // String attribute → type discriminant = 1 (`AnyValueKey::String`). assert!( encoded.windows(b"k_str".len()).any(|w| w == b"k_str"), @@ -1336,7 +1350,7 @@ mod v1_payload_tests { chunks: vec![make_chunk(vec![span], [0u8; 16])], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); // bin8 marker `0xc4` followed by length `0x02` and the bytes themselves. let want = [0xc4u8, 0x02, 0xde, 0xad]; assert!( @@ -1372,7 +1386,7 @@ mod v1_payload_tests { chunks: vec![make_chunk(vec![span], [0u8; 16])], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); // The keys and the nested key must all appear at least once. for s in &[b"list" as &[u8], b"kv", b"a", b"nk"] { assert!( @@ -1396,7 +1410,7 @@ mod v1_payload_tests { chunks: vec![make_chunk(vec![make_span("svc", "op", 1)], [0u8; 16])], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); for s in &[ b"python" as &[u8], b"3.11", @@ -1427,7 +1441,7 @@ mod v1_payload_tests { chunks: vec![chunk], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); assert!( encoded.windows(b"lambda".len()).any(|w| w == b"lambda"), "chunk origin should appear" @@ -1449,7 +1463,7 @@ mod v1_payload_tests { chunks: vec![chunk], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); // DROPPED_TRACE (0x05) + msgpack true marker (0xc3) let want = [chunk_key::DROPPED_TRACE, 0xc3]; assert!( @@ -1470,7 +1484,7 @@ mod v1_payload_tests { chunks: vec![chunk], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); assert!( !encoded.contains(&chunk_key::DROPPED_TRACE), "DROPPED_TRACE key should not be emitted when false" @@ -1491,7 +1505,7 @@ mod v1_payload_tests { chunks: vec![chunk], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); // ATTRIBUTES (0x03) + msgpack fixarray header for 3 elements (0x93) let want = [chunk_key::ATTRIBUTES, 0x93]; assert!( @@ -1529,7 +1543,7 @@ mod v1_payload_tests { chunks: vec![make_chunk(vec![span], [0u8; 16])], ..Default::default() }; - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); let want = [0x10u8, expected_byte]; assert!( encoded.windows(2).any(|w| w == want), @@ -1555,8 +1569,8 @@ mod v1_payload_tests { chunks: vec![make_chunk(vec![make_span("shared", "op1", 1)], [0u8; 16])], ..Default::default() }; - let two = to_vec_from_payload_v1(&chunk_with_two); - let one = to_vec_from_payload_v1(&single); + let two = to_vec_from_v1(&chunk_with_two); + let one = to_vec_from_v1(&single); let shared_occurrences = two .windows(b"shared".len()) .filter(|w| *w == b"shared") diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs b/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs index 6b33f925e1..a1b580101c 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v1/span_v04.rs @@ -1,9 +1,8 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -// Naming convention: parent module (`v1`) = output wire format, file suffix (`_v04`) = input -// span type. This file encodes a [`crate::span::v04::Span`] into the V1 msgpack wire format -// (cross-format upgrade). +//! Upgrade encoder: `crate::span::v04::Span` → V1 msgpack wire. +//! (Convention documented in [`crate::msgpack_encoder`].) use crate::span::v04::{AttributeAnyValue, AttributeArrayValue, Span, SpanEvent, SpanLink}; use crate::span::TraceData; diff --git a/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs index 9b1b6e8d52..e63a68b212 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v1/span_v1.rs @@ -1,8 +1,8 @@ // Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -// Naming convention: parent module (`v1`) = output wire format, file suffix (`_v1`) = input span -// type. This file encodes a [`crate::span::v1::Span`] into the V1 msgpack wire format (native). +//! Native V1 span encoder: `crate::span::v1::Span` → V1 msgpack wire. +//! (Convention documented in [`crate::msgpack_encoder`].) use crate::span::v1::{AttributeValue, Span, SpanEvent, SpanLink}; use crate::span::vec_map::VecMap; diff --git a/libdd-trace-utils/src/send_data/mod.rs b/libdd-trace-utils/src/send_data/mod.rs index 371c4909d5..5fce403865 100644 --- a/libdd-trace-utils/src/send_data/mod.rs +++ b/libdd-trace-utils/src/send_data/mod.rs @@ -384,7 +384,7 @@ impl SendData { headers.insert(DATADOG_TRACE_COUNT, chunks.into()); headers.insert(CONTENT_TYPE, APPLICATION_MSGPACK); - let payload = msgpack_encoder::v04::to_vec_v04(payload); + let payload = msgpack_encoder::v04::to_vec_from_v04(payload); futures.push(self.send_payload( capabilities, @@ -534,7 +534,7 @@ mod tests { total } TracerPayloadCollection::V04(payloads) => { - msgpack_encoder::v04::to_encoded_byte_len_v04(payloads) as usize + msgpack_encoder::v04::to_encoded_byte_len_from_v04(payloads) as usize } TracerPayloadCollection::V05(payloads) => rmp_serde::to_vec(payloads).unwrap().len(), } diff --git a/libdd-trace-utils/src/span/v1/mod.rs b/libdd-trace-utils/src/span/v1/mod.rs index 459cbc5438..42e64f76bf 100644 --- a/libdd-trace-utils/src/span/v1/mod.rs +++ b/libdd-trace-utils/src/span/v1/mod.rs @@ -112,6 +112,7 @@ pub struct TraceChunk { /// A V1 tracer payload: tracer-level metadata and the trace chunks it carries. #[derive(Debug, Default)] pub struct TracerPayload { + pub container_id: T::Text, pub language_name: T::Text, pub language_version: T::Text, pub tracer_version: T::Text, diff --git a/libdd-trace-utils/src/tracer_metadata.rs b/libdd-trace-utils/src/tracer_metadata.rs index 7d67ee953d..5fe93098ab 100644 --- a/libdd-trace-utils/src/tracer_metadata.rs +++ b/libdd-trace-utils/src/tracer_metadata.rs @@ -16,6 +16,7 @@ pub struct TracerMetadata { pub language_version: String, pub language_interpreter: String, pub language_interpreter_vendor: String, + pub container_id: String, pub git_commit_sha: String, pub process_tags: String, pub client_computed_stats: bool, @@ -30,6 +31,7 @@ impl<'a> From<&'a TracerMetadata> for TracerHeaderTags<'a> { tracer_version: &tags.tracer_version, lang_interpreter: &tags.language_interpreter, lang_vendor: &tags.language_interpreter_vendor, + container_id: &tags.container_id, client_computed_stats: tags.client_computed_stats, client_computed_top_level: tags.client_computed_top_level, ..Default::default() diff --git a/libdd-trace-utils/tests/test_send_data.rs b/libdd-trace-utils/tests/test_send_data.rs index 7adcafd615..a3b6546426 100644 --- a/libdd-trace-utils/tests/test_send_data.rs +++ b/libdd-trace-utils/tests/test_send_data.rs @@ -396,7 +396,7 @@ mod tracing_integration_tests { // ───────────────────────── V1 integration tests ────────────────────────── // // These tests cover the v1::Span encoder end-to-end: the payload is built directly from the - // `TracerPayload` data model in Rust, encoded with `to_vec_from_payload_v1`, POSTed to the + // `TracerPayload` data model in Rust, encoded with `to_vec_from_v1`, POSTed to the // `dd-apm-test-agent`'s `/v1.0/traces`, and validated via snapshot. The test-agent is the V1 // decoder, so this exercises the full round-trip without us having to maintain one in this // crate. @@ -521,15 +521,16 @@ mod tracing_integration_tests { app_version: bs_v1("1.2.3"), attributes: VecMap::new(), chunks: vec![chunk], + ..Default::default() } } /// End-to-end round-trip: builds a V1 payload directly from `TracerPayload`, encodes it - /// with `to_vec_from_payload_v1`, POSTs to the test-agent, and asserts the snapshot. + /// with `to_vec_from_v1`, POSTs to the test-agent, and asserts the snapshot. #[cfg_attr(miri, ignore)] #[tokio::test] async fn compare_v1_native_trace_snapshot_test() { - use libdd_trace_utils::msgpack_encoder::v1::to_vec_from_payload_v1; + use libdd_trace_utils::msgpack_encoder::v1::to_vec_from_v1; let relative_snapshot_path = "libdd-trace-utils/tests/snapshots/"; let snapshot_name = "compare_send_data_v1_native_trace_snapshot_test"; @@ -542,7 +543,7 @@ mod tracing_integration_tests { test_agent.start_session(snapshot_name, None).await; let payload = make_v1_payload("test_send_data_v1_native_snapshot"); - let encoded = to_vec_from_payload_v1(&payload); + let encoded = to_vec_from_v1(&payload); post_msgpack_traces(uri, encoded).await; @@ -556,7 +557,7 @@ mod tracing_integration_tests { #[cfg_attr(miri, ignore)] #[tokio::test] async fn compare_v04_and_v1_encoders_snapshot_test() { - use libdd_trace_utils::msgpack_encoder::v1::{to_vec_from_payload_v1, to_vec_v04}; + use libdd_trace_utils::msgpack_encoder::v1::{to_vec_from_v04, to_vec_from_v1}; use libdd_trace_utils::span::v04::SpanBytes as V04SpanBytes; use libdd_trace_utils::span::v1::{ AttributeValue, SpanBytes as V1SpanBytes, SpanKind, TraceChunkBytes, TracerPayloadBytes, @@ -619,8 +620,8 @@ mod tracing_integration_tests { }; // ── POST both into the same session ──────────────────────────────────────── - let bytes_v04 = to_vec_v04(&v04_traces, &metadata); - let bytes_v1 = to_vec_from_payload_v1(&v1_payload); + let bytes_v04 = to_vec_from_v04(&v04_traces, &metadata); + let bytes_v1 = to_vec_from_v1(&v1_payload); post_msgpack_traces(uri.clone(), bytes_v04).await; post_msgpack_traces(uri, bytes_v1).await; @@ -630,7 +631,7 @@ mod tracing_integration_tests { } /// Round-trip: the v1::Span → v0.4 downgrade encoder - /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec_v1`]) must produce a v0.4 + /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec_from_v1`]) must produce a v0.4 /// payload that decodes to the same canonical form as the native v0.4 encoder /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec`]) when fed equivalent input. /// @@ -640,8 +641,8 @@ mod tracing_integration_tests { #[cfg_attr(miri, ignore)] #[tokio::test] async fn compare_v04_native_and_v1_to_v04_encoders_snapshot_test() { - use libdd_trace_utils::msgpack_encoder::v04::to_vec_v04 as to_vec_v04_native; - use libdd_trace_utils::msgpack_encoder::v04::to_vec_v1; + use libdd_trace_utils::msgpack_encoder::v04::to_vec_from_v04 as to_vec_v04_native; + use libdd_trace_utils::msgpack_encoder::v04::to_vec_from_v1; use libdd_trace_utils::span::v04::SpanBytes as V04SpanBytes; use libdd_trace_utils::span::v1::{ AttributeValue, SpanBytes as V1SpanBytes, SpanKind, TraceChunkBytes, TracerPayloadBytes, @@ -706,7 +707,7 @@ mod tracing_integration_tests { // ── Encode each via its respective encoder and POST both ─────────────────── let bytes_v04 = to_vec_v04_native(&v04_traces); - let bytes_v1_to_v04 = to_vec_v1(&v1_payload); + let bytes_v1_to_v04 = to_vec_from_v1(&v1_payload); post_msgpack_traces(uri.clone(), bytes_v04).await; post_msgpack_traces(uri, bytes_v1_to_v04).await; From 51379b2af5027de72d044e3e16bee5f69b281aba Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Thu, 2 Jul 2026 14:43:04 +0200 Subject: [PATCH 10/23] fix: comments --- .../src/msgpack_encoder/v04/mod.rs | 19 +- .../src/msgpack_encoder/v04/span_v1.rs | 455 +++++++++++------- 2 files changed, 309 insertions(+), 165 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs index 22e6cd1473..652f008e99 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs @@ -231,7 +231,10 @@ pub fn to_encoded_byte_len_from_v04]>>(traces: & /// Encodes a [`TracerPayload`] in the v0.4 wire format (downgrade path used when the agent /// does not advertise `/v1.0/traces`). The output is a msgpack array of traces, where each /// trace is itself a msgpack array of v0.4-shaped spans — matching the existing v0.4 wire -/// format produced by [`to_vec`]. +/// format produced by [`to_vec`]. Payload-level `env`/`app_version`/`attributes` are propagated +/// into every span (see [`span_v1`]'s mapping table); `payload.hostname` has no v0.4 body +/// equivalent (the agent gets hostname from the `Datadog-Meta-Hostname` header instead) and is +/// intentionally dropped here. fn encode_payload_from_v1( writer: &mut W, payload: &TracerPayload, @@ -240,12 +243,24 @@ fn encode_payload_from_v1( write_array_len(writer, payload.chunks.len() as u32)?; for chunk in &payload.chunks { + // v0.4 has no wire-level equivalent of `dropped_trace`; the closest historical signal + // is `USER_REJECT` (priority -1), which tells the agent the sampler rejected this trace + // without dropping the spans themselves. Only force it when the chunk doesn't already + // carry a negative (reject-like) priority. + let priority = if chunk.dropped_trace { + Some(chunk.priority.filter(|&p| p < 0).unwrap_or(-1)) + } else { + chunk.priority + }; let ctx = ChunkContext { trace_id: &chunk.trace_id, - priority: chunk.priority, + priority, origin: &chunk.origin, sampling_mechanism: chunk.sampling_mechanism, attributes: &chunk.attributes, + payload_env: &payload.env, + payload_app_version: &payload.app_version, + payload_attributes: &payload.attributes, }; write_array_len(writer, chunk.spans.len() as u32)?; for span in &chunk.spans { diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs index 2b16399328..7576303ad9 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs @@ -9,26 +9,33 @@ //! //! | v1::Span field / attribute | v0.4 field | //! |---------------------------------------|---------------------------------------------| -//! | `env` / `version` / `component` | `meta["env"]` / `meta["version"]` / ... | +//! | `env` / `version` / `component` | `meta["env"]` / `meta["version"]` / ... (`env`/`version` fall back to the payload-level `env`/`app_version` when unset on the span) | //! | `span_kind` | `meta["span.kind"]` (lowercase string) | //! | `AttributeValue::String` / `Bool` | `meta[k]` (`"true"` / `"false"` for bool) | //! | `AttributeValue::Float` / `Int` | `metrics[k]` (Int cast to `f64`) | -//! | `AttributeValue::Bytes` / `KeyValue` | `meta_struct[k]` (re-encoded as msgpack) | -//! | `AttributeValue::List` | `meta_struct[k]` (re-encoded as msgpack) | +//! | `AttributeValue::Bytes` | `meta_struct[k]` (raw bytes) | +//! | `AttributeValue::List` | flattened into `meta`/`metrics[k.0]`, `[k.1]`, ... (per element type) | +//! | `AttributeValue::KeyValue` | flattened into `meta`/`metrics[k.a]`, `[k.a.b]`, ... (per member, recursively) | //! | `error: bool` | `error: i32` (`true → 1`, `false → 0`) | //! | Chunk `trace_id: [u8; 16]` | `trace_id: u64` (low 64) + `meta["_dd.p.tid"]` (hex of high 64, when non-zero) | //! | Chunk `origin` | `meta["_dd.origin"]` | //! | Chunk `priority` | `metrics["_sampling_priority_v1"]` | //! | Chunk `sampling_mechanism` | `meta["_dd.p.dm"]` (`"-{mechanism}"`) | //! | Chunk `attributes` | Applied to every span in the chunk | +//! | Payload `env` / `app_version` | Fallback for `meta["env"]` / `meta["version"]` when the span leaves them unset | +//! | Payload `attributes` | Applied to every span, lowest precedence (span > chunk > payload) | +//! | Chunk `dropped_trace: true` | Forces `metrics["_sampling_priority_v1"] = -1` (USER_REJECT) unless the chunk's own priority is already negative | +//! +//! An attribute sharing a name with one of the dedicated fields above (`env`, `version`, +//! `component`, `span.kind`, `_dd.p.tid`, `_dd.origin`, `_dd.p.dm`, `_sampling_priority_v1`) is +//! dropped: the dedicated field always wins, so each key is written at most once. use crate::span::v1::{AttributeValue, Span, SpanEvent, SpanKind, SpanLink}; use crate::span::vec_map::VecMap; use crate::span::TraceData; -use libdd_common::ResultInfallibleExt; use rmp::encode::{ write_array_len, write_bin, write_bool, write_f64, write_i64, write_map_len, write_sint, - write_str, write_u32, write_u64, write_u8, ByteBuf, RmpWrite, ValueWriteError, + write_str, write_u32, write_u64, write_u8, RmpWrite, ValueWriteError, }; use std::borrow::Borrow; use std::collections::HashSet; @@ -42,14 +49,35 @@ fn write_bool_as_str( write_str(writer, if b { "true" } else { "false" }) } +/// Reserved v0.4 `meta`/`metrics` key names written from dedicated typed fields (`span.env`, +/// chunk `origin`, ...) rather than from the attribute maps. An attribute sharing one of these +/// names would otherwise collide with the dedicated field's entry on the wire; the dedicated +/// field always wins and the same-named attribute is dropped — see `encode_span`. +const PROMOTED_ATTR_KEYS: &[&str] = &[ + "env", + "version", + "component", + "span.kind", + "_dd.p.tid", + "_dd.origin", + "_dd.p.dm", + "_sampling_priority_v1", +]; + /// Chunk-level context propagated into every span when downgrading to v0.4. Built once per -/// chunk by the top-level encoder and passed by reference to `encode_span_v1_to_v04`. +/// chunk by the top-level encoder and passed by reference to `encode_span_v1_to_v04`. Also +/// carries payload-level fields (`payload_env`, `payload_app_version`, `payload_attributes`), +/// which apply as a fallback when the span itself doesn't set the equivalent field — v0.4 has +/// neither a chunk nor a payload concept, so both levels collapse onto every span. pub(super) struct ChunkContext<'a, T: TraceData> { pub trace_id: &'a [u8; 16], pub priority: Option, pub origin: &'a T::Text, pub sampling_mechanism: Option, pub attributes: &'a VecMap>, + pub payload_env: &'a T::Text, + pub payload_app_version: &'a T::Text, + pub payload_attributes: &'a VecMap>, } /// Maps a `SpanKind` to its v0.4 `span.kind` meta string. Returns `None` for `Internal` so @@ -86,14 +114,36 @@ struct BucketCounts { meta_struct: u32, } -impl BucketCounts { - /// Routes a typed attribute into its bucket per the mapping table in the module docs. - fn count_attr(&mut self, v: &AttributeValue) { - match v { - AttributeValue::String(_) | AttributeValue::Bool(_) => self.meta += 1, - AttributeValue::Float(_) | AttributeValue::Int(_) => self.metrics += 1, - AttributeValue::Bytes(_) | AttributeValue::KeyValue(_) | AttributeValue::List(_) => { - self.meta_struct += 1 +/// Recursively flattens a `List`/`KeyValue` attribute into dotted-key leaf entries for the v0.4 +/// `meta` (string-valued) and `metrics` (numeric) maps — matching how intake/the UI expect nested +/// V1 attributes to be exploded: list elements become `key.0`, `key.1`, ... and `KeyValue` +/// members become `key.` (recursively, for nested `KeyValue`/`List` values). Scalars +/// (`String`/`Bool`/`Int`/`Float`) are leaves in their own right and produce a single entry under +/// `key`. `Bytes` has no flattened form; callers must route it to `meta_struct` separately. +fn flatten_attr_into( + key: String, + v: &AttributeValue, + meta_out: &mut Vec<(String, String)>, + metrics_out: &mut Vec<(String, f64)>, +) { + match v { + AttributeValue::String(s) => meta_out.push((key, s.borrow().to_owned())), + AttributeValue::Bool(b) => { + meta_out.push((key, if *b { "true" } else { "false" }.to_owned())) + } + AttributeValue::Int(i) => metrics_out.push((key, *i as f64)), + AttributeValue::Float(f) => metrics_out.push((key, *f)), + AttributeValue::Bytes(_) => { + // Callers filter `Bytes` out before recursing; unreachable in practice. + } + AttributeValue::List(items) => { + for (i, item) in items.iter().enumerate() { + flatten_attr_into(format!("{key}.{i}"), item, meta_out, metrics_out); + } + } + AttributeValue::KeyValue(map) => { + for (k, v) in map.defensive_dedup().iter() { + flatten_attr_into(format!("{key}.{}", k.borrow()), v, meta_out, metrics_out); } } } @@ -126,36 +176,83 @@ pub(super) fn encode_span( ) -> Result<(), ValueWriteError> { let span_attrs_dd = span.attributes.defensive_dedup(); let chunk_attrs_dd = chunk.attributes.defensive_dedup(); - - // Merge span + chunk attributes upfront with explicit "span overrides chunk" precedence. - // We don't rely on msgpack map last-write-wins decoding here: the v0.4 / msgpack specs do - // not formalize behavior for duplicate map keys, so we emit each key exactly once. + let payload_attrs_dd = chunk.payload_attributes.defensive_dedup(); + + // Merge span + chunk + payload attributes upfront with explicit "span overrides chunk + // overrides payload" precedence. We don't rely on msgpack map last-write-wins decoding + // here: the v0.4 / msgpack specs do not formalize behavior for duplicate map keys, so we + // emit each key exactly once. + // + // Attributes sharing a name with a "promoted" dedicated field (`env`, `_dd.origin`, ...) + // are dropped here: the dedicated field always wins so we never emit that key twice. let span_keys: HashSet<&T::Text> = span_attrs_dd.iter().map(|(k, _)| k).collect(); + let mut seen_keys: HashSet<&T::Text> = span_keys.clone(); + let chunk_only: Vec<(&T::Text, &AttributeValue)> = chunk_attrs_dd + .iter() + .filter(|(k, _)| !seen_keys.contains(k)) + .collect(); + seen_keys.extend(chunk_attrs_dd.iter().map(|(k, _)| k)); + let payload_only: Vec<(&T::Text, &AttributeValue)> = payload_attrs_dd + .iter() + .filter(|(k, _)| !seen_keys.contains(k)) + .collect(); let merged_attrs: Vec<(&T::Text, &AttributeValue)> = span_attrs_dd .iter() - .chain( - chunk_attrs_dd - .iter() - .filter(|(k, _)| !span_keys.contains(k)), - ) + .chain(chunk_only) + .chain(payload_only) + .filter(|(k, _)| !PROMOTED_ATTR_KEYS.contains(&(*k).borrow())) .collect(); let (trace_id_low, trace_id_high) = split_trace_id(chunk.trace_id); let kind_meta = span_kind_to_meta(span.span_kind); + // `env`/`version` fall back to the payload-level value when the span doesn't set its own — + // mirrors how a v1 tracer can set these once at the payload level instead of duplicating + // them on every span/chunk. + let env: &str = if !span.env.borrow().is_empty() { + span.env.borrow() + } else { + chunk.payload_env.borrow() + }; + let version: &str = if !span.version.borrow().is_empty() { + span.version.borrow() + } else { + chunk.payload_app_version.borrow() + }; + + // Flatten every attribute into `meta` (string-valued) / `metrics` (numeric) leaf entries. + // `List` and `KeyValue` have no v0.4 wire representation as a single value, so they are + // exploded into dotted keys (`key.0`, `key.a.b`) the same way intake/the UI expect nested V1 + // attributes — see the mapping table in the module docs. `Bytes` keeps going to + // `meta_struct` since it has no flattened form. + let mut meta_leaves: Vec<(String, String)> = Vec::new(); + let mut metrics_leaves: Vec<(String, f64)> = Vec::new(); + let mut bytes_attrs: Vec<(&T::Text, &T::Bytes)> = Vec::new(); + for &(k, v) in &merged_attrs { + match v { + AttributeValue::Bytes(b) => bytes_attrs.push((k, b)), + _ => flatten_attr_into( + k.borrow().to_owned(), + v, + &mut meta_leaves, + &mut metrics_leaves, + ), + } + } + // First pass: count bucket sizes so each msgpack map header carries the exact length. let mut counts = BucketCounts::default(); - counts.meta += !span.env.borrow().is_empty() as u32; - counts.meta += !span.version.borrow().is_empty() as u32; + counts.meta += !env.is_empty() as u32; + counts.meta += !version.is_empty() as u32; counts.meta += !span.component.borrow().is_empty() as u32; counts.meta += kind_meta.is_some() as u32; counts.meta += (trace_id_high != 0) as u32; counts.meta += !chunk.origin.borrow().is_empty() as u32; counts.meta += chunk.sampling_mechanism.is_some() as u32; + counts.meta += meta_leaves.len() as u32; counts.metrics += chunk.priority.is_some() as u32; - for (_, v) in &merged_attrs { - counts.count_attr(v); - } + counts.metrics += metrics_leaves.len() as u32; + counts.meta_struct += bytes_attrs.len() as u32; let span_len = 7 // service, name, resource, trace_id, span_id, start, duration (always) + (!span.r#type.borrow().is_empty()) as u32 @@ -204,13 +301,13 @@ pub(super) fn encode_span( write_const_msgpack_str!(writer, "meta")?; write_map_len(writer, counts.meta)?; - if !span.env.borrow().is_empty() { + if !env.is_empty() { write_const_msgpack_str!(writer, "env")?; - write_str(writer, span.env.borrow())?; + write_str(writer, env)?; } - if !span.version.borrow().is_empty() { + if !version.is_empty() { write_const_msgpack_str!(writer, "version")?; - write_str(writer, span.version.borrow())?; + write_str(writer, version)?; } if !span.component.borrow().is_empty() { write_const_msgpack_str!(writer, "component")?; @@ -233,18 +330,9 @@ pub(super) fn encode_span( write_const_msgpack_str!(writer, "_dd.p.dm")?; write_str(writer, &format!("-{mechanism}"))?; } - for &(k, v) in &merged_attrs { - match v { - AttributeValue::String(s) => { - write_str(writer, k.borrow())?; - write_str(writer, s.borrow())?; - } - AttributeValue::Bool(b) => { - write_str(writer, k.borrow())?; - write_bool_as_str(writer, *b)?; - } - _ => {} - } + for (k, v) in &meta_leaves { + write_str(writer, k)?; + write_str(writer, v)?; } } @@ -256,18 +344,9 @@ pub(super) fn encode_span( write_const_msgpack_str!(writer, "_sampling_priority_v1")?; write_f64(writer, priority as f64)?; } - for &(k, v) in &merged_attrs { - match v { - AttributeValue::Float(f) => { - write_str(writer, k.borrow())?; - write_f64(writer, *f)?; - } - AttributeValue::Int(i) => { - write_str(writer, k.borrow())?; - write_f64(writer, *i as f64)?; - } - _ => {} - } + for (k, v) in &metrics_leaves { + write_str(writer, k)?; + write_f64(writer, *v)?; } } @@ -280,8 +359,9 @@ pub(super) fn encode_span( write_const_msgpack_str!(writer, "meta_struct")?; write_map_len(writer, counts.meta_struct)?; - for &(k, v) in &merged_attrs { - write_meta_struct_entry(writer, k, v)?; + for &(k, b) in &bytes_attrs { + write_str(writer, k.borrow())?; + write_bin(writer, b.borrow())?; } } @@ -295,74 +375,6 @@ pub(super) fn encode_span( Ok(()) } -/// Writes a single `meta_struct` entry. `Bytes` passes through as a msgpack `bin`. `KeyValue` -/// and `List` are recursively re-encoded as standalone msgpack (no V1 type tags, no string -/// interning) into a scratch buffer, then emitted as the `bin` payload — matching the v0.4 -/// `meta_struct` contract of `String → msgpack-encoded bytes`. -fn write_meta_struct_entry( - writer: &mut W, - k: &T::Text, - v: &AttributeValue, -) -> Result<(), ValueWriteError> { - match v { - AttributeValue::Bytes(b) => { - write_str(writer, k.borrow())?; - write_bin(writer, b.borrow())?; - } - AttributeValue::KeyValue(_) | AttributeValue::List(_) => { - write_str(writer, k.borrow())?; - let mut buf = ByteBuf::with_capacity(0); - encode_attribute_value(&mut buf, v) - .map_err(super::super::flatten_value_write_infallible) - .unwrap_infallible(); - write_bin(writer, buf.as_ref())?; - } - _ => {} - } - Ok(()) -} - -/// Recursively encodes a [`v1::AttributeValue`] as a self-contained msgpack value using natural -/// types — `str`/`bool`/`int`/`f64`/`bin`/`array`/`map` — without the V1 `[type, value]` framing -/// or its streaming string table. Used to build the body of a v0.4 `meta_struct` `bin` payload. -fn encode_attribute_value( - writer: &mut W, - v: &AttributeValue, -) -> Result<(), ValueWriteError> { - match v { - AttributeValue::String(s) => { - write_str(writer, s.borrow())?; - } - AttributeValue::Bool(b) => { - write_bool(writer, *b).map_err(ValueWriteError::InvalidDataWrite)?; - } - AttributeValue::Int(i) => { - write_sint(writer, *i)?; - } - AttributeValue::Float(f) => { - write_f64(writer, *f)?; - } - AttributeValue::Bytes(b) => { - write_bin(writer, b.borrow())?; - } - AttributeValue::List(items) => { - write_array_len(writer, items.len() as u32)?; - for item in items { - encode_attribute_value(writer, item)?; - } - } - AttributeValue::KeyValue(map) => { - let deduped = map.defensive_dedup(); - write_map_len(writer, deduped.len() as u32)?; - for (k, v) in deduped.iter() { - write_str(writer, k.borrow())?; - encode_attribute_value(writer, v)?; - } - } - } - Ok(()) -} - /// Encodes [`v1::SpanLink`](crate::span::v1::SpanLink)s into the v0.4 msgpack wire format /// (downgrade: v1 input → v0.4 output). The 128-bit `trace_id` is split into /// `(trace_id, trace_id_high)` u64s. Typed link attributes are downgraded to strings; @@ -715,6 +727,26 @@ mod tests { assert_eq!(map_get(meta, "span.kind").unwrap().as_str(), Some("server")); } + #[test] + fn attribute_sharing_a_promoted_key_name_is_dropped_in_favor_of_the_dedicated_field() { + let mut attrs: VecMap = VecMap::new(); + attrs.insert(bs("env"), AttributeValue::String(bs("staging"))); + attrs.insert(bs("http.method"), AttributeValue::String(bs("GET"))); + let span = SpanBytes { + env: bs("prod"), + attributes: attrs, + ..minimal_span() + }; + let payload = minimal_payload([0u8; 16], span); + let traces = encode_and_decode(&payload); + let meta = map_get(&traces[0][0], "meta").expect("meta present"); + + // The dedicated `span.env` field wins; the colliding attribute is dropped rather than + // producing a duplicate `"env"` key on the wire. + assert_eq!(map_get(meta, "env").unwrap().as_str(), Some("prod")); + assert_eq!(map_get(meta, "http.method").unwrap().as_str(), Some("GET")); + } + #[test] fn span_kind_internal_is_not_emitted() { // Internal is the default and is implied by the absence of `meta["span.kind"]`. @@ -858,17 +890,8 @@ mod tests { ); } - /// Extracts the standalone msgpack value packed inside a `meta_struct` `bin` payload. - fn meta_struct_inner(meta_struct: &Value, key: &str) -> Value { - let bin = match map_get(meta_struct, key) { - Some(Value::Binary(b)) => b, - other => panic!("expected meta_struct[{key}] to be a Binary, got {other:?}"), - }; - rmpv::decode::read_value(&mut bin.as_slice()).expect("inner msgpack decode failed") - } - #[test] - fn list_attribute_routes_to_meta_struct_as_nested_msgpack_array() { + fn list_attribute_is_flattened_into_dotted_meta_and_metrics_keys() { let mut attrs: VecMap = VecMap::new(); attrs.insert( bs("ids"), @@ -886,21 +909,19 @@ mod tests { }, ); let traces = encode_and_decode(&payload); - let ms = map_get(&traces[0][0], "meta_struct").expect("meta_struct present"); + let span = &traces[0][0]; + assert!(map_get(span, "meta_struct").is_none()); - let inner = meta_struct_inner(ms, "ids"); - let items = match inner { - Value::Array(a) => a, - other => panic!("expected msgpack array inside bin, got {other:?}"), - }; - assert_eq!(items.len(), 3); - assert_eq!(items[0].as_i64(), Some(1)); - assert_eq!(items[1].as_i64(), Some(2)); - assert_eq!(items[2].as_str(), Some("three")); + let metrics = map_get(span, "metrics").expect("metrics present"); + assert_eq!(map_get(metrics, "ids.0").unwrap().as_f64(), Some(1.0)); + assert_eq!(map_get(metrics, "ids.1").unwrap().as_f64(), Some(2.0)); + + let meta = map_get(span, "meta").expect("meta present"); + assert_eq!(map_get(meta, "ids.2").unwrap().as_str(), Some("three")); } #[test] - fn keyvalue_attribute_routes_to_meta_struct_as_nested_msgpack_map() { + fn keyvalue_attribute_is_flattened_into_dotted_meta_and_metrics_keys() { let mut inner_kv: VecMap = VecMap::new(); inner_kv.insert(bs("user_id"), AttributeValue::Int(42)); inner_kv.insert(bs("name"), AttributeValue::String(bs("alice"))); @@ -917,17 +938,22 @@ mod tests { }, ); let traces = encode_and_decode(&payload); - let ms = map_get(&traces[0][0], "meta_struct").expect("meta_struct present"); + let span = &traces[0][0]; + assert!(map_get(span, "meta_struct").is_none()); + + let metrics = map_get(span, "metrics").expect("metrics present"); + assert_eq!( + map_get(metrics, "user.user_id").unwrap().as_f64(), + Some(42.0) + ); - let inner = meta_struct_inner(ms, "user"); - assert_eq!(inner.as_map().expect("inner is map").len(), 3); - assert_eq!(map_get(&inner, "user_id").unwrap().as_i64(), Some(42)); - assert_eq!(map_get(&inner, "name").unwrap().as_str(), Some("alice")); - assert_eq!(map_get(&inner, "active").unwrap().as_bool(), Some(true)); + let meta = map_get(span, "meta").expect("meta present"); + assert_eq!(map_get(meta, "user.name").unwrap().as_str(), Some("alice")); + assert_eq!(map_get(meta, "user.active").unwrap().as_str(), Some("true")); } #[test] - fn nested_keyvalue_and_list_recurse_through_meta_struct() { + fn nested_keyvalue_and_list_recurse_into_dotted_keys() { // Build: {"outer": KeyValue { "items": List [String "a", KeyValue {"k": Int 1}] }} let mut nested_kv: VecMap = VecMap::new(); nested_kv.insert(bs("k"), AttributeValue::Int(1)); @@ -952,16 +978,17 @@ mod tests { }, ); let traces = encode_and_decode(&payload); - let ms = map_get(&traces[0][0], "meta_struct").expect("meta_struct present"); + let span = &traces[0][0]; + assert!(map_get(span, "meta_struct").is_none()); + + let meta = map_get(span, "meta").expect("meta present"); + assert_eq!(map_get(meta, "outer.items.0").unwrap().as_str(), Some("a")); - let outer = meta_struct_inner(ms, "outer"); - let items = map_get(&outer, "items").expect("items present"); - let items_arr = items.as_array().expect("items is array"); - assert_eq!(items_arr.len(), 2); - assert_eq!(items_arr[0].as_str(), Some("a")); - let nested_map = items_arr[1].as_map().expect("nested KeyValue is map"); - assert_eq!(nested_map.len(), 1); - assert_eq!(map_get(&items_arr[1], "k").unwrap().as_i64(), Some(1)); + let metrics = map_get(span, "metrics").expect("metrics present"); + assert_eq!( + map_get(metrics, "outer.items.1.k").unwrap().as_f64(), + Some(1.0) + ); } #[test] @@ -1029,6 +1056,108 @@ mod tests { } } + #[test] + fn payload_env_and_app_version_are_used_when_span_leaves_them_unset() { + let payload = TracerPayloadBytes { + env: bs("prod"), + app_version: bs("2.0.0"), + chunks: vec![TraceChunkBytes { + trace_id: [0u8; 16], + spans: vec![minimal_span()], + ..Default::default() + }], + ..Default::default() + }; + let traces = encode_and_decode(&payload); + let meta = map_get(&traces[0][0], "meta").expect("meta present"); + assert_eq!(map_get(meta, "env").unwrap().as_str(), Some("prod")); + assert_eq!(map_get(meta, "version").unwrap().as_str(), Some("2.0.0")); + } + + #[test] + fn span_env_takes_precedence_over_payload_env() { + let payload = TracerPayloadBytes { + env: bs("prod"), + chunks: vec![TraceChunkBytes { + trace_id: [0u8; 16], + spans: vec![SpanBytes { + env: bs("staging"), + ..minimal_span() + }], + ..Default::default() + }], + ..Default::default() + }; + let traces = encode_and_decode(&payload); + let meta = map_get(&traces[0][0], "meta").expect("meta present"); + assert_eq!(map_get(meta, "env").unwrap().as_str(), Some("staging")); + } + + #[test] + fn payload_attributes_are_propagated_with_lowest_precedence() { + let mut payload_attrs: VecMap = VecMap::new(); + payload_attrs.insert(bs("region"), AttributeValue::String(bs("us-east-1"))); + payload_attrs.insert(bs("shared"), AttributeValue::String(bs("payload"))); + + let mut chunk_attrs: VecMap = VecMap::new(); + chunk_attrs.insert(bs("shared"), AttributeValue::String(bs("chunk"))); + + let payload = TracerPayloadBytes { + attributes: payload_attrs, + chunks: vec![TraceChunkBytes { + trace_id: [0u8; 16], + attributes: chunk_attrs, + spans: vec![minimal_span()], + ..Default::default() + }], + ..Default::default() + }; + let traces = encode_and_decode(&payload); + let meta = map_get(&traces[0][0], "meta").expect("meta present"); + assert_eq!(map_get(meta, "region").unwrap().as_str(), Some("us-east-1")); + // Chunk value wins over the payload's same-named attribute. + assert_eq!(map_get(meta, "shared").unwrap().as_str(), Some("chunk")); + } + + #[test] + fn dropped_trace_forces_user_reject_priority() { + let payload = TracerPayloadBytes { + chunks: vec![TraceChunkBytes { + trace_id: [0u8; 16], + dropped_trace: true, + spans: vec![minimal_span()], + ..Default::default() + }], + ..Default::default() + }; + let traces = encode_and_decode(&payload); + let metrics = map_get(&traces[0][0], "metrics").expect("metrics present"); + assert_eq!( + map_get(metrics, "_sampling_priority_v1").unwrap().as_f64(), + Some(-1.0) + ); + } + + #[test] + fn dropped_trace_keeps_existing_negative_priority() { + let payload = TracerPayloadBytes { + chunks: vec![TraceChunkBytes { + trace_id: [0u8; 16], + dropped_trace: true, + priority: Some(-2), + spans: vec![minimal_span()], + ..Default::default() + }], + ..Default::default() + }; + let traces = encode_and_decode(&payload); + let metrics = map_get(&traces[0][0], "metrics").expect("metrics present"); + assert_eq!( + map_get(metrics, "_sampling_priority_v1").unwrap().as_f64(), + Some(-2.0) + ); + } + #[test] fn empty_payload_encodes_as_empty_top_level_array() { let payload = TracerPayloadBytes::default(); From a077baa587f16009d00a3a3a0c5ea7ae0ad3eeec Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Thu, 2 Jul 2026 15:51:09 +0200 Subject: [PATCH 11/23] fix: clippy --- libdd-data-pipeline/src/trace_exporter/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libdd-data-pipeline/src/trace_exporter/mod.rs b/libdd-data-pipeline/src/trace_exporter/mod.rs index 0d0bd5a0c0..39ae528337 100644 --- a/libdd-data-pipeline/src/trace_exporter/mod.rs +++ b/libdd-data-pipeline/src/trace_exporter/mod.rs @@ -2413,7 +2413,7 @@ mod tests { error: 0, ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); let result = exporter.send(data.as_ref()); assert!( @@ -2470,7 +2470,7 @@ mod tests { duration: 1, ..Default::default() }]]; - let data = msgpack_encoder::v04::to_vec(&traces); + let data = msgpack_encoder::v04::to_vec_from_v04(&traces); exporter.send(data.as_ref()).unwrap(); mock_intake.assert(); } From 67e02c2a1f5f71e4afc6481927b37f7ef6ec8c30 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Thu, 2 Jul 2026 16:47:17 +0200 Subject: [PATCH 12/23] fix: macro --- .../src/msgpack_encoder/v04/span_v1.rs | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs index 7576303ad9..037bfa8f94 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs @@ -500,6 +500,15 @@ fn is_supported_event_attr(v: &AttributeValue) -> bool { ) } +macro_rules! write_type { + ($writer:expr, $int_type:expr, $str_type:expr) => {{ + write_map_len($writer, 2)?; + write_const_msgpack_str!($writer, "type")?; + write_u8($writer, $int_type)?; + write_str($writer, $str_type)?; + }}; +} + /// Writes a v0.4 event-attribute value as `{"type": , "..._value": ...}`. Scalars produce a /// 2-entry map; `List` produces `{"type": 4, "array_value": {"values": [...]}}` with each /// element written via `write_event_array_element`. @@ -509,42 +518,27 @@ fn write_event_attr_value( ) -> Result<(), ValueWriteError> { match v { AttributeValue::String(s) => { - write_map_len(writer, 2)?; - write_const_msgpack_str!(writer, "type")?; - write_u8(writer, 0)?; - write_const_msgpack_str!(writer, "string_value")?; + write_type!(writer, 0, "string_value"); write_str(writer, s.borrow())?; } AttributeValue::Bool(b) => { - write_map_len(writer, 2)?; - write_const_msgpack_str!(writer, "type")?; - write_u8(writer, 1)?; - write_const_msgpack_str!(writer, "bool_value")?; + write_type!(writer, 1, "bool_value"); write_bool(writer, *b).map_err(ValueWriteError::InvalidDataWrite)?; } AttributeValue::Int(i) => { - write_map_len(writer, 2)?; - write_const_msgpack_str!(writer, "type")?; - write_u8(writer, 2)?; - write_const_msgpack_str!(writer, "int_value")?; + write_type!(writer, 2, "int_value"); write_sint(writer, *i)?; } AttributeValue::Float(f) => { - write_map_len(writer, 2)?; - write_const_msgpack_str!(writer, "type")?; - write_u8(writer, 3)?; - write_const_msgpack_str!(writer, "double_value")?; + write_type!(writer, 3, "double_value"); write_f64(writer, *f)?; } AttributeValue::List(arr) => { + write_type!(writer, 4, "array_value"); // Only scalar elements survive the downgrade; nested structural entries are // skipped because v0.4 array elements must themselves be scalar. let scalar_elems = arr.iter().filter(|e| is_scalar_array_elem(e)); let elem_count = scalar_elems.clone().count() as u32; - write_map_len(writer, 2)?; - write_const_msgpack_str!(writer, "type")?; - write_u8(writer, 4)?; - write_const_msgpack_str!(writer, "array_value")?; write_map_len(writer, 1)?; write_const_msgpack_str!(writer, "values")?; write_array_len(writer, elem_count)?; From bca3f2427612a2ae36e2c05eb7dd715082623fc8 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Wed, 8 Jul 2026 15:33:59 +0200 Subject: [PATCH 13/23] fix: comments --- .../src/msgpack_decoder/v1/mod.rs | 45 ++++++++++--- .../src/msgpack_decoder/v1/span.rs | 65 +++++++++++-------- libdd-trace-utils/src/span/mod.rs | 3 + libdd-trace-utils/src/span/v1/mod.rs | 2 +- libdd-trace-utils/src/trace_utils.rs | 11 ++-- libdd-trace-utils/src/tracer_payload.rs | 22 +++++-- 6 files changed, 102 insertions(+), 46 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs index a91568bd08..c364b8d24f 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs @@ -205,6 +205,17 @@ where Ok((payload, consumed)) } +/// Consumes and discards the msgpack value at the current buffer position, regardless of its +/// type. Used to skip unknown keys for forward compatibility: if the V1 format gains new fields, +/// older decoders shouldn't reject the whole payload just because they don't recognize a key. +pub(super) fn skip_unknown_value( + buf: &mut Buffer, +) -> Result<(), DecodeError> { + rmpv::decode::read_value(buf.as_mut_slice()) + .map_err(|_| DecodeError::InvalidFormat("Failed to skip unknown V1 value".to_owned()))?; + Ok(()) +} + /// Decodes the top-level V1 payload map: tracer metadata fields + chunks array. fn decode_payload( buf: &mut Buffer, @@ -240,11 +251,7 @@ where trace_key::ATTRIBUTES => { payload.attributes = span::read_attributes_map(buf, table)?; } - unknown => { - return Err(DecodeError::InvalidFormat(format!( - "Unknown V1 payload key: {unknown}" - ))); - } + _unknown => skip_unknown_value(buf)?, } } @@ -342,11 +349,7 @@ where ) })?; } - unknown => { - return Err(DecodeError::InvalidFormat(format!( - "Unknown V1 chunk key: {unknown}" - ))); - } + _unknown => skip_unknown_value(buf)?, } } @@ -501,6 +504,28 @@ mod tests { assert!(matches!(err, DecodeError::InvalidFormat(_))); } + #[test] + fn truncated_trace_id_is_rejected_not_panicking() { + // Payload map with 1 entry: chunks -> [ chunk map with 1 entry: trace_id -> bin(16) ]. + // The bin declares 16 bytes but only 4 are actually present, so the owned decoder's + // `try_slice_and_advance` must reject this instead of indexing out of bounds. + let bytes = vec![ + 0x81, + trace_key::CHUNKS, + 0x91, // array len 1 + 0x81, // chunk fixmap len 1 + chunk_key::TRACE_ID, + 0xc4, // bin8 marker + 0x10, // declared length: 16 bytes + 0x01, + 0x02, + 0x03, + 0x04, // only 4 bytes actually present + ]; + let err = from_bytes(Bytes::from(bytes)).expect_err("truncated trace_id must error"); + assert!(matches!(err, DecodeError::InvalidFormat(_))); + } + #[test] fn string_interning_resolves_across_chunks() { // Two chunks sharing the same service name. The decoded service strings must both diff --git a/libdd-trace-utils/src/msgpack_decoder/v1/span.rs b/libdd-trace-utils/src/msgpack_decoder/v1/span.rs index 16822f959e..969e62937a 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v1/span.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v1/span.rs @@ -5,10 +5,10 @@ // a V1 msgpack span into a [`crate::span::v1::Span`]. use super::{ - read_interned_string, span_event_key, span_key, span_link_key, StringTable, - ANY_VALUE_KEY_ARRAY, ANY_VALUE_KEY_BOOL, ANY_VALUE_KEY_BYTES, ANY_VALUE_KEY_DOUBLE, - ANY_VALUE_KEY_INT64, ANY_VALUE_KEY_KEY_VALUE_LIST, ANY_VALUE_KEY_STRING, FLAT_ATTR_STRIDE, - TYPED_VALUE_STRIDE, + read_interned_string, skip_unknown_value, span_event_key, span_key, span_link_key, + StringTable, ANY_VALUE_KEY_ARRAY, ANY_VALUE_KEY_BOOL, ANY_VALUE_KEY_BYTES, + ANY_VALUE_KEY_DOUBLE, ANY_VALUE_KEY_INT64, ANY_VALUE_KEY_KEY_VALUE_LIST, ANY_VALUE_KEY_STRING, + FLAT_ATTR_STRIDE, TYPED_VALUE_STRIDE, }; use crate::msgpack_decoder::decode::buffer::Buffer; use crate::msgpack_decoder::decode::error::DecodeError; @@ -33,6 +33,8 @@ where .map_err(|_| DecodeError::InvalidFormat("Unable to read V1 span map len".to_owned()))?; let mut span = Span::::default(); + let mut has_span_id = false; + let mut has_start = false; for _ in 0..map_len { let key = decode::read_int::(buf.as_mut_slice()) @@ -45,7 +47,8 @@ where span_key::SPAN_ID => { span.span_id = decode::read_int(buf.as_mut_slice()).map_err(|_| { DecodeError::InvalidFormat("V1 span_id u64 read failure".to_owned()) - })? + })?; + has_span_id = true; } span_key::PARENT_ID => { span.parent_id = decode::read_int(buf.as_mut_slice()).map_err(|_| { @@ -53,14 +56,25 @@ where })? } span_key::START => { - span.start = decode::read_int::(buf.as_mut_slice()).map_err(|_| { + let start: u64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { DecodeError::InvalidFormat("V1 span start u64 read failure".to_owned()) - })? as i64; + })?; + span.start = i64::try_from(start).map_err(|_| { + DecodeError::InvalidFormat(format!( + "V1 span start {start} exceeds i64::MAX" + )) + })?; + has_start = true; } span_key::DURATION => { - span.duration = decode::read_int::(buf.as_mut_slice()).map_err(|_| { + let duration: u64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { DecodeError::InvalidFormat("V1 span duration u64 read failure".to_owned()) - })? as i64; + })?; + span.duration = i64::try_from(duration).map_err(|_| { + DecodeError::InvalidFormat(format!( + "V1 span duration {duration} exceeds i64::MAX" + )) + })?; } span_key::ERROR => { span.error = decode::read_bool(buf.as_mut_slice()).map_err(|_| { @@ -87,15 +101,20 @@ where _ => SpanKind::Internal, }; } - unknown => { - return Err(DecodeError::InvalidFormat(format!( - "Unknown V1 span key: {unknown}" - ))); - } + _unknown => skip_unknown_value(buf)?, } } - span.attributes.mark_deduped(); + if !has_span_id { + return Err(DecodeError::InvalidFormat( + "V1 span is missing span_id".to_owned(), + )); + } + if !has_start { + return Err(DecodeError::InvalidFormat( + "V1 span is missing start".to_owned(), + )); + } Ok(span) } @@ -127,8 +146,8 @@ where map.insert(key, value); } - map.mark_deduped(); - + // V1 attributes are a flat array of triplets, not a real msgpack map, so duplicate keys are + // possible; leave `deduped` false so re-encoding applies the defensive last-write-wins dedup. Ok(map) } @@ -266,11 +285,7 @@ where })?; link.flags = v as u32; } - unknown => { - return Err(DecodeError::InvalidFormat(format!( - "Unknown V1 span_link key: {unknown}" - ))); - } + _unknown => skip_unknown_value(buf)?, } } Ok(link) @@ -319,11 +334,7 @@ where span_event_key::ATTRIBUTES => { event.attributes = read_attributes_map(buf, table)?; } - unknown => { - return Err(DecodeError::InvalidFormat(format!( - "Unknown V1 span_event key: {unknown}" - ))); - } + _unknown => skip_unknown_value(buf)?, } } Ok(event) diff --git a/libdd-trace-utils/src/span/mod.rs b/libdd-trace-utils/src/span/mod.rs index 1a122efe99..f453c440bc 100644 --- a/libdd-trace-utils/src/span/mod.rs +++ b/libdd-trace-utils/src/span/mod.rs @@ -88,6 +88,9 @@ impl DeserializableTraceData for BytesData { #[inline] fn try_slice_and_advance(buf: &mut Bytes, bytes: usize) -> Option { + if bytes > buf.len() { + return None; + } let data = buf.slice_ref(&buf[0..bytes])?; unsafe { // SAFETY: forwarding the buffer requires that buf is borrowed from static. diff --git a/libdd-trace-utils/src/span/v1/mod.rs b/libdd-trace-utils/src/span/v1/mod.rs index 459cbc5438..122ac2f6c4 100644 --- a/libdd-trace-utils/src/span/v1/mod.rs +++ b/libdd-trace-utils/src/span/v1/mod.rs @@ -34,7 +34,7 @@ impl SpanKind { /// Typed V1 attribute value. /// Replaces v0.4's split `meta` / `metrics` / `meta_struct` maps. -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum AttributeValue { String(T::Text), Float(f64), diff --git a/libdd-trace-utils/src/trace_utils.rs b/libdd-trace-utils/src/trace_utils.rs index 6fd439a2ee..15cfaeded7 100644 --- a/libdd-trace-utils/src/trace_utils.rs +++ b/libdd-trace-utils/src/trace_utils.rs @@ -354,10 +354,13 @@ pub fn coalesce_send_data(mut data: Vec) -> Vec { // has similar results. The primary goal here is avoiding many small requests. // TODO: maybe make the MAX_PAYLOAD_SIZE configurable? if a.size + b.size < MAX_PAYLOAD_SIZE / 2 { - // Note: dedup_by drops a, and retains b. - b.tracer_payloads.append(&mut a.tracer_payloads); - b.size += a.size; - return true; + // Note: dedup_by drops a, and retains b. Only drop a if the append actually + // merged its data into b; otherwise keep both entries (e.g. diverging V1 + // tracer metadata) so a's traces aren't silently lost. + if b.tracer_payloads.append(&mut a.tracer_payloads) { + b.size += a.size; + return true; + } } } false diff --git a/libdd-trace-utils/src/tracer_payload.rs b/libdd-trace-utils/src/tracer_payload.rs index 99677c7ddb..6560ed9b11 100644 --- a/libdd-trace-utils/src/tracer_payload.rs +++ b/libdd-trace-utils/src/tracer_payload.rs @@ -85,17 +85,27 @@ impl TracerPayloadCollection { /// let mut col2 = TracerPayloadCollection::V07(vec![TracerPayload::default()]); /// col1.append(&mut col2); /// ``` - pub fn append(&mut self, other: &mut Self) { + /// + /// # Returns + /// + /// `true` if `other`'s data was merged into `self`, `false` if the append was skipped (e.g. + /// diverging V1 tracer metadata). Callers that rely on `other` being fully drained must check + /// this return value rather than assuming success. + pub fn append(&mut self, other: &mut Self) -> bool { match self { TracerPayloadCollection::V07(dest) => { if let TracerPayloadCollection::V07(src) = other { - dest.append(src) + dest.append(src); + return true; } + false } TracerPayloadCollection::V04(dest) => { if let TracerPayloadCollection::V04(src) = other { - dest.append(src) + dest.append(src); + return true; } + false } TracerPayloadCollection::V1(dest) => { if let TracerPayloadCollection::V1(src) = other { @@ -106,8 +116,10 @@ impl TracerPayloadCollection { // (logging a warning) rather than silently dropping `src`'s metadata. if metadata_matches_v1(dest, src) { dest.chunks.append(&mut src.chunks); + return true; } } + false } // TODO: Properly handle non-OK states to prevent possible panics (APMSP-18190). #[allow(clippy::unimplemented)] @@ -268,7 +280,8 @@ pub fn decode_to_trace_chunks( } } -/// Returns `true` iff every tracer-level metadata string field of `src` matches `dest`. +/// Returns `true` iff every tracer-level metadata field (string fields and attributes) of `src` +/// matches `dest`. /// /// V1 payloads carry tracer metadata (env, hostname, language, …) inside the payload itself, so /// merging two payloads whose metadata diverges would silently drop one set of values. Callers @@ -290,6 +303,7 @@ fn metadata_matches_v1( ("env", dest.env == src.env), ("hostname", dest.hostname == src.hostname), ("app_version", dest.app_version == src.app_version), + ("attributes", dest.attributes == src.attributes), ] .into_iter() .filter_map(|(label, eq)| (!eq).then_some(label)) From dbbec42f00fa4f9ee1f2cffdb3c9a7f143d9da46 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Wed, 8 Jul 2026 16:07:41 +0200 Subject: [PATCH 14/23] fix: clippy --- libdd-trace-utils/src/msgpack_decoder/v1/mod.rs | 2 ++ libdd-trace-utils/src/msgpack_decoder/v1/span.rs | 12 +++++------- libdd-trace-utils/src/tracer_payload.rs | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs index d2913957b3..de43b468e2 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs @@ -14,6 +14,7 @@ use std::borrow::Borrow; // (`msgpack_encoder::v1::{trace_key, chunk_key, SpanKey, SpanLinkKey, SpanEventKey, AnyValueKey}`). pub(super) mod trace_key { + pub const CONTAINER_ID: u8 = 2; pub const LANGUAGE_NAME: u8 = 3; pub const LANGUAGE_VERSION: u8 = 4; pub const TRACER_VERSION: u8 = 5; @@ -239,6 +240,7 @@ where payload.chunks = decode_chunks(buf, table)?; saw_chunks = true; } + trace_key::CONTAINER_ID => payload.container_id = read_interned_string(buf, table)?, trace_key::LANGUAGE_NAME => payload.language_name = read_interned_string(buf, table)?, trace_key::LANGUAGE_VERSION => { payload.language_version = read_interned_string(buf, table)? diff --git a/libdd-trace-utils/src/msgpack_decoder/v1/span.rs b/libdd-trace-utils/src/msgpack_decoder/v1/span.rs index 969e62937a..739d5eb6b5 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v1/span.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v1/span.rs @@ -5,10 +5,10 @@ // a V1 msgpack span into a [`crate::span::v1::Span`]. use super::{ - read_interned_string, skip_unknown_value, span_event_key, span_key, span_link_key, - StringTable, ANY_VALUE_KEY_ARRAY, ANY_VALUE_KEY_BOOL, ANY_VALUE_KEY_BYTES, - ANY_VALUE_KEY_DOUBLE, ANY_VALUE_KEY_INT64, ANY_VALUE_KEY_KEY_VALUE_LIST, ANY_VALUE_KEY_STRING, - FLAT_ATTR_STRIDE, TYPED_VALUE_STRIDE, + read_interned_string, skip_unknown_value, span_event_key, span_key, span_link_key, StringTable, + ANY_VALUE_KEY_ARRAY, ANY_VALUE_KEY_BOOL, ANY_VALUE_KEY_BYTES, ANY_VALUE_KEY_DOUBLE, + ANY_VALUE_KEY_INT64, ANY_VALUE_KEY_KEY_VALUE_LIST, ANY_VALUE_KEY_STRING, FLAT_ATTR_STRIDE, + TYPED_VALUE_STRIDE, }; use crate::msgpack_decoder::decode::buffer::Buffer; use crate::msgpack_decoder::decode::error::DecodeError; @@ -60,9 +60,7 @@ where DecodeError::InvalidFormat("V1 span start u64 read failure".to_owned()) })?; span.start = i64::try_from(start).map_err(|_| { - DecodeError::InvalidFormat(format!( - "V1 span start {start} exceeds i64::MAX" - )) + DecodeError::InvalidFormat(format!("V1 span start {start} exceeds i64::MAX")) })?; has_start = true; } diff --git a/libdd-trace-utils/src/tracer_payload.rs b/libdd-trace-utils/src/tracer_payload.rs index 6560ed9b11..d931640938 100644 --- a/libdd-trace-utils/src/tracer_payload.rs +++ b/libdd-trace-utils/src/tracer_payload.rs @@ -32,7 +32,7 @@ pub enum TraceChunks { /// Collection of TraceChunkSpan with de-duplicated strings. V05((SharedDict, Vec>)), /// Collection of v0.4 spans to be serialized as a V1 msgpack payload. - V1(v1::TracerPayload), + V1(Box>), } impl TraceChunks { @@ -66,7 +66,7 @@ pub enum TracerPayloadCollection { /// Collection of TraceChunkSpan with de-duplicated strings. V05((SharedDictBytes, Vec>)), // /// V0.4-shaped spans that must be serialized as a V1 msgpack payload on send. - V1(v1::TracerPayload), + V1(Box>), } impl TracerPayloadCollection { @@ -275,7 +275,7 @@ pub fn decode_to_trace_chunks( let (data, size) = msgpack_decoder::v1::from_bytes(data).map_err(|e| { anyhow::format_err!("Error deserializing trace from request body: {e}") })?; - Ok((TraceChunks::V1(data), size)) + Ok((TraceChunks::V1(Box::new(data)), size)) } } } @@ -293,6 +293,7 @@ fn metadata_matches_v1( src: &v1::TracerPayload, ) -> bool { let differing: Vec<&'static str> = [ + ("container_id", dest.container_id == src.container_id), ("language_name", dest.language_name == src.language_name), ( "language_version", From 1af6df1ac825d2efcfde721a81d286a4229ca4ad Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Wed, 8 Jul 2026 16:35:42 +0200 Subject: [PATCH 15/23] fix: test --- libdd-trace-utils/src/span/vec_map.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/libdd-trace-utils/src/span/vec_map.rs b/libdd-trace-utils/src/span/vec_map.rs index de02dcabb1..bdd4367ce1 100644 --- a/libdd-trace-utils/src/span/vec_map.rs +++ b/libdd-trace-utils/src/span/vec_map.rs @@ -61,10 +61,9 @@ impl Default for VecMap { } } -// This implementation allocates, which isn't expected for equality testing (not allocating would be -// rather tedious though). It's fine for tests (where `PartialEq` is currently needed), so we -// cfg-gate it to avoid unintended usage in prod. -#[cfg(any(test, feature = "test-utils"))] +// This implementation allocates, which isn't ideal for equality testing (not allocating would be +// rather tedious though), but is needed both by production code (e.g. comparing V1 tracer metadata +// before coalescing) and by tests. impl PartialEq for VecMap { fn eq(&self, other: &Self) -> bool { let lhs: HashMap<&K, &V> = self.data.iter().map(|(k, v)| (k, v)).collect(); @@ -73,7 +72,6 @@ impl PartialEq for VecMap { } } -#[cfg(any(test, feature = "test-utils"))] impl Eq for VecMap {} impl VecMap { From dcf375e95cef29d23a5da01f8aa39774f588dbba Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Mon, 13 Jul 2026 10:40:31 +0200 Subject: [PATCH 16/23] fix: comments --- .../src/msgpack_encoder/v04/mod.rs | 77 +++++++-- .../src/msgpack_encoder/v04/span_v1.rs | 163 ++++++++++++++---- libdd-trace-utils/tests/test_send_data.rs | 2 +- 3 files changed, 191 insertions(+), 51 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs index 652f008e99..c025e9141a 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs @@ -46,7 +46,15 @@ const fn msgpack_const_string_encoding(s: &str) -> [u storage[0] = 0xdb; 4 }; - copy_to_slice(storage.split_at_mut(1).1, &len.to_be_bytes(), len_bytes); + let len_be_bytes = len.to_be_bytes(); + // `len_be_bytes` holds `len` as 8 big-endian bytes; the marker only needs the low-order + // `len_bytes` of those (e.g. for a str8 length of 200, that's byte `[200]`, not `[0]`), so + // skip the leading zero bytes rather than copying from the front. + copy_to_slice( + storage.split_at_mut(1).1, + len_be_bytes.split_at(8 - len_bytes).1, + len_bytes, + ); copy_to_slice(storage.split_at_mut(1 + len_bytes).1, s.as_bytes(), s.len()); storage } @@ -231,9 +239,9 @@ pub fn to_encoded_byte_len_from_v04]>>(traces: & /// Encodes a [`TracerPayload`] in the v0.4 wire format (downgrade path used when the agent /// does not advertise `/v1.0/traces`). The output is a msgpack array of traces, where each /// trace is itself a msgpack array of v0.4-shaped spans — matching the existing v0.4 wire -/// format produced by [`to_vec`]. Payload-level `env`/`app_version`/`attributes` are propagated -/// into every span (see [`span_v1`]'s mapping table); `payload.hostname` has no v0.4 body -/// equivalent (the agent gets hostname from the `Datadog-Meta-Hostname` header instead) and is +/// format produced by [`to_vec_from_v04`]. Payload-level `env`/`app_version`/`attributes` are +/// propagated into every span (see [`span_v1`]'s mapping table); `payload.hostname` has no v0.4 +/// body equivalent (the agent gets hostname from the `Datadog-Meta-Hostname` header instead) and is /// intentionally dropped here. fn encode_payload_from_v1( writer: &mut W, @@ -252,16 +260,16 @@ fn encode_payload_from_v1( } else { chunk.priority }; - let ctx = ChunkContext { - trace_id: &chunk.trace_id, + let ctx = ChunkContext::new( + &chunk.trace_id, priority, - origin: &chunk.origin, - sampling_mechanism: chunk.sampling_mechanism, - attributes: &chunk.attributes, - payload_env: &payload.env, - payload_app_version: &payload.app_version, - payload_attributes: &payload.attributes, - }; + &chunk.origin, + chunk.sampling_mechanism, + &chunk.attributes, + &payload.env, + &payload.app_version, + &payload.attributes, + ); write_array_len(writer, chunk.spans.len() as u32)?; for span in &chunk.spans { encode_span(writer, span, &ctx)?; @@ -273,9 +281,9 @@ fn encode_payload_from_v1( /// Serializes a [`TracerPayload`] (V1 data model) as a v0.4 msgpack payload. /// /// Used by the trace exporter when the agent has not advertised `/v1.0/traces` via `/info`. -/// The output is byte-compatible with [`to_vec`] for equivalent data — chunk-level fields are -/// propagated to every span and typed attributes are bucketed into the v0.4 `meta` / -/// `metrics` / `meta_struct` maps per [`span_v1_to_v04`]'s mapping table. +/// The output is byte-compatible with [`to_vec_from_v04`] for equivalent data — chunk-level fields +/// are propagated to every span and typed attributes are bucketed into the v0.4 `meta` / +/// `metrics` / `meta_struct` maps per [`span_v1`]'s mapping table. pub fn to_vec_from_v1(payload: &TracerPayload) -> Vec { to_vec_with_capacity_from_v1(payload, 0) } @@ -314,3 +322,40 @@ pub fn to_encoded_byte_len_from_v1(payload: &TracerPayload) -> let _ = encode_payload_from_v1(&mut counter, payload); counter.0 } + +#[cfg(test)] +mod tests { + //! Regression tests for [`msgpack_const_string_encoding`] across every msgpack string + //! length-marker boundary (fixstr / str8 / str16), since the length only fits in the + //! low-order bytes of `len.to_be_bytes()` and it's easy to accidentally copy from the + //! high-order (zero) end instead. + use super::msgpack_const_string_encoding; + + fn encode(s: &str) -> [u8; N] { + msgpack_const_string_encoding::(s) + } + + #[test] + fn fixstr_boundary_31_bytes() { + let s = "a".repeat(31); + let bytes: [u8; 32] = encode(&s); + let value = rmpv::decode::read_value(&mut &bytes[..]).expect("decode failed"); + assert_eq!(value.as_str(), Some(s.as_str())); + } + + #[test] + fn str8_boundary_200_bytes() { + let s = "b".repeat(200); + let bytes: [u8; 202] = encode(&s); + let value = rmpv::decode::read_value(&mut &bytes[..]).expect("decode failed"); + assert_eq!(value.as_str(), Some(s.as_str())); + } + + #[test] + fn str16_boundary_300_bytes() { + let s = "c".repeat(300); + let bytes: [u8; 303] = encode(&s); + let value = rmpv::decode::read_value(&mut &bytes[..]).expect("decode failed"); + assert_eq!(value.as_str(), Some(s.as_str())); + } +} diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs index 037bfa8f94..82816921f1 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs @@ -31,7 +31,7 @@ //! dropped: the dedicated field always wins, so each key is written at most once. use crate::span::v1::{AttributeValue, Span, SpanEvent, SpanKind, SpanLink}; -use crate::span::vec_map::VecMap; +use crate::span::vec_map::{DedupedVecMap, VecMap}; use crate::span::TraceData; use rmp::encode::{ write_array_len, write_bin, write_bool, write_f64, write_i64, write_map_len, write_sint, @@ -39,6 +39,7 @@ use rmp::encode::{ }; use std::borrow::Borrow; use std::collections::HashSet; +use std::fmt::Write as _; /// Writes a `bool` as the v0.4 string representation (`"true"` / `"false"`). Used wherever a /// typed V1 `Bool` attribute is downgraded into v0.4 `meta` (which is `String → String` only). @@ -69,15 +70,43 @@ const PROMOTED_ATTR_KEYS: &[&str] = &[ /// carries payload-level fields (`payload_env`, `payload_app_version`, `payload_attributes`), /// which apply as a fallback when the span itself doesn't set the equivalent field — v0.4 has /// neither a chunk nor a payload concept, so both levels collapse onto every span. +/// +/// `chunk_attrs_dd` / `payload_attrs_dd` are deduped once here rather than per span: unlike the +/// span's own attributes, they're identical for every span in the chunk. pub(super) struct ChunkContext<'a, T: TraceData> { pub trace_id: &'a [u8; 16], pub priority: Option, pub origin: &'a T::Text, pub sampling_mechanism: Option, - pub attributes: &'a VecMap>, pub payload_env: &'a T::Text, pub payload_app_version: &'a T::Text, - pub payload_attributes: &'a VecMap>, + pub chunk_attrs_dd: DedupedVecMap<'a, T::Text, AttributeValue>, + pub payload_attrs_dd: DedupedVecMap<'a, T::Text, AttributeValue>, +} + +impl<'a, T: TraceData> ChunkContext<'a, T> { + #[allow(clippy::too_many_arguments)] + pub fn new( + trace_id: &'a [u8; 16], + priority: Option, + origin: &'a T::Text, + sampling_mechanism: Option, + attributes: &'a VecMap>, + payload_env: &'a T::Text, + payload_app_version: &'a T::Text, + payload_attributes: &'a VecMap>, + ) -> Self { + Self { + trace_id, + priority, + origin, + sampling_mechanism, + payload_env, + payload_app_version, + chunk_attrs_dd: attributes.defensive_dedup(), + payload_attrs_dd: payload_attributes.defensive_dedup(), + } + } } /// Maps a `SpanKind` to its v0.4 `span.kind` meta string. Returns `None` for `Internal` so @@ -114,6 +143,26 @@ struct BucketCounts { meta_struct: u32, } +/// Drops entries whose key was already seen, keeping the first occurrence. Two distinct +/// attributes can flatten to the same dotted key (e.g. a literal `"a.0"` attribute alongside a +/// `List` attribute named `"a"`, whose first element also flattens to `"a.0"`); msgpack doesn't +/// forbid duplicate map keys, but doesn't define decoder precedence for them either, so we +/// enforce "each key written at most once" ourselves rather than leaving it to chance. +fn dedup_first_wins(mut leaves: Vec<(String, V)>) -> Vec<(String, V)> { + // Same two-pass technique as `VecMap::dedup`: collect a keep/drop bitmap using borrowed + // `&str`s (no per-key clone), then `retain` in place (no full re-collect into a new `Vec`). + let keep: Vec = { + let mut seen: HashSet<&str> = HashSet::with_capacity(leaves.len()); + leaves + .iter() + .map(|(k, _)| seen.insert(k.as_str())) + .collect() + }; + let mut keep = keep.into_iter(); + leaves.retain(|_| keep.next().unwrap_or(false)); + leaves +} + /// Recursively flattens a `List`/`KeyValue` attribute into dotted-key leaf entries for the v0.4 /// `meta` (string-valued) and `metrics` (numeric) maps — matching how intake/the UI expect nested /// V1 attributes to be exploded: list elements become `key.0`, `key.1`, ... and `KeyValue` @@ -121,29 +170,40 @@ struct BucketCounts { /// (`String`/`Bool`/`Int`/`Float`) are leaves in their own right and produce a single entry under /// `key`. `Bytes` has no flattened form; callers must route it to `meta_struct` separately. fn flatten_attr_into( - key: String, + key: &mut String, v: &AttributeValue, meta_out: &mut Vec<(String, String)>, metrics_out: &mut Vec<(String, f64)>, ) { match v { - AttributeValue::String(s) => meta_out.push((key, s.borrow().to_owned())), + AttributeValue::String(s) => meta_out.push((key.clone(), s.borrow().to_owned())), AttributeValue::Bool(b) => { - meta_out.push((key, if *b { "true" } else { "false" }.to_owned())) + meta_out.push((key.clone(), if *b { "true" } else { "false" }.to_owned())) } - AttributeValue::Int(i) => metrics_out.push((key, *i as f64)), - AttributeValue::Float(f) => metrics_out.push((key, *f)), + AttributeValue::Int(i) => metrics_out.push((key.clone(), *i as f64)), + AttributeValue::Float(f) => metrics_out.push((key.clone(), *f)), AttributeValue::Bytes(_) => { // Callers filter `Bytes` out before recursing; unreachable in practice. } AttributeValue::List(items) => { + // Reuse `key`'s buffer across siblings instead of allocating a new `String` per + // recursion level: append the suffix, recurse, then truncate back before the next + // sibling. Only leaves actually need an owned `String` (via `.clone()` above). + let base_len = key.len(); for (i, item) in items.iter().enumerate() { - flatten_attr_into(format!("{key}.{i}"), item, meta_out, metrics_out); + key.push('.'); + let _ = write!(key, "{i}"); + flatten_attr_into(key, item, meta_out, metrics_out); + key.truncate(base_len); } } AttributeValue::KeyValue(map) => { + let base_len = key.len(); for (k, v) in map.defensive_dedup().iter() { - flatten_attr_into(format!("{key}.{}", k.borrow()), v, meta_out, metrics_out); + key.push('.'); + key.push_str(k.borrow()); + flatten_attr_into(key, v, meta_out, metrics_out); + key.truncate(base_len); } } } @@ -175,8 +235,6 @@ pub(super) fn encode_span( chunk: &ChunkContext<'_, T>, ) -> Result<(), ValueWriteError> { let span_attrs_dd = span.attributes.defensive_dedup(); - let chunk_attrs_dd = chunk.attributes.defensive_dedup(); - let payload_attrs_dd = chunk.payload_attributes.defensive_dedup(); // Merge span + chunk + payload attributes upfront with explicit "span overrides chunk // overrides payload" precedence. We don't rely on msgpack map last-write-wins decoding @@ -185,23 +243,24 @@ pub(super) fn encode_span( // // Attributes sharing a name with a "promoted" dedicated field (`env`, `_dd.origin`, ...) // are dropped here: the dedicated field always wins so we never emit that key twice. - let span_keys: HashSet<&T::Text> = span_attrs_dd.iter().map(|(k, _)| k).collect(); - let mut seen_keys: HashSet<&T::Text> = span_keys.clone(); - let chunk_only: Vec<(&T::Text, &AttributeValue)> = chunk_attrs_dd - .iter() - .filter(|(k, _)| !seen_keys.contains(k)) - .collect(); - seen_keys.extend(chunk_attrs_dd.iter().map(|(k, _)| k)); - let payload_only: Vec<(&T::Text, &AttributeValue)> = payload_attrs_dd - .iter() - .filter(|(k, _)| !seen_keys.contains(k)) - .collect(); - let merged_attrs: Vec<(&T::Text, &AttributeValue)> = span_attrs_dd + // + // This is a hot path (once per span), so precedence is resolved by chaining filtered + // iterators over the already-deduped maps rather than materializing intermediate + // `Vec`/`HashSet`s of keys — `chunk_attrs_dd`/`payload_attrs_dd` are small enough in + // practice that the linear "is this key already present" scans below are cheaper than an + // allocation. + let merged_attrs = span_attrs_dd .iter() - .chain(chunk_only) - .chain(payload_only) .filter(|(k, _)| !PROMOTED_ATTR_KEYS.contains(&(*k).borrow())) - .collect(); + .chain(chunk.chunk_attrs_dd.iter().filter(|(k, _)| { + !PROMOTED_ATTR_KEYS.contains(&(*k).borrow()) + && !span_attrs_dd.iter().any(|(k2, _)| k2 == *k) + })) + .chain(chunk.payload_attrs_dd.iter().filter(|(k, _)| { + !PROMOTED_ATTR_KEYS.contains(&(*k).borrow()) + && !span_attrs_dd.iter().any(|(k2, _)| k2 == *k) + && !chunk.chunk_attrs_dd.iter().any(|(k2, _)| k2 == *k) + })); let (trace_id_low, trace_id_high) = split_trace_id(chunk.trace_id); let kind_meta = span_kind_to_meta(span.span_kind); @@ -228,17 +287,21 @@ pub(super) fn encode_span( let mut meta_leaves: Vec<(String, String)> = Vec::new(); let mut metrics_leaves: Vec<(String, f64)> = Vec::new(); let mut bytes_attrs: Vec<(&T::Text, &T::Bytes)> = Vec::new(); - for &(k, v) in &merged_attrs { + let mut key_buf = String::new(); + for (k, v) in merged_attrs { match v { AttributeValue::Bytes(b) => bytes_attrs.push((k, b)), - _ => flatten_attr_into( - k.borrow().to_owned(), - v, - &mut meta_leaves, - &mut metrics_leaves, - ), + _ => { + key_buf.clear(); + key_buf.push_str(k.borrow()); + flatten_attr_into(&mut key_buf, v, &mut meta_leaves, &mut metrics_leaves); + } } } + // Two distinct attributes can flatten to the same dotted key (see `dedup_first_wins`); + // `meta` and `metrics` are separate wire maps, so dedup independently within each bucket. + let meta_leaves = dedup_first_wins(meta_leaves); + let metrics_leaves = dedup_first_wins(metrics_leaves); // First pass: count bucket sizes so each msgpack map header carries the exact length. let mut counts = BucketCounts::default(); @@ -741,6 +804,38 @@ mod tests { assert_eq!(map_get(meta, "http.method").unwrap().as_str(), Some("GET")); } + #[test] + fn flattened_attribute_colliding_with_another_attribute_keeps_first_wins() { + // "a" is a List whose first element flattens to "a.0"; "a.0" is also a literal + // attribute key. Both flatten to the same dotted key "a.0" — only one must survive + // on the wire, keeping whichever attribute came first in insertion order. + let mut attrs: VecMap = VecMap::new(); + attrs.insert( + bs("a"), + AttributeValue::List(vec![AttributeValue::String(bs("from-list"))]), + ); + attrs.insert(bs("a.0"), AttributeValue::String(bs("from-literal"))); + // Mark deduped (as attributes are expected to be by the time they reach the encoder) + // so iteration order is deterministic rather than depending on `defensive_dedup`'s + // fallback `HashMap` order. + attrs.dedup(); + let span = SpanBytes { + attributes: attrs, + ..minimal_span() + }; + let payload = minimal_payload([0u8; 16], span); + let traces = encode_and_decode(&payload); + let meta = map_get(&traces[0][0], "meta").expect("meta present"); + let meta_entries = meta.as_map().expect("meta must be a map"); + + // Only one "a.0" entry must survive on the wire, regardless of which attribute wins. + let a0_count = meta_entries + .iter() + .filter(|(k, _)| k.as_str() == Some("a.0")) + .count(); + assert_eq!(a0_count, 1, "duplicate \"a.0\" key written to the wire"); + } + #[test] fn span_kind_internal_is_not_emitted() { // Internal is the default and is implied by the absence of `meta["span.kind"]`. diff --git a/libdd-trace-utils/tests/test_send_data.rs b/libdd-trace-utils/tests/test_send_data.rs index a3b6546426..a0267d0e44 100644 --- a/libdd-trace-utils/tests/test_send_data.rs +++ b/libdd-trace-utils/tests/test_send_data.rs @@ -633,7 +633,7 @@ mod tracing_integration_tests { /// Round-trip: the v1::Span → v0.4 downgrade encoder /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec_from_v1`]) must produce a v0.4 /// payload that decodes to the same canonical form as the native v0.4 encoder - /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec`]) when fed equivalent input. + /// ([`libdd_trace_utils::msgpack_encoder::v04::to_vec_from_v04`]) when fed equivalent input. /// /// Both encodings are POSTed to `/v0.4/traces` in the same session with distinct /// `trace_id`s, and the checked-in snapshot records the two traces side by side. Any From 6c37c8814c60d36def8287731a297ebaf43177b7 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Mon, 13 Jul 2026 15:00:54 +0200 Subject: [PATCH 17/23] fix: merge --- libdd-trace-utils/src/tracer_payload.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/libdd-trace-utils/src/tracer_payload.rs b/libdd-trace-utils/src/tracer_payload.rs index 366ea41378..ed43aa6162 100644 --- a/libdd-trace-utils/src/tracer_payload.rs +++ b/libdd-trace-utils/src/tracer_payload.rs @@ -1,7 +1,6 @@ // Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use crate::span::v05::dict::SharedDict; use crate::span::{v04, v05, v1, BytesData, SharedDictBytes, TraceData}; use crate::trace_utils::convert_trace_chunks_v04_to_v05; use crate::{msgpack_decoder, trace_utils::cmp_send_data_payloads}; From 23914228987d44fa9db181c0582bf7fffc0d20d2 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Wed, 15 Jul 2026 13:37:30 +0200 Subject: [PATCH 18/23] fix: comments --- .../src/msgpack_decoder/decode/buffer.rs | 5 ++ .../src/msgpack_decoder/v1/mod.rs | 63 +++++++++++-------- .../src/msgpack_decoder/v1/span.rs | 8 +-- libdd-trace-utils/src/tracer_payload.rs | 52 +++++++-------- 4 files changed, 70 insertions(+), 58 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_decoder/decode/buffer.rs b/libdd-trace-utils/src/msgpack_decoder/decode/buffer.rs index 9b859613f8..4eb05ed936 100644 --- a/libdd-trace-utils/src/msgpack_decoder/decode/buffer.rs +++ b/libdd-trace-utils/src/msgpack_decoder/decode/buffer.rs @@ -40,6 +40,11 @@ impl Buffer { T::get_mut_slice(&mut self.0) } + /// Returns an immutable reference to the underlying slice, without advancing the buffer. + pub fn as_slice(&self) -> &[u8] { + self.0.borrow() + } + /// Tries to extract a slice of `bytes` from the buffer and advances the buffer. pub fn try_slice_and_advance(&mut self, bytes: usize) -> Option { T::try_slice_and_advance(&mut self.0, bytes) diff --git a/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs index de43b468e2..9513c8feb0 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs @@ -83,6 +83,9 @@ pub(super) const TYPED_VALUE_STRIDE: u32 = 2; /// Number of msgpack items consumed per `[key, type, value]` triplet in a typed attributes map. pub(super) const FLAT_ATTR_STRIDE: u32 = 3; +/// Length in bytes of a V1 chunk's `trace_id` field (128-bit trace ID). +pub(super) const TRACE_ID_LEN: u32 = 16; + /// Streaming string intern table built up as the payload is decoded. /// /// V1 strings are encoded inline the first time they appear (as msgpack `str`), and as a @@ -118,10 +121,9 @@ where }) } - /// Records a freshly-read inline string and returns it (cloned for reuse). - fn record(&mut self, s: T::Text) -> T::Text { + /// Records a freshly-read inline string in the table. + fn record(&mut self, s: &T::Text) { self.seen.push(s.clone()); - s } } @@ -137,7 +139,7 @@ pub(super) fn read_interned_string( where T::Text: Clone, { - let slice: &[u8] = buf.as_mut_slice(); + let slice: &[u8] = buf.as_slice(); let marker_byte = *slice.first().ok_or_else(|| { DecodeError::InvalidFormat( "Unexpected end of V1 buffer when reading interned string".to_owned(), @@ -149,21 +151,23 @@ where // str8/str16/str32 : 0xd9, 0xda, 0xdb // fixint (positive): 0x00..=0x7f // uint8/16/32/64 : 0xcc, 0xcd, 0xce, 0xcf - let is_string = matches!(marker_byte, 0xa0..=0xbf | 0xd9 | 0xda | 0xdb); - let is_uint = matches!(marker_byte, 0x00..=0x7f | 0xcc | 0xcd | 0xce | 0xcf); - - if is_string { - let s = buf.read_string()?; - Ok(table.record(s)) - } else if is_uint { - let id: u64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { - DecodeError::InvalidFormat("V1 interned string reference uint read failure".to_owned()) - })?; - table.resolve(id) - } else { - Err(DecodeError::InvalidFormat(format!( + match marker_byte { + 0xa0..=0xbf | 0xd9 | 0xda | 0xdb => { + let s = buf.read_string()?; + table.record(&s); + Ok(s) + } + 0x00..=0x7f | 0xcc | 0xcd | 0xce | 0xcf => { + let id: u64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { + DecodeError::InvalidFormat( + "V1 interned string reference uint read failure".to_owned(), + ) + })?; + table.resolve(id) + } + _ => Err(DecodeError::InvalidFormat(format!( "Unexpected msgpack marker 0x{marker_byte:02x} for V1 interned string" - ))) + ))), } } @@ -192,7 +196,6 @@ pub fn from_slice(data: &[u8]) -> Result<(TracerPayloadSlice<'_>, usize), Decode } /// Generic over the deserialization mode (owned `BytesData` or borrowed `SliceData`). -#[allow(clippy::type_complexity)] pub fn from_buffer( data: &mut Buffer, ) -> Result<(TracerPayload, usize), DecodeError> @@ -303,14 +306,16 @@ where let len = decode::read_bin_len(buf.as_mut_slice()).map_err(|_| { DecodeError::InvalidFormat("V1 chunk trace_id bin len read failure".to_owned()) })?; - if len != 16 { + if len != TRACE_ID_LEN { return Err(DecodeError::InvalidFormat(format!( - "V1 chunk trace_id must be 16 bytes, got {len}" + "V1 chunk trace_id must be {TRACE_ID_LEN} bytes, got {len}" ))); } - let bytes = buf.try_slice_and_advance(16).ok_or_else(|| { - DecodeError::InvalidFormat("V1 chunk trace_id payload truncated".to_owned()) - })?; + let bytes = buf + .try_slice_and_advance(TRACE_ID_LEN as usize) + .ok_or_else(|| { + DecodeError::InvalidFormat("V1 chunk trace_id payload truncated".to_owned()) + })?; let slice: &[u8] = bytes.borrow(); chunk.trace_id.copy_from_slice(slice); saw_trace_id = true; @@ -331,7 +336,9 @@ where let v: i64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { DecodeError::InvalidFormat("V1 chunk priority read failure".to_owned()) })?; - chunk.priority = Some(v as i32); + chunk.priority = Some(i32::try_from(v).map_err(|_| { + DecodeError::InvalidFormat(format!("V1 chunk priority {v} exceeds i32 range")) + })?); } chunk_key::SAMPLING_MECHANISM => { let v: u64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { @@ -339,7 +346,11 @@ where "V1 chunk sampling_mechanism read failure".to_owned(), ) })?; - chunk.sampling_mechanism = Some(v as u32); + chunk.sampling_mechanism = Some(u32::try_from(v).map_err(|_| { + DecodeError::InvalidFormat(format!( + "V1 chunk sampling_mechanism {v} exceeds u32::MAX" + )) + })?); } chunk_key::ATTRIBUTES => { chunk.attributes = span::read_attributes_map(buf, table)?; diff --git a/libdd-trace-utils/src/msgpack_decoder/v1/span.rs b/libdd-trace-utils/src/msgpack_decoder/v1/span.rs index 739d5eb6b5..992baec146 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v1/span.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v1/span.rs @@ -262,9 +262,7 @@ where slice.len() ))); } - let mut arr = [0u8; 16]; - arr.copy_from_slice(slice); - link.trace_id = arr; + link.trace_id.copy_from_slice(slice); } span_link_key::SPAN_ID => { link.span_id = decode::read_int(buf.as_mut_slice()).map_err(|_| { @@ -281,7 +279,9 @@ where let v: u64 = decode::read_int(buf.as_mut_slice()).map_err(|_| { DecodeError::InvalidFormat("V1 span_link flags read failure".to_owned()) })?; - link.flags = v as u32; + link.flags = u32::try_from(v).map_err(|_| { + DecodeError::InvalidFormat(format!("V1 span_link flags {v} exceeds u32::MAX")) + })?; } _unknown => skip_unknown_value(buf)?, } diff --git a/libdd-trace-utils/src/tracer_payload.rs b/libdd-trace-utils/src/tracer_payload.rs index ed43aa6162..121e317f45 100644 --- a/libdd-trace-utils/src/tracer_payload.rs +++ b/libdd-trace-utils/src/tracer_payload.rs @@ -95,38 +95,34 @@ impl TracerPayloadCollection { /// diverging V1 tracer metadata). Callers that rely on `other` being fully drained must check /// this return value rather than assuming success. pub fn append(&mut self, other: &mut Self) -> bool { - match self { - TracerPayloadCollection::V07(dest) => { - if let TracerPayloadCollection::V07(src) = other { - dest.append(src); - return true; - } - false + match (self, other) { + (TracerPayloadCollection::V07(dest), TracerPayloadCollection::V07(src)) => { + dest.append(src); + true } - TracerPayloadCollection::V04(dest) => { - if let TracerPayloadCollection::V04(src) = other { - dest.append(src); - return true; - } - false + (TracerPayloadCollection::V04(dest), TracerPayloadCollection::V04(src)) => { + dest.append(src); + true } - TracerPayloadCollection::V1(dest) => { - if let TracerPayloadCollection::V1(src) = other { - // Same-target SendData entries are coalesced by - // trace_utils::coalesce_send_data, so both V1 payloads - // typically share tracer-level metadata. If all metadata - // fields match we append `src`'s chunks into `dest`; if any diverge we no-op - // (logging a warning) rather than silently dropping `src`'s metadata. - if metadata_matches_v1(dest, src) { - dest.chunks.append(&mut src.chunks); - return true; - } + (TracerPayloadCollection::V1(dest), TracerPayloadCollection::V1(src)) => { + // Same-target SendData entries are coalesced by + // trace_utils::coalesce_send_data, so both V1 payloads + // typically share tracer-level metadata. If all metadata + // fields match we append `src`'s chunks into `dest`; if any diverge we no-op + // (logging a warning) rather than silently dropping `src`'s metadata. + if metadata_matches_v1(dest, src) { + dest.chunks.append(&mut src.chunks); + true + } else { + false } - false } // TODO: Properly handle non-OK states to prevent possible panics (APMSP-18190). #[allow(clippy::unimplemented)] - TracerPayloadCollection::V05(_) => unimplemented!("Append for V05 not implemented"), + (TracerPayloadCollection::V05(_), _) => { + unimplemented!("Append for V05 not implemented") + } + _ => false, } } @@ -283,8 +279,8 @@ pub fn decode_to_trace_chunks( } } -/// Returns `true` iff every tracer-level metadata field (string fields and attributes) of `src` -/// matches `dest`. +/// Returns `true` if and only if every tracer-level metadata field (string fields and attributes) +/// of `src` matches `dest`. /// /// V1 payloads carry tracer metadata (env, hostname, language, …) inside the payload itself, so /// merging two payloads whose metadata diverges would silently drop one set of values. Callers From 41e75b1f6d33f41bee16ab918630f6936444f7b2 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Wed, 15 Jul 2026 13:54:51 +0200 Subject: [PATCH 19/23] fix: comments --- libdd-data-pipeline/src/trace_exporter/builder.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/libdd-data-pipeline/src/trace_exporter/builder.rs b/libdd-data-pipeline/src/trace_exporter/builder.rs index f6a40500e2..868e05b407 100644 --- a/libdd-data-pipeline/src/trace_exporter/builder.rs +++ b/libdd-data-pipeline/src/trace_exporter/builder.rs @@ -64,6 +64,7 @@ pub struct TraceExporterBuilder { language_interpreter_vendor: String, git_commit_sha: String, process_tags: String, + container_id: String, input_format: TraceExporterInputFormat, output_format: TraceExporterOutputFormat, dogstatsd_url: Option, @@ -134,6 +135,7 @@ impl TraceExporterBuilder { language_interpreter_vendor: String::new(), git_commit_sha: String::new(), process_tags: String::new(), + container_id: String::new(), input_format: TraceExporterInputFormat::default(), output_format: TraceExporterOutputFormat::default(), dogstatsd_url: None, @@ -234,6 +236,12 @@ impl TraceExporterBuilder { self } + /// Set the `Datadog-Container-Id` header + pub fn set_container_id(&mut self, container_id: &str) -> &mut Self { + container_id.clone_into(&mut self.container_id); + self + } + /// Set the `Datadog-Meta-Tracer-Version` header pub fn set_tracer_version(&mut self, tracer_version: &str) -> &mut Self { tracer_version.clone_into(&mut self.tracer_version); @@ -808,7 +816,7 @@ impl TraceExporterBuilder { app_version: self.app_version, runtime_id, service: self.service, - container_id: String::new(), + container_id: self.container_id, }, input_format: self.input_format, output_format: self.output_format, From cbca66abbd55c4f1d29aa1c8e5eb91eea4699af1 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Wed, 15 Jul 2026 14:49:00 +0200 Subject: [PATCH 20/23] fix: comments --- Cargo.lock | 1 + libdd-trace-utils/Cargo.toml | 1 + libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs | 10 ++++++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8057cea510..89335512f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3540,6 +3540,7 @@ dependencies = [ "httpmock", "hyper", "indexmap 2.12.1", + "itoa", "libdd-capabilities", "libdd-capabilities-impl", "libdd-common", diff --git a/libdd-trace-utils/Cargo.toml b/libdd-trace-utils/Cargo.toml index cf4462a47e..0323782f2a 100644 --- a/libdd-trace-utils/Cargo.toml +++ b/libdd-trace-utils/Cargo.toml @@ -27,6 +27,7 @@ required-features = ["bench-internals"] anyhow = "1.0" base64 = "0.22" hex = "0.4" +itoa = "1.0" hyper = { workspace = true, optional = true, default-features = false } "http" = "1" "http-body" = "1" diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs index 82816921f1..991e6eaebb 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/span_v1.rs @@ -383,7 +383,12 @@ pub(super) fn encode_span( if trace_id_high != 0 { // Lower-case hex without `0x` prefix — the agent expects this format. write_const_msgpack_str!(writer, "_dd.p.tid")?; - write_str(writer, &format!("{trace_id_high:016x}"))?; + let mut buf = [0u8; 16]; + let hex_str = hex::encode_to_slice(trace_id_high.to_be_bytes(), &mut buf) + .ok() + .and_then(|_| std::str::from_utf8(&buf).ok()) + .unwrap_or_default(); + write_str(writer, hex_str)?; } if !chunk.origin.borrow().is_empty() { write_const_msgpack_str!(writer, "_dd.origin")?; @@ -391,7 +396,8 @@ pub(super) fn encode_span( } if let Some(mechanism) = chunk.sampling_mechanism { write_const_msgpack_str!(writer, "_dd.p.dm")?; - write_str(writer, &format!("-{mechanism}"))?; + let mut buf = itoa::Buffer::new(); + write_str(writer, buf.format(-(mechanism as i64)))?; } for (k, v) in &meta_leaves { write_str(writer, k)?; From 196e0f34239e31b76803f8d82a5abde8c25f15b7 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Thu, 16 Jul 2026 13:55:25 +0200 Subject: [PATCH 21/23] fix: comments --- .../src/msgpack_decoder/decode/buffer.rs | 8 ++++++++ libdd-trace-utils/src/msgpack_decoder/v1/mod.rs | 4 ++-- .../src/msgpack_decoder/v1/span.rs | 17 +++++------------ .../src/msgpack_encoder/v04/mod.rs | 14 ++++++-------- libdd-trace-utils/src/span/v1/mod.rs | 13 +++++++++++++ libdd-trace-utils/src/tracer_payload.rs | 1 - 6 files changed, 34 insertions(+), 23 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_decoder/decode/buffer.rs b/libdd-trace-utils/src/msgpack_decoder/decode/buffer.rs index 4eb05ed936..12f7dbc5c0 100644 --- a/libdd-trace-utils/src/msgpack_decoder/decode/buffer.rs +++ b/libdd-trace-utils/src/msgpack_decoder/decode/buffer.rs @@ -57,6 +57,14 @@ impl Buffer { pub fn read_string(&mut self) -> Result { T::read_string(&mut self.0) } + + /// Caps a decoded element count at the bytes remaining in the buffer. Each msgpack + /// element needs >=1 byte on the wire, so a length prefix can't legitimately exceed + /// the remaining bytes — this prevents a malicious count (e.g. 0xFFFFFFFF) from + /// forcing a huge pre-allocation before any element is read. + pub fn capped_capacity(&self, count: usize) -> usize { + count.min(self.len()) + } } impl Deref for Buffer { diff --git a/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs index 9513c8feb0..b7ee7c45d7 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v1/mod.rs @@ -278,7 +278,7 @@ where { let count = decode::read_array_len(buf.as_mut_slice()) .map_err(|_| DecodeError::InvalidFormat("V1 chunks array len read failure".to_owned()))?; - let mut chunks = Vec::with_capacity(count as usize); + let mut chunks = Vec::with_capacity(buf.capped_capacity(count as usize)); for _ in 0..count { chunks.push(decode_chunk(buf, table)?); } @@ -324,7 +324,7 @@ where let count = decode::read_array_len(buf.as_mut_slice()).map_err(|_| { DecodeError::InvalidFormat("V1 chunk spans array len read failure".to_owned()) })?; - let mut spans = Vec::with_capacity(count as usize); + let mut spans = Vec::with_capacity(buf.capped_capacity(count as usize)); for _ in 0..count { spans.push(span::decode_span(buf, table)?); } diff --git a/libdd-trace-utils/src/msgpack_decoder/v1/span.rs b/libdd-trace-utils/src/msgpack_decoder/v1/span.rs index 992baec146..62fd2837bc 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v1/span.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v1/span.rs @@ -90,14 +90,7 @@ where let kind: u32 = decode::read_int(buf.as_mut_slice()).map_err(|_| { DecodeError::InvalidFormat("V1 span_kind u32 read failure".to_owned()) })?; - // OTEL spec: unset / unknown → Internal. - span.span_kind = match kind { - 2 => SpanKind::Server, - 3 => SpanKind::Client, - 4 => SpanKind::Producer, - 5 => SpanKind::Consumer, - _ => SpanKind::Internal, - }; + span.span_kind = SpanKind::from(kind); } _unknown => skip_unknown_value(buf)?, } @@ -136,7 +129,7 @@ where } let entries = (flat_len / FLAT_ATTR_STRIDE) as usize; - let mut map = VecMap::with_capacity(entries); + let mut map = VecMap::with_capacity(buf.capped_capacity(entries)); for _ in 0..entries { let key = read_interned_string(buf, table)?; @@ -197,7 +190,7 @@ where ))); } let n = (array_len_with_stride / TYPED_VALUE_STRIDE) as usize; - let mut items = Vec::with_capacity(n); + let mut items = Vec::with_capacity(buf.capped_capacity(n)); for _ in 0..n { items.push(read_typed_attribute_value(buf, table)?); } @@ -230,7 +223,7 @@ where { let count = decode::read_array_len(buf.as_mut_slice()) .map_err(|_| DecodeError::InvalidFormat("V1 span_links len read failure".to_owned()))?; - let mut links = ThinVec::with_capacity(count as usize); + let mut links = ThinVec::with_capacity(buf.capped_capacity(count as usize)); for _ in 0..count { links.push(decode_span_link(buf, table)?); } @@ -298,7 +291,7 @@ where { let count = decode::read_array_len(buf.as_mut_slice()) .map_err(|_| DecodeError::InvalidFormat("V1 span_events len read failure".to_owned()))?; - let mut events = ThinVec::with_capacity(count as usize); + let mut events = ThinVec::with_capacity(buf.capped_capacity(count as usize)); for _ in 0..count { events.push(decode_span_event(buf, table)?); } diff --git a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs index c025e9141a..a151a7c450 100644 --- a/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_encoder/v04/mod.rs @@ -8,14 +8,12 @@ use libdd_common::ResultInfallibleExt; use rmp::encode::{write_array_len, ByteBuf, RmpWrite, ValueWriteError}; const fn msgpack_string_encoding_len(s: &str) -> usize { - let length_marker_len = if s.len() < 32 { - 1 - } else if s.len() < 256 { - 2 - } else if s.len() <= (u16::MAX as usize) { - 3 - } else { - 5 + const U16_MAX: usize = u16::MAX as usize; + let length_marker_len = match s.len() { + 0..32 => 1, + 32..256 => 2, + 256..=U16_MAX => 3, + _ => 5, }; length_marker_len + s.len() } diff --git a/libdd-trace-utils/src/span/v1/mod.rs b/libdd-trace-utils/src/span/v1/mod.rs index 43aff8e293..1ff9fd0eac 100644 --- a/libdd-trace-utils/src/span/v1/mod.rs +++ b/libdd-trace-utils/src/span/v1/mod.rs @@ -32,6 +32,19 @@ impl SpanKind { } } +impl From for SpanKind { + /// OTEL SpanKind wire value → enum; unset/unknown → Internal (per OTEL spec). + fn from(kind: u32) -> Self { + match kind { + 2 => SpanKind::Server, + 3 => SpanKind::Client, + 4 => SpanKind::Producer, + 5 => SpanKind::Consumer, + _ => SpanKind::Internal, + } + } +} + /// Typed V1 attribute value. /// Replaces v0.4's split `meta` / `metrics` / `meta_struct` maps. #[derive(Debug, PartialEq)] diff --git a/libdd-trace-utils/src/tracer_payload.rs b/libdd-trace-utils/src/tracer_payload.rs index 121e317f45..30b54d0c7d 100644 --- a/libdd-trace-utils/src/tracer_payload.rs +++ b/libdd-trace-utils/src/tracer_payload.rs @@ -4,7 +4,6 @@ use crate::span::{v04, v05, v1, BytesData, SharedDictBytes, TraceData}; use crate::trace_utils::convert_trace_chunks_v04_to_v05; use crate::{msgpack_decoder, trace_utils::cmp_send_data_payloads}; -use anyhow::Ok; use libdd_trace_protobuf::pb; use std::cmp::Ordering; use std::iter::Iterator; From d418c618858a875f145a550e761fb61f4d547151 Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Thu, 16 Jul 2026 15:59:21 +0200 Subject: [PATCH 22/23] fix: comments --- libdd-trace-utils/src/span/v1/mod.rs | 20 ++++++++++++++++++- libdd-trace-utils/src/span/vec_map.rs | 26 +++++++++++++++++++------ libdd-trace-utils/src/tracer_payload.rs | 2 +- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/libdd-trace-utils/src/span/v1/mod.rs b/libdd-trace-utils/src/span/v1/mod.rs index 1ff9fd0eac..575447c31c 100644 --- a/libdd-trace-utils/src/span/v1/mod.rs +++ b/libdd-trace-utils/src/span/v1/mod.rs @@ -47,7 +47,7 @@ impl From for SpanKind { /// Typed V1 attribute value. /// Replaces v0.4's split `meta` / `metrics` / `meta_struct` maps. -#[derive(Debug, PartialEq)] +#[derive(Debug)] pub enum AttributeValue { String(T::Text), Float(f64), @@ -58,6 +58,24 @@ pub enum AttributeValue { List(Vec>), } +// Implemented manually rather than derived: `VecMap`'s `PartialEq` is gated to +// test/test-utils (see its definition) to keep its allocation cost out of casual `==`, so the +// `KeyValue` variant compares via `slow_compare` instead of relying on that trait impl. +impl PartialEq for AttributeValue { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (AttributeValue::String(a), AttributeValue::String(b)) => a == b, + (AttributeValue::Float(a), AttributeValue::Float(b)) => a == b, + (AttributeValue::Int(a), AttributeValue::Int(b)) => a == b, + (AttributeValue::Bool(a), AttributeValue::Bool(b)) => a == b, + (AttributeValue::Bytes(a), AttributeValue::Bytes(b)) => a == b, + (AttributeValue::KeyValue(a), AttributeValue::KeyValue(b)) => a.slow_compare(b), + (AttributeValue::List(a), AttributeValue::List(b)) => a == b, + _ => false, + } + } +} + /// The generic representation of a V1 span. /// /// `T: TraceData` carries the associated text type `T::Text` used for every string field in the diff --git a/libdd-trace-utils/src/span/vec_map.rs b/libdd-trace-utils/src/span/vec_map.rs index bdd4367ce1..984e3a459f 100644 --- a/libdd-trace-utils/src/span/vec_map.rs +++ b/libdd-trace-utils/src/span/vec_map.rs @@ -61,17 +61,17 @@ impl Default for VecMap { } } -// This implementation allocates, which isn't ideal for equality testing (not allocating would be -// rather tedious though), but is needed both by production code (e.g. comparing V1 tracer metadata -// before coalescing) and by tests. +// Only enabled for tests: this allocates (builds two `HashMap`s), which would be surprising +// behind a plain `==` in production code. Production callers should use `slow_compare` instead, +// so the cost is visible at the call site. +#[cfg(any(test, feature = "test-utils"))] impl PartialEq for VecMap { fn eq(&self, other: &Self) -> bool { - let lhs: HashMap<&K, &V> = self.data.iter().map(|(k, v)| (k, v)).collect(); - let rhs: HashMap<&K, &V> = other.data.iter().map(|(k, v)| (k, v)).collect(); - lhs == rhs + self.slow_compare(other) } } +#[cfg(any(test, feature = "test-utils"))] impl Eq for VecMap {} impl VecMap { @@ -136,6 +136,20 @@ impl VecMap { self.data.iter().any(|(k, _)| k.borrow() == key) } + /// Compares two maps for equality, ignoring insertion order and duplicate entries (the last + /// value for a given key wins on both sides). This allocates two intermediate `HashMap`s, so + /// it's exposed as a named method rather than [PartialEq]/[Eq], to keep that cost visible at + /// the call site instead of hiding it behind `==`. + pub fn slow_compare(&self, other: &Self) -> bool + where + K: Eq + Hash, + V: PartialEq, + { + let lhs: HashMap<&K, &V> = self.data.iter().map(|(k, v)| (k, v)).collect(); + let rhs: HashMap<&K, &V> = other.data.iter().map(|(k, v)| (k, v)).collect(); + lhs == rhs + } + /// Remove all entries matching this key from the map. This method uses [Vec::retain], and is /// thus potentially costly (like any removal in a vector-like datastructure). // Note: we might implement a tombstone or option-based deletion later, if removal is a bit too diff --git a/libdd-trace-utils/src/tracer_payload.rs b/libdd-trace-utils/src/tracer_payload.rs index 30b54d0c7d..9d035a89f8 100644 --- a/libdd-trace-utils/src/tracer_payload.rs +++ b/libdd-trace-utils/src/tracer_payload.rs @@ -302,7 +302,7 @@ fn metadata_matches_v1( ("env", dest.env == src.env), ("hostname", dest.hostname == src.hostname), ("app_version", dest.app_version == src.app_version), - ("attributes", dest.attributes == src.attributes), + ("attributes", dest.attributes.slow_compare(&src.attributes)), ] .into_iter() .filter_map(|(label, eq)| (!eq).then_some(label)) From 7c1161e0dc76ff01272d4e1a5fc15128a2b7bd6e Mon Sep 17 00:00:00 2001 From: Anais Raison Date: Fri, 17 Jul 2026 14:46:06 +0200 Subject: [PATCH 23/23] fix: comments --- libdd-trace-utils/src/tracer_payload.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libdd-trace-utils/src/tracer_payload.rs b/libdd-trace-utils/src/tracer_payload.rs index 9d035a89f8..b68c343d6c 100644 --- a/libdd-trace-utils/src/tracer_payload.rs +++ b/libdd-trace-utils/src/tracer_payload.rs @@ -290,7 +290,7 @@ fn metadata_matches_v1( dest: &v1::TracerPayload, src: &v1::TracerPayload, ) -> bool { - let differing: Vec<&'static str> = [ + let fields = [ ("container_id", dest.container_id == src.container_id), ("language_name", dest.language_name == src.language_name), ( @@ -303,15 +303,15 @@ fn metadata_matches_v1( ("hostname", dest.hostname == src.hostname), ("app_version", dest.app_version == src.app_version), ("attributes", dest.attributes.slow_compare(&src.attributes)), - ] - .into_iter() - .filter_map(|(label, eq)| (!eq).then_some(label)) - .collect(); + ]; - if !differing.is_empty() { + if fields.iter().any(|(_, eq)| !eq) { warn!( "Skipping V1 TracerPayload append: diverging metadata fields {:?}", - differing + fields + .iter() + .filter_map(|(label, eq)| (!eq).then_some(*label)) + .collect::>() ); return false; }