diff --git a/Cargo.lock b/Cargo.lock index 2917cbba83..3a13c9258b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3046,6 +3046,7 @@ dependencies = [ "tokio-util", "tracing", "uuid", + "zstd", ] [[package]] diff --git a/builder/Cargo.toml b/builder/Cargo.toml index 4c07580c7d..0fbf801709 100644 --- a/builder/Cargo.toml +++ b/builder/Cargo.toml @@ -25,6 +25,9 @@ crashtracker = [] profiling = [] telemetry = [] data-pipeline = [] +# Enable zstd compression for the agentless trace intake sender (opt-in: links a C +# library and increases artifact size). +data-pipeline-compression = ["data-pipeline"] symbolizer = [] library-config = [] log = [] diff --git a/builder/src/bin/release.rs b/builder/src/bin/release.rs index ff0904ee7e..be54a3b5e1 100644 --- a/builder/src/bin/release.rs +++ b/builder/src/bin/release.rs @@ -62,6 +62,8 @@ pub fn main() { f.push("ddtelemetry-ffi".to_string()); #[cfg(feature = "data-pipeline")] f.push("data-pipeline-ffi".to_string()); + #[cfg(feature = "data-pipeline-compression")] + f.push("data-pipeline-compression".to_string()); #[cfg(feature = "crashtracker")] f.push("crashtracker-ffi".to_string()); #[cfg(feature = "symbolizer")] diff --git a/libdd-data-pipeline-ffi/Cargo.toml b/libdd-data-pipeline-ffi/Cargo.toml index 2ddb404544..8cf2bf9571 100644 --- a/libdd-data-pipeline-ffi/Cargo.toml +++ b/libdd-data-pipeline-ffi/Cargo.toml @@ -20,6 +20,8 @@ default = ["cbindgen", "catch_panic"] catch_panic = [] cbindgen = ["build_common/cbindgen", "libdd-common-ffi/cbindgen"] regex-lite = ["libdd-data-pipeline/regex-lite"] +# Enable zstd compression for the agentless trace intake sender. +compression = ["libdd-data-pipeline/compression"] [build-dependencies] build_common = { path = "../build-common" } diff --git a/libdd-data-pipeline/Cargo.toml b/libdd-data-pipeline/Cargo.toml index 6425345d22..3c696b167d 100644 --- a/libdd-data-pipeline/Cargo.toml +++ b/libdd-data-pipeline/Cargo.toml @@ -83,6 +83,7 @@ tokio = { version = "1.23", features = [ "test-util", ], default-features = false } duplicate = "2.0.1" +zstd = { version = "0.13", default-features = false } [features] default = ["https", "telemetry"] @@ -111,3 +112,5 @@ fips = [ ] test-utils = [] regex-lite = ["libdd-common/regex-lite"] +# Enable zstd compression for the agentless trace intake sender. +compression = ["libdd-trace-utils/compression"] diff --git a/libdd-data-pipeline/src/agentless/exporter.rs b/libdd-data-pipeline/src/agentless/exporter.rs index bc512433bc..5ac9d05209 100644 --- a/libdd-data-pipeline/src/agentless/exporter.rs +++ b/libdd-data-pipeline/src/agentless/exporter.rs @@ -9,7 +9,7 @@ use http::HeaderMap; use libdd_capabilities::{HttpClientCapability, SleepCapability}; use libdd_common::Endpoint; use libdd_trace_utils::send_with_retry::{ - send_with_retry, RetryBackoffType, RetryStrategy, SendWithRetryError, + send_with_retry, CompressionStrategy, RetryBackoffType, RetryStrategy, SendWithRetryError, }; use tracing::error; @@ -46,7 +46,21 @@ pub async fn send_agentless_traces_http Ok(()), Err(e) => Err(map_send_error(e)), } diff --git a/libdd-data-pipeline/src/otlp/exporter.rs b/libdd-data-pipeline/src/otlp/exporter.rs index 31c952010a..1297a7a25e 100644 --- a/libdd-data-pipeline/src/otlp/exporter.rs +++ b/libdd-data-pipeline/src/otlp/exporter.rs @@ -9,7 +9,7 @@ use http::HeaderMap; use libdd_capabilities::{HttpClientCapability, SleepCapability}; use libdd_common::Endpoint; use libdd_trace_utils::send_with_retry::{ - send_with_retry, RetryBackoffType, RetryStrategy, SendWithRetryError, + send_with_retry, CompressionStrategy, RetryBackoffType, RetryStrategy, SendWithRetryError, }; use std::time::Duration; @@ -67,7 +67,16 @@ pub(crate) async fn send_otlp_http( None, ); - match send_with_retry(capabilities, &target, body, &headers, &retry_strategy).await { + match send_with_retry( + capabilities, + &target, + body, + &headers, + &retry_strategy, + CompressionStrategy::None, + ) + .await + { Ok(_) => Ok(()), Err(e) => Err(map_send_error(e).await), } diff --git a/libdd-data-pipeline/src/trace_exporter/mod.rs b/libdd-data-pipeline/src/trace_exporter/mod.rs index 08f69e8b99..e6f1570b1e 100644 --- a/libdd-data-pipeline/src/trace_exporter/mod.rs +++ b/libdd-data-pipeline/src/trace_exporter/mod.rs @@ -48,7 +48,7 @@ use libdd_shared_runtime::BlockingRuntime; use libdd_shared_runtime::{SharedRuntime, WorkerHandle}; use libdd_trace_utils::msgpack_decoder; use libdd_trace_utils::send_with_retry::{ - send_with_retry, RetryStrategy, SendWithRetryError, SendWithRetryResult, + send_with_retry, CompressionStrategy, RetryStrategy, SendWithRetryError, SendWithRetryResult, }; use libdd_trace_utils::span::{v04::Span, TraceData}; use libdd_trace_utils::trace_utils::TracerHeaderTags; @@ -733,6 +733,7 @@ impl< mp_payload, &headers, &strategy, + CompressionStrategy::None, ) .await; @@ -2270,16 +2271,32 @@ mod tests { fn test_agentless_export_body_shape() { let server = MockServer::start(); let mock_intake = server.mock(|when, then| { - when.method(POST) - .path("/v1/input") - .body_includes("\"traces\":") - .body_includes("\"spans\":") - .body_includes("\"hostname\":\"h-1\"") - .body_includes("\"languageName\":\"nodejs\"") - .body_includes("\"_dd.compute_stats\":\"1\"") - .body_includes("\"_top_level\":1") - .body_includes("\"_trace_root\":1") - .body_includes("\"parent_id\":\"0000000000000000\""); + fn check_body(body: &str) -> bool { + body.contains("\"traces\":") + && body.contains("\"spans\":") + && body.contains("\"hostname\":\"h-1\"") + && body.contains("\"languageName\":\"nodejs\"") + && body.contains("\"_dd.compute_stats\":\"1\"") + && body.contains("\"_top_level\":1") + && body.contains("\"_trace_root\":1") + && body.contains("\"parent_id\":\"0000000000000000\"") + } + let when = when.method(POST).path("/v1/input"); + #[cfg(feature = "compression")] + let when = when.header("content-encoding", "zstd").is_true(|req| { + let Ok(body) = zstd::decode_all(req.body_ref()) else { + return false; + }; + let body = String::from_utf8(body).unwrap(); + check_body(&body) + }); + #[cfg(not(feature = "compression"))] + let when = when + .is_true(|req| { + let body = String::from_utf8(req.body_vec()).unwrap(); + check_body(&body) + }); + let _ = when; then.status(200).body(""); }); diff --git a/libdd-profiling-ffi/Cargo.toml b/libdd-profiling-ffi/Cargo.toml index e1b40f7063..977a5d70de 100644 --- a/libdd-profiling-ffi/Cargo.toml +++ b/libdd-profiling-ffi/Cargo.toml @@ -23,6 +23,8 @@ ddtelemetry-ffi = ["dep:libdd-telemetry-ffi"] datadog-log-ffi = ["dep:libdd-log-ffi"] symbolizer = ["symbolizer-ffi"] data-pipeline-ffi = ["dep:libdd-data-pipeline-ffi"] +# Enable zstd compression for the agentless trace intake sender. +data-pipeline-compression = ["data-pipeline-ffi", "libdd-data-pipeline-ffi/compression"] crashtracker-ffi = ["dep:libdd-crashtracker-ffi"] # Enables the in-process collection of crash-info crashtracker-collector = ["crashtracker-ffi", "libdd-crashtracker-ffi/collector"] diff --git a/libdd-trace-stats/src/stats_exporter.rs b/libdd-trace-stats/src/stats_exporter.rs index c771fd5b2b..0aac3e8ec4 100644 --- a/libdd-trace-stats/src/stats_exporter.rs +++ b/libdd-trace-stats/src/stats_exporter.rs @@ -17,7 +17,7 @@ use libdd_capabilities::{HttpClientCapability, MaybeSend, SleepCapability}; use libdd_common::{tag, Endpoint}; use libdd_shared_runtime::Worker; use libdd_trace_protobuf::pb; -use libdd_trace_utils::send_with_retry::{send_with_retry, RetryStrategy}; +use libdd_trace_utils::send_with_retry::{send_with_retry, CompressionStrategy, RetryStrategy}; use libdd_trace_utils::trace_utils::TracerHeaderTags; use libdd_trace_utils::tracer_metadata::TracerMetadata; use std::fmt::Debug; @@ -224,6 +224,7 @@ impl body, &headers, &RetryStrategy::default(), + CompressionStrategy::None, ) .await; diff --git a/libdd-trace-utils/Cargo.toml b/libdd-trace-utils/Cargo.toml index cf4462a47e..1d9c7a5840 100644 --- a/libdd-trace-utils/Cargo.toml +++ b/libdd-trace-utils/Cargo.toml @@ -76,7 +76,9 @@ getrandom = { version = "0.2", features = ["js"] } [dev-dependencies] libdd-capabilities-impl = { version = "3.0.0", path = "../libdd-capabilities-impl" } -libdd-common = { path = "../libdd-common", default-features = false, features = ["bench-utils"] } +libdd-common = { path = "../libdd-common", default-features = false, features = [ + "bench-utils", +] } bolero = "0.13" criterion = "0.5.1" httpmock = { version = "0.8.0-alpha.1" } @@ -88,7 +90,7 @@ tempfile = "3.3.0" [features] default = ["https"] https = ["libdd-common/https", "libdd-capabilities-impl/https"] -mini_agent = ["compression", "libdd-common/use_webpki_roots"] +mini_agent = ["compression", "libdd-common/use_webpki_roots", "dep:flate2"] test-utils = [ "hyper/server", "httpmock", @@ -99,6 +101,6 @@ test-utils = [ change-buffer = [] # Opt-in switch for crate-internal microbenchmarks (e.g. `vec_map_bench`). bench-internals = [] -compression = ["zstd", "flate2"] +compression = ["zstd"] # FIPS mode uses the FIPS-compliant cryptographic provider (Unix only) fips = ["libdd-common/fips", "libdd-capabilities-impl/fips"] diff --git a/libdd-trace-utils/src/send_data/mod.rs b/libdd-trace-utils/src/send_data/mod.rs index 53d14d8ecd..d6e7b9db47 100644 --- a/libdd-trace-utils/src/send_data/mod.rs +++ b/libdd-trace-utils/src/send_data/mod.rs @@ -4,7 +4,10 @@ pub mod send_data_result; use crate::msgpack_encoder; -use crate::send_with_retry::{send_with_retry, RetryStrategy, SendWithRetryResult}; +use crate::send_with_retry::compression::{add_headers, compress}; +use crate::send_with_retry::{ + send_with_retry, CompressionStrategy, RetryStrategy, SendWithRetryResult, +}; use crate::trace_utils::TracerHeaderTags; use crate::tracer_payload::TracerPayloadCollection; use anyhow::{anyhow, Context}; @@ -22,10 +25,6 @@ use libdd_common::{ use libdd_trace_protobuf::pb::{AgentPayload, TracerPayload}; use send_data_result::SendDataResult; use std::collections::HashMap; -#[cfg(feature = "compression")] -use std::io::Write; -#[cfg(feature = "compression")] -use zstd::stream::write::Encoder; #[derive(Debug)] /// `SendData` is a structure that holds the data to be sent to a target endpoint. @@ -71,15 +70,7 @@ pub struct SendData { target: Endpoint, headers: HeaderMap, retry_strategy: RetryStrategy, - #[cfg(feature = "compression")] - compression: Compression, -} - -#[cfg(feature = "compression")] -#[derive(Debug, Clone)] -pub enum Compression { - Zstd(i32), - None, + compression: CompressionStrategy, } pub struct SendDataBuilder { @@ -88,8 +79,7 @@ pub struct SendDataBuilder { target: Endpoint, headers: HeaderMap, retry_strategy: RetryStrategy, - #[cfg(feature = "compression")] - compression: Compression, + compression: CompressionStrategy, } impl SendDataBuilder { @@ -107,13 +97,11 @@ impl SendDataBuilder { target: target.clone(), headers, retry_strategy: RetryStrategy::default(), - #[cfg(feature = "compression")] - compression: Compression::None, + compression: CompressionStrategy::None, } } - #[cfg(feature = "compression")] - pub fn with_compression(mut self, compression: Compression) -> SendDataBuilder { + pub fn with_compression(mut self, compression: CompressionStrategy) -> SendDataBuilder { self.compression = compression; self } @@ -135,7 +123,6 @@ impl SendDataBuilder { target: self.target, headers: self.headers, retry_strategy: self.retry_strategy, - #[cfg(feature = "compression")] compression: self.compression, } } @@ -169,8 +156,7 @@ impl SendData { target: target.clone(), headers, retry_strategy: RetryStrategy::default(), - #[cfg(feature = "compression")] - compression: Compression::None, + compression: CompressionStrategy::None, } } @@ -248,9 +234,14 @@ impl SendData { capabilities: &C, chunks: u64, payload: Vec, - headers: HeaderMap, + mut headers: HeaderMap, endpoint: Option<&Endpoint>, + compression_strategy: CompressionStrategy, ) -> (SendWithRetryResult, u64, u64) { + // Compress here (rather than inside `send_with_retry`) so that the reported + // `bytes_sent` metric reflects the number of bytes actually put on the wire. + let (payload, compression_strategy) = compress(payload, compression_strategy); + add_headers(&mut headers, compression_strategy); #[allow(clippy::unwrap_used)] let payload_len = u64::try_from(payload.len()).unwrap(); ( @@ -260,6 +251,7 @@ impl SendData { payload, &headers, &self.retry_strategy, + CompressionStrategy::None, ) .await, payload_len, @@ -271,31 +263,6 @@ impl SendData { self.target.api_key.is_some() } - #[cfg(feature = "compression")] - fn compress_payload(&self, payload: Vec, headers: &mut HeaderMap) -> Vec { - match self.compression { - Compression::Zstd(level) => { - let result = (|| -> std::io::Result> { - let mut encoder = Encoder::new(Vec::new(), level)?; - encoder.write_all(&payload)?; - encoder.finish() - })(); - - match result { - Ok(compressed_payload) => { - headers.insert( - http::header::CONTENT_ENCODING, - HeaderValue::from_static("zstd"), - ); - compressed_payload - } - Err(_) => payload, - } - } - _ => payload, - } - } - async fn send_with_protobuf( &self, capabilities: &C, @@ -317,22 +284,16 @@ impl SendData { }; let mut request_headers = self.headers.clone(); - #[cfg(feature = "compression")] - let final_payload = - self.compress_payload(serialized_trace_payload, &mut request_headers); - - #[cfg(not(feature = "compression"))] - let final_payload = serialized_trace_payload; - request_headers.insert(CONTENT_TYPE, APPLICATION_PROTOBUF); let (response, bytes_sent, chunks) = self .send_payload( capabilities, chunks, - final_payload, + serialized_trace_payload, request_headers, endpoint.as_ref(), + self.compression, ) .await; @@ -373,6 +334,7 @@ impl SendData { payload, headers, endpoint.as_ref(), + CompressionStrategy::None, )); } } @@ -392,6 +354,7 @@ impl SendData { payload, headers, endpoint.as_ref(), + CompressionStrategy::None, )); } TracerPayloadCollection::V05(payload) => { @@ -413,6 +376,7 @@ impl SendData { payload, headers, endpoint.as_ref(), + CompressionStrategy::None, )); } } diff --git a/libdd-trace-utils/src/send_with_retry/compression.rs b/libdd-trace-utils/src/send_with_retry/compression.rs new file mode 100644 index 0000000000..447a7ebaef --- /dev/null +++ b/libdd-trace-utils/src/send_with_retry/compression.rs @@ -0,0 +1,50 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(feature = "compression")] +use std::io::Write as _; + +#[cfg(feature = "compression")] +const CONTENT_ENCODING_ZSTD: http::HeaderValue = http::HeaderValue::from_static("zstd"); + +#[derive(Clone, Copy, Debug)] +pub enum CompressionStrategy { + None, + #[cfg(feature = "compression")] + Zstd { + level: i32, + }, +} + +/// Returns the compressed data, and the actual compression strategy used. +/// If an error happens during compression, defaults to [`CompressionStrategy::None`] +pub fn compress(data: Vec, strategy: CompressionStrategy) -> (Vec, CompressionStrategy) { + match strategy { + CompressionStrategy::None => (data, CompressionStrategy::None), + #[cfg(feature = "compression")] + CompressionStrategy::Zstd { level } => { + // Start with an initial buffer + // Allocate 1/10th of the original buffer, so we shouldn't add too + // much memory usage, and no less than 256 bytes + let writer = Vec::with_capacity((data.len() / 10).max(256)); + zstd::Encoder::new(writer, level) + .and_then(|mut e| { + e.write_all(&data)?; + Ok((e.finish()?, strategy)) + }) + .unwrap_or((data, CompressionStrategy::None)) + } + } +} + +pub fn add_headers(headers: &mut http::HeaderMap, strategy: CompressionStrategy) { + match strategy { + CompressionStrategy::None => { + let _ = headers; + } + #[cfg(feature = "compression")] + CompressionStrategy::Zstd { .. } => { + headers.insert(http::header::CONTENT_ENCODING, CONTENT_ENCODING_ZSTD); + } + } +} diff --git a/libdd-trace-utils/src/send_with_retry/mod.rs b/libdd-trace-utils/src/send_with_retry/mod.rs index 88cedb1764..30f35db752 100644 --- a/libdd-trace-utils/src/send_with_retry/mod.rs +++ b/libdd-trace-utils/src/send_with_retry/mod.rs @@ -7,6 +7,9 @@ mod retry_strategy; pub use retry_strategy::{RetryBackoffType, RetryStrategy}; +pub(crate) mod compression; +pub use compression::CompressionStrategy; + use bytes::Bytes; use http::HeaderMap; use libdd_capabilities::{HttpClientCapability, HttpError, SleepCapability}; @@ -83,7 +86,15 @@ impl std::error::Error for SendWithRetryError {} /// ); /// let retry_strategy = RetryStrategy::new(3, 10, RetryBackoffType::Exponential, Some(5)); /// let capabilities = libdd_capabilities_impl::NativeCapabilities::new_client(); -/// send_with_retry(&capabilities, &target, payload, &headers, &retry_strategy).await +/// send_with_retry( +/// &capabilities, +/// &target, +/// payload, +/// &headers, +/// &retry_strategy, +/// CompressionStrategy::None, +/// ) +/// .await /// # } /// ``` pub async fn send_with_retry( @@ -92,6 +103,7 @@ pub async fn send_with_retry( payload: Vec, headers: &HeaderMap, retry_strategy: &RetryStrategy, + compression_strategy: CompressionStrategy, ) -> SendWithRetryResult { let mut request_attempt = 0; let timeout = Duration::from_millis(target.timeout_ms); @@ -103,7 +115,9 @@ pub async fn send_with_retry( "Sending with retry" ); - let payload = Bytes::from(payload); + let (compressed, compression_strategy) = compression::compress(payload, compression_strategy); + let payload = Bytes::from(compressed); + loop { request_attempt += 1; @@ -121,6 +135,10 @@ pub async fn send_with_retry( for (key, value) in headers { builder = builder.header(key, value); } + // headers_mut is only None if the builder is in an error state + if let Some(h) = builder.headers_mut() { + compression::add_headers(h, compression_strategy); + } let req = match builder.body(payload.clone()) { Ok(r) => r, Err(_) => { @@ -281,6 +299,7 @@ mod tests { vec![0, 1, 2, 3], &HeaderMap::new(), &strategy, + CompressionStrategy::None, ) .await; assert!(result.is_err(), "Expected an error result"); @@ -330,6 +349,7 @@ mod tests { vec![0, 1, 2, 3], &HeaderMap::new(), &strategy, + CompressionStrategy::None, ) .await; assert!( @@ -375,6 +395,7 @@ mod tests { vec![0, 1, 2, 3], &HeaderMap::new(), &strategy, + CompressionStrategy::None, ) .await; assert!( @@ -424,6 +445,7 @@ mod tests { vec![0, 1, 2, 3], &HeaderMap::new(), &strategy, + CompressionStrategy::None, ) .await; assert!(