[Metrics] Datapoints follow OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE

.schema/pgdog.schema.json

@@ -175,7 +175,8 @@
         "endpoint": null,
         "headers": {},
         "namespace": null,
-        "push_interval": 0
+        "push_interval": 10000,
+        "temporality_preference": "Cumulative"
       }
     },
     "plugins": {
@@ -1598,10 +1599,42 @@
           "format": "uint64",
           "default": 10000,
           "minimum": 0
+        },
+        "temporality_preference": {
+          "description": "Describes how the exported metric points should be described.\n\nSee https://opentelemetry.io/docs/specs/otel/metrics/data-model/#metric-points\n\n_Default:_ `Cumulative`, or `Delta` when `datadog_api_key` is set.\n\nEnv: `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE`",
+          "anyOf": [
+            {
+              "$ref": "#/$defs/OtelTemporalityPreference"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": "Cumulative"
         }
       },
       "additionalProperties": false
     },
+    "OtelTemporalityPreference": {
+      "description": "Aggregation temporality used when exporting OTLP metric points.",
+      "oneOf": [
+        {
+          "description": "Points report the value accumulated since the exporter started.",
+          "type": "string",
+          "const": "Cumulative"
+        },
+        {
+          "description": "Points report the change since the last export.",
+          "type": "string",
+          "const": "Delta"
+        },
+        {
+          "description": "Delta for sums, cumulative for histograms; minimizes exporter memory.",
+          "type": "string",
+          "const": "LowMemory"
+        }
+      ]
+    },
     "PassthroughAuth": {
       "description": "toggle automatic creation of connection pools given the user name, database and password.\n\nSee [passthrough authentication](https://docs.pgdog.dev/features/authentication/#passthrough-authentication).\n\n<https://docs.pgdog.dev/configuration/pgdog.toml/general/#passthrough_auth>",
       "oneOf": [

CONTRIBUTING.md

@@ -24,3 +24,4 @@ Contributions are welcome. If you see a bug, feel free to submit a PR with a fix
 1. Please format your code with `cargo fmt`.
 2. If you're feeling generous, `cargo clippy` as well.
 3. Please write and include tests. This is production software used in one of the most important areas of the stack.
+4. If changes have been made to configuration schemas, run `cargo run -p pgdog-jsonschema`

pgdog-config/src/core.rs

@@ -6,6 +6,7 @@ use std::fs::read_to_string;
 use std::path::{Path, PathBuf};
 use tracing::{error, info, warn};
 
+use crate::otel_temporality::OtelTemporalityPreference;
 use crate::sharding::ShardedSchema;
 use crate::util::random_string;
 use crate::{
@@ -120,6 +121,7 @@ impl ConfigAndUsers {
         self.config.check();
         self.users.check(&self.config);
         self.validate_server_auth()?;
+        self.warn_if_data_dog_cumulative();
         Ok(())
     }
 
@@ -149,6 +151,33 @@ impl ConfigAndUsers {
         Ok(())
     }
 
+    fn warn_if_data_dog_cumulative(&self) {
+        match (
+            &self.config.otel.datadog_api_key,
+            &self.config.otel.temporality_preference,
+        ) {
+            (Some(_datadog_present), Some(OtelTemporalityPreference::Cumulative))
+                if std::env::var("IGNORE_DATADOG_CUMULATIVE_WARNING")
+                    .ok()
+                    .as_deref()
+                    != Some("1") =>
+            {
+                warn!(
+                    "Sending Cumulative OTLP sums/histograms to Datadog is stateful and lossy: \
+                        all points on a timeseries must reach the same Agent/exporter (constraining \
+                        how you scale collectors), the first point of a new series may be dropped \
+                        (causing gaps on restart), and histogram min/max may be missing or \
+                        approximated. See \
+                        https://docs.datadoghq.com/opentelemetry/guide/otlp_delta_temporality/?tab=python#implications-of-using-cumulative-aggregation-temporality. \
+                        Set IGNORE_DATADOG_CUMULATIVE_WARNING=1 to silence."
+                );
+            }
+            _ => {
+                // valid
+            }
+        }
+    }
+
     /// Prepared statements are enabled.
     pub fn prepared_statements(&self) -> PreparedStatements {
         // Disable prepared statements automatically in session mode
@@ -284,6 +313,7 @@ pub struct Config {
     ///
     /// <https://docs.pgdog.dev/configuration/pgdog.toml/otel/>
     #[serde(default)]
+    #[schemars(default = "Otel::schema_default")]
     pub otel: Otel,
 
     /// HashiCorp Vault settings, required for users configured with `server_auth = "vault"`.

pgdog-config/src/lib.rs

@@ -8,6 +8,7 @@ pub mod general;
 pub mod memory;
 pub mod networking;
 pub mod otel;
+pub mod otel_temporality;
 pub mod overrides;
 pub mod pooling;
 pub mod replication;

pgdog-config/src/otel.rs

@@ -4,6 +4,8 @@ use std::env;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 
+use crate::otel_temporality::OtelTemporalityPreference;
+
 /// OpenTelemetry push exporter settings.
 ///
 /// When `endpoint` is set, PgDog periodically POSTs OTLP JSON metrics
@@ -60,10 +62,31 @@ pub struct Otel {
     ///
     /// Env: `OTEL_METRIC_EXPORT_INTERVAL`
     #[serde(default = "Otel::push_interval")]
+    #[schemars(default = "Otel::schema_default_push_interval")]
     pub push_interval: u64,
+
+    /// Describes how the exported metric points should be described.
+    ///
+    /// See https://opentelemetry.io/docs/specs/otel/metrics/data-model/#metric-points
+    ///
+    /// _Default:_ `Cumulative`, or `Delta` when `datadog_api_key` is set.
+    ///
+    /// Env: `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE`
+    #[serde(default = "Otel::temporality_preference")]
+    #[schemars(default = "Otel::schema_default_temporality_preference")]
+    pub temporality_preference: Option<OtelTemporalityPreference>,
 }
 
 impl Otel {
+    pub fn effective_temporality_preference(&self) -> OtelTemporalityPreference {
+        self.temporality_preference
+            .unwrap_or(if self.datadog_api_key.is_some() {
+                OtelTemporalityPreference::Delta
+            } else {
+                OtelTemporalityPreference::Cumulative
+            })
+    }
+
     fn env_option_string(env_var: &str) -> Option<String> {
         env::var(env_var).ok().filter(|s| !s.is_empty())
     }
@@ -99,6 +122,31 @@ impl Otel {
             .and_then(|v| v.parse().ok())
             .unwrap_or(10_000)
     }
+
+    fn temporality_preference() -> Option<OtelTemporalityPreference> {
+        env::var("OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE")
+            .ok()
+            .and_then(|v| v.parse().ok())
+    }
+
+    fn schema_default_push_interval() -> u64 {
+        10_000
+    }
+
+    fn schema_default_temporality_preference() -> Option<OtelTemporalityPreference> {
+        Some(OtelTemporalityPreference::Cumulative)
+    }
+
+    /// Schema-only default for the whole `Otel` object, used so the top-level
+    /// `default` block in the generated JSON schema matches the per-field
+    /// documented defaults instead of the raw derived `Default` (0 / null).
+    pub fn schema_default() -> Self {
+        Self {
+            push_interval: Self::schema_default_push_interval(),
+            temporality_preference: Self::schema_default_temporality_preference(),
+            ..Self::default()
+        }
+    }
 }
 
 #[cfg(test)]
@@ -133,6 +181,32 @@ mod test {
         assert!(otel.endpoint.is_none());
         assert!(otel.datadog_api_key.is_none());
         assert_eq!(otel.push_interval, 10_000);
+        assert!(otel.temporality_preference.is_none());
+    }
+
+    #[test]
+    fn endpoint_toml_wins_over_env() {
+        let _guard = set_env_var("OTEL_EXPORTER_OTLP_ENDPOINT", "https://env.example/v1");
+        let toml = r#"endpoint = "https://toml.example/v1""#;
+        let otel: Otel = toml::from_str(toml).expect("parse");
+        assert_eq!(otel.endpoint.as_deref(), Some("https://toml.example/v1"));
+    }
+
+    #[test]
+    fn push_interval_env_used_when_toml_absent() {
+        let _guard = set_env_var("OTEL_METRIC_EXPORT_INTERVAL", "7500");
+        let otel: Otel = toml::from_str("").expect("parse");
+        assert_eq!(otel.push_interval, 7500);
+    }
+
+    #[test]
+    fn temporality_preference_env_parsed() {
+        let _guard = set_env_var("OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", "Delta");
+        let otel: Otel = toml::from_str("").expect("parse");
+        assert_eq!(
+            otel.temporality_preference,
+            Some(OtelTemporalityPreference::Delta)
+        );
     }
 
     #[test]
@@ -143,6 +217,7 @@ mod test {
             namespace = "pgdog_"
             datadog_api_key = "my-key"
             push_interval = 5000
+            temporality_preference = "Delta"
 
             [otel.headers]
             Authorization = "Bearer token"
@@ -156,12 +231,37 @@ mod test {
         assert_eq!(config.otel.namespace.as_deref(), Some("pgdog_"));
         assert_eq!(config.otel.datadog_api_key.as_deref(), Some("my-key"));
         assert_eq!(config.otel.push_interval, 5000);
+        assert_eq!(
+            config.otel.temporality_preference,
+            Some(OtelTemporalityPreference::Delta)
+        );
         assert_eq!(
             config.otel.headers.get("Authorization").unwrap(),
             "Bearer token"
         );
     }
 
+    #[test]
+    fn effective_temporality_defaults_to_delta_with_datadog_key() {
+        let mut otel = Otel::default();
+        assert_eq!(
+            otel.effective_temporality_preference(),
+            OtelTemporalityPreference::Cumulative
+        );
+
+        otel.datadog_api_key = Some("abc".into());
+        assert_eq!(
+            otel.effective_temporality_preference(),
+            OtelTemporalityPreference::Delta
+        );
+
+        otel.temporality_preference = Some(OtelTemporalityPreference::Cumulative);
+        assert_eq!(
+            otel.effective_temporality_preference(),
+            OtelTemporalityPreference::Cumulative
+        );
+    }
+
     #[test]
     fn namespace_from_env() {
         let _guard = set_env_var("PGDOG_OTEL_NAMESPACE", "pgdog_");