From 7e193655d6aa45c57f79024adaccff8ea4411d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 13 May 2024 14:07:54 -0400 Subject: [PATCH 001/112] chore(bottlecap): make config a folder module (#242) * remove `config.rs` file * create `config/mod.rs` * move to `config/flush_strategy.rs` * move to `config/log_level.rs` * update imports * fmt --- flush_strategy.rs | 53 +++++++++ log_level.rs | 42 +++++++ mod.rs | 274 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 369 insertions(+) create mode 100644 flush_strategy.rs create mode 100644 log_level.rs create mode 100644 mod.rs diff --git a/flush_strategy.rs b/flush_strategy.rs new file mode 100644 index 00000000..f01ab502 --- /dev/null +++ b/flush_strategy.rs @@ -0,0 +1,53 @@ +use serde::{Deserialize, Deserializer}; +use tracing::debug; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct PeriodicStrategy { + pub interval: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FlushStrategy { + Default, + End, + Periodically(PeriodicStrategy), +} + +// Deserialize for FlushStrategy +// Flush Strategy can be either "end" or "periodically," +impl<'de> Deserialize<'de> for FlushStrategy { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + if value.as_str() == "end" { + Ok(FlushStrategy::End) + } else { + let mut split_value = value.as_str().split(','); + // "periodically,60000" + match split_value.next() { + Some(first_value) if first_value.starts_with("periodically") => { + let interval = split_value.next(); + // "60000" + if let Some(interval) = interval { + if let Ok(parsed_interval) = interval.parse() { + return Ok(FlushStrategy::Periodically(PeriodicStrategy { + interval: parsed_interval, + })); + } + debug!("invalid flush interval: {}, using default", interval); + Ok(FlushStrategy::Default) + } else { + debug!("invalid flush strategy: {}, using default", value); + Ok(FlushStrategy::Default) + } + } + _ => { + debug!("invalid flush strategy: {}, using default", value); + Ok(FlushStrategy::Default) + } + } + } + } +} diff --git a/log_level.rs b/log_level.rs new file mode 100644 index 00000000..1c8e347b --- /dev/null +++ b/log_level.rs @@ -0,0 +1,42 @@ +use serde::Deserialize; + +#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Default)] +pub enum LogLevel { + /// Designates very serious errors. + Error, + /// Designates hazardous situations. + Warn, + /// Designates useful information. + #[default] + Info, + /// Designates lower priority information. + Debug, + /// Designates very low priority, often extremely verbose, information. + Trace, +} + +impl AsRef for LogLevel { + fn as_ref(&self) -> &str { + match self { + LogLevel::Error => "ERROR", + LogLevel::Warn => "WARN", + LogLevel::Info => "INFO", + LogLevel::Debug => "DEBUG", + LogLevel::Trace => "TRACE", + } + } +} + +impl LogLevel { + /// Construct a `log::LevelFilter` from a `LogLevel` + #[must_use] + pub fn as_level_filter(self) -> log::LevelFilter { + match self { + LogLevel::Error => log::LevelFilter::Error, + LogLevel::Warn => log::LevelFilter::Warn, + LogLevel::Info => log::LevelFilter::Info, + LogLevel::Debug => log::LevelFilter::Debug, + LogLevel::Trace => log::LevelFilter::Trace, + } + } +} diff --git a/mod.rs b/mod.rs new file mode 100644 index 00000000..945838c4 --- /dev/null +++ b/mod.rs @@ -0,0 +1,274 @@ +pub mod flush_strategy; +pub mod log_level; + +use std::path::Path; + +use serde::Deserialize; + +use figment::{ + providers::{Env, Format, Yaml}, + Figment, +}; + +use crate::config::flush_strategy::FlushStrategy; +use crate::config::log_level::LogLevel; + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(default)] +pub struct Config { + pub site: String, + pub api_key: String, + pub env: Option, + pub service: Option, + pub version: Option, + pub tags: Option, + pub log_level: LogLevel, + pub serverless_logs_enabled: bool, + pub apm_enabled: bool, + pub lambda_handler: String, + pub serverless_flush_strategy: FlushStrategy, +} + +impl Default for Config { + fn default() -> Self { + Config { + // General + site: "datadoghq.com".to_string(), + api_key: String::default(), + serverless_flush_strategy: FlushStrategy::Default, + // Unified Tagging + env: None, + service: None, + version: None, + tags: None, + // Logs + log_level: LogLevel::default(), + serverless_logs_enabled: true, + // APM + apm_enabled: false, + lambda_handler: String::default(), + } + } +} + +#[derive(Debug, PartialEq)] +#[allow(clippy::module_name_repetitions)] +pub enum ConfigError { + ParseError(String), + UnsupportedField(String), +} + +#[allow(clippy::module_name_repetitions)] +pub fn get_config(config_directory: &Path) -> Result { + let path = config_directory.join("datadog.yaml"); + let figment = Figment::new() + .merge(Yaml::file(path)) + .merge(Env::prefixed("DATADOG_")) + .merge(Env::prefixed("DD_")); + + let config = figment.extract().map_err(|err| match err.kind { + figment::error::Kind::UnknownField(field, _) => ConfigError::UnsupportedField(field), + _ => ConfigError::ParseError(err.to_string()), + })?; + + Ok(config) +} + +#[cfg(test)] +pub mod tests { + use super::*; + + use crate::config::flush_strategy::PeriodicStrategy; + + #[test] + fn test_reject_unknown_fields_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + unknown_field: true + ", + )?; + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("unknown_field".to_string()) + ); + Ok(()) + }); + } + + #[test] + fn test_reject_unknown_fields_env() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_UNKNOWN_FIELD", "true"); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("unknown_field".to_string()) + ); + Ok(()) + }); + } + + #[test] + fn test_precedence() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + apm_enabled: true + ", + )?; + jail.set_env("DD_APM_ENABLED", "false"); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + apm_enabled: false, + ..Config::default() + } + ); + Ok(()) + }); + } + + #[test] + fn test_parse_config_file() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + apm_enabled: true + ", + )?; + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + apm_enabled: true, + ..Config::default() + } + ); + Ok(()) + }); + } + + #[test] + fn test_parse_env() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_APM_ENABLED", "true"); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + apm_enabled: true, + ..Config::default() + } + ); + Ok(()) + }); + } + + #[test] + fn test_parse_default() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!(config, Config::default()); + Ok(()) + }); + } + + #[test] + fn test_parse_flush_strategy_default() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + ..Config::default() + } + ); + Ok(()) + }); + } + + #[test] + fn test_parse_flush_strategy_end() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + serverless_flush_strategy: FlushStrategy::End, + ..Config::default() + } + ); + Ok(()) + }); + } + + #[test] + fn test_parse_flush_strategy_periodically() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { + interval: 100_000 + }), + ..Config::default() + } + ); + Ok(()) + }); + } + + #[test] + fn test_parse_flush_strategy_invalid() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + ..Config::default() + } + ); + Ok(()) + }); + } + + #[test] + fn test_parse_flush_strategy_invalid_periodic() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env( + "DD_SERVERLESS_FLUSH_STRATEGY", + "periodically,invalid_interval", + ); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + ..Config::default() + } + ); + Ok(()) + }); + } +} From cba7b973f3de6ad7e32bc8d140207378d4a3d45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 16 May 2024 16:28:52 -0400 Subject: [PATCH 002/112] feat(bottlecap): add logs processing rules (#243) * add logs processing rules field * add `regex` crate * add `processing_rules.rs` config module * use `processing_rule` module instead * update logs `processor` to use compiled rules * update unit test --- mod.rs | 87 ++++++++++++++++++++++++++++++++++++++++++++-- processing_rule.rs | 49 ++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 processing_rule.rs diff --git a/mod.rs b/mod.rs index 945838c4..c8c4b60c 100644 --- a/mod.rs +++ b/mod.rs @@ -1,17 +1,18 @@ pub mod flush_strategy; pub mod log_level; +pub mod processing_rule; use std::path::Path; -use serde::Deserialize; - use figment::{ providers::{Env, Format, Yaml}, Figment, }; +use serde::Deserialize; use crate::config::flush_strategy::FlushStrategy; use crate::config::log_level::LogLevel; +use crate::config::processing_rule::{deserialize_processing_rules, ProcessingRule}; #[derive(Debug, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] @@ -25,6 +26,8 @@ pub struct Config { pub tags: Option, pub log_level: LogLevel, pub serverless_logs_enabled: bool, + #[serde(deserialize_with = "deserialize_processing_rules")] + pub logs_config_processing_rules: Option>, pub apm_enabled: bool, pub lambda_handler: String, pub serverless_flush_strategy: FlushStrategy, @@ -45,6 +48,8 @@ impl Default for Config { // Logs log_level: LogLevel::default(), serverless_logs_enabled: true, + // TODO(duncanista): Add serializer for YAML + logs_config_processing_rules: None, // APM apm_enabled: false, lambda_handler: String::default(), @@ -80,6 +85,7 @@ pub mod tests { use super::*; use crate::config::flush_strategy::PeriodicStrategy; + use crate::config::processing_rule; #[test] fn test_reject_unknown_fields_yaml() { @@ -271,4 +277,81 @@ pub mod tests { Ok(()) }); } + + #[test] + fn test_parse_logs_config_processing_rules_from_env() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env( + "DD_LOGS_CONFIG_PROCESSING_RULES", + r#"[{"type":"exclude_at_match","name":"exclude","pattern":"exclude"}]"#, + ); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + logs_config_processing_rules: Some(vec![ProcessingRule { + kind: processing_rule::Kind::ExcludeAtMatch, + name: "exclude".to_string(), + pattern: "exclude".to_string(), + replace_placeholder: None + }]), + ..Config::default() + } + ); + Ok(()) + }); + } + + #[test] + fn test_parse_logs_config_processing_rules_from_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + // TODO(duncanista): Update to use YAML configuration field `logs_config`: `processing_rules`: - ... + jail.create_file( + "datadog.yaml", + r#" + logs_config_processing_rules: + - type: exclude_at_match + name: "exclude" + pattern: "exclude" + - type: include_at_match + name: "include" + pattern: "include" + - type: mask_sequences + name: "mask" + pattern: "mask" + replace_placeholder: "REPLACED" + "#, + )?; + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + logs_config_processing_rules: Some(vec![ + ProcessingRule { + kind: processing_rule::Kind::ExcludeAtMatch, + name: "exclude".to_string(), + pattern: "exclude".to_string(), + replace_placeholder: None + }, + ProcessingRule { + kind: processing_rule::Kind::IncludeAtMatch, + name: "include".to_string(), + pattern: "include".to_string(), + replace_placeholder: None + }, + ProcessingRule { + kind: processing_rule::Kind::MaskSequences, + name: "mask".to_string(), + pattern: "mask".to_string(), + replace_placeholder: Some("REPLACED".to_string()) + } + ]), + ..Config::default() + } + ); + Ok(()) + }); + } } diff --git a/processing_rule.rs b/processing_rule.rs new file mode 100644 index 00000000..f494d02f --- /dev/null +++ b/processing_rule.rs @@ -0,0 +1,49 @@ +use serde::{Deserialize, Deserializer}; +use serde_json::Value as JsonValue; + +#[derive(Clone, Copy, Debug, PartialEq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Kind { + ExcludeAtMatch, + IncludeAtMatch, + MaskSequences, +} + +#[derive(Clone, Debug, PartialEq, Deserialize)] +pub struct ProcessingRule { + #[serde(rename = "type")] + pub kind: Kind, + pub name: String, + pub pattern: String, + pub replace_placeholder: Option, +} + +pub fn deserialize_processing_rules<'de, D>( + deserializer: D, +) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + // Deserialize the JSON value using serde_json::Value + let value: JsonValue = Deserialize::deserialize(deserializer)?; + + match value { + JsonValue::String(s) => { + let values: Vec = serde_json::from_str(&s).map_err(|e| { + serde::de::Error::custom(format!("Failed to deserialize processing rules: {e}")) + })?; + Ok(Some(values)) + } + JsonValue::Array(a) => { + let mut values = Vec::new(); + for v in a { + let rule: ProcessingRule = serde_json::from_value(v).map_err(|e| { + serde::de::Error::custom(format!("Failed to deserialize processing rule: {e}")) + })?; + values.push(rule); + } + Ok(Some(values)) + } + _ => Ok(None), + } +} From 800b31354118384a767d9858e4921d722379094d Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Thu, 30 May 2024 16:19:18 -0400 Subject: [PATCH 003/112] Svls 4825 support encrypted keys manual (#258) * add plumbing for aws secret manager * strip as much deps as possible * fix test * remove unused warning * reorg runner for bottlecap * fix overwriting of arch * add full error to the panic * avoid building the go agent all the time * rename module * speed up build * add simple scripts to build and publish * remove deleted call * remove changes from common scripts * resolve import conflicts * wrong file pushed * make sure permissions are right * move secret parsing after log activation * add some stat to build * add manual req for secret (still broken) * rebuild after conflict on cargo loc * automate update and call * change headers and fix signature * fix typo and small refactor * remove useless thread spawn * small refactors on deploy scripts * use access key always for signatures * the secret has to be used to sign * fix: missing newline in request * use only manual decrypt * add timed steps * add scripts to force restarts * fix launch script * refactor decrypt * cargo format and clippy * fix clippy error add formatting/clippy functinos --------- Co-authored-by: AJ Stuyvenberg --- mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mod.rs b/mod.rs index c8c4b60c..04416ba0 100644 --- a/mod.rs +++ b/mod.rs @@ -14,12 +14,13 @@ use crate::config::flush_strategy::FlushStrategy; use crate::config::log_level::LogLevel; use crate::config::processing_rule::{deserialize_processing_rules, ProcessingRule}; -#[derive(Debug, PartialEq, Deserialize)] +#[derive(Debug, PartialEq, Deserialize, Clone)] #[serde(deny_unknown_fields)] #[serde(default)] pub struct Config { pub site: String, pub api_key: String, + pub api_key_secret_arn: String, pub env: Option, pub service: Option, pub version: Option, @@ -39,6 +40,7 @@ impl Default for Config { // General site: "datadoghq.com".to_string(), api_key: String::default(), + api_key_secret_arn: String::default(), serverless_flush_strategy: FlushStrategy::Default, // Unified Tagging env: None, From aaea185c41e771f40fd5879ca988b133a53930fe Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:19:51 -0400 Subject: [PATCH 004/112] add kms handling (#261) * add kms handling * fix return value * fix test * fix kms * remove committed test file * rename * format * fmt after fix * fix conflicts * await async stuff * formatting * bubble up error converting to sdt * use box dyn for generic errors * reforamt * address other comments * remove old build file added with conflict --- mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mod.rs b/mod.rs index 04416ba0..fcab5f77 100644 --- a/mod.rs +++ b/mod.rs @@ -21,6 +21,7 @@ pub struct Config { pub site: String, pub api_key: String, pub api_key_secret_arn: String, + pub kms_api_key: String, pub env: Option, pub service: Option, pub version: Option, @@ -41,6 +42,7 @@ impl Default for Config { site: "datadoghq.com".to_string(), api_key: String::default(), api_key_secret_arn: String::default(), + kms_api_key: String::default(), serverless_flush_strategy: FlushStrategy::Default, // Unified Tagging env: None, From 8e5f60aa46a96598b2dd56c728e0e33fd4488926 Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:41:56 -0400 Subject: [PATCH 005/112] Svls 4978 handle secrets error (#271) * add kms handling * fix return value * fix test * fix kms * remove committed test file * rename * format * fmt after fix * fix conflicts * await async stuff * formatting * bubble up error converting to sdt * use box dyn for generic errors * reforamt * address other comments * remove old build file added with conflict * do not pass around the whole config for just the secret * fix scope and just bubble up erros * reformat * renaming * without api key, just call next loop * fix types and format * fix folder path * fix cd and returns * resolve conflicts * formatter --- mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mod.rs b/mod.rs index fcab5f77..590ec9e2 100644 --- a/mod.rs +++ b/mod.rs @@ -84,6 +84,15 @@ pub fn get_config(config_directory: &Path) -> Result { Ok(config) } +#[allow(clippy::module_name_repetitions)] +pub struct AwsConfig { + pub region: String, + pub aws_access_key_id: String, + pub aws_secret_access_key: String, + pub aws_session_token: String, + pub function_name: String, +} + #[cfg(test)] pub mod tests { use super::*; From 910e1ac236901263394c62af56a0e14e6db216c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 27 Jun 2024 15:48:02 -0400 Subject: [PATCH 006/112] chore(bottlecap): log failover reason (#292) * print failover reason as json string * fmt * update key to be more verbose --- mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mod.rs b/mod.rs index 590ec9e2..d456505e 100644 --- a/mod.rs +++ b/mod.rs @@ -77,7 +77,10 @@ pub fn get_config(config_directory: &Path) -> Result { .merge(Env::prefixed("DD_")); let config = figment.extract().map_err(|err| match err.kind { - figment::error::Kind::UnknownField(field, _) => ConfigError::UnsupportedField(field), + figment::error::Kind::UnknownField(field, _) => { + println!("{{\"DD_EXTENSION_FAILOVER_REASON\":\"{field}\"}}"); + ConfigError::UnsupportedField(field) + } _ => ConfigError::ParseError(err.to_string()), })?; From aff2036d1d6ffc9bce9eadc9c229fe2d4de53b0c Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Fri, 28 Jun 2024 11:29:10 -0400 Subject: [PATCH 007/112] Add APM tracing support (#294) * wip: tracing * feat: tracing WIP * feat: rename mini agent to trace agent * feat: fmt * feat: Fix formatting after rename * fix: remove extra tokio task * feat: allow tracing * feat: working v5 traces * feat: Update to use my branch of libdatadog so we have v5 support * feat: Update w/ libdatadog to pass trace encoding version * feat: update w/ merged libdatadog changes * feat: Refactor trace agent, reduce code duplication, enum for trace version. Pass trace provider. Manual stats flushing. Custom create endpoint until we clean up that code in libdatadog. * feat: Unify config, remove trace config. Tests pass * feat: fmt * feat: fmt * clippy fixes * parse time * feat: clippy again * feat: revert dockerfile * feat: no-default-features * feat: Remove utils, take only what we need * feat: fmt moves the import * feat: replace info with debug. Replace log with tracing lib * feat: more debug * feat: Remove call to trace utils --- mod.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mod.rs b/mod.rs index d456505e..0b6da119 100644 --- a/mod.rs +++ b/mod.rs @@ -17,6 +17,7 @@ use crate::config::processing_rule::{deserialize_processing_rules, ProcessingRul #[derive(Debug, PartialEq, Deserialize, Clone)] #[serde(deny_unknown_fields)] #[serde(default)] +#[allow(clippy::struct_excessive_bools)] pub struct Config { pub site: String, pub api_key: String, @@ -33,6 +34,13 @@ pub struct Config { pub apm_enabled: bool, pub lambda_handler: String, pub serverless_flush_strategy: FlushStrategy, + pub trace_enabled: bool, + pub serverless_trace_enabled: bool, + pub capture_lambda_payload: bool, + // Deprecated or ignored, just here so we don't failover + pub flush_to_log: bool, + pub logs_injection: bool, + pub merge_xray_traces: bool, } impl Default for Config { @@ -57,6 +65,12 @@ impl Default for Config { // APM apm_enabled: false, lambda_handler: String::default(), + serverless_trace_enabled: true, + trace_enabled: true, + capture_lambda_payload: false, + flush_to_log: false, + logs_injection: false, + merge_xray_traces: false, } } } From 0c1fc3839fe6a7ce3a648b94f8615c72b0304db0 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 2 Jul 2024 13:11:06 -0400 Subject: [PATCH 008/112] feat: Allow appsec but in a disabled-only state until we add support for the runtime proxy (#296) * feat: Allow appsec but in a disabled-only state until we add support for the runtime proxy * feat: Log failover reason * fix: serverless_appsec_enabled. Also log the reason --- mod.rs | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/mod.rs b/mod.rs index 0b6da119..01205735 100644 --- a/mod.rs +++ b/mod.rs @@ -41,6 +41,7 @@ pub struct Config { pub flush_to_log: bool, pub logs_injection: bool, pub merge_xray_traces: bool, + pub serverless_appsec_enabled: bool, } impl Default for Config { @@ -71,6 +72,7 @@ impl Default for Config { flush_to_log: false, logs_injection: false, merge_xray_traces: false, + serverless_appsec_enabled: false, } } } @@ -82,6 +84,10 @@ pub enum ConfigError { UnsupportedField(String), } +fn log_failover_reason(reason: &str) { + println!("{{\"DD_EXTENSION_FAILOVER_REASON\":\"{reason}\"}}"); +} + #[allow(clippy::module_name_repetitions)] pub fn get_config(config_directory: &Path) -> Result { let path = config_directory.join("datadog.yaml"); @@ -90,14 +96,19 @@ pub fn get_config(config_directory: &Path) -> Result { .merge(Env::prefixed("DATADOG_")) .merge(Env::prefixed("DD_")); - let config = figment.extract().map_err(|err| match err.kind { + let config: Config = figment.extract().map_err(|err| match err.kind { figment::error::Kind::UnknownField(field, _) => { - println!("{{\"DD_EXTENSION_FAILOVER_REASON\":\"{field}\"}}"); + log_failover_reason(&field.clone()); ConfigError::UnsupportedField(field) } _ => ConfigError::ParseError(err.to_string()), })?; + if config.serverless_appsec_enabled { + log_failover_reason("appsec_enabled"); + return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); + } + Ok(config) } @@ -136,6 +147,25 @@ pub mod tests { }); } + #[test] + fn test_allowed_but_disabled() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + appsec_enabled: true + ", + )?; + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("appsec_enabled".to_string()) + ); + Ok(()) + }); + } + #[test] fn test_reject_unknown_fields_env() { figment::Jail::expect_with(|jail| { From 35257ddfb7b8b1a90670f00cf5cbbba41d165b90 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 8 Jul 2024 10:45:22 -0400 Subject: [PATCH 009/112] feat: Require DD_EXTENSION_VERSION: next (#302) * feat: Require DD_EXTENSION_VERSION: next * feat: add tests, fix metric tests * feat: revert metrics test byte changes * feat: fmt * feat: remove ref --- log_level.rs | 1 + mod.rs | 56 +++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/log_level.rs b/log_level.rs index 1c8e347b..77d9e724 100644 --- a/log_level.rs +++ b/log_level.rs @@ -1,6 +1,7 @@ use serde::Deserialize; #[derive(Clone, Copy, Debug, PartialEq, Deserialize, Default)] +#[serde(rename_all = "lowercase")] pub enum LogLevel { /// Designates very serious errors. Error, diff --git a/mod.rs b/mod.rs index 01205735..e8549711 100644 --- a/mod.rs +++ b/mod.rs @@ -42,6 +42,7 @@ pub struct Config { pub logs_injection: bool, pub merge_xray_traces: bool, pub serverless_appsec_enabled: bool, + pub extension_version: Option, } impl Default for Config { @@ -73,6 +74,7 @@ impl Default for Config { logs_injection: false, merge_xray_traces: false, serverless_appsec_enabled: false, + extension_version: None, } } } @@ -109,6 +111,16 @@ pub fn get_config(config_directory: &Path) -> Result { return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); } + match config.extension_version.as_deref() { + Some("next") => {} + Some(_) | None => { + log_failover_reason("extension_version"); + return Err(ConfigError::UnsupportedField( + "extension_version".to_string(), + )); + } + } + Ok(config) } @@ -180,6 +192,18 @@ pub mod tests { }); } + #[test] + fn test_reject_without_opt_in() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("extension_version".to_string()) + ); + Ok(()) + }); + } #[test] fn test_precedence() { figment::Jail::expect_with(|jail| { @@ -187,7 +211,8 @@ pub mod tests { jail.create_file( "datadog.yaml", r" - apm_enabled: true + apm_enabled: true, + extension_version: next ", )?; jail.set_env("DD_APM_ENABLED", "false"); @@ -196,6 +221,7 @@ pub mod tests { config, Config { apm_enabled: false, + extension_version: Some("next".to_string()), ..Config::default() } ); @@ -211,6 +237,7 @@ pub mod tests { "datadog.yaml", r" apm_enabled: true + extension_version: next ", )?; let config = get_config(Path::new("")).expect("should parse config"); @@ -218,6 +245,7 @@ pub mod tests { config, Config { apm_enabled: true, + extension_version: Some("next".to_string()), ..Config::default() } ); @@ -230,11 +258,13 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_APM_ENABLED", "true"); + jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, Config { apm_enabled: true, + extension_version: Some("next".to_string()), ..Config::default() } ); @@ -246,20 +276,12 @@ pub mod tests { fn test_parse_default() { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!(config, Config::default()); - Ok(()) - }); - } - - #[test] - fn test_parse_flush_strategy_default() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); + jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, Config { + extension_version: Some("next".to_string()), ..Config::default() } ); @@ -272,11 +294,13 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); + jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, Config { serverless_flush_strategy: FlushStrategy::End, + extension_version: Some("next".to_string()), ..Config::default() } ); @@ -289,6 +313,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); + jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, @@ -296,6 +321,7 @@ pub mod tests { serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }), + extension_version: Some("next".to_string()), ..Config::default() } ); @@ -308,10 +334,12 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); + jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, Config { + extension_version: Some("next".to_string()), ..Config::default() } ); @@ -327,10 +355,12 @@ pub mod tests { "DD_SERVERLESS_FLUSH_STRATEGY", "periodically,invalid_interval", ); + jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, Config { + extension_version: Some("next".to_string()), ..Config::default() } ); @@ -346,6 +376,7 @@ pub mod tests { "DD_LOGS_CONFIG_PROCESSING_RULES", r#"[{"type":"exclude_at_match","name":"exclude","pattern":"exclude"}]"#, ); + jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, @@ -356,6 +387,7 @@ pub mod tests { pattern: "exclude".to_string(), replace_placeholder: None }]), + extension_version: Some("next".to_string()), ..Config::default() } ); @@ -371,6 +403,7 @@ pub mod tests { jail.create_file( "datadog.yaml", r#" + extension_version: next logs_config_processing_rules: - type: exclude_at_match name: "exclude" @@ -408,6 +441,7 @@ pub mod tests { replace_placeholder: Some("REPLACED".to_string()) } ]), + extension_version: Some("next".to_string()), ..Config::default() } ); From 9c471989600b9e245d7029ae2660efab9194dfa7 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 9 Jul 2024 11:43:06 -0400 Subject: [PATCH 010/112] feat: honor enhanced metrics bool (#307) * feat: honor enhanced metrics bool * feat: add test * feat: refactor to log instead of return result * fix: clippy --- mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mod.rs b/mod.rs index e8549711..e8b6a16e 100644 --- a/mod.rs +++ b/mod.rs @@ -43,6 +43,7 @@ pub struct Config { pub merge_xray_traces: bool, pub serverless_appsec_enabled: bool, pub extension_version: Option, + pub enhanced_metrics: bool, } impl Default for Config { @@ -75,6 +76,7 @@ impl Default for Config { merge_xray_traces: false, serverless_appsec_enabled: false, extension_version: None, + enhanced_metrics: true, } } } From 38497639a29c372f25859e1bbb2b9079a66e0b1c Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 10 Jul 2024 16:32:26 -0400 Subject: [PATCH 011/112] feat: warn by default (#316) --- log_level.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log_level.rs b/log_level.rs index 77d9e724..03f1ddb3 100644 --- a/log_level.rs +++ b/log_level.rs @@ -6,9 +6,9 @@ pub enum LogLevel { /// Designates very serious errors. Error, /// Designates hazardous situations. + #[default] Warn, /// Designates useful information. - #[default] Info, /// Designates lower priority information. Debug, From 95d1ea535b324324a738b8e6656711658abb4968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 18 Jul 2024 17:39:51 -0400 Subject: [PATCH 012/112] chore(bottlecap): fallback on `datadog.yaml` usage (#326) * fallback on `datadog.yaml` presence * add comment --- mod.rs | 138 ++++++++------------------------------------------------- 1 file changed, 19 insertions(+), 119 deletions(-) diff --git a/mod.rs b/mod.rs index e8b6a16e..784dd4e0 100644 --- a/mod.rs +++ b/mod.rs @@ -4,11 +4,9 @@ pub mod processing_rule; use std::path::Path; -use figment::{ - providers::{Env, Format, Yaml}, - Figment, -}; +use figment::{providers::Env, Figment}; use serde::Deserialize; +use tracing::debug; use crate::config::flush_strategy::FlushStrategy; use crate::config::log_level::LogLevel; @@ -96,7 +94,7 @@ fn log_failover_reason(reason: &str) { pub fn get_config(config_directory: &Path) -> Result { let path = config_directory.join("datadog.yaml"); let figment = Figment::new() - .merge(Yaml::file(path)) + // .merge(Yaml::file(path)) .merge(Env::prefixed("DATADOG_")) .merge(Env::prefixed("DD_")); @@ -108,6 +106,13 @@ pub fn get_config(config_directory: &Path) -> Result { _ => ConfigError::ParseError(err.to_string()), })?; + // TODO(duncanista): revert to using datadog.yaml when we have a proper serializer + if path.exists() { + log_failover_reason("datadog_yaml"); + debug!("datadog.yaml is not supported, use environment variables instead"); + return Err(ConfigError::UnsupportedField("datadog_yaml".to_string())); + } + if config.serverless_appsec_enabled { log_failover_reason("appsec_enabled"); return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); @@ -143,38 +148,35 @@ pub mod tests { use crate::config::processing_rule; #[test] - fn test_reject_unknown_fields_yaml() { + fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.create_file( - "datadog.yaml", - r" - unknown_field: true - ", - )?; + jail.set_env("DD_APPSEC_ENABLED", "true"); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); assert_eq!( config, - ConfigError::UnsupportedField("unknown_field".to_string()) + ConfigError::UnsupportedField("appsec_enabled".to_string()) ); Ok(()) }); } #[test] - fn test_allowed_but_disabled() { + fn test_reject_datadog_yaml() { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.create_file( "datadog.yaml", r" - appsec_enabled: true + apm_enabled: true + extension_version: next ", )?; - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + let config = get_config(Path::new("")).expect_err("should reject datadog.yaml file"); assert_eq!( config, - ConfigError::UnsupportedField("appsec_enabled".to_string()) + ConfigError::UnsupportedField("datadog_yaml".to_string()) ); Ok(()) }); @@ -206,54 +208,6 @@ pub mod tests { Ok(()) }); } - #[test] - fn test_precedence() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.create_file( - "datadog.yaml", - r" - apm_enabled: true, - extension_version: next - ", - )?; - jail.set_env("DD_APM_ENABLED", "false"); - let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!( - config, - Config { - apm_enabled: false, - extension_version: Some("next".to_string()), - ..Config::default() - } - ); - Ok(()) - }); - } - - #[test] - fn test_parse_config_file() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.create_file( - "datadog.yaml", - r" - apm_enabled: true - extension_version: next - ", - )?; - let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!( - config, - Config { - apm_enabled: true, - extension_version: Some("next".to_string()), - ..Config::default() - } - ); - Ok(()) - }); - } #[test] fn test_parse_env() { @@ -396,58 +350,4 @@ pub mod tests { Ok(()) }); } - - #[test] - fn test_parse_logs_config_processing_rules_from_yaml() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - // TODO(duncanista): Update to use YAML configuration field `logs_config`: `processing_rules`: - ... - jail.create_file( - "datadog.yaml", - r#" - extension_version: next - logs_config_processing_rules: - - type: exclude_at_match - name: "exclude" - pattern: "exclude" - - type: include_at_match - name: "include" - pattern: "include" - - type: mask_sequences - name: "mask" - pattern: "mask" - replace_placeholder: "REPLACED" - "#, - )?; - let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!( - config, - Config { - logs_config_processing_rules: Some(vec![ - ProcessingRule { - kind: processing_rule::Kind::ExcludeAtMatch, - name: "exclude".to_string(), - pattern: "exclude".to_string(), - replace_placeholder: None - }, - ProcessingRule { - kind: processing_rule::Kind::IncludeAtMatch, - name: "include".to_string(), - pattern: "include".to_string(), - replace_placeholder: None - }, - ProcessingRule { - kind: processing_rule::Kind::MaskSequences, - name: "mask".to_string(), - pattern: "mask".to_string(), - replace_placeholder: Some("REPLACED".to_string()) - } - ]), - extension_version: Some("next".to_string()), - ..Config::default() - } - ); - Ok(()) - }); - } } From 64916b2a63cdc8faa634aeb3eeca04204d833b2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Fri, 19 Jul 2024 11:35:07 -0400 Subject: [PATCH 013/112] fix(bottlecap): filter debug logs from external crates (#329) * remove `tracing-log` instead, use the `tracing-subscriber` `tracing-log` feature * capitalize debugs * remove unnecessary file * update log formatter prefix * update log filter * fmt --- flush_strategy.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flush_strategy.rs b/flush_strategy.rs index f01ab502..1dae5bf8 100644 --- a/flush_strategy.rs +++ b/flush_strategy.rs @@ -36,15 +36,15 @@ impl<'de> Deserialize<'de> for FlushStrategy { interval: parsed_interval, })); } - debug!("invalid flush interval: {}, using default", interval); + debug!("Invalid flush interval: {}, using default", interval); Ok(FlushStrategy::Default) } else { - debug!("invalid flush strategy: {}, using default", value); + debug!("Invalid flush strategy: {}, using default", value); Ok(FlushStrategy::Default) } } _ => { - debug!("invalid flush strategy: {}, using default", value); + debug!("Invalid flush strategy: {}, using default", value); Ok(FlushStrategy::Default) } } From 06cf049d87ead51034607d11a7f84809fff5abad Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 22 Jul 2024 14:01:26 -0400 Subject: [PATCH 014/112] chore(bottlecap): switch flushing strategy to race (#318) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: race flush * refactor: periodic only when configured * fmt * when flushing strategy is default, set periodic flush tick to `1s` * on `End`, never flush until the end of the invocation * remove `tokio_unstable` feature for building * remove debug comment * remove `invocation_times` mod * update `flush_control.rs` * use `flush_control` in main * allow `end,` strategy allows to flush periodically over a given amount of seconds and at the end * update `debug` comment for flushing * simplify logic for flush strategy parsing * remove log that could spam debug * refactor code and add unit test --------- Co-authored-by: jordan gonzález <30836115+duncanista@users.noreply.github.com> Co-authored-by: alexgallotta <5581237+alexgallotta@users.noreply.github.com> --- flush_strategy.rs | 83 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/flush_strategy.rs b/flush_strategy.rs index 1dae5bf8..525ec422 100644 --- a/flush_strategy.rs +++ b/flush_strategy.rs @@ -10,11 +10,12 @@ pub struct PeriodicStrategy { pub enum FlushStrategy { Default, End, + EndPeriodically(PeriodicStrategy), Periodically(PeriodicStrategy), } // Deserialize for FlushStrategy -// Flush Strategy can be either "end" or "periodically," +// Flush Strategy can be either "end", "end,", or "periodically," impl<'de> Deserialize<'de> for FlushStrategy { fn deserialize(deserializer: D) -> Result where @@ -26,22 +27,22 @@ impl<'de> Deserialize<'de> for FlushStrategy { } else { let mut split_value = value.as_str().split(','); // "periodically,60000" - match split_value.next() { - Some(first_value) if first_value.starts_with("periodically") => { - let interval = split_value.next(); - // "60000" - if let Some(interval) = interval { - if let Ok(parsed_interval) = interval.parse() { - return Ok(FlushStrategy::Periodically(PeriodicStrategy { - interval: parsed_interval, - })); - } - debug!("Invalid flush interval: {}, using default", interval); - Ok(FlushStrategy::Default) - } else { - debug!("Invalid flush strategy: {}, using default", value); - Ok(FlushStrategy::Default) - } + // "end,1000" + let strategy = split_value.next(); + let interval: Option = split_value.next().and_then(|v| v.parse().ok()); + + match (strategy, interval) { + (Some("periodically"), Some(interval)) => { + Ok(FlushStrategy::Periodically(PeriodicStrategy { interval })) + } + (Some("end"), Some(interval)) => { + Ok(FlushStrategy::EndPeriodically(PeriodicStrategy { + interval, + })) + } + (Some(strategy), _) => { + debug!("Invalid flush interval: {}, using default", strategy); + Ok(FlushStrategy::Default) } _ => { debug!("Invalid flush strategy: {}, using default", value); @@ -51,3 +52,51 @@ impl<'de> Deserialize<'de> for FlushStrategy { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deserialize_end() { + let flush_strategy: FlushStrategy = serde_json::from_str("\"end\"").unwrap(); + assert_eq!(flush_strategy, FlushStrategy::End); + } + + #[test] + fn deserialize_periodically() { + let flush_strategy: FlushStrategy = serde_json::from_str("\"periodically,60000\"").unwrap(); + assert_eq!( + flush_strategy, + FlushStrategy::Periodically(PeriodicStrategy { interval: 60000 }) + ); + } + + #[test] + fn deserialize_end_periodically() { + let flush_strategy: FlushStrategy = serde_json::from_str("\"end,1000\"").unwrap(); + assert_eq!( + flush_strategy, + FlushStrategy::EndPeriodically(PeriodicStrategy { interval: 1000 }) + ); + } + + #[test] + fn deserialize_invalid() { + let flush_strategy: FlushStrategy = serde_json::from_str("\"invalid\"").unwrap(); + assert_eq!(flush_strategy, FlushStrategy::Default); + } + + #[test] + fn deserialize_invalid_interval() { + let flush_strategy: FlushStrategy = + serde_json::from_str("\"periodically,invalid\"").unwrap(); + assert_eq!(flush_strategy, FlushStrategy::Default); + } + + #[test] + fn deserialize_invalid_end_interval() { + let flush_strategy: FlushStrategy = serde_json::from_str("\"end,invalid\"").unwrap(); + assert_eq!(flush_strategy, FlushStrategy::Default); + } +} From 3777933a7f086b4f14cd4a5d11ecdaf9e894c431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 23 Jul 2024 03:47:45 -0400 Subject: [PATCH 015/112] remove log that might confuse customers (#333) --- mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/mod.rs b/mod.rs index 784dd4e0..d7682df6 100644 --- a/mod.rs +++ b/mod.rs @@ -6,7 +6,6 @@ use std::path::Path; use figment::{providers::Env, Figment}; use serde::Deserialize; -use tracing::debug; use crate::config::flush_strategy::FlushStrategy; use crate::config::log_level::LogLevel; @@ -109,7 +108,6 @@ pub fn get_config(config_directory: &Path) -> Result { // TODO(duncanista): revert to using datadog.yaml when we have a proper serializer if path.exists() { log_failover_reason("datadog_yaml"); - debug!("datadog.yaml is not supported, use environment variables instead"); return Err(ConfigError::UnsupportedField("datadog_yaml".to_string())); } From be361c09a7fd910f3fe2d0c3bb4209a22bb3187f Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Thu, 25 Jul 2024 09:16:30 -0400 Subject: [PATCH 016/112] Fix dogstatsd multiline (#335) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: add invalid string and multi line distro test with empty newline * test: move unit test to appropriate package * fix: do not error log for empty and new line strings --------- Co-authored-by: jordan gonzález <30836115+duncanista@users.noreply.github.com> --- flush_strategy.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/flush_strategy.rs b/flush_strategy.rs index 525ec422..dd4fea2c 100644 --- a/flush_strategy.rs +++ b/flush_strategy.rs @@ -54,6 +54,7 @@ impl<'de> Deserialize<'de> for FlushStrategy { } #[cfg(test)] +#[allow(clippy::unwrap_used)] mod tests { use super::*; From 5872915164cf40eeccda21a8dfd731ffebbfa577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 25 Jul 2024 11:07:27 -0400 Subject: [PATCH 017/112] add env vars to be ignored (#337) --- mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mod.rs b/mod.rs index d7682df6..26b18061 100644 --- a/mod.rs +++ b/mod.rs @@ -41,6 +41,9 @@ pub struct Config { pub serverless_appsec_enabled: bool, pub extension_version: Option, pub enhanced_metrics: bool, + pub cold_start_tracing: bool, + pub min_cold_start_duration: String, + pub cold_start_trace_skip_lib: String, } impl Default for Config { @@ -67,11 +70,16 @@ impl Default for Config { lambda_handler: String::default(), serverless_trace_enabled: true, trace_enabled: true, + // Ignored by the extension for now capture_lambda_payload: false, flush_to_log: false, logs_injection: false, merge_xray_traces: false, serverless_appsec_enabled: false, + cold_start_tracing: true, + min_cold_start_duration: String::default(), + cold_start_trace_skip_lib: String::default(), + // Failover extension_version: None, enhanced_metrics: true, } From 29a1ebefb59b9ae79b905ed6f9e23eee2a9a8685 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 1 Aug 2024 16:15:41 -0400 Subject: [PATCH 018/112] feat: Open up more env vars which we don't rely on (#344) --- mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mod.rs b/mod.rs index 26b18061..1dbdc9d7 100644 --- a/mod.rs +++ b/mod.rs @@ -29,6 +29,7 @@ pub struct Config { #[serde(deserialize_with = "deserialize_processing_rules")] pub logs_config_processing_rules: Option>, pub apm_enabled: bool, + pub apm_replace_tags: Option, pub lambda_handler: String, pub serverless_flush_strategy: FlushStrategy, pub trace_enabled: bool, @@ -44,6 +45,8 @@ pub struct Config { pub cold_start_tracing: bool, pub min_cold_start_duration: String, pub cold_start_trace_skip_lib: String, + pub service_mapping: Option, + pub data_streams_enabled: bool, } impl Default for Config { @@ -67,6 +70,7 @@ impl Default for Config { logs_config_processing_rules: None, // APM apm_enabled: false, + apm_replace_tags: None, lambda_handler: String::default(), serverless_trace_enabled: true, trace_enabled: true, @@ -82,6 +86,8 @@ impl Default for Config { // Failover extension_version: None, enhanced_metrics: true, + service_mapping: None, + data_streams_enabled: false, } } } From 9cbafbebcd1ca09b4b64bc3721264b63b0f219d9 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 5 Aug 2024 13:08:45 -0400 Subject: [PATCH 019/112] feat: Allow trace disabled plugins (#348) * feat: Allow trace disabled plugins * feat: trace debug --- mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mod.rs b/mod.rs index 1dbdc9d7..da752591 100644 --- a/mod.rs +++ b/mod.rs @@ -47,6 +47,8 @@ pub struct Config { pub cold_start_trace_skip_lib: String, pub service_mapping: Option, pub data_streams_enabled: bool, + pub trace_disabled_plugins: Option, + pub trace_debug: bool, } impl Default for Config { @@ -71,6 +73,7 @@ impl Default for Config { // APM apm_enabled: false, apm_replace_tags: None, + trace_disabled_plugins: None, lambda_handler: String::default(), serverless_trace_enabled: true, trace_enabled: true, @@ -88,6 +91,7 @@ impl Default for Config { enhanced_metrics: true, service_mapping: None, data_streams_enabled: false, + trace_debug: false, } } } From 02fc013969172174388632debe97ecb901d72049 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Fri, 9 Aug 2024 10:21:27 -0400 Subject: [PATCH 020/112] feat: Allowlist additional env vars (#354) * feat: Allowlist additional env vars * fix: fmt * feat: and repo url --- mod.rs | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/mod.rs b/mod.rs index da752591..cfa2e147 100644 --- a/mod.rs +++ b/mod.rs @@ -35,7 +35,8 @@ pub struct Config { pub trace_enabled: bool, pub serverless_trace_enabled: bool, pub capture_lambda_payload: bool, - // Deprecated or ignored, just here so we don't failover + // ALL ENV VARS below are deprecated or not used by the extension + // just here so we don't failover pub flush_to_log: bool, pub logs_injection: bool, pub merge_xray_traces: bool, @@ -49,6 +50,15 @@ pub struct Config { pub data_streams_enabled: bool, pub trace_disabled_plugins: Option, pub trace_debug: bool, + pub profiling_enabled: bool, + pub git_commit_sha: Option, + pub logs_enabled: bool, + pub app_key: Option, + pub trace_sample_rate: Option, + pub dotnet_tracer_home: Option, + pub trace_managed_services: bool, + pub runtime_metrics_enabled: bool, + pub git_repository_url: Option, } impl Default for Config { @@ -67,6 +77,8 @@ impl Default for Config { tags: None, // Logs log_level: LogLevel::default(), + logs_enabled: false, // IGNORED, this is the main agent config, but never used in + // severless serverless_logs_enabled: true, // TODO(duncanista): Add serializer for YAML logs_config_processing_rules: None, @@ -92,6 +104,14 @@ impl Default for Config { service_mapping: None, data_streams_enabled: false, trace_debug: false, + profiling_enabled: false, + git_commit_sha: None, + app_key: None, + trace_sample_rate: None, + dotnet_tracer_home: None, + trace_managed_services: true, + runtime_metrics_enabled: false, + git_repository_url: None, } } } @@ -134,6 +154,13 @@ pub fn get_config(config_directory: &Path) -> Result { return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); } + if config.profiling_enabled { + log_failover_reason("profiling_enabled"); + return Err(ConfigError::UnsupportedField( + "profiling_enabled".to_string(), + )); + } + match config.extension_version.as_deref() { Some("next") => {} Some(_) | None => { From 0f5d14aef47fc3dc32f5fc29cbdba105f8e32e01 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 13 Aug 2024 10:55:22 -0400 Subject: [PATCH 021/112] aj/allow apm replace tags array (#358) * fix: allow objects to be ignored * feat: specs --- mod.rs | 26 +++++++++++++++++++++++++- object_ignore.rs | 15 +++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 object_ignore.rs diff --git a/mod.rs b/mod.rs index cfa2e147..e366a595 100644 --- a/mod.rs +++ b/mod.rs @@ -1,5 +1,6 @@ pub mod flush_strategy; pub mod log_level; +pub mod object_ignore; pub mod processing_rule; use std::path::Path; @@ -9,6 +10,7 @@ use serde::Deserialize; use crate::config::flush_strategy::FlushStrategy; use crate::config::log_level::LogLevel; +use crate::config::object_ignore::ObjectIgnore; use crate::config::processing_rule::{deserialize_processing_rules, ProcessingRule}; #[derive(Debug, PartialEq, Deserialize, Clone)] @@ -29,7 +31,7 @@ pub struct Config { #[serde(deserialize_with = "deserialize_processing_rules")] pub logs_config_processing_rules: Option>, pub apm_enabled: bool, - pub apm_replace_tags: Option, + pub apm_replace_tags: Option, pub lambda_handler: String, pub serverless_flush_strategy: FlushStrategy, pub trace_enabled: bool, @@ -393,4 +395,26 @@ pub mod tests { Ok(()) }); } + + #[test] + fn test_ignore_apm_replace_tags() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env( + "DD_APM_REPLACE_TAGS", + r#"[{"name":"resource.name","pattern":"(.*)/(foo[:%].+)","repl":"$1/{foo}"}]"#, + ); + jail.set_env("DD_EXTENSION_VERSION", "next"); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + apm_replace_tags: Some(ObjectIgnore::Ignore), + extension_version: Some("next".to_string()), + ..Config::default() + } + ); + Ok(()) + }); + } } diff --git a/object_ignore.rs b/object_ignore.rs new file mode 100644 index 00000000..700ab07d --- /dev/null +++ b/object_ignore.rs @@ -0,0 +1,15 @@ +use serde::{Deserialize, Deserializer}; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ObjectIgnore { + Ignore, +} + +impl<'de> Deserialize<'de> for ObjectIgnore { + fn deserialize(_deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(ObjectIgnore::Ignore) + } +} From 35079ae57c31c094f77291701662e7dc0e18a7fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Fri, 16 Aug 2024 10:25:11 -0400 Subject: [PATCH 022/112] fix(bottlecap): set explicit deny list and allow yaml usage (#363) * set explicit deny list also allow `datadog.yaml` usage * add unit test for parsing rule from yaml * remove `object_ignore.rs` * remove import * remove logging failover reason when user is not opt-in --- mod.rs | 206 ++++++++++++++++++++++++++--------------------- object_ignore.rs | 15 ---- 2 files changed, 116 insertions(+), 105 deletions(-) delete mode 100644 object_ignore.rs diff --git a/mod.rs b/mod.rs index e366a595..5d8e9f51 100644 --- a/mod.rs +++ b/mod.rs @@ -1,20 +1,36 @@ pub mod flush_strategy; pub mod log_level; -pub mod object_ignore; pub mod processing_rule; use std::path::Path; +use figment::providers::{Format, Yaml}; use figment::{providers::Env, Figment}; use serde::Deserialize; use crate::config::flush_strategy::FlushStrategy; use crate::config::log_level::LogLevel; -use crate::config::object_ignore::ObjectIgnore; use crate::config::processing_rule::{deserialize_processing_rules, ProcessingRule}; +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct YamlLogsConfig { + #[serde(deserialize_with = "deserialize_processing_rules")] + processing_rules: Option>, +} + +/// `YamlConfig` is a struct that represents some of the fields in the datadog.yaml file. +/// +/// It is used to deserialize the datadog.yaml file into a struct that can be merged with the Config struct. +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct YamlConfig { + pub logs_config: YamlLogsConfig, +} + #[derive(Debug, PartialEq, Deserialize, Clone)] -#[serde(deny_unknown_fields)] #[serde(default)] #[allow(clippy::struct_excessive_bools)] pub struct Config { @@ -27,47 +43,22 @@ pub struct Config { pub version: Option, pub tags: Option, pub log_level: LogLevel, - pub serverless_logs_enabled: bool, #[serde(deserialize_with = "deserialize_processing_rules")] pub logs_config_processing_rules: Option>, - pub apm_enabled: bool, - pub apm_replace_tags: Option, - pub lambda_handler: String, pub serverless_flush_strategy: FlushStrategy, - pub trace_enabled: bool, - pub serverless_trace_enabled: bool, - pub capture_lambda_payload: bool, - // ALL ENV VARS below are deprecated or not used by the extension - // just here so we don't failover - pub flush_to_log: bool, - pub logs_injection: bool, - pub merge_xray_traces: bool, - pub serverless_appsec_enabled: bool, - pub extension_version: Option, pub enhanced_metrics: bool, - pub cold_start_tracing: bool, - pub min_cold_start_duration: String, - pub cold_start_trace_skip_lib: String, - pub service_mapping: Option, - pub data_streams_enabled: bool, - pub trace_disabled_plugins: Option, - pub trace_debug: bool, + // Failover + pub extension_version: Option, + pub serverless_appsec_enabled: bool, + pub appsec_enabled: bool, pub profiling_enabled: bool, - pub git_commit_sha: Option, - pub logs_enabled: bool, - pub app_key: Option, - pub trace_sample_rate: Option, - pub dotnet_tracer_home: Option, - pub trace_managed_services: bool, - pub runtime_metrics_enabled: bool, - pub git_repository_url: Option, } impl Default for Config { fn default() -> Self { Config { // General - site: "datadoghq.com".to_string(), + site: String::default(), api_key: String::default(), api_key_secret_arn: String::default(), kms_api_key: String::default(), @@ -79,41 +70,14 @@ impl Default for Config { tags: None, // Logs log_level: LogLevel::default(), - logs_enabled: false, // IGNORED, this is the main agent config, but never used in - // severless - serverless_logs_enabled: true, - // TODO(duncanista): Add serializer for YAML logs_config_processing_rules: None, - // APM - apm_enabled: false, - apm_replace_tags: None, - trace_disabled_plugins: None, - lambda_handler: String::default(), - serverless_trace_enabled: true, - trace_enabled: true, - // Ignored by the extension for now - capture_lambda_payload: false, - flush_to_log: false, - logs_injection: false, - merge_xray_traces: false, - serverless_appsec_enabled: false, - cold_start_tracing: true, - min_cold_start_duration: String::default(), - cold_start_trace_skip_lib: String::default(), + // Metrics + enhanced_metrics: true, // Failover extension_version: None, - enhanced_metrics: true, - service_mapping: None, - data_streams_enabled: false, - trace_debug: false, + serverless_appsec_enabled: false, + appsec_enabled: false, profiling_enabled: false, - git_commit_sha: None, - app_key: None, - trace_sample_rate: None, - dotnet_tracer_home: None, - trace_managed_services: true, - runtime_metrics_enabled: false, - git_repository_url: None, } } } @@ -132,26 +96,34 @@ fn log_failover_reason(reason: &str) { #[allow(clippy::module_name_repetitions)] pub fn get_config(config_directory: &Path) -> Result { let path = config_directory.join("datadog.yaml"); + + // Get default config fields (and ENV specific ones) let figment = Figment::new() - // .merge(Yaml::file(path)) + .merge(Yaml::file(&path)) .merge(Env::prefixed("DATADOG_")) .merge(Env::prefixed("DD_")); - let config: Config = figment.extract().map_err(|err| match err.kind { - figment::error::Kind::UnknownField(field, _) => { - log_failover_reason(&field.clone()); - ConfigError::UnsupportedField(field) - } - _ => ConfigError::ParseError(err.to_string()), - })?; + // Get YAML nested fields + let yaml_figment = Figment::new().merge(Yaml::file(&path)); + + let (mut config, yaml_config): (Config, YamlConfig) = + match (figment.extract(), yaml_figment.extract()) { + (Ok(env_config), Ok(yaml_config)) => (env_config, yaml_config), + (_, Err(err)) | (Err(err), _) => return Err(ConfigError::ParseError(err.to_string())), + }; + + // Set site if empty + if config.site.is_empty() { + config.site = "datadoghq.com".to_string(); + } - // TODO(duncanista): revert to using datadog.yaml when we have a proper serializer - if path.exists() { - log_failover_reason("datadog_yaml"); - return Err(ConfigError::UnsupportedField("datadog_yaml".to_string())); + // Merge YAML nested fields + if let Some(processing_rules) = yaml_config.logs_config.processing_rules { + config.logs_config_processing_rules = Some(processing_rules); } - if config.serverless_appsec_enabled { + // Failover + if config.serverless_appsec_enabled || config.appsec_enabled { log_failover_reason("appsec_enabled"); return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); } @@ -166,7 +138,6 @@ pub fn get_config(config_directory: &Path) -> Result { match config.extension_version.as_deref() { Some("next") => {} Some(_) | None => { - log_failover_reason("extension_version"); return Err(ConfigError::UnsupportedField( "extension_version".to_string(), )); @@ -196,7 +167,7 @@ pub mod tests { fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_APPSEC_ENABLED", "true"); + jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); let config = get_config(Path::new("")).expect_err("should reject unknown fields"); assert_eq!( @@ -208,34 +179,48 @@ pub mod tests { } #[test] - fn test_reject_datadog_yaml() { + fn test_precedence() { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.create_file( "datadog.yaml", r" - apm_enabled: true + site: datadoghq.eu, extension_version: next ", )?; - let config = get_config(Path::new("")).expect_err("should reject datadog.yaml file"); + jail.set_env("DD_SITE", "datad0g.com"); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, - ConfigError::UnsupportedField("datadog_yaml".to_string()) + Config { + site: "datad0g.com".to_string(), + extension_version: Some("next".to_string()), + ..Config::default() + } ); Ok(()) }); } #[test] - fn test_reject_unknown_fields_env() { + fn test_parse_config_file() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_UNKNOWN_FIELD", "true"); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + jail.create_file( + "datadog.yaml", + r" + extension_version: next + ", + )?; + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, - ConfigError::UnsupportedField("unknown_field".to_string()) + Config { + extension_version: Some("next".to_string()), + site: "datadoghq.com".to_string(), + ..Config::default() + } ); Ok(()) }); @@ -258,13 +243,13 @@ pub mod tests { fn test_parse_env() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_APM_ENABLED", "true"); + jail.set_env("DD_SITE", "datadoghq.eu"); jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, Config { - apm_enabled: true, + site: "datadoghq.eu".to_string(), extension_version: Some("next".to_string()), ..Config::default() } @@ -282,6 +267,7 @@ pub mod tests { assert_eq!( config, Config { + site: "datadoghq.com".to_string(), extension_version: Some("next".to_string()), ..Config::default() } @@ -301,6 +287,7 @@ pub mod tests { config, Config { serverless_flush_strategy: FlushStrategy::End, + site: "datadoghq.com".to_string(), extension_version: Some("next".to_string()), ..Config::default() } @@ -322,6 +309,7 @@ pub mod tests { serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }), + site: "datadoghq.com".to_string(), extension_version: Some("next".to_string()), ..Config::default() } @@ -340,6 +328,7 @@ pub mod tests { assert_eq!( config, Config { + site: "datadoghq.com".to_string(), extension_version: Some("next".to_string()), ..Config::default() } @@ -361,6 +350,7 @@ pub mod tests { assert_eq!( config, Config { + site: "datadoghq.com".to_string(), extension_version: Some("next".to_string()), ..Config::default() } @@ -388,6 +378,42 @@ pub mod tests { pattern: "exclude".to_string(), replace_placeholder: None }]), + site: "datadoghq.com".to_string(), + extension_version: Some("next".to_string()), + ..Config::default() + } + ); + Ok(()) + }); + } + + #[test] + fn test_parse_logs_config_processing_rules_from_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + extension_version: next + site: datadoghq.com + logs_config: + processing_rules: + - type: exclude_at_match + name: exclude + pattern: exclude + ", + )?; + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + logs_config_processing_rules: Some(vec![ProcessingRule { + kind: processing_rule::Kind::ExcludeAtMatch, + name: "exclude".to_string(), + pattern: "exclude".to_string(), + replace_placeholder: None + }]), + site: "datadoghq.com".to_string(), extension_version: Some("next".to_string()), ..Config::default() } @@ -409,7 +435,7 @@ pub mod tests { assert_eq!( config, Config { - apm_replace_tags: Some(ObjectIgnore::Ignore), + site: "datadoghq.com".to_string(), extension_version: Some("next".to_string()), ..Config::default() } diff --git a/object_ignore.rs b/object_ignore.rs deleted file mode 100644 index 700ab07d..00000000 --- a/object_ignore.rs +++ /dev/null @@ -1,15 +0,0 @@ -use serde::{Deserialize, Deserializer}; - -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum ObjectIgnore { - Ignore, -} - -impl<'de> Deserialize<'de> for ObjectIgnore { - fn deserialize(_deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(ObjectIgnore::Ignore) - } -} From 62642d3936acad072c7c0ac1ab862f48ccbe6c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 22 Aug 2024 13:48:12 -0400 Subject: [PATCH 023/112] chore(bottlecap): fast failover (#371) * failover fast * typo * failover on `/opt/datadog_wrapper` set --- mod.rs | 172 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 114 insertions(+), 58 deletions(-) diff --git a/mod.rs b/mod.rs index 5d8e9f51..748a381f 100644 --- a/mod.rs +++ b/mod.rs @@ -12,6 +12,19 @@ use crate::config::flush_strategy::FlushStrategy; use crate::config::log_level::LogLevel; use crate::config::processing_rule::{deserialize_processing_rules, ProcessingRule}; +/// `FailoverConfig` is a struct that represents fields that are not supported in the extension yet. +/// +/// `extension_version` is expected to be set to "next" to enable the optimized extension. +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct FailoverConfig { + extension_version: Option, + serverless_appsec_enabled: bool, + appsec_enabled: bool, + profiling_enabled: bool, +} + #[derive(Debug, PartialEq, Deserialize, Clone, Default)] #[serde(default)] #[allow(clippy::module_name_repetitions)] @@ -47,11 +60,6 @@ pub struct Config { pub logs_config_processing_rules: Option>, pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, - // Failover - pub extension_version: Option, - pub serverless_appsec_enabled: bool, - pub appsec_enabled: bool, - pub profiling_enabled: bool, } impl Default for Config { @@ -74,10 +82,6 @@ impl Default for Config { // Metrics enhanced_metrics: true, // Failover - extension_version: None, - serverless_appsec_enabled: false, - appsec_enabled: false, - profiling_enabled: false, } } } @@ -93,6 +97,53 @@ fn log_failover_reason(reason: &str) { println!("{{\"DD_EXTENSION_FAILOVER_REASON\":\"{reason}\"}}"); } +fn failsover(figment: &Figment) -> Result<(), ConfigError> { + let failover_config: FailoverConfig = match figment.extract() { + Ok(failover_config) => failover_config, + Err(err) => { + println!("Failed to parse Datadog config: {err}"); + return Err(ConfigError::ParseError(err.to_string())); + } + }; + + let opted_in = match failover_config.extension_version.as_deref() { + Some("next") => true, + // Only log when the field is present but its not "next" + Some(_) => { + log_failover_reason("extension_version"); + false + } + _ => false, + }; + + if !opted_in { + return Err(ConfigError::UnsupportedField( + "extension_version".to_string(), + )); + } + + let datadog_wrapper_set = + std::env::var("AWS_LAMBDA_EXEC_WRAPPER").unwrap_or_default() == "/opt/datadog_wrapper"; + if datadog_wrapper_set { + log_failover_reason("datadog_wrapper"); + return Err(ConfigError::UnsupportedField("datadog_wrapper".to_string())); + } + + if failover_config.serverless_appsec_enabled || failover_config.appsec_enabled { + log_failover_reason("appsec_enabled"); + return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); + } + + if failover_config.profiling_enabled { + log_failover_reason("profiling_enabled"); + return Err(ConfigError::UnsupportedField( + "profiling_enabled".to_string(), + )); + } + + Ok(()) +} + #[allow(clippy::module_name_repetitions)] pub fn get_config(config_directory: &Path) -> Result { let path = config_directory.join("datadog.yaml"); @@ -104,12 +155,18 @@ pub fn get_config(config_directory: &Path) -> Result { .merge(Env::prefixed("DD_")); // Get YAML nested fields - let yaml_figment = Figment::new().merge(Yaml::file(&path)); + let yaml_figment = Figment::from(Yaml::file(&path)); + + // Failover + failsover(&figment)?; let (mut config, yaml_config): (Config, YamlConfig) = match (figment.extract(), yaml_figment.extract()) { (Ok(env_config), Ok(yaml_config)) => (env_config, yaml_config), - (_, Err(err)) | (Err(err), _) => return Err(ConfigError::ParseError(err.to_string())), + (_, Err(err)) | (Err(err), _) => { + println!("Failed to parse Datadog config: {err}"); + return Err(ConfigError::ParseError(err.to_string())); + } }; // Set site if empty @@ -118,29 +175,11 @@ pub fn get_config(config_directory: &Path) -> Result { } // Merge YAML nested fields - if let Some(processing_rules) = yaml_config.logs_config.processing_rules { - config.logs_config_processing_rules = Some(processing_rules); - } - - // Failover - if config.serverless_appsec_enabled || config.appsec_enabled { - log_failover_reason("appsec_enabled"); - return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); - } - - if config.profiling_enabled { - log_failover_reason("profiling_enabled"); - return Err(ConfigError::UnsupportedField( - "profiling_enabled".to_string(), - )); - } - - match config.extension_version.as_deref() { - Some("next") => {} - Some(_) | None => { - return Err(ConfigError::UnsupportedField( - "extension_version".to_string(), - )); + // + // Set logs_config_processing_rules if not defined in env + if config.logs_config_processing_rules.is_none() { + if let Some(processing_rules) = yaml_config.logs_config.processing_rules { + config.logs_config_processing_rules = Some(processing_rules); } } @@ -163,10 +202,40 @@ pub mod tests { use crate::config::flush_strategy::PeriodicStrategy; use crate::config::processing_rule; + #[test] + fn test_reject_without_opt_in() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("extension_version".to_string()) + ); + Ok(()) + }); + } + + #[test] + fn test_reject_datadog_wrapper() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_EXTENSION_VERSION", "next"); + jail.set_env("AWS_LAMBDA_EXEC_WRAPPER", "/opt/datadog_wrapper"); + + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("datadog_wrapper".to_string()) + ); + Ok(()) + }); + } + #[test] fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { jail.clear_env(); + jail.set_env("DD_EXTENSION_VERSION", "next"); jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); let config = get_config(Path::new("")).expect_err("should reject unknown fields"); @@ -195,7 +264,6 @@ pub mod tests { config, Config { site: "datad0g.com".to_string(), - extension_version: Some("next".to_string()), ..Config::default() } ); @@ -217,7 +285,6 @@ pub mod tests { assert_eq!( config, Config { - extension_version: Some("next".to_string()), site: "datadoghq.com".to_string(), ..Config::default() } @@ -226,19 +293,6 @@ pub mod tests { }); } - #[test] - fn test_reject_without_opt_in() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!( - config, - ConfigError::UnsupportedField("extension_version".to_string()) - ); - Ok(()) - }); - } - #[test] fn test_parse_env() { figment::Jail::expect_with(|jail| { @@ -250,7 +304,6 @@ pub mod tests { config, Config { site: "datadoghq.eu".to_string(), - extension_version: Some("next".to_string()), ..Config::default() } ); @@ -268,7 +321,6 @@ pub mod tests { config, Config { site: "datadoghq.com".to_string(), - extension_version: Some("next".to_string()), ..Config::default() } ); @@ -288,7 +340,6 @@ pub mod tests { Config { serverless_flush_strategy: FlushStrategy::End, site: "datadoghq.com".to_string(), - extension_version: Some("next".to_string()), ..Config::default() } ); @@ -310,7 +361,6 @@ pub mod tests { interval: 100_000 }), site: "datadoghq.com".to_string(), - extension_version: Some("next".to_string()), ..Config::default() } ); @@ -329,7 +379,6 @@ pub mod tests { config, Config { site: "datadoghq.com".to_string(), - extension_version: Some("next".to_string()), ..Config::default() } ); @@ -351,7 +400,6 @@ pub mod tests { config, Config { site: "datadoghq.com".to_string(), - extension_version: Some("next".to_string()), ..Config::default() } ); @@ -367,6 +415,17 @@ pub mod tests { "DD_LOGS_CONFIG_PROCESSING_RULES", r#"[{"type":"exclude_at_match","name":"exclude","pattern":"exclude"}]"#, ); + jail.create_file( + "datadog.yaml", + r" + extension_version: next + logs_config: + processing_rules: + - type: exclude_at_match + name: exclude-me-yaml + pattern: exclude-me-yaml + ", + )?; jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( @@ -379,7 +438,6 @@ pub mod tests { replace_placeholder: None }]), site: "datadoghq.com".to_string(), - extension_version: Some("next".to_string()), ..Config::default() } ); @@ -414,7 +472,6 @@ pub mod tests { replace_placeholder: None }]), site: "datadoghq.com".to_string(), - extension_version: Some("next".to_string()), ..Config::default() } ); @@ -436,7 +493,6 @@ pub mod tests { config, Config { site: "datadoghq.com".to_string(), - extension_version: Some("next".to_string()), ..Config::default() } ); From ca8d916582aedc715285ecb602929149fe5e2b23 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Fri, 23 Aug 2024 10:10:15 -0400 Subject: [PATCH 024/112] aj/fix log level casing (#372) * feat: serde's rename_all isn't working, use a custom deserializer to lowercase loglevels * feat: default is warn * feat: Allow reptition to clear up imports * feat: rebase --- log_level.rs | 23 +++++++++++++++++++++-- mod.rs | 22 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/log_level.rs b/log_level.rs index 03f1ddb3..86c255e1 100644 --- a/log_level.rs +++ b/log_level.rs @@ -1,7 +1,7 @@ -use serde::Deserialize; +use serde::{Deserialize, Deserializer}; +use tracing::error; #[derive(Clone, Copy, Debug, PartialEq, Deserialize, Default)] -#[serde(rename_all = "lowercase")] pub enum LogLevel { /// Designates very serious errors. Error, @@ -41,3 +41,22 @@ impl LogLevel { } } } + +#[allow(clippy::module_name_repetitions)] +pub fn deserialize_log_level<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s: String = Deserialize::deserialize(deserializer)?; + match s.to_lowercase().as_str() { + "error" => Ok(LogLevel::Error), + "warn" => Ok(LogLevel::Warn), + "info" => Ok(LogLevel::Info), + "debug" => Ok(LogLevel::Debug), + "trace" => Ok(LogLevel::Trace), + _ => { + error!("Unknown log level: {}, using warn", s); + Ok(LogLevel::Warn) + } + } +} diff --git a/mod.rs b/mod.rs index 748a381f..a9571b77 100644 --- a/mod.rs +++ b/mod.rs @@ -9,7 +9,7 @@ use figment::{providers::Env, Figment}; use serde::Deserialize; use crate::config::flush_strategy::FlushStrategy; -use crate::config::log_level::LogLevel; +use crate::config::log_level::{deserialize_log_level, LogLevel}; use crate::config::processing_rule::{deserialize_processing_rules, ProcessingRule}; /// `FailoverConfig` is a struct that represents fields that are not supported in the extension yet. @@ -55,6 +55,7 @@ pub struct Config { pub service: Option, pub version: Option, pub tags: Option, + #[serde(deserialize_with = "deserialize_log_level")] pub log_level: LogLevel, #[serde(deserialize_with = "deserialize_processing_rules")] pub logs_config_processing_rules: Option>, @@ -311,6 +312,25 @@ pub mod tests { }); } + #[test] + fn test_parse_log_level() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_LOG_LEVEL", "TRACE"); + jail.set_env("DD_EXTENSION_VERSION", "next"); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( + config, + Config { + log_level: LogLevel::Trace, + site: "datadoghq.com".to_string(), + ..Config::default() + } + ); + Ok(()) + }); + } + #[test] fn test_parse_default() { figment::Jail::expect_with(|jail| { From 372bc94a15b92b441ea87a8713dfde7f36269bd3 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 10 Sep 2024 16:27:26 -0400 Subject: [PATCH 025/112] feat: failover on dd proxy (#391) --- mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mod.rs b/mod.rs index a9571b77..e94a112c 100644 --- a/mod.rs +++ b/mod.rs @@ -23,6 +23,8 @@ pub struct FailoverConfig { serverless_appsec_enabled: bool, appsec_enabled: bool, profiling_enabled: bool, + http_proxy: Option, + https_proxy: Option, } #[derive(Debug, PartialEq, Deserialize, Clone, Default)] @@ -142,6 +144,11 @@ fn failsover(figment: &Figment) -> Result<(), ConfigError> { )); } + if failover_config.http_proxy.is_some() || failover_config.https_proxy.is_some() { + log_failover_reason("http_proxy"); + return Err(ConfigError::UnsupportedField("http_proxy".to_string())); + } + Ok(()) } From 5e0a1df85972c14d5581b89f66caf3a4bca5c274 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 8 Oct 2024 10:33:20 -0400 Subject: [PATCH 026/112] feat: support HTTPS_PROXY (#381) * feat: support DD_HTTP_PROXY and DD_HTTPS_PROXY * fix: remove import * fix: fmt * feat: Revert fqdn changes to enable testing * feat: Use let instead of repeated instantiation * feat: Rip out proxy stuff we dont need but make sure we dont proxy the telemetry or runtime APIs with system proxies * feat: remove debug * fix: no debugs for hyper/h2 * fix: revert cargo changes * feat: Pin libdatadog deps to v13.1 * fix: rebase with dogstatsd 13.1 * fix: use main for dsdrs * fix: remove unwrap * fix: fmt * fix: licenses * increase size boo * fix: size ugh * fix: install_default() in tests --- mod.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/mod.rs b/mod.rs index e94a112c..f3da1904 100644 --- a/mod.rs +++ b/mod.rs @@ -23,8 +23,6 @@ pub struct FailoverConfig { serverless_appsec_enabled: bool, appsec_enabled: bool, profiling_enabled: bool, - http_proxy: Option, - https_proxy: Option, } #[derive(Debug, PartialEq, Deserialize, Clone, Default)] @@ -63,6 +61,7 @@ pub struct Config { pub logs_config_processing_rules: Option>, pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, + pub https_proxy: Option, } impl Default for Config { @@ -85,6 +84,7 @@ impl Default for Config { // Metrics enhanced_metrics: true, // Failover + https_proxy: None, } } } @@ -144,11 +144,6 @@ fn failsover(figment: &Figment) -> Result<(), ConfigError> { )); } - if failover_config.http_proxy.is_some() || failover_config.https_proxy.is_some() { - log_failover_reason("http_proxy"); - return Err(ConfigError::UnsupportedField("http_proxy".to_string())); - } - Ok(()) } @@ -160,7 +155,8 @@ pub fn get_config(config_directory: &Path) -> Result { let figment = Figment::new() .merge(Yaml::file(&path)) .merge(Env::prefixed("DATADOG_")) - .merge(Env::prefixed("DD_")); + .merge(Env::prefixed("DD_")) + .merge(Env::raw().only(&["HTTPS_PROXY"])); // Get YAML nested fields let yaml_figment = Figment::from(Yaml::file(&path)); From ce86674080aebdfa05d7f805839ed35cb13d860d Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 23 Oct 2024 14:18:13 -0400 Subject: [PATCH 027/112] aj/honor both proxies in order (#410) * feat: Honor priority order of DD_PROXY_HTTPS over HTTPS_PROXY * feat: fmt * fix: Prefer Ok over some + ok * Feat: Use tags for proxy support in libdatadog * fix: no proxy for tests * fix: license * all this for a comma --- mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mod.rs b/mod.rs index f3da1904..c2b2655d 100644 --- a/mod.rs +++ b/mod.rs @@ -173,6 +173,11 @@ pub fn get_config(config_directory: &Path) -> Result { } }; + // Prefer DD_PROXY_HTTPS over HTTPS_PROXY + // No else needed as HTTPS_PROXY is handled by reqwest and built into trace client + if let Ok(https_proxy) = std::env::var("DD_PROXY_HTTPS") { + config.https_proxy = Some(https_proxy); + } // Set site if empty if config.site.is_empty() { config.site = "datadoghq.com".to_string(); From e36d3d862bccba2951c76d98033603b1f3656c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 26 Aug 2024 17:18:14 -0400 Subject: [PATCH 028/112] accept `datadog_wrapper` --- mod.rs | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/mod.rs b/mod.rs index c2b2655d..3785843b 100644 --- a/mod.rs +++ b/mod.rs @@ -125,13 +125,6 @@ fn failsover(figment: &Figment) -> Result<(), ConfigError> { )); } - let datadog_wrapper_set = - std::env::var("AWS_LAMBDA_EXEC_WRAPPER").unwrap_or_default() == "/opt/datadog_wrapper"; - if datadog_wrapper_set { - log_failover_reason("datadog_wrapper"); - return Err(ConfigError::UnsupportedField("datadog_wrapper".to_string())); - } - if failover_config.serverless_appsec_enabled || failover_config.appsec_enabled { log_failover_reason("appsec_enabled"); return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); @@ -224,22 +217,6 @@ pub mod tests { }); } - #[test] - fn test_reject_datadog_wrapper() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_EXTENSION_VERSION", "next"); - jail.set_env("AWS_LAMBDA_EXEC_WRAPPER", "/opt/datadog_wrapper"); - - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!( - config, - ConfigError::UnsupportedField("datadog_wrapper".to_string()) - ); - Ok(()) - }); - } - #[test] fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { From 3ee2eaf0ae2d6deed9d5a0807abcb02ef986ad39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 26 Aug 2024 17:19:00 -0400 Subject: [PATCH 029/112] Revert "accept `datadog_wrapper`" This reverts commit 9560657582f2f22c8e68af5d0bb9d7d2b0765650. --- mod.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/mod.rs b/mod.rs index 3785843b..c2b2655d 100644 --- a/mod.rs +++ b/mod.rs @@ -125,6 +125,13 @@ fn failsover(figment: &Figment) -> Result<(), ConfigError> { )); } + let datadog_wrapper_set = + std::env::var("AWS_LAMBDA_EXEC_WRAPPER").unwrap_or_default() == "/opt/datadog_wrapper"; + if datadog_wrapper_set { + log_failover_reason("datadog_wrapper"); + return Err(ConfigError::UnsupportedField("datadog_wrapper".to_string())); + } + if failover_config.serverless_appsec_enabled || failover_config.appsec_enabled { log_failover_reason("appsec_enabled"); return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); @@ -217,6 +224,22 @@ pub mod tests { }); } + #[test] + fn test_reject_datadog_wrapper() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_EXTENSION_VERSION", "next"); + jail.set_env("AWS_LAMBDA_EXEC_WRAPPER", "/opt/datadog_wrapper"); + + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("datadog_wrapper".to_string()) + ); + Ok(()) + }); + } + #[test] fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { From b2a67df834713381c1a792ece6b4d5c8eedcda90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:42:08 -0400 Subject: [PATCH 030/112] accept `datadog_wrapper` (#373) --- mod.rs | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/mod.rs b/mod.rs index c2b2655d..3785843b 100644 --- a/mod.rs +++ b/mod.rs @@ -125,13 +125,6 @@ fn failsover(figment: &Figment) -> Result<(), ConfigError> { )); } - let datadog_wrapper_set = - std::env::var("AWS_LAMBDA_EXEC_WRAPPER").unwrap_or_default() == "/opt/datadog_wrapper"; - if datadog_wrapper_set { - log_failover_reason("datadog_wrapper"); - return Err(ConfigError::UnsupportedField("datadog_wrapper".to_string())); - } - if failover_config.serverless_appsec_enabled || failover_config.appsec_enabled { log_failover_reason("appsec_enabled"); return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); @@ -224,22 +217,6 @@ pub mod tests { }); } - #[test] - fn test_reject_datadog_wrapper() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_EXTENSION_VERSION", "next"); - jail.set_env("AWS_LAMBDA_EXEC_WRAPPER", "/opt/datadog_wrapper"); - - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!( - config, - ConfigError::UnsupportedField("datadog_wrapper".to_string()) - ); - Ok(()) - }); - } - #[test] fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { From 7a48458e06a1f0cb4d98c254356fc36b9ee7b382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 9 Oct 2024 14:59:24 -0400 Subject: [PATCH 031/112] feat(bottlecap): create Inferred Spans baseline + infer API Gateway HTTP spans (#405) * add `Trigger` trait for inferred spans * add `ApiGatewayHttpEvent` trigger * add `SpanInferrer` * make `invocation::processor` to use `SpanInferrer` * send `aws_config` to `invocation::processor` * use incoming payload for `invocation::processor` for span inferring * add `api_gateway_http_event.json` for testing * add `api_gateway_proxy_event.json` for testing * fix: Convert tag hashmap to sorted vector of tags * fix: fmt --------- Co-authored-by: AJ Stuyvenberg --- mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mod.rs b/mod.rs index 3785843b..43f7c254 100644 --- a/mod.rs +++ b/mod.rs @@ -189,6 +189,7 @@ pub fn get_config(config_directory: &Path) -> Result { } #[allow(clippy::module_name_repetitions)] +#[derive(Debug, Clone)] pub struct AwsConfig { pub region: String, pub aws_access_key_id: String, From 9f960d06a26324dca3ef2886fa82b816771dcb41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:58:06 -0400 Subject: [PATCH 032/112] feat(bottlecap): Add Composite Trace Propagator (#413) * add `trace_propagation_style.rs` * add Trace Propagation to `config.rs` also updated unit tests, as we have custom behavior, we should check only the fields we care about in the tests * add `links` to `SpanContext` * add composite propagator also known as our internal http propagator, but in reality, http doesnt make any sense to me, its just a composite propagator which we used based on our configuration * update `TextMapPropagator`s to comply with interface also updated the naming * fmt * add unit testing for `config.rs` * add `PartialEq` to `SpanContext` * correct logic from `text_map_propagator.rs` logic was wrong in some parts, this was discovered through unit tests * add unit tests for `DatadogCompositePropagator` also corrected some logic --- mod.rs | 191 ++++++++++++++++++++----------------- trace_propagation_style.rs | 58 +++++++++++ 2 files changed, 160 insertions(+), 89 deletions(-) create mode 100644 trace_propagation_style.rs diff --git a/mod.rs b/mod.rs index 43f7c254..597e954f 100644 --- a/mod.rs +++ b/mod.rs @@ -1,12 +1,15 @@ pub mod flush_strategy; pub mod log_level; pub mod processing_rule; +pub mod trace_propagation_style; use std::path::Path; +use std::vec; use figment::providers::{Format, Yaml}; use figment::{providers::Env, Figment}; use serde::Deserialize; +use trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}; use crate::config::flush_strategy::FlushStrategy; use crate::config::log_level::{deserialize_log_level, LogLevel}; @@ -62,6 +65,13 @@ pub struct Config { pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, pub https_proxy: Option, + // Trace Propagation + #[serde(deserialize_with = "deserialize_trace_propagation_style")] + pub trace_propagation_style: Vec, + #[serde(deserialize_with = "deserialize_trace_propagation_style")] + pub trace_propagation_style_extract: Vec, + pub trace_propagation_extract_first: bool, + pub trace_propagation_http_baggage_enabled: bool, } impl Default for Config { @@ -85,6 +95,14 @@ impl Default for Config { enhanced_metrics: true, // Failover https_proxy: None, + // Trace Propagation + trace_propagation_style: vec![ + TracePropagationStyle::Datadog, + TracePropagationStyle::TraceContext, + ], + trace_propagation_style_extract: vec![], + trace_propagation_extract_first: false, + trace_propagation_http_baggage_enabled: false, } } } @@ -185,6 +203,15 @@ pub fn get_config(config_directory: &Path) -> Result { } } + // Trace Propagation + // + // If not set by the user, set defaults + if config.trace_propagation_style_extract.is_empty() { + config + .trace_propagation_style_extract + .clone_from(&config.trace_propagation_style); + } + Ok(config) } @@ -247,13 +274,7 @@ pub mod tests { )?; jail.set_env("DD_SITE", "datad0g.com"); let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!( - config, - Config { - site: "datad0g.com".to_string(), - ..Config::default() - } - ); + assert_eq!(config.site, "datad0g.com"); Ok(()) }); } @@ -269,13 +290,7 @@ pub mod tests { ", )?; let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!( - config, - Config { - site: "datadoghq.com".to_string(), - ..Config::default() - } - ); + assert_eq!(config.site, "datadoghq.com"); Ok(()) }); } @@ -287,13 +302,7 @@ pub mod tests { jail.set_env("DD_SITE", "datadoghq.eu"); jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!( - config, - Config { - site: "datadoghq.eu".to_string(), - ..Config::default() - } - ); + assert_eq!(config.site, "datadoghq.eu"); Ok(()) }); } @@ -305,14 +314,7 @@ pub mod tests { jail.set_env("DD_LOG_LEVEL", "TRACE"); jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!( - config, - Config { - log_level: LogLevel::Trace, - site: "datadoghq.com".to_string(), - ..Config::default() - } - ); + assert_eq!(config.log_level, LogLevel::Trace); Ok(()) }); } @@ -327,6 +329,10 @@ pub mod tests { config, Config { site: "datadoghq.com".to_string(), + trace_propagation_style_extract: vec![ + TracePropagationStyle::Datadog, + TracePropagationStyle::TraceContext + ], ..Config::default() } ); @@ -341,14 +347,7 @@ pub mod tests { jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!( - config, - Config { - serverless_flush_strategy: FlushStrategy::End, - site: "datadoghq.com".to_string(), - ..Config::default() - } - ); + assert_eq!(config.serverless_flush_strategy, FlushStrategy::End); Ok(()) }); } @@ -361,14 +360,8 @@ pub mod tests { jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( - config, - Config { - serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { - interval: 100_000 - }), - site: "datadoghq.com".to_string(), - ..Config::default() - } + config.serverless_flush_strategy, + FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }) ); Ok(()) }); @@ -381,13 +374,7 @@ pub mod tests { jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!( - config, - Config { - site: "datadoghq.com".to_string(), - ..Config::default() - } - ); + assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); } @@ -402,13 +389,7 @@ pub mod tests { ); jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!( - config, - Config { - site: "datadoghq.com".to_string(), - ..Config::default() - } - ); + assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); } @@ -435,17 +416,13 @@ pub mod tests { jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( - config, - Config { - logs_config_processing_rules: Some(vec![ProcessingRule { - kind: processing_rule::Kind::ExcludeAtMatch, - name: "exclude".to_string(), - pattern: "exclude".to_string(), - replace_placeholder: None - }]), - site: "datadoghq.com".to_string(), - ..Config::default() - } + config.logs_config_processing_rules, + Some(vec![ProcessingRule { + kind: processing_rule::Kind::ExcludeAtMatch, + name: "exclude".to_string(), + pattern: "exclude".to_string(), + replace_placeholder: None + }]) ); Ok(()) }); @@ -469,39 +446,75 @@ pub mod tests { )?; let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( - config, - Config { - logs_config_processing_rules: Some(vec![ProcessingRule { - kind: processing_rule::Kind::ExcludeAtMatch, - name: "exclude".to_string(), - pattern: "exclude".to_string(), - replace_placeholder: None - }]), - site: "datadoghq.com".to_string(), - ..Config::default() - } + config.logs_config_processing_rules, + Some(vec![ProcessingRule { + kind: processing_rule::Kind::ExcludeAtMatch, + name: "exclude".to_string(), + pattern: "exclude".to_string(), + replace_placeholder: None + }]), ); Ok(()) }); } #[test] - fn test_ignore_apm_replace_tags() { + fn test_parse_trace_propagation_style() { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env( - "DD_APM_REPLACE_TAGS", - r#"[{"name":"resource.name","pattern":"(.*)/(foo[:%].+)","repl":"$1/{foo}"}]"#, + "DD_TRACE_PROPAGATION_STYLE", + "datadog,tracecontext,b3,b3multi", ); jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); + + let expected_styles = vec![ + TracePropagationStyle::Datadog, + TracePropagationStyle::TraceContext, + TracePropagationStyle::B3, + TracePropagationStyle::B3Multi, + ]; + assert_eq!(config.trace_propagation_style, expected_styles); + assert_eq!(config.trace_propagation_style_extract, expected_styles); + Ok(()) + }); + } + + #[test] + fn test_parse_trace_propagation_style_extract() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "datadog"); + jail.set_env("DD_EXTENSION_VERSION", "next"); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!( - config, - Config { - site: "datadoghq.com".to_string(), - ..Config::default() - } + config.trace_propagation_style, + vec![ + TracePropagationStyle::Datadog, + TracePropagationStyle::TraceContext, + ] + ); + assert_eq!( + config.trace_propagation_style_extract, + vec![TracePropagationStyle::Datadog] + ); + Ok(()) + }); + } + + #[test] + fn test_ignore_apm_replace_tags() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env( + "DD_APM_REPLACE_TAGS", + r#"[{"name":"resource.name","pattern":"(.*)/(foo[:%].+)","repl":"$1/{foo}"}]"#, ); + jail.set_env("DD_EXTENSION_VERSION", "next"); + let config = get_config(Path::new("")); + assert!(config.is_ok()); Ok(()) }); } diff --git a/trace_propagation_style.rs b/trace_propagation_style.rs new file mode 100644 index 00000000..6ebc9dc7 --- /dev/null +++ b/trace_propagation_style.rs @@ -0,0 +1,58 @@ +use std::{fmt::Display, str::FromStr}; + +use serde::{Deserialize, Deserializer}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TracePropagationStyle { + Datadog, + B3Multi, + B3, + TraceContext, + None, +} + +impl FromStr for TracePropagationStyle { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "datadog" => Ok(TracePropagationStyle::Datadog), + "b3multi" => Ok(TracePropagationStyle::B3Multi), + "b3" => Ok(TracePropagationStyle::B3), + "tracecontext" => Ok(TracePropagationStyle::TraceContext), + "none" => Ok(TracePropagationStyle::None), + _ => Err(format!("Unknown trace propagation style: {s}")), + } + } +} + +impl Display for TracePropagationStyle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let style = match self { + TracePropagationStyle::Datadog => "datadog", + TracePropagationStyle::B3Multi => "b3multi", + TracePropagationStyle::B3 => "b3", + TracePropagationStyle::TraceContext => "tracecontext", + TracePropagationStyle::None => "none", + }; + write!(f, "{style}") + } +} + +#[allow(clippy::module_name_repetitions)] +pub fn deserialize_trace_propagation_style<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s: String = String::deserialize(deserializer)?; + + s.split(',') + .map(|style| { + TracePropagationStyle::from_str(style.trim()).map_err(|e| { + serde::de::Error::custom(format!("Failed to deserialize propagation style: {e}")) + }) + }) + .collect() +} From 53ef766f6bdec6f01425bd7d12d474b451cbf7ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:32:48 -0500 Subject: [PATCH 033/112] feat(bottlecap): add capture lambda payload (#454) * add `tag_span_from_value` * add `capture_lambda_payload` config * add unit testing for `tag_span_from_value` * update listener `end_invocation_handler` parsing should not be handled here * add capture lambda payload feature also parse body properly, and handle `statusCode` --- mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mod.rs b/mod.rs index 597e954f..df238991 100644 --- a/mod.rs +++ b/mod.rs @@ -65,6 +65,8 @@ pub struct Config { pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, pub https_proxy: Option, + pub capture_lambda_payload: bool, + pub capture_lambda_payload_max_depth: u32, // Trace Propagation #[serde(deserialize_with = "deserialize_trace_propagation_style")] pub trace_propagation_style: Vec, @@ -93,8 +95,9 @@ impl Default for Config { logs_config_processing_rules: None, // Metrics enhanced_metrics: true, - // Failover https_proxy: None, + capture_lambda_payload: false, + capture_lambda_payload_max_depth: 10, // Trace Propagation trace_propagation_style: vec![ TracePropagationStyle::Datadog, From 13c2b9d82b05461bf6c1e37586d6e4c3f3f84121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 14 Nov 2024 19:51:30 -0500 Subject: [PATCH 034/112] feat(bottlecap): add Cold Start Span + Tags (#450) * add some helper functions to `invocation::lifecycle` mod * create cold start span on processor * move `generate_span_id` to father module * send `platform_init_start` data to processor * send `PlatformInitStart` to main bus * update cold start `parent_id` * fix start time of cold start span * enhanced metrics now have a `dynamic_value_tags` for tags which we have to calculate at points in time * `AwsConfig` now has a `sandbox_init_time` value * add `is_empty` to `ContextBuffer` * calculate init tags on invoke also add a method to reset processor invocation state * restart init tags on set * set tags properly for proactive init * fix unit test * remove debug line * make sure `cold_start` tag is only set in one place --- mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mod.rs b/mod.rs index df238991..d1cdd8e5 100644 --- a/mod.rs +++ b/mod.rs @@ -4,6 +4,7 @@ pub mod processing_rule; pub mod trace_propagation_style; use std::path::Path; +use std::time::Instant; use std::vec; use figment::providers::{Format, Yaml}; @@ -226,6 +227,7 @@ pub struct AwsConfig { pub aws_secret_access_key: String, pub aws_session_token: String, pub function_name: String, + pub sandbox_init_time: Instant, } #[cfg(test)] From 2853dfdd12be9e095a5678f4741dfed6e75defa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Fri, 15 Nov 2024 12:48:07 -0500 Subject: [PATCH 035/112] feat(bottlecap): support service mapping and `peer.service` tag (#455) * add some helper functions to `invocation::lifecycle` mod * create cold start span on processor * move `generate_span_id` to father module * send `platform_init_start` data to processor * send `PlatformInitStart` to main bus * update cold start `parent_id` * fix start time of cold start span * enhanced metrics now have a `dynamic_value_tags` for tags which we have to calculate at points in time * `AwsConfig` now has a `sandbox_init_time` value * add `is_empty` to `ContextBuffer` * calculate init tags on invoke also add a method to reset processor invocation state * restart init tags on set * set tags properly for proactive init * fix unit test * remove debug line * make sure `cold_start` tag is only set in one place * add service mapping config serializer * add `service_mapping.rs` * add `ServiceNameResolver` interface for service mapping * implement interface in every trigger * send `service_mapping` lookup table to span enricher * create `SpanInferrer` with `service_mapping` config * fmt --- mod.rs | 6 ++++++ service_mapping.rs | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 service_mapping.rs diff --git a/mod.rs b/mod.rs index d1cdd8e5..54feab25 100644 --- a/mod.rs +++ b/mod.rs @@ -1,8 +1,10 @@ pub mod flush_strategy; pub mod log_level; pub mod processing_rule; +pub mod service_mapping; pub mod trace_propagation_style; +use std::collections::HashMap; use std::path::Path; use std::time::Instant; use std::vec; @@ -15,6 +17,7 @@ use trace_propagation_style::{deserialize_trace_propagation_style, TracePropagat use crate::config::flush_strategy::FlushStrategy; use crate::config::log_level::{deserialize_log_level, LogLevel}; use crate::config::processing_rule::{deserialize_processing_rules, ProcessingRule}; +use crate::config::service_mapping::deserialize_service_mapping; /// `FailoverConfig` is a struct that represents fields that are not supported in the extension yet. /// @@ -68,6 +71,8 @@ pub struct Config { pub https_proxy: Option, pub capture_lambda_payload: bool, pub capture_lambda_payload_max_depth: u32, + #[serde(deserialize_with = "deserialize_service_mapping")] + pub service_mapping: HashMap, // Trace Propagation #[serde(deserialize_with = "deserialize_trace_propagation_style")] pub trace_propagation_style: Vec, @@ -99,6 +104,7 @@ impl Default for Config { https_proxy: None, capture_lambda_payload: false, capture_lambda_payload_max_depth: 10, + service_mapping: HashMap::new(), // Trace Propagation trace_propagation_style: vec![ TracePropagationStyle::Datadog, diff --git a/service_mapping.rs b/service_mapping.rs new file mode 100644 index 00000000..4deda11f --- /dev/null +++ b/service_mapping.rs @@ -0,0 +1,35 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Deserializer}; +use tracing::debug; + +#[allow(clippy::module_name_repetitions)] +pub fn deserialize_service_mapping<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s: String = String::deserialize(deserializer)?; + + let map = s + .split(',') + .map(|pair| { + let mut split = pair.split(':'); + + let service = split.next(); + let to_map = split.next(); + + if let (Some(service), Some(to_map)) = (service, to_map) { + Ok((service.trim().to_string(), to_map.trim().to_string())) + } else { + debug!("Ignoring invalid service mapping pair: {pair}"); + Err(serde::de::Error::custom(format!( + "Failed to deserialize service mapping for pair: {pair}" + ))) + } + }) + .collect(); + + map +} From 9757c8c9833cacafe2a040eb81c424f1cd397dd9 Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:41:22 -0500 Subject: [PATCH 036/112] rename failover to fallback (#465) --- mod.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/mod.rs b/mod.rs index 54feab25..f26c8e9b 100644 --- a/mod.rs +++ b/mod.rs @@ -19,13 +19,13 @@ use crate::config::log_level::{deserialize_log_level, LogLevel}; use crate::config::processing_rule::{deserialize_processing_rules, ProcessingRule}; use crate::config::service_mapping::deserialize_service_mapping; -/// `FailoverConfig` is a struct that represents fields that are not supported in the extension yet. +/// `FallbackConfig` is a struct that represents fields that are not supported in the extension yet. /// /// `extension_version` is expected to be set to "next" to enable the optimized extension. #[derive(Debug, PartialEq, Deserialize, Clone, Default)] #[serde(default)] #[allow(clippy::module_name_repetitions)] -pub struct FailoverConfig { +pub struct FallbackConfig { extension_version: Option, serverless_appsec_enabled: bool, appsec_enabled: bool, @@ -124,24 +124,24 @@ pub enum ConfigError { UnsupportedField(String), } -fn log_failover_reason(reason: &str) { - println!("{{\"DD_EXTENSION_FAILOVER_REASON\":\"{reason}\"}}"); +fn log_fallback_reason(reason: &str) { + println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); } -fn failsover(figment: &Figment) -> Result<(), ConfigError> { - let failover_config: FailoverConfig = match figment.extract() { - Ok(failover_config) => failover_config, +fn fallback(figment: &Figment) -> Result<(), ConfigError> { + let fallback_config: FallbackConfig = match figment.extract() { + Ok(fallback_config) => fallback_config, Err(err) => { println!("Failed to parse Datadog config: {err}"); return Err(ConfigError::ParseError(err.to_string())); } }; - let opted_in = match failover_config.extension_version.as_deref() { + let opted_in = match fallback_config.extension_version.as_deref() { Some("next") => true, // Only log when the field is present but its not "next" Some(_) => { - log_failover_reason("extension_version"); + log_fallback_reason("extension_version"); false } _ => false, @@ -153,13 +153,13 @@ fn failsover(figment: &Figment) -> Result<(), ConfigError> { )); } - if failover_config.serverless_appsec_enabled || failover_config.appsec_enabled { - log_failover_reason("appsec_enabled"); + if fallback_config.serverless_appsec_enabled || fallback_config.appsec_enabled { + log_fallback_reason("appsec_enabled"); return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); } - if failover_config.profiling_enabled { - log_failover_reason("profiling_enabled"); + if fallback_config.profiling_enabled { + log_fallback_reason("profiling_enabled"); return Err(ConfigError::UnsupportedField( "profiling_enabled".to_string(), )); @@ -182,8 +182,7 @@ pub fn get_config(config_directory: &Path) -> Result { // Get YAML nested fields let yaml_figment = Figment::from(Yaml::file(&path)); - // Failover - failsover(&figment)?; + fallback(&figment)?; let (mut config, yaml_config): (Config, YamlConfig) = match (figment.extract(), yaml_figment.extract()) { From 696b4b253fdec5701213613cb0eb0752ccb99e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:27:44 -0800 Subject: [PATCH 037/112] fix(bottlecap): fallback when otel set (#470) * fallback on otel * add unit test --- mod.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/mod.rs b/mod.rs index f26c8e9b..c79a76c8 100644 --- a/mod.rs +++ b/mod.rs @@ -25,11 +25,16 @@ use crate::config::service_mapping::deserialize_service_mapping; #[derive(Debug, PartialEq, Deserialize, Clone, Default)] #[serde(default)] #[allow(clippy::module_name_repetitions)] +#[allow(clippy::struct_excessive_bools)] pub struct FallbackConfig { extension_version: Option, serverless_appsec_enabled: bool, appsec_enabled: bool, profiling_enabled: bool, + // otel + trace_otel_enabled: bool, + otlp_config_receiver_protocols_http_endpoint: Option, + otlp_config_receiver_protocols_grpc_endpoint: Option, } #[derive(Debug, PartialEq, Deserialize, Clone, Default)] @@ -165,6 +170,18 @@ fn fallback(figment: &Figment) -> Result<(), ConfigError> { )); } + if fallback_config.trace_otel_enabled + || fallback_config + .otlp_config_receiver_protocols_http_endpoint + .is_some() + || fallback_config + .otlp_config_receiver_protocols_grpc_endpoint + .is_some() + { + log_fallback_reason("otel"); + return Err(ConfigError::UnsupportedField("otel".to_string())); + } + Ok(()) } @@ -255,6 +272,22 @@ pub mod tests { }); } + #[test] + fn test_fallback_on_otel() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_EXTENSION_VERSION", "next"); + jail.set_env( + "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT", + "localhost:4138", + ); + + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); + Ok(()) + }); + } + #[test] fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { From 456ddd1d85d72ff7e6bba80d66c590d973b98066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 25 Nov 2024 08:17:40 -0800 Subject: [PATCH 038/112] feat(bottlecap): fallback on opted out only (#473) * fallback on opted out only * log on opted out --- mod.rs | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/mod.rs b/mod.rs index c79a76c8..db255e22 100644 --- a/mod.rs +++ b/mod.rs @@ -21,7 +21,7 @@ use crate::config::service_mapping::deserialize_service_mapping; /// `FallbackConfig` is a struct that represents fields that are not supported in the extension yet. /// -/// `extension_version` is expected to be set to "next" to enable the optimized extension. +/// If `extension_version` is set to "legacy", the Go extension will be launched. #[derive(Debug, PartialEq, Deserialize, Clone, Default)] #[serde(default)] #[allow(clippy::module_name_repetitions)] @@ -142,17 +142,15 @@ fn fallback(figment: &Figment) -> Result<(), ConfigError> { } }; - let opted_in = match fallback_config.extension_version.as_deref() { - Some("next") => true, - // Only log when the field is present but its not "next" - Some(_) => { - log_fallback_reason("extension_version"); - false - } + // Customer explicitly opted out of the Next Gen extension + let opted_out = match fallback_config.extension_version.as_deref() { + Some("legacy") => true, + // We want customers using the `next` to not be affected _ => false, }; - if !opted_in { + if opted_out { + log_fallback_reason("extension_version"); return Err(ConfigError::UnsupportedField( "extension_version".to_string(), )); @@ -260,9 +258,10 @@ pub mod tests { use crate::config::processing_rule; #[test] - fn test_reject_without_opt_in() { + fn test_reject_on_opted_out() { figment::Jail::expect_with(|jail| { jail.clear_env(); + jail.set_env("DD_EXTENSION_VERSION", "legacy"); let config = get_config(Path::new("")).expect_err("should reject unknown fields"); assert_eq!( config, @@ -276,7 +275,6 @@ pub mod tests { fn test_fallback_on_otel() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_EXTENSION_VERSION", "next"); jail.set_env( "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT", "localhost:4138", @@ -292,7 +290,6 @@ pub mod tests { fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_EXTENSION_VERSION", "next"); jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); let config = get_config(Path::new("")).expect_err("should reject unknown fields"); @@ -312,7 +309,6 @@ pub mod tests { "datadog.yaml", r" site: datadoghq.eu, - extension_version: next ", )?; jail.set_env("DD_SITE", "datad0g.com"); @@ -343,7 +339,6 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SITE", "datadoghq.eu"); - jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.site, "datadoghq.eu"); Ok(()) @@ -355,7 +350,6 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_LOG_LEVEL", "TRACE"); - jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.log_level, LogLevel::Trace); Ok(()) @@ -366,7 +360,6 @@ pub mod tests { fn test_parse_default() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, @@ -388,7 +381,6 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); - jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::End); Ok(()) @@ -400,7 +392,6 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); - jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.serverless_flush_strategy, @@ -415,7 +406,6 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); - jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) @@ -430,7 +420,6 @@ pub mod tests { "DD_SERVERLESS_FLUSH_STRATEGY", "periodically,invalid_interval", ); - jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) @@ -456,7 +445,6 @@ pub mod tests { pattern: exclude-me-yaml ", )?; - jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.logs_config_processing_rules, @@ -478,7 +466,6 @@ pub mod tests { jail.create_file( "datadog.yaml", r" - extension_version: next site: datadoghq.com logs_config: processing_rules: @@ -529,7 +516,6 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "datadog"); - jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( @@ -555,7 +541,6 @@ pub mod tests { "DD_APM_REPLACE_TAGS", r#"[{"name":"resource.name","pattern":"(.*)/(foo[:%].+)","repl":"$1/{foo}"}]"#, ); - jail.set_env("DD_EXTENSION_VERSION", "next"); let config = get_config(Path::new("")); assert!(config.is_ok()); Ok(()) From 4d0b266cc8ff19dfd4878f53fb1dbb2f1bc2e0e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 25 Nov 2024 08:38:34 -0800 Subject: [PATCH 039/112] fix(bottlecap): fallback on yaml otel config (#474) * fallback on opted out only * fallback on yaml otel config * switch `legacy` to `compatibility` --- mod.rs | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/mod.rs b/mod.rs index db255e22..3af745a1 100644 --- a/mod.rs +++ b/mod.rs @@ -12,16 +12,19 @@ use std::vec; use figment::providers::{Format, Yaml}; use figment::{providers::Env, Figment}; use serde::Deserialize; +use serde_json::Value; use trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}; -use crate::config::flush_strategy::FlushStrategy; -use crate::config::log_level::{deserialize_log_level, LogLevel}; -use crate::config::processing_rule::{deserialize_processing_rules, ProcessingRule}; -use crate::config::service_mapping::deserialize_service_mapping; +use crate::config::{ + flush_strategy::FlushStrategy, + log_level::{deserialize_log_level, LogLevel}, + processing_rule::{deserialize_processing_rules, ProcessingRule}, + service_mapping::deserialize_service_mapping, +}; /// `FallbackConfig` is a struct that represents fields that are not supported in the extension yet. /// -/// If `extension_version` is set to "legacy", the Go extension will be launched. +/// If `extension_version` is set to `compatibility`, the Go extension will be launched. #[derive(Debug, PartialEq, Deserialize, Clone, Default)] #[serde(default)] #[allow(clippy::module_name_repetitions)] @@ -35,6 +38,8 @@ pub struct FallbackConfig { trace_otel_enabled: bool, otlp_config_receiver_protocols_http_endpoint: Option, otlp_config_receiver_protocols_grpc_endpoint: Option, + // YAML otel, we don't care about the content + otlp_config: Option, } #[derive(Debug, PartialEq, Deserialize, Clone, Default)] @@ -144,7 +149,7 @@ fn fallback(figment: &Figment) -> Result<(), ConfigError> { // Customer explicitly opted out of the Next Gen extension let opted_out = match fallback_config.extension_version.as_deref() { - Some("legacy") => true, + Some("compatibility") => true, // We want customers using the `next` to not be affected _ => false, }; @@ -168,6 +173,7 @@ fn fallback(figment: &Figment) -> Result<(), ConfigError> { )); } + // OTEL env if fallback_config.trace_otel_enabled || fallback_config .otlp_config_receiver_protocols_http_endpoint @@ -180,6 +186,12 @@ fn fallback(figment: &Figment) -> Result<(), ConfigError> { return Err(ConfigError::UnsupportedField("otel".to_string())); } + // OTEL YAML + if fallback_config.otlp_config.is_some() { + log_fallback_reason("otel"); + return Err(ConfigError::UnsupportedField("otel".to_string())); + } + Ok(()) } @@ -261,7 +273,7 @@ pub mod tests { fn test_reject_on_opted_out() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_EXTENSION_VERSION", "legacy"); + jail.set_env("DD_EXTENSION_VERSION", "compatibility"); let config = get_config(Path::new("")).expect_err("should reject unknown fields"); assert_eq!( config, @@ -286,6 +298,47 @@ pub mod tests { }); } + #[test] + fn test_fallback_on_otel_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + otlp_config: + receiver: + protocols: + http: + endpoint: localhost:4138 + ", + )?; + + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); + Ok(()) + }); + } + + #[test] + fn test_fallback_on_otel_yaml_empty_section() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + otlp_config: + receiver: + protocols: + http: + ", + )?; + + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); + Ok(()) + }); + } + #[test] fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { @@ -325,7 +378,6 @@ pub mod tests { jail.create_file( "datadog.yaml", r" - extension_version: next ", )?; let config = get_config(Path::new("")).expect("should parse config"); From a1593524e2ee0a8c742f362f474e7838b72842c6 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 25 Nov 2024 12:15:14 -0500 Subject: [PATCH 040/112] feat: honor serverless_logs (#475) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: honor serverless_logs * fmt --------- Co-authored-by: jordan gonzález <30836115+duncanista@users.noreply.github.com> --- mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mod.rs b/mod.rs index 3af745a1..56d00523 100644 --- a/mod.rs +++ b/mod.rs @@ -83,6 +83,7 @@ pub struct Config { pub capture_lambda_payload_max_depth: u32, #[serde(deserialize_with = "deserialize_service_mapping")] pub service_mapping: HashMap, + pub serverless_logs_enabled: bool, // Trace Propagation #[serde(deserialize_with = "deserialize_trace_propagation_style")] pub trace_propagation_style: Vec, @@ -115,6 +116,7 @@ impl Default for Config { capture_lambda_payload: false, capture_lambda_payload_max_depth: 10, service_mapping: HashMap::new(), + serverless_logs_enabled: true, // Trace Propagation trace_propagation_style: vec![ TracePropagationStyle::Datadog, From ba02d9caccdacaa95b8d96605efa32646514901b Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 25 Nov 2024 18:39:23 -0500 Subject: [PATCH 041/112] feat: Flush timeouts (#480) --- mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mod.rs b/mod.rs index 56d00523..7ccada39 100644 --- a/mod.rs +++ b/mod.rs @@ -78,6 +78,7 @@ pub struct Config { pub logs_config_processing_rules: Option>, pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, + pub flush_timeout: u64, pub https_proxy: Option, pub capture_lambda_payload: bool, pub capture_lambda_payload_max_depth: u32, @@ -102,6 +103,7 @@ impl Default for Config { api_key_secret_arn: String::default(), kms_api_key: String::default(), serverless_flush_strategy: FlushStrategy::Default, + flush_timeout: 5, // Unified Tagging env: None, service: None, From fd31c5c024331cfcb32691e7d48cc4830322a927 Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Thu, 5 Dec 2024 13:18:05 -0500 Subject: [PATCH 042/112] fix version parsing for number (#492) --- mod.rs | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/mod.rs b/mod.rs index 7ccada39..14c3ca24 100644 --- a/mod.rs +++ b/mod.rs @@ -11,7 +11,7 @@ use std::vec; use figment::providers::{Format, Yaml}; use figment::{providers::Env, Figment}; -use serde::Deserialize; +use serde::{Deserialize, Deserializer}; use serde_json::Value; use trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}; @@ -70,6 +70,7 @@ pub struct Config { pub kms_api_key: String, pub env: Option, pub service: Option, + #[serde(deserialize_with = "deserialize_string_or_int")] pub version: Option, pub tags: Option, #[serde(deserialize_with = "deserialize_log_level")] @@ -255,6 +256,24 @@ pub fn get_config(config_directory: &Path) -> Result { Ok(config) } +fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Value::deserialize(deserializer)?; + match value { + Value::String(s) => { + if s.trim().is_empty() { + Ok(None) + } else { + Ok(Some(s)) + } + } + Value::Number(n) => Ok(Some(n.to_string())), + _ => Err(serde::de::Error::custom("expected a string or an integer")), + } +} + #[allow(clippy::module_name_repetitions)] #[derive(Debug, Clone)] pub struct AwsConfig { @@ -482,6 +501,17 @@ pub mod tests { }); } + #[test] + fn parse_dd_version() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_VERSION", "123"); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!(config.version.expect("failed to parse DD_VERSION"), "123"); + Ok(()) + }); + } + #[test] fn test_parse_logs_config_processing_rules_from_env() { figment::Jail::expect_with(|jail| { From cd370f02253169068f8ef03eb63f42d970fa78e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:02:36 -0600 Subject: [PATCH 043/112] fix: fallback on intake urls (#495) * fallback on `dd_url`, `dd_url`, and, apm and logs intake urls * fix env var for apm url * grammar --- mod.rs | 104 +++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 83 insertions(+), 21 deletions(-) diff --git a/mod.rs b/mod.rs index 14c3ca24..02d19bb1 100644 --- a/mod.rs +++ b/mod.rs @@ -38,10 +38,24 @@ pub struct FallbackConfig { trace_otel_enabled: bool, otlp_config_receiver_protocols_http_endpoint: Option, otlp_config_receiver_protocols_grpc_endpoint: Option, - // YAML otel, we don't care about the content - otlp_config: Option, + // intake urls + url: Option, + dd_url: Option, + logs_config_logs_dd_url: Option, + // APM, as opposed to logs, does not use the `apm_config` prefix for env vars + apm_dd_url: Option, } +/// `FallbackYamlConfig` is a struct that represents fields in `datadog.yaml` not yet supported in the extension yet. +/// +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct FallbackYamlConfig { + logs_config: Option, + apm_config: Option, + otlp_config: Option, +} #[derive(Debug, PartialEq, Deserialize, Clone, Default)] #[serde(default)] #[allow(clippy::module_name_repetitions)] @@ -143,17 +157,18 @@ fn log_fallback_reason(reason: &str) { println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); } -fn fallback(figment: &Figment) -> Result<(), ConfigError> { - let fallback_config: FallbackConfig = match figment.extract() { - Ok(fallback_config) => fallback_config, - Err(err) => { - println!("Failed to parse Datadog config: {err}"); - return Err(ConfigError::ParseError(err.to_string())); - } - }; +fn fallback(figment: &Figment, yaml_figment: &Figment) -> Result<(), ConfigError> { + let (config, yaml_config): (FallbackConfig, FallbackYamlConfig) = + match (figment.extract(), yaml_figment.extract()) { + (Ok(env_config), Ok(yaml_config)) => (env_config, yaml_config), + (_, Err(err)) | (Err(err), _) => { + println!("Failed to parse Datadog config: {err}"); + return Err(ConfigError::ParseError(err.to_string())); + } + }; // Customer explicitly opted out of the Next Gen extension - let opted_out = match fallback_config.extension_version.as_deref() { + let opted_out = match config.extension_version.as_deref() { Some("compatibility") => true, // We want customers using the `next` to not be affected _ => false, @@ -166,12 +181,12 @@ fn fallback(figment: &Figment) -> Result<(), ConfigError> { )); } - if fallback_config.serverless_appsec_enabled || fallback_config.appsec_enabled { + if config.serverless_appsec_enabled || config.appsec_enabled { log_fallback_reason("appsec_enabled"); return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); } - if fallback_config.profiling_enabled { + if config.profiling_enabled { log_fallback_reason("profiling_enabled"); return Err(ConfigError::UnsupportedField( "profiling_enabled".to_string(), @@ -179,22 +194,33 @@ fn fallback(figment: &Figment) -> Result<(), ConfigError> { } // OTEL env - if fallback_config.trace_otel_enabled - || fallback_config + if config.trace_otel_enabled + || config .otlp_config_receiver_protocols_http_endpoint .is_some() - || fallback_config + || config .otlp_config_receiver_protocols_grpc_endpoint .is_some() + || yaml_config.otlp_config.is_some() { log_fallback_reason("otel"); return Err(ConfigError::UnsupportedField("otel".to_string())); } - // OTEL YAML - if fallback_config.otlp_config.is_some() { - log_fallback_reason("otel"); - return Err(ConfigError::UnsupportedField("otel".to_string())); + // Intake URLs + if config.url.is_some() + || config.dd_url.is_some() + || config.logs_config_logs_dd_url.is_some() + || config.apm_dd_url.is_some() + || yaml_config + .logs_config + .is_some_and(|c| c.get("logs_dd_url").is_some()) + || yaml_config + .apm_config + .is_some_and(|c| c.get("apm_dd_url").is_some()) + { + log_fallback_reason("intake_urls"); + return Err(ConfigError::UnsupportedField("intake_urls".to_string())); } Ok(()) @@ -214,7 +240,7 @@ pub fn get_config(config_directory: &Path) -> Result { // Get YAML nested fields let yaml_figment = Figment::from(Yaml::file(&path)); - fallback(&figment)?; + fallback(&figment, &yaml_figment)?; let (mut config, yaml_config): (Config, YamlConfig) = match (figment.extract(), yaml_figment.extract()) { @@ -362,6 +388,42 @@ pub mod tests { }); } + #[test] + fn test_fallback_on_intake_urls() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_APM_DD_URL", "some_url"); + + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("intake_urls".to_string()) + ); + Ok(()) + }); + } + + #[test] + fn test_fallback_on_intake_urls_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + apm_config: + apm_dd_url: some_url + ", + )?; + + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("intake_urls".to_string()) + ); + Ok(()) + }); + } + #[test] fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { From c01c4332f6e61e5320c6e93689b23dd0cb6a4680 Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Thu, 19 Dec 2024 16:41:50 -0500 Subject: [PATCH 044/112] set dogstatsd timeout (#497) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * set dogstatsd timeout * add todo for other edge case * add comment on jitter. Likely not required for lambda * fmt * update license * update sha for dogstatsd --------- Co-authored-by: jordan gonzález <30836115+duncanista@users.noreply.github.com> --- mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod.rs b/mod.rs index 02d19bb1..e7775e26 100644 --- a/mod.rs +++ b/mod.rs @@ -93,7 +93,7 @@ pub struct Config { pub logs_config_processing_rules: Option>, pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, - pub flush_timeout: u64, + pub flush_timeout: u64, //TODO go agent adds jitter too pub https_proxy: Option, pub capture_lambda_payload: bool, pub capture_lambda_payload_max_depth: u32, From 0d8cc2c08fca6446a60c068504dde03fcb19cd70 Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:30:28 -0500 Subject: [PATCH 045/112] fix: set right domain and arn by region on secrets manager (#511) * check whether the region is in China and use the appropriated domain * correct arn for lambda in chinese regions * fix: typo in china arn * fix: reuse function to detect right aws partition and support gov too * nest and rearrange imports * fix imports again --- mod.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mod.rs b/mod.rs index e7775e26..598ba1e2 100644 --- a/mod.rs +++ b/mod.rs @@ -311,6 +311,15 @@ pub struct AwsConfig { pub sandbox_init_time: Instant, } +#[must_use] +pub fn get_aws_partition_by_region(region: &str) -> String { + match region { + r if r.starts_with("us-gov-") => "aws-us-gov".to_string(), + r if r.starts_with("cn-") => "aws-cn".to_string(), + _ => "aws".to_string(), + } +} + #[cfg(test)] pub mod tests { use super::*; From 19ae289d0691b9f97d3d73b19d44b4624d5c7f19 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 23 Jan 2025 14:19:21 -0500 Subject: [PATCH 046/112] fix: Honor noproxy and skip proxying if ddsite is in the noproxy list (#520) * fix: Honor noproxy and skip proxying if ddsite is in the noproxy list * feat: specs * feat: Oneline check, add comment --- mod.rs | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/mod.rs b/mod.rs index 598ba1e2..1c34cd3f 100644 --- a/mod.rs +++ b/mod.rs @@ -14,6 +14,7 @@ use figment::{providers::Env, Figment}; use serde::{Deserialize, Deserializer}; use serde_json::Value; use trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}; +use tracing::debug; use crate::config::{ flush_strategy::FlushStrategy, @@ -260,7 +261,13 @@ pub fn get_config(config_directory: &Path) -> Result { if config.site.is_empty() { config.site = "datadoghq.com".to_string(); } - + // TODO(astuyve) + // Bit of a hack as we're not checking individual privatelink setups + // potentially a user could use the proxy for logs and privatelink for APM + if std::env::var("NO_PROXY").map_or(false, |no_proxy| no_proxy.contains(&config.site)) { + debug!("NO_PROXY contains DD_SITE, disabling proxy"); + config.https_proxy = None; + } // Merge YAML nested fields // // Set logs_config_processing_rules if not defined in env @@ -522,6 +529,33 @@ pub mod tests { }); } + #[test] + fn test_proxy_config() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_PROXY_HTTPS", "my-proxy:3128"); + let config = get_config(Path::new("")).expect("should parse config"); + assert_eq!(config.https_proxy, Some("my-proxy:3128".to_string())); + Ok(()) + }); + } + + #[test] + fn test_noproxy_config() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SITE", "datadoghq.eu"); + jail.set_env("DD_PROXY_HTTPS", "my-proxy:3128"); + jail.set_env( + "NO_PROXY", + "127.0.0.1,localhost,172.16.0.0/12,us-east-1.amazonaws.com,datadoghq.eu", + ); + let config = get_config(Path::new("")).expect("should parse noproxy"); + assert_eq!(config.https_proxy, None); + Ok(()) + }); + } + #[test] fn test_parse_flush_strategy_end() { figment::Jail::expect_with(|jail| { From 88657fc617f7126337142a38066b10d7aa567231 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Fri, 24 Jan 2025 10:35:52 -0500 Subject: [PATCH 047/112] Support proxy yaml config (#523) * fix: Honor noproxy and skip proxying if ddsite is in the noproxy list * feat: specs * feat: yaml proxy had a different format * feat: Oneline check, add comment * feat: Support nonstandard proxy config * feat: specs * fix: bad merge whoops --- mod.rs | 85 +++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 72 insertions(+), 13 deletions(-) diff --git a/mod.rs b/mod.rs index 1c34cd3f..533ec603 100644 --- a/mod.rs +++ b/mod.rs @@ -14,7 +14,6 @@ use figment::{providers::Env, Figment}; use serde::{Deserialize, Deserializer}; use serde_json::Value; use trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}; -use tracing::debug; use crate::config::{ flush_strategy::FlushStrategy, @@ -73,6 +72,15 @@ pub struct YamlLogsConfig { #[allow(clippy::module_name_repetitions)] pub struct YamlConfig { pub logs_config: YamlLogsConfig, + pub proxy: YamlProxyConfig, +} + +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct YamlProxyConfig { + pub https: Option, + pub no_proxy: Option>, } #[derive(Debug, PartialEq, Deserialize, Clone)] @@ -251,23 +259,33 @@ pub fn get_config(config_directory: &Path) -> Result { return Err(ConfigError::ParseError(err.to_string())); } }; - - // Prefer DD_PROXY_HTTPS over HTTPS_PROXY - // No else needed as HTTPS_PROXY is handled by reqwest and built into trace client - if let Ok(https_proxy) = std::env::var("DD_PROXY_HTTPS") { - config.https_proxy = Some(https_proxy); - } // Set site if empty if config.site.is_empty() { config.site = "datadoghq.com".to_string(); } - // TODO(astuyve) - // Bit of a hack as we're not checking individual privatelink setups - // potentially a user could use the proxy for logs and privatelink for APM - if std::env::var("NO_PROXY").map_or(false, |no_proxy| no_proxy.contains(&config.site)) { - debug!("NO_PROXY contains DD_SITE, disabling proxy"); - config.https_proxy = None; + + // NOTE: Must happen after config.site is set + // Prefer DD_PROXY_HTTPS over HTTPS_PROXY + // No else needed as HTTPS_PROXY is handled by reqwest and built into trace client + if let Ok(https_proxy) = std::env::var("DD_PROXY_HTTPS").or_else(|_| { + yaml_config + .proxy + .https + .clone() + .ok_or(std::env::VarError::NotPresent) + }) { + if std::env::var("NO_PROXY").map_or(false, |no_proxy| no_proxy.contains(&config.site)) + || yaml_config + .proxy + .no_proxy + .map_or(false, |no_proxy| no_proxy.contains(&config.site)) + { + config.https_proxy = None; + } else { + config.https_proxy = Some(https_proxy); + } } + // Merge YAML nested fields // // Set logs_config_processing_rules if not defined in env @@ -556,6 +574,47 @@ pub mod tests { }); } + #[test] + fn test_proxy_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + proxy: + https: my-proxy:3128 + ", + )?; + + let config = get_config(Path::new("")).expect("should parse weird proxy config"); + assert_eq!(config.https_proxy, Some("my-proxy:3128".to_string())); + Ok(()) + }); + } + + #[test] + fn test_no_proxy_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + proxy: + https: my-proxy:3128 + no_proxy: + - datadoghq.com + ", + )?; + + let config = get_config(Path::new("")).expect("should parse weird proxy config"); + assert_eq!(config.https_proxy, None); + // Assertion to ensure config.site runs before proxy + // because we chenck that noproxy contains the site + assert_eq!(config.site, "datadoghq.com"); + Ok(()) + }); + } + #[test] fn test_parse_flush_strategy_end() { figment::Jail::expect_with(|jail| { From f17c088720ab3cbc8d7027e81e7567c5873dda8d Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 29 Jan 2025 20:18:36 -0500 Subject: [PATCH 048/112] feat: Support snapstart's vended credentials (#532) * feat: Support snapstart's vended credentials * feat: Add snapstart events * fix: specs * feat: Mutable config as we consume it entirely by the secrets module. * fix: needless borrow --- mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mod.rs b/mod.rs index 533ec603..e46f059d 100644 --- a/mod.rs +++ b/mod.rs @@ -334,6 +334,8 @@ pub struct AwsConfig { pub aws_session_token: String, pub function_name: String, pub sandbox_init_time: Instant, + pub aws_container_credentials_full_uri: String, + pub aws_container_authorization_token: String, } #[must_use] From c487b96f28327ee37a08d2cf1644ba8ceaf21077 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 20 Feb 2025 13:07:02 -0500 Subject: [PATCH 049/112] feat: add zstd and compress (#558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add zstd and compress * hack: skip clippy for a sec * feat: Honor logs config settings. * fix: dont set zstd header unless we compress * fmt * clippy * fmt * fix: ints * licenses * remove debug code * wtf clippy and fmt, pick one --------- Co-authored-by: jordan gonzález <30836115+duncanista@users.noreply.github.com> --- mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mod.rs b/mod.rs index e46f059d..603e4e9e 100644 --- a/mod.rs +++ b/mod.rs @@ -100,6 +100,8 @@ pub struct Config { pub log_level: LogLevel, #[serde(deserialize_with = "deserialize_processing_rules")] pub logs_config_processing_rules: Option>, + pub logs_config_use_compression: bool, + pub logs_config_compression_level: i32, pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, pub flush_timeout: u64, //TODO go agent adds jitter too @@ -136,6 +138,8 @@ impl Default for Config { // Logs log_level: LogLevel::default(), logs_config_processing_rules: None, + logs_config_use_compression: true, + logs_config_compression_level: 6, // Metrics enhanced_metrics: true, https_proxy: None, From 39016fdd484fa5026e2ce9850ab8d7cd35fe8c0d Mon Sep 17 00:00:00 2001 From: alexgallotta <5581237+alexgallotta@users.noreply.github.com> Date: Thu, 20 Feb 2025 14:02:47 -0500 Subject: [PATCH 050/112] Svls 6036 respect timeouts (#537) * log shipping times * set flush timeout for traces * remove retries * fix conflicts * address comments --- mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mod.rs b/mod.rs index 603e4e9e..85a71b14 100644 --- a/mod.rs +++ b/mod.rs @@ -104,6 +104,7 @@ pub struct Config { pub logs_config_compression_level: i32, pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, + //flush timeout in seconds pub flush_timeout: u64, //TODO go agent adds jitter too pub https_proxy: Option, pub capture_lambda_payload: bool, From 29bd463ad76621f377e33f8f967f7b845d26fd72 Mon Sep 17 00:00:00 2001 From: Nicholas Hulston Date: Wed, 26 Feb 2025 14:21:12 -0500 Subject: [PATCH 051/112] Fallback on gov regions (#550) --- mod.rs | 85 ++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/mod.rs b/mod.rs index 85a71b14..76ce1da9 100644 --- a/mod.rs +++ b/mod.rs @@ -171,7 +171,7 @@ fn log_fallback_reason(reason: &str) { println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); } -fn fallback(figment: &Figment, yaml_figment: &Figment) -> Result<(), ConfigError> { +fn fallback(figment: &Figment, yaml_figment: &Figment, region: &str) -> Result<(), ConfigError> { let (config, yaml_config): (FallbackConfig, FallbackYamlConfig) = match (figment.extract(), yaml_figment.extract()) { (Ok(env_config), Ok(yaml_config)) => (env_config, yaml_config), @@ -237,11 +237,17 @@ fn fallback(figment: &Figment, yaml_figment: &Figment) -> Result<(), ConfigError return Err(ConfigError::UnsupportedField("intake_urls".to_string())); } + // Govcloud Regions + if region.starts_with("us-gov-") { + log_fallback_reason("gov_region"); + return Err(ConfigError::UnsupportedField("gov_region".to_string())); + } + Ok(()) } #[allow(clippy::module_name_repetitions)] -pub fn get_config(config_directory: &Path) -> Result { +pub fn get_config(config_directory: &Path, region: &str) -> Result { let path = config_directory.join("datadog.yaml"); // Get default config fields (and ENV specific ones) @@ -254,7 +260,7 @@ pub fn get_config(config_directory: &Path) -> Result { // Get YAML nested fields let yaml_figment = Figment::from(Yaml::file(&path)); - fallback(&figment, &yaml_figment)?; + fallback(&figment, &yaml_figment, region)?; let (mut config, yaml_config): (Config, YamlConfig) = match (figment.extract(), yaml_figment.extract()) { @@ -359,12 +365,15 @@ pub mod tests { use crate::config::flush_strategy::PeriodicStrategy; use crate::config::processing_rule; + const MOCK_REGION: &str = "us-east-1"; + #[test] fn test_reject_on_opted_out() { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_EXTENSION_VERSION", "compatibility"); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + let config = + get_config(Path::new(""), MOCK_REGION).expect_err("should reject unknown fields"); assert_eq!( config, ConfigError::UnsupportedField("extension_version".to_string()) @@ -372,6 +381,16 @@ pub mod tests { Ok(()) }); } + #[test] + fn test_reject_on_gov_region() { + let mock_gov_region = "us-gov-east-1"; + let config = + get_config(Path::new(""), mock_gov_region).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("gov_region".to_string()) + ); + } #[test] fn test_fallback_on_otel() { @@ -382,7 +401,8 @@ pub mod tests { "localhost:4138", ); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + let config = + get_config(Path::new(""), MOCK_REGION).expect_err("should reject unknown fields"); assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); Ok(()) }); @@ -403,7 +423,8 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + let config = + get_config(Path::new(""), MOCK_REGION).expect_err("should reject unknown fields"); assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); Ok(()) }); @@ -423,7 +444,8 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + let config = + get_config(Path::new(""), MOCK_REGION).expect_err("should reject unknown fields"); assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); Ok(()) }); @@ -435,7 +457,8 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_APM_DD_URL", "some_url"); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + let config = + get_config(Path::new(""), MOCK_REGION).expect_err("should reject unknown fields"); assert_eq!( config, ConfigError::UnsupportedField("intake_urls".to_string()) @@ -456,7 +479,8 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + let config = + get_config(Path::new(""), MOCK_REGION).expect_err("should reject unknown fields"); assert_eq!( config, ConfigError::UnsupportedField("intake_urls".to_string()) @@ -471,7 +495,8 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + let config = + get_config(Path::new(""), MOCK_REGION).expect_err("should reject unknown fields"); assert_eq!( config, ConfigError::UnsupportedField("appsec_enabled".to_string()) @@ -491,7 +516,7 @@ pub mod tests { ", )?; jail.set_env("DD_SITE", "datad0g.com"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!(config.site, "datad0g.com"); Ok(()) }); @@ -506,7 +531,7 @@ pub mod tests { r" ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!(config.site, "datadoghq.com"); Ok(()) }); @@ -517,7 +542,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SITE", "datadoghq.eu"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!(config.site, "datadoghq.eu"); Ok(()) }); @@ -528,7 +553,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_LOG_LEVEL", "TRACE"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!(config.log_level, LogLevel::Trace); Ok(()) }); @@ -538,7 +563,7 @@ pub mod tests { fn test_parse_default() { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!( config, Config { @@ -559,7 +584,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_PROXY_HTTPS", "my-proxy:3128"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!(config.https_proxy, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -575,7 +600,7 @@ pub mod tests { "NO_PROXY", "127.0.0.1,localhost,172.16.0.0/12,us-east-1.amazonaws.com,datadoghq.eu", ); - let config = get_config(Path::new("")).expect("should parse noproxy"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse noproxy"); assert_eq!(config.https_proxy, None); Ok(()) }); @@ -593,7 +618,8 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect("should parse weird proxy config"); + let config = + get_config(Path::new(""), MOCK_REGION).expect("should parse weird proxy config"); assert_eq!(config.https_proxy, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -613,7 +639,8 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect("should parse weird proxy config"); + let config = + get_config(Path::new(""), MOCK_REGION).expect("should parse weird proxy config"); assert_eq!(config.https_proxy, None); // Assertion to ensure config.site runs before proxy // because we chenck that noproxy contains the site @@ -627,7 +654,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::End); Ok(()) }); @@ -638,7 +665,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!( config.serverless_flush_strategy, FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }) @@ -652,7 +679,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -666,7 +693,7 @@ pub mod tests { "DD_SERVERLESS_FLUSH_STRATEGY", "periodically,invalid_interval", ); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -677,7 +704,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_VERSION", "123"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!(config.version.expect("failed to parse DD_VERSION"), "123"); Ok(()) }); @@ -702,7 +729,7 @@ pub mod tests { pattern: exclude-me-yaml ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -731,7 +758,7 @@ pub mod tests { pattern: exclude ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -754,7 +781,7 @@ pub mod tests { "datadog,tracecontext,b3,b3multi", ); jail.set_env("DD_EXTENSION_VERSION", "next"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); let expected_styles = vec![ TracePropagationStyle::Datadog, @@ -773,7 +800,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "datadog"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!( config.trace_propagation_style, @@ -798,7 +825,7 @@ pub mod tests { "DD_APM_REPLACE_TAGS", r#"[{"name":"resource.name","pattern":"(.*)/(foo[:%].+)","repl":"$1/{foo}"}]"#, ); - let config = get_config(Path::new("")); + let config = get_config(Path::new(""), MOCK_REGION); assert!(config.is_ok()); Ok(()) }); From a7b45a1ae0dbfe6099aed240734dbd5d3b2f8c15 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 5 Mar 2025 09:51:41 -0500 Subject: [PATCH 052/112] Aj/support pci and custom endpoints (#585) * feat: logs_config_logs_dd_url * feat: apm pci endpoints * feat: metrics * feat: support metrics using dogstatsd methods * fix: use the right var * tests: use server url override * feat: refactor into flusher method * feat: clippy --- mod.rs | 137 +++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 95 insertions(+), 42 deletions(-) diff --git a/mod.rs b/mod.rs index 76ce1da9..4cd0ebe7 100644 --- a/mod.rs +++ b/mod.rs @@ -4,6 +4,7 @@ pub mod processing_rule; pub mod service_mapping; pub mod trace_propagation_style; +use datadog_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; use std::collections::HashMap; use std::path::Path; use std::time::Instant; @@ -38,12 +39,6 @@ pub struct FallbackConfig { trace_otel_enabled: bool, otlp_config_receiver_protocols_http_endpoint: Option, otlp_config_receiver_protocols_grpc_endpoint: Option, - // intake urls - url: Option, - dd_url: Option, - logs_config_logs_dd_url: Option, - // APM, as opposed to logs, does not use the `apm_config` prefix for env vars - apm_dd_url: Option, } /// `FallbackYamlConfig` is a struct that represents fields in `datadog.yaml` not yet supported in the extension yet. @@ -52,8 +47,6 @@ pub struct FallbackConfig { #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct FallbackYamlConfig { - logs_config: Option, - apm_config: Option, otlp_config: Option, } #[derive(Debug, PartialEq, Deserialize, Clone, Default)] @@ -102,6 +95,7 @@ pub struct Config { pub logs_config_processing_rules: Option>, pub logs_config_use_compression: bool, pub logs_config_compression_level: i32, + pub logs_config_logs_dd_url: String, pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, //flush timeout in seconds @@ -119,6 +113,10 @@ pub struct Config { pub trace_propagation_style_extract: Vec, pub trace_propagation_extract_first: bool, pub trace_propagation_http_baggage_enabled: bool, + pub apm_config_apm_dd_url: String, + // Metrics overrides + pub dd_url: String, + pub url: String, } impl Default for Config { @@ -141,6 +139,7 @@ impl Default for Config { logs_config_processing_rules: None, logs_config_use_compression: true, logs_config_compression_level: 6, + logs_config_logs_dd_url: String::default(), // Metrics enhanced_metrics: true, https_proxy: None, @@ -156,6 +155,9 @@ impl Default for Config { trace_propagation_style_extract: vec![], trace_propagation_extract_first: false, trace_propagation_http_baggage_enabled: false, + apm_config_apm_dd_url: String::default(), + dd_url: String::default(), + url: String::default(), } } } @@ -221,22 +223,6 @@ fn fallback(figment: &Figment, yaml_figment: &Figment, region: &str) -> Result<( return Err(ConfigError::UnsupportedField("otel".to_string())); } - // Intake URLs - if config.url.is_some() - || config.dd_url.is_some() - || config.logs_config_logs_dd_url.is_some() - || config.apm_dd_url.is_some() - || yaml_config - .logs_config - .is_some_and(|c| c.get("logs_dd_url").is_some()) - || yaml_config - .apm_config - .is_some_and(|c| c.get("apm_dd_url").is_some()) - { - log_fallback_reason("intake_urls"); - return Err(ConfigError::UnsupportedField("intake_urls".to_string())); - } - // Govcloud Regions if region.starts_with("us-gov-") { log_fallback_reason("gov_region"); @@ -314,10 +300,27 @@ pub fn get_config(config_directory: &Path, region: &str) -> Result String { + format!("https://http-intake.logs.{site}") +} + fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -452,43 +455,90 @@ pub mod tests { } #[test] - fn test_fallback_on_intake_urls() { + fn test_default_logs_intake_url() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_APM_DD_URL", "some_url"); - let config = - get_config(Path::new(""), MOCK_REGION).expect_err("should reject unknown fields"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!( - config, - ConfigError::UnsupportedField("intake_urls".to_string()) + config.logs_config_logs_dd_url, + "https://http-intake.logs.datadoghq.com".to_string() ); Ok(()) }); } #[test] - fn test_fallback_on_intake_urls_yaml() { + fn test_support_pci_logs_intake_url() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.create_file( - "datadog.yaml", - r" - apm_config: - apm_dd_url: some_url - ", - )?; + jail.set_env( + "DD_LOGS_CONFIG_LOGS_DD_URL", + "agent-http-intake-pci.logs.datadoghq.com:443", + ); - let config = - get_config(Path::new(""), MOCK_REGION).expect_err("should reject unknown fields"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!( - config, - ConfigError::UnsupportedField("intake_urls".to_string()) + config.logs_config_logs_dd_url, + "agent-http-intake-pci.logs.datadoghq.com:443".to_string() + ); + Ok(()) + }); + } + + #[test] + fn test_support_pci_traces_intake_url() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env( + "DD_APM_CONFIG_APM_DD_URL", + "https://trace-pci.agent.datadoghq.com", + ); + + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); + assert_eq!( + config.apm_config_apm_dd_url, + "https://trace-pci.agent.datadoghq.com/api/v0.2/traces".to_string() ); Ok(()) }); } + #[test] + fn test_support_dd_dd_url() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_DD_URL", "custom_proxy:3128"); + + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); + assert_eq!(config.dd_url, "custom_proxy:3128".to_string()); + Ok(()) + }); + } + + #[test] + fn test_support_dd_url() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_URL", "custom_proxy:3128"); + + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); + assert_eq!(config.url, "custom_proxy:3128".to_string()); + Ok(()) + }); + } + + #[test] + fn test_dd_dd_url_default() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); + assert_eq!(config.dd_url, String::new()); + Ok(()) + }); + } + #[test] fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { @@ -572,6 +622,9 @@ pub mod tests { TracePropagationStyle::Datadog, TracePropagationStyle::TraceContext ], + logs_config_logs_dd_url: "https://http-intake.logs.datadoghq.com".to_string(), + apm_config_apm_dd_url: trace_intake_url("datadoghq.com").to_string(), + dd_url: String::new(), // We add the prefix in main.rs ..Config::default() } ); From d65b65827d14789c769a1f84dbf730a8813b57d9 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 11 Mar 2025 21:33:13 -0400 Subject: [PATCH 053/112] Aj/yaml apm replace tags (#602) * feat: yaml APM replace tags rule parsing * feat: Custom deserializer for replace tags. yaml -> JSON so we can rely on the same method because ReplaceRule is totally private * remove aj * feat: merge w/ libdatadog main * feat: Parse http obfuscation config from yaml * feat: licenses --- apm_replace_rule.rs | 62 ++++++++++++++++++++++++++ mod.rs | 105 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 apm_replace_rule.rs diff --git a/apm_replace_rule.rs b/apm_replace_rule.rs new file mode 100644 index 00000000..843fd07a --- /dev/null +++ b/apm_replace_rule.rs @@ -0,0 +1,62 @@ +use datadog_trace_obfuscation::replacer::{parse_rules_from_string, ReplaceRule}; +use serde::de::{Deserializer, SeqAccess, Visitor}; +use serde::{Deserialize, Serialize}; +use serde_json; +use std::fmt; + +#[derive(Deserialize, Serialize)] +struct ReplaceRuleYaml { + name: String, + pattern: String, + repl: String, +} + +struct StringOrReplaceRulesVisitor; + +impl<'de> Visitor<'de> for StringOrReplaceRulesVisitor { + type Value = String; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON string or YAML sequence of replace rules") + } + + // Handle existing JSON strings + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + // Validate it's at least valid JSON + let _: serde_json::Value = + serde_json::from_str(value).map_err(|_| E::custom("Expected valid JSON string"))?; + Ok(value.to_string()) + } + + // Convert YAML sequences to JSON strings + fn visit_seq(self, mut seq: A) -> Result + where + A: SeqAccess<'de>, + { + let mut rules = Vec::new(); + while let Some(rule) = seq.next_element::()? { + rules.push(rule); + } + // Serialize to JSON string for compatibility with parse_rules_from_string + serde_json::to_string(&rules).map_err(|e| { + serde::de::Error::custom(format!("Failed to serialize rules to JSON: {e}")) + }) + } +} + +pub fn deserialize_apm_replace_rules<'de, D>( + deserializer: D, +) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let json_string = deserializer.deserialize_any(StringOrReplaceRulesVisitor)?; + + let rules = parse_rules_from_string(&json_string) + .map_err(|e| serde::de::Error::custom(format!("Parse error: {e}")))?; + + Ok(Some(rules)) +} diff --git a/mod.rs b/mod.rs index 4cd0ebe7..ffb00372 100644 --- a/mod.rs +++ b/mod.rs @@ -1,9 +1,11 @@ +pub mod apm_replace_rule; pub mod flush_strategy; pub mod log_level; pub mod processing_rule; pub mod service_mapping; pub mod trace_propagation_style; +use datadog_trace_obfuscation::replacer::ReplaceRule; use datadog_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; use std::collections::HashMap; use std::path::Path; @@ -17,6 +19,7 @@ use serde_json::Value; use trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}; use crate::config::{ + apm_replace_rule::deserialize_apm_replace_rules, flush_strategy::FlushStrategy, log_level::{deserialize_log_level, LogLevel}, processing_rule::{deserialize_processing_rules, ProcessingRule}, @@ -57,6 +60,30 @@ pub struct YamlLogsConfig { processing_rules: Option>, } +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct YamlApmConfig { + #[serde(deserialize_with = "deserialize_apm_replace_rules")] + replace_tags: Option>, + obfuscation: Option, +} + +#[derive(Debug, PartialEq, Deserialize, Clone, Copy, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct ApmObfuscation { + http: ApmHttpObfuscation, +} + +#[derive(Debug, PartialEq, Deserialize, Clone, Copy, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct ApmHttpObfuscation { + remove_query_string: bool, + remove_paths_with_digits: bool, +} + /// `YamlConfig` is a struct that represents some of the fields in the datadog.yaml file. /// /// It is used to deserialize the datadog.yaml file into a struct that can be merged with the Config struct. @@ -65,6 +92,7 @@ pub struct YamlLogsConfig { #[allow(clippy::module_name_repetitions)] pub struct YamlConfig { pub logs_config: YamlLogsConfig, + pub apm_config: YamlApmConfig, pub proxy: YamlProxyConfig, } @@ -114,6 +142,10 @@ pub struct Config { pub trace_propagation_extract_first: bool, pub trace_propagation_http_baggage_enabled: bool, pub apm_config_apm_dd_url: String, + #[serde(deserialize_with = "deserialize_apm_replace_rules")] + pub apm_config_replace_tags: Option>, + pub apm_config_obfuscation_http_remove_query_string: bool, + pub apm_config_obfuscation_http_remove_paths_with_digits: bool, // Metrics overrides pub dd_url: String, pub url: String, @@ -156,6 +188,9 @@ impl Default for Config { trace_propagation_extract_first: false, trace_propagation_http_baggage_enabled: false, apm_config_apm_dd_url: String::default(), + apm_config_replace_tags: None, + apm_config_obfuscation_http_remove_query_string: false, + apm_config_obfuscation_http_remove_paths_with_digits: false, dd_url: String::default(), url: String::default(), } @@ -311,6 +346,26 @@ pub fn get_config(config_directory: &Path, region: &str) -> Result String { #[cfg(test)] pub mod tests { + use datadog_trace_obfuscation::replacer::parse_rules_from_string; + use super::*; use crate::config::flush_strategy::PeriodicStrategy; @@ -825,6 +882,54 @@ pub mod tests { }); } + #[test] + fn test_parse_apm_replace_tags_from_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + site: datadoghq.com + apm_config: + replace_tags: + - name: '*' + pattern: 'foo' + repl: 'REDACTED' + ", + )?; + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); + let rule = parse_rules_from_string( + r#"[ + {"name": "*", "pattern": "foo", "repl": "REDACTED"} + ]"#, + ) + .expect("can't parse rules"); + assert_eq!(config.apm_config_replace_tags, Some(rule),); + Ok(()) + }); + } + + #[test] + fn test_parse_apm_http_obfuscation_from_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + site: datadoghq.com + apm_config: + obfuscation: + http: + remove_query_string: true + remove_paths_with_digits: true + ", + )?; + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); + assert!(config.apm_config_obfuscation_http_remove_query_string,); + assert!(config.apm_config_obfuscation_http_remove_paths_with_digits,); + Ok(()) + }); + } #[test] fn test_parse_trace_propagation_style() { figment::Jail::expect_with(|jail| { From 49a4f0a31f7ee0e64dbcdf19badc0c4df7355b0d Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 13 Mar 2025 13:59:53 -0400 Subject: [PATCH 054/112] feat: parse env and service as strings or ints (#608) * feat: parse env and service as strings or ints * feat: add service test * fmt --- mod.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mod.rs b/mod.rs index ffb00372..fdcfc438 100644 --- a/mod.rs +++ b/mod.rs @@ -112,7 +112,9 @@ pub struct Config { pub api_key: String, pub api_key_secret_arn: String, pub kms_api_key: String, + #[serde(deserialize_with = "deserialize_string_or_int")] pub env: Option, + #[serde(deserialize_with = "deserialize_string_or_int")] pub service: Option, #[serde(deserialize_with = "deserialize_string_or_int")] pub version: Option, @@ -810,12 +812,19 @@ pub mod tests { } #[test] - fn parse_dd_version() { + fn parse_number_or_string_env_vars() { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_VERSION", "123"); + jail.set_env("DD_ENV", "123456890"); + jail.set_env("DD_SERVICE", "123456"); let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!(config.version.expect("failed to parse DD_VERSION"), "123"); + assert_eq!(config.env.expect("failed to parse DD_ENV"), "123456890"); + assert_eq!( + config.service.expect("failed to parse DD_SERVICE"), + "123456" + ); Ok(()) }); } From 8ea81d9908d4e5c46ee2a579f38a7e5ab85a304c Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 26 Mar 2025 13:06:58 -0400 Subject: [PATCH 055/112] Add DSM and Profiling endpoints (#622) - **feat: Support DSM proxy endpoint** - **feat: profiling support** - **feat: add additional tags** --- mod.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mod.rs b/mod.rs index fdcfc438..53400ec2 100644 --- a/mod.rs +++ b/mod.rs @@ -37,7 +37,6 @@ pub struct FallbackConfig { extension_version: Option, serverless_appsec_enabled: bool, appsec_enabled: bool, - profiling_enabled: bool, // otel trace_otel_enabled: bool, otlp_config_receiver_protocols_http_endpoint: Option, @@ -239,13 +238,6 @@ fn fallback(figment: &Figment, yaml_figment: &Figment, region: &str) -> Result<( return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); } - if config.profiling_enabled { - log_fallback_reason("profiling_enabled"); - return Err(ConfigError::UnsupportedField( - "profiling_enabled".to_string(), - )); - } - // OTEL env if config.trace_otel_enabled || config From 9947edaf02dad476655835e9d4a76f6b25e5370e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 16 Apr 2025 16:22:00 -0400 Subject: [PATCH 056/112] chore(config): parse config only twice (#651) # What? Removes `FallbackConfig` and `FallbackYamlConfig` in favor of the existing configurations. # How? 1. Using only the known places where we are going to fallback from the available configs. 2. Moved environment variables and yaml config to its own file for readability. # Notes - Added fallbacks for OTLP (in preparation for that PR, allowed some fields to not fallback). --- env.rs | 209 +++++++++++++++++++++++++++++++ mod.rs | 376 ++++++++++++++++++-------------------------------------- yaml.rs | 120 ++++++++++++++++++ 3 files changed, 449 insertions(+), 256 deletions(-) create mode 100644 env.rs create mode 100644 yaml.rs diff --git a/env.rs b/env.rs new file mode 100644 index 00000000..8ad7f437 --- /dev/null +++ b/env.rs @@ -0,0 +1,209 @@ +use serde::{Deserialize, Deserializer}; +use std::collections::HashMap; +use std::vec; + +use datadog_trace_obfuscation::replacer::ReplaceRule; +use serde_json::Value; + +use crate::config::{ + apm_replace_rule::deserialize_apm_replace_rules, + flush_strategy::FlushStrategy, + log_level::{deserialize_log_level, LogLevel}, + processing_rule::{deserialize_processing_rules, ProcessingRule}, + service_mapping::deserialize_service_mapping, + trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}, +}; + +#[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(default)] +#[allow(clippy::struct_excessive_bools)] +pub struct Config { + pub site: String, + pub api_key: String, + pub api_key_secret_arn: String, + pub kms_api_key: String, + #[serde(deserialize_with = "deserialize_string_or_int")] + pub env: Option, + #[serde(deserialize_with = "deserialize_string_or_int")] + pub service: Option, + #[serde(deserialize_with = "deserialize_string_or_int")] + pub version: Option, + pub tags: Option, + #[serde(deserialize_with = "deserialize_log_level")] + pub log_level: LogLevel, + #[serde(deserialize_with = "deserialize_processing_rules")] + pub logs_config_processing_rules: Option>, + pub logs_config_use_compression: bool, + pub logs_config_compression_level: i32, + pub logs_config_logs_dd_url: String, + pub serverless_flush_strategy: FlushStrategy, + pub enhanced_metrics: bool, + /// Flush timeout in seconds + pub flush_timeout: u64, //TODO go agent adds jitter too + pub https_proxy: Option, + pub capture_lambda_payload: bool, + pub capture_lambda_payload_max_depth: u32, + #[serde(deserialize_with = "deserialize_service_mapping")] + pub service_mapping: HashMap, + pub serverless_logs_enabled: bool, + // Trace Propagation + #[serde(deserialize_with = "deserialize_trace_propagation_style")] + pub trace_propagation_style: Vec, + #[serde(deserialize_with = "deserialize_trace_propagation_style")] + pub trace_propagation_style_extract: Vec, + pub trace_propagation_extract_first: bool, + pub trace_propagation_http_baggage_enabled: bool, + // APM + pub apm_config_apm_dd_url: String, + #[serde(deserialize_with = "deserialize_apm_replace_rules")] + pub apm_config_replace_tags: Option>, + pub apm_config_obfuscation_http_remove_query_string: bool, + pub apm_config_obfuscation_http_remove_paths_with_digits: bool, + pub apm_features: Vec, + pub apm_ignore_resources: Vec, + // Metrics overrides + pub dd_url: String, + pub url: String, + // OTLP + // + // - Traces + pub otlp_config_traces_enabled: bool, + pub otlp_config_traces_span_name_as_resource_name: bool, + pub otlp_config_traces_span_name_remappings: HashMap, + pub otlp_config_ignore_missing_datadog_fields: bool, + // - Receiver / HTTP + pub otlp_config_receiver_protocols_http_endpoint: Option, + // + // + // Fallback Config + pub extension_version: Option, + // AppSec + pub serverless_appsec_enabled: bool, + pub appsec_enabled: bool, + // OTLP + // + // - Receiver / GRPC + pub otlp_config_receiver_protocols_grpc_endpoint: Option, + pub otlp_config_receiver_protocols_grpc_transport: Option, + pub otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib: Option, + // - Metrics + pub otlp_config_metrics_enabled: bool, + pub otlp_config_metrics_resource_attributes_as_tags: bool, + pub otlp_config_metrics_instrumentation_scope_metadata_as_tags: bool, + pub otlp_config_metrics_tag_cardinality: Option, + pub otlp_config_metrics_delta_ttl: Option, + pub otlp_config_metrics_histograms_mode: Option, + pub otlp_config_metrics_histograms_send_count_sum_metrics: bool, + pub otlp_config_metrics_histograms_send_aggregation_metrics: bool, + pub otlp_config_metrics_sums_cumulative_monotonic_mode: Option, + pub otlp_config_metrics_sums_initial_cumulativ_monotonic_value: Option, + pub otlp_config_metrics_summaries_mode: Option, + // - Traces + pub otlp_config_traces_probabilistic_sampler_sampling_percentage: Option, + // - Logs + pub otlp_config_logs_enabled: bool, +} + +impl Default for Config { + fn default() -> Self { + Config { + // General + site: String::default(), + api_key: String::default(), + api_key_secret_arn: String::default(), + kms_api_key: String::default(), + serverless_flush_strategy: FlushStrategy::Default, + flush_timeout: 5, + // Unified Tagging + env: None, + service: None, + version: None, + tags: None, + // Logs + log_level: LogLevel::default(), + logs_config_processing_rules: None, + logs_config_use_compression: true, + logs_config_compression_level: 6, + logs_config_logs_dd_url: String::default(), + // Metrics + enhanced_metrics: true, + https_proxy: None, + capture_lambda_payload: false, + capture_lambda_payload_max_depth: 10, + service_mapping: HashMap::new(), + serverless_logs_enabled: true, + // Trace Propagation + trace_propagation_style: vec![ + TracePropagationStyle::Datadog, + TracePropagationStyle::TraceContext, + ], + trace_propagation_style_extract: vec![], + trace_propagation_extract_first: false, + trace_propagation_http_baggage_enabled: false, + // APM + apm_config_apm_dd_url: String::default(), + apm_config_replace_tags: None, + apm_config_obfuscation_http_remove_query_string: false, + apm_config_obfuscation_http_remove_paths_with_digits: false, + apm_features: vec![], + apm_ignore_resources: vec![], + dd_url: String::default(), + url: String::default(), + // OTLP + // + // - Receiver + otlp_config_receiver_protocols_http_endpoint: None, + // - Traces + otlp_config_traces_enabled: true, + otlp_config_ignore_missing_datadog_fields: false, + otlp_config_traces_span_name_as_resource_name: false, + otlp_config_traces_span_name_remappings: HashMap::new(), + // + // Fallback Config (NOT SUPPORTED yet) + extension_version: None, + // AppSec + serverless_appsec_enabled: false, + appsec_enabled: false, + // OTLP + // + // - Receiver + otlp_config_receiver_protocols_grpc_endpoint: None, + otlp_config_receiver_protocols_grpc_transport: None, + otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib: None, + // - Metrics + otlp_config_metrics_enabled: false, // TODO(duncanista): Go Agent default is to true + otlp_config_metrics_resource_attributes_as_tags: false, + otlp_config_metrics_instrumentation_scope_metadata_as_tags: false, + otlp_config_metrics_tag_cardinality: None, + otlp_config_metrics_delta_ttl: None, + otlp_config_metrics_histograms_mode: None, + otlp_config_metrics_histograms_send_count_sum_metrics: false, + otlp_config_metrics_histograms_send_aggregation_metrics: false, + otlp_config_metrics_sums_cumulative_monotonic_mode: None, + otlp_config_metrics_sums_initial_cumulativ_monotonic_value: None, + otlp_config_metrics_summaries_mode: None, + // - Traces + otlp_config_traces_probabilistic_sampler_sampling_percentage: None, + // - Logs + otlp_config_logs_enabled: false, + } + } +} + +fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Value::deserialize(deserializer)?; + match value { + Value::String(s) => { + if s.trim().is_empty() { + Ok(None) + } else { + Ok(Some(s)) + } + } + Value::Number(n) => Ok(Some(n.to_string())), + _ => Err(serde::de::Error::custom("expected a string or an integer")), + } +} diff --git a/mod.rs b/mod.rs index 53400ec2..b5cb3555 100644 --- a/mod.rs +++ b/mod.rs @@ -1,202 +1,27 @@ pub mod apm_replace_rule; +pub mod env; pub mod flush_strategy; pub mod log_level; pub mod processing_rule; pub mod service_mapping; pub mod trace_propagation_style; +pub mod yaml; -use datadog_trace_obfuscation::replacer::ReplaceRule; use datadog_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; -use std::collections::HashMap; use std::path::Path; use std::time::Instant; -use std::vec; use figment::providers::{Format, Yaml}; use figment::{providers::Env, Figment}; -use serde::{Deserialize, Deserializer}; -use serde_json::Value; -use trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}; use crate::config::{ apm_replace_rule::deserialize_apm_replace_rules, - flush_strategy::FlushStrategy, - log_level::{deserialize_log_level, LogLevel}, + env::Config as EnvConfig, processing_rule::{deserialize_processing_rules, ProcessingRule}, - service_mapping::deserialize_service_mapping, + yaml::Config as YamlConfig, }; -/// `FallbackConfig` is a struct that represents fields that are not supported in the extension yet. -/// -/// If `extension_version` is set to `compatibility`, the Go extension will be launched. -#[derive(Debug, PartialEq, Deserialize, Clone, Default)] -#[serde(default)] -#[allow(clippy::module_name_repetitions)] -#[allow(clippy::struct_excessive_bools)] -pub struct FallbackConfig { - extension_version: Option, - serverless_appsec_enabled: bool, - appsec_enabled: bool, - // otel - trace_otel_enabled: bool, - otlp_config_receiver_protocols_http_endpoint: Option, - otlp_config_receiver_protocols_grpc_endpoint: Option, -} - -/// `FallbackYamlConfig` is a struct that represents fields in `datadog.yaml` not yet supported in the extension yet. -/// -#[derive(Debug, PartialEq, Deserialize, Clone, Default)] -#[serde(default)] -#[allow(clippy::module_name_repetitions)] -pub struct FallbackYamlConfig { - otlp_config: Option, -} -#[derive(Debug, PartialEq, Deserialize, Clone, Default)] -#[serde(default)] -#[allow(clippy::module_name_repetitions)] -pub struct YamlLogsConfig { - #[serde(deserialize_with = "deserialize_processing_rules")] - processing_rules: Option>, -} - -#[derive(Debug, PartialEq, Deserialize, Clone, Default)] -#[serde(default)] -#[allow(clippy::module_name_repetitions)] -pub struct YamlApmConfig { - #[serde(deserialize_with = "deserialize_apm_replace_rules")] - replace_tags: Option>, - obfuscation: Option, -} - -#[derive(Debug, PartialEq, Deserialize, Clone, Copy, Default)] -#[serde(default)] -#[allow(clippy::module_name_repetitions)] -pub struct ApmObfuscation { - http: ApmHttpObfuscation, -} - -#[derive(Debug, PartialEq, Deserialize, Clone, Copy, Default)] -#[serde(default)] -#[allow(clippy::module_name_repetitions)] -pub struct ApmHttpObfuscation { - remove_query_string: bool, - remove_paths_with_digits: bool, -} - -/// `YamlConfig` is a struct that represents some of the fields in the datadog.yaml file. -/// -/// It is used to deserialize the datadog.yaml file into a struct that can be merged with the Config struct. -#[derive(Debug, PartialEq, Deserialize, Clone, Default)] -#[serde(default)] -#[allow(clippy::module_name_repetitions)] -pub struct YamlConfig { - pub logs_config: YamlLogsConfig, - pub apm_config: YamlApmConfig, - pub proxy: YamlProxyConfig, -} - -#[derive(Debug, PartialEq, Deserialize, Clone, Default)] -#[serde(default)] -#[allow(clippy::module_name_repetitions)] -pub struct YamlProxyConfig { - pub https: Option, - pub no_proxy: Option>, -} - -#[derive(Debug, PartialEq, Deserialize, Clone)] -#[serde(default)] -#[allow(clippy::struct_excessive_bools)] -pub struct Config { - pub site: String, - pub api_key: String, - pub api_key_secret_arn: String, - pub kms_api_key: String, - #[serde(deserialize_with = "deserialize_string_or_int")] - pub env: Option, - #[serde(deserialize_with = "deserialize_string_or_int")] - pub service: Option, - #[serde(deserialize_with = "deserialize_string_or_int")] - pub version: Option, - pub tags: Option, - #[serde(deserialize_with = "deserialize_log_level")] - pub log_level: LogLevel, - #[serde(deserialize_with = "deserialize_processing_rules")] - pub logs_config_processing_rules: Option>, - pub logs_config_use_compression: bool, - pub logs_config_compression_level: i32, - pub logs_config_logs_dd_url: String, - pub serverless_flush_strategy: FlushStrategy, - pub enhanced_metrics: bool, - //flush timeout in seconds - pub flush_timeout: u64, //TODO go agent adds jitter too - pub https_proxy: Option, - pub capture_lambda_payload: bool, - pub capture_lambda_payload_max_depth: u32, - #[serde(deserialize_with = "deserialize_service_mapping")] - pub service_mapping: HashMap, - pub serverless_logs_enabled: bool, - // Trace Propagation - #[serde(deserialize_with = "deserialize_trace_propagation_style")] - pub trace_propagation_style: Vec, - #[serde(deserialize_with = "deserialize_trace_propagation_style")] - pub trace_propagation_style_extract: Vec, - pub trace_propagation_extract_first: bool, - pub trace_propagation_http_baggage_enabled: bool, - pub apm_config_apm_dd_url: String, - #[serde(deserialize_with = "deserialize_apm_replace_rules")] - pub apm_config_replace_tags: Option>, - pub apm_config_obfuscation_http_remove_query_string: bool, - pub apm_config_obfuscation_http_remove_paths_with_digits: bool, - // Metrics overrides - pub dd_url: String, - pub url: String, -} - -impl Default for Config { - fn default() -> Self { - Config { - // General - site: String::default(), - api_key: String::default(), - api_key_secret_arn: String::default(), - kms_api_key: String::default(), - serverless_flush_strategy: FlushStrategy::Default, - flush_timeout: 5, - // Unified Tagging - env: None, - service: None, - version: None, - tags: None, - // Logs - log_level: LogLevel::default(), - logs_config_processing_rules: None, - logs_config_use_compression: true, - logs_config_compression_level: 6, - logs_config_logs_dd_url: String::default(), - // Metrics - enhanced_metrics: true, - https_proxy: None, - capture_lambda_payload: false, - capture_lambda_payload_max_depth: 10, - service_mapping: HashMap::new(), - serverless_logs_enabled: true, - // Trace Propagation - trace_propagation_style: vec![ - TracePropagationStyle::Datadog, - TracePropagationStyle::TraceContext, - ], - trace_propagation_style_extract: vec![], - trace_propagation_extract_first: false, - trace_propagation_http_baggage_enabled: false, - apm_config_apm_dd_url: String::default(), - apm_config_replace_tags: None, - apm_config_obfuscation_http_remove_query_string: false, - apm_config_obfuscation_http_remove_paths_with_digits: false, - dd_url: String::default(), - url: String::default(), - } - } -} +pub type Config = EnvConfig; #[derive(Debug, PartialEq)] #[allow(clippy::module_name_repetitions)] @@ -209,16 +34,7 @@ fn log_fallback_reason(reason: &str) { println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); } -fn fallback(figment: &Figment, yaml_figment: &Figment, region: &str) -> Result<(), ConfigError> { - let (config, yaml_config): (FallbackConfig, FallbackYamlConfig) = - match (figment.extract(), yaml_figment.extract()) { - (Ok(env_config), Ok(yaml_config)) => (env_config, yaml_config), - (_, Err(err)) | (Err(err), _) => { - println!("Failed to parse Datadog config: {err}"); - return Err(ConfigError::ParseError(err.to_string())); - } - }; - +fn fallback(config: &EnvConfig, yaml_config: &YamlConfig, region: &str) -> Result<(), ConfigError> { // Customer explicitly opted out of the Next Gen extension let opted_out = match config.extension_version.as_deref() { Some("compatibility") => true, @@ -238,16 +54,45 @@ fn fallback(figment: &Figment, yaml_figment: &Figment, region: &str) -> Result<( return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); } - // OTEL env - if config.trace_otel_enabled + // OTLP + let has_otlp_env_config = config + .otlp_config_receiver_protocols_grpc_endpoint + .is_some() || config - .otlp_config_receiver_protocols_http_endpoint + .otlp_config_receiver_protocols_grpc_transport .is_some() || config - .otlp_config_receiver_protocols_grpc_endpoint + .otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib .is_some() - || yaml_config.otlp_config.is_some() - { + || config.otlp_config_metrics_enabled + || config.otlp_config_metrics_resource_attributes_as_tags + || config.otlp_config_metrics_instrumentation_scope_metadata_as_tags + || config.otlp_config_metrics_tag_cardinality.is_some() + || config.otlp_config_metrics_delta_ttl.is_some() + || config.otlp_config_metrics_histograms_mode.is_some() + || config.otlp_config_metrics_histograms_send_count_sum_metrics + || config.otlp_config_metrics_histograms_send_aggregation_metrics + || config + .otlp_config_metrics_sums_cumulative_monotonic_mode + .is_some() + || config + .otlp_config_metrics_sums_initial_cumulativ_monotonic_value + .is_some() + || config.otlp_config_metrics_summaries_mode.is_some() + || config + .otlp_config_traces_probabilistic_sampler_sampling_percentage + .is_some() + || config.otlp_config_logs_enabled; + + let has_otlp_yaml_config = yaml_config.otlp_config.receiver.protocols.grpc.is_some() + || yaml_config + .otlp_config + .traces + .probabilistic_sampler + .is_some() + || yaml_config.otlp_config.logs.is_some(); + + if has_otlp_env_config || has_otlp_yaml_config { log_fallback_reason("otel"); return Err(ConfigError::UnsupportedField("otel".to_string())); } @@ -262,7 +107,7 @@ fn fallback(figment: &Figment, yaml_figment: &Figment, region: &str) -> Result<( } #[allow(clippy::module_name_repetitions)] -pub fn get_config(config_directory: &Path, region: &str) -> Result { +pub fn get_config(config_directory: &Path, region: &str) -> Result { let path = config_directory.join("datadog.yaml"); // Get default config fields (and ENV specific ones) @@ -275,9 +120,7 @@ pub fn get_config(config_directory: &Path, region: &str) -> Result (env_config, yaml_config), (_, Err(err)) | (Err(err), _) => { @@ -285,6 +128,9 @@ pub fn get_config(config_directory: &Path, region: &str) -> Result Result Result Result Result String { format!("https://http-intake.logs.{site}") } -fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let value = Value::deserialize(deserializer)?; - match value { - Value::String(s) => { - if s.trim().is_empty() { - Ok(None) - } else { - Ok(Some(s)) - } - } - Value::Number(n) => Ok(Some(n.to_string())), - _ => Err(serde::de::Error::custom("expected a string or an integer")), - } -} - #[allow(clippy::module_name_repetitions)] #[derive(Debug, Clone)] pub struct AwsConfig { @@ -416,8 +299,10 @@ pub mod tests { use super::*; - use crate::config::flush_strategy::PeriodicStrategy; + use crate::config::flush_strategy::{FlushStrategy, PeriodicStrategy}; + use crate::config::log_level::LogLevel; use crate::config::processing_rule; + use crate::config::trace_propagation_style::TracePropagationStyle; const MOCK_REGION: &str = "us-east-1"; @@ -451,7 +336,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env( - "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT", + "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", "localhost:4138", ); @@ -472,7 +357,7 @@ pub mod tests { otlp_config: receiver: protocols: - http: + grpc: endpoint: localhost:4138 ", )?; @@ -484,27 +369,6 @@ pub mod tests { }); } - #[test] - fn test_fallback_on_otel_yaml_empty_section() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.create_file( - "datadog.yaml", - r" - otlp_config: - receiver: - protocols: - http: - ", - )?; - - let config = - get_config(Path::new(""), MOCK_REGION).expect_err("should reject unknown fields"); - assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); - Ok(()) - }); - } - #[test] fn test_default_logs_intake_url() { figment::Jail::expect_with(|jail| { @@ -667,7 +531,7 @@ pub mod tests { let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); assert_eq!( config, - Config { + EnvConfig { site: "datadoghq.com".to_string(), trace_propagation_style_extract: vec![ TracePropagationStyle::Datadog, @@ -676,7 +540,7 @@ pub mod tests { logs_config_logs_dd_url: "https://http-intake.logs.datadoghq.com".to_string(), apm_config_apm_dd_url: trace_intake_url("datadoghq.com").to_string(), dd_url: String::new(), // We add the prefix in main.rs - ..Config::default() + ..EnvConfig::default() } ); Ok(()) diff --git a/yaml.rs b/yaml.rs new file mode 100644 index 00000000..484e14a9 --- /dev/null +++ b/yaml.rs @@ -0,0 +1,120 @@ +use std::collections::HashMap; + +use crate::config::{deserialize_apm_replace_rules, deserialize_processing_rules, ProcessingRule}; +use datadog_trace_obfuscation::replacer::ReplaceRule; +use serde::Deserialize; +use serde_json::Value; + +/// `Config` is a struct that represents some of the fields in the `datadog.yaml` file. +/// +/// It is used to deserialize the `datadog.yaml` file into a struct that can be merged with the `Config` struct. +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct Config { + pub logs_config: LogsConfig, + pub apm_config: ApmConfig, + pub proxy: ProxyConfig, + pub otlp_config: OtlpConfig, +} + +/// Logs Config +/// + +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct LogsConfig { + #[serde(deserialize_with = "deserialize_processing_rules")] + pub processing_rules: Option>, +} + +/// APM Config +/// + +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct ApmConfig { + #[serde(deserialize_with = "deserialize_apm_replace_rules")] + pub replace_tags: Option>, + pub obfuscation: Option, +} + +#[derive(Debug, PartialEq, Deserialize, Clone, Copy, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct ApmObfuscation { + pub http: ApmHttpObfuscation, +} + +#[derive(Debug, PartialEq, Deserialize, Clone, Copy, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct ApmHttpObfuscation { + pub remove_query_string: bool, + pub remove_paths_with_digits: bool, +} + +/// Proxy Config +/// + +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct ProxyConfig { + pub https: Option, + pub no_proxy: Option>, +} + +/// OTLP Config +/// + +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct OtlpConfig { + pub receiver: OtlpReceiverConfig, + pub traces: OtlpTracesConfig, + + // NOT SUPPORTED + pub metrics: Option, + pub logs: Option, +} + +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct OtlpReceiverConfig { + pub protocols: OtlpReceiverProtocolsConfig, +} + +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct OtlpReceiverProtocolsConfig { + pub http: OtlpReceiverHttpConfig, + + // NOT SUPPORTED + pub grpc: Option, +} + +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct OtlpReceiverHttpConfig { + pub endpoint: String, +} + +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct OtlpTracesConfig { + pub enabled: bool, + pub span_name_as_resource_name: bool, + pub span_name_remappings: HashMap, + pub ignore_missing_datadog_fields: bool, + + // NOT SUPORTED + pub probabilistic_sampler: Option, +} From ed3d2712e94c9db7f45537266fe08bb7880c6a89 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 21 Apr 2025 16:51:42 -0400 Subject: [PATCH 057/112] fix: Parse DD_APM_REPLACE_TAGS env var (#656) Fixes an issue where we didn't parse `DD_APM_REPLACE_TAGS` because the yaml block includes an additional `config` word after APM, which is not present in the env var. As usual, env vars override config file settings --- env.rs | 3 +++ mod.rs | 37 ++++++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/env.rs b/env.rs index 8ad7f437..a915d5a8 100644 --- a/env.rs +++ b/env.rs @@ -56,6 +56,8 @@ pub struct Config { // APM pub apm_config_apm_dd_url: String, #[serde(deserialize_with = "deserialize_apm_replace_rules")] + pub apm_replace_tags: Option>, + #[serde(deserialize_with = "deserialize_apm_replace_rules")] pub apm_config_replace_tags: Option>, pub apm_config_obfuscation_http_remove_query_string: bool, pub apm_config_obfuscation_http_remove_paths_with_digits: bool, @@ -142,6 +144,7 @@ impl Default for Config { trace_propagation_http_baggage_enabled: false, // APM apm_config_apm_dd_url: String::default(), + apm_replace_tags: None, apm_config_replace_tags: None, apm_config_obfuscation_http_remove_query_string: false, apm_config_obfuscation_http_remove_paths_with_digits: false, diff --git a/mod.rs b/mod.rs index b5cb3555..0d6ed14c 100644 --- a/mod.rs +++ b/mod.rs @@ -191,9 +191,9 @@ fn merge_config(config: &mut EnvConfig, yaml_config: &YamlConfig) { trace_intake_url_prefixed(config.apm_config_apm_dd_url.as_str()); } - if config.apm_config_replace_tags.is_none() { + if config.apm_replace_tags.is_none() { if let Some(rules) = yaml_config.apm_config.replace_tags.as_ref() { - config.apm_config_replace_tags = Some(rules.clone()); + config.apm_replace_tags = Some(rules.clone()); } } @@ -769,7 +769,38 @@ pub mod tests { ]"#, ) .expect("can't parse rules"); - assert_eq!(config.apm_config_replace_tags, Some(rule),); + assert_eq!(config.apm_replace_tags, Some(rule),); + Ok(()) + }); + } + + #[test] + fn test_apm_tags_env_overrides_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env( + "DD_APM_REPLACE_TAGS", + r#"[{"name":"*","pattern":"foo","repl":"REDACTED-ENV"}]"#, + ); + jail.create_file( + "datadog.yaml", + r" + site: datadoghq.com + apm_config: + replace_tags: + - name: '*' + pattern: 'foo' + repl: 'REDACTED-YAML' + ", + )?; + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); + let rule = parse_rules_from_string( + r#"[ + {"name": "*", "pattern": "foo", "repl": "REDACTED-ENV"} + ]"#, + ) + .expect("can't parse rules"); + assert_eq!(config.apm_replace_tags, Some(rule),); Ok(()) }); } From 99a5ea0aff8c9948569446845963c23571708e15 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 24 Apr 2025 11:56:26 -0400 Subject: [PATCH 058/112] feat: Optionally disable proc enhanced metrics (#663) Fixes #648 For customers using very very fast/small lambda functions (usually just rust), there can be a small 1-2ms increase in runtime duration when collecing metrics like open file descriptors or tmp file usage. We still enable these by default, but customers can now optionally disable them --- env.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/env.rs b/env.rs index a915d5a8..e45ee87b 100644 --- a/env.rs +++ b/env.rs @@ -38,6 +38,7 @@ pub struct Config { pub logs_config_logs_dd_url: String, pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, + pub lambda_proc_enhanced_metrics: bool, /// Flush timeout in seconds pub flush_timeout: u64, //TODO go agent adds jitter too pub https_proxy: Option, @@ -129,6 +130,7 @@ impl Default for Config { logs_config_logs_dd_url: String::default(), // Metrics enhanced_metrics: true, + lambda_proc_enhanced_metrics: true, https_proxy: None, capture_lambda_payload: false, capture_lambda_payload_max_depth: 10, From a5aec3ca6a6f3717cfccb04a8f4397bb86c24fbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 24 Apr 2025 12:43:25 -0400 Subject: [PATCH 059/112] fix(config): serialize booleans from anything (#657) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What? Serializes any boolean with values `0|1|true|TRUE|False|false` to its boolean part. # How? Using `serde-aux` crate to leverage the unit testing and ownership. # Motivation Some products at Datadog allow this values as they coalesce them – [SVLS-6687](https://datadoghq.atlassian.net/browse/SVLS-6687) [SVLS-6687]: https://datadoghq.atlassian.net/browse/SVLS-6687?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ --- env.rs | 20 ++++++++++++++++++++ mod.rs | 17 +++++++++++++++++ yaml.rs | 6 ++++++ 3 files changed, 43 insertions(+) diff --git a/env.rs b/env.rs index e45ee87b..b3e4f159 100644 --- a/env.rs +++ b/env.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::vec; use datadog_trace_obfuscation::replacer::ReplaceRule; +use serde_aux::field_attributes::deserialize_bool_from_anything; use serde_json::Value; use crate::config::{ @@ -33,26 +34,32 @@ pub struct Config { pub log_level: LogLevel, #[serde(deserialize_with = "deserialize_processing_rules")] pub logs_config_processing_rules: Option>, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub logs_config_use_compression: bool, pub logs_config_compression_level: i32, pub logs_config_logs_dd_url: String, pub serverless_flush_strategy: FlushStrategy, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub enhanced_metrics: bool, pub lambda_proc_enhanced_metrics: bool, /// Flush timeout in seconds pub flush_timeout: u64, //TODO go agent adds jitter too pub https_proxy: Option, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub capture_lambda_payload: bool, pub capture_lambda_payload_max_depth: u32, #[serde(deserialize_with = "deserialize_service_mapping")] pub service_mapping: HashMap, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub serverless_logs_enabled: bool, // Trace Propagation #[serde(deserialize_with = "deserialize_trace_propagation_style")] pub trace_propagation_style: Vec, #[serde(deserialize_with = "deserialize_trace_propagation_style")] pub trace_propagation_style_extract: Vec, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub trace_propagation_extract_first: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub trace_propagation_http_baggage_enabled: bool, // APM pub apm_config_apm_dd_url: String, @@ -60,7 +67,9 @@ pub struct Config { pub apm_replace_tags: Option>, #[serde(deserialize_with = "deserialize_apm_replace_rules")] pub apm_config_replace_tags: Option>, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub apm_config_obfuscation_http_remove_query_string: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub apm_config_obfuscation_http_remove_paths_with_digits: bool, pub apm_features: Vec, pub apm_ignore_resources: Vec, @@ -70,9 +79,12 @@ pub struct Config { // OTLP // // - Traces + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub otlp_config_traces_enabled: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub otlp_config_traces_span_name_as_resource_name: bool, pub otlp_config_traces_span_name_remappings: HashMap, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub otlp_config_ignore_missing_datadog_fields: bool, // - Receiver / HTTP pub otlp_config_receiver_protocols_http_endpoint: Option, @@ -81,7 +93,9 @@ pub struct Config { // Fallback Config pub extension_version: Option, // AppSec + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub serverless_appsec_enabled: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub appsec_enabled: bool, // OTLP // @@ -90,13 +104,18 @@ pub struct Config { pub otlp_config_receiver_protocols_grpc_transport: Option, pub otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib: Option, // - Metrics + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub otlp_config_metrics_enabled: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub otlp_config_metrics_resource_attributes_as_tags: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub otlp_config_metrics_instrumentation_scope_metadata_as_tags: bool, pub otlp_config_metrics_tag_cardinality: Option, pub otlp_config_metrics_delta_ttl: Option, pub otlp_config_metrics_histograms_mode: Option, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub otlp_config_metrics_histograms_send_count_sum_metrics: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub otlp_config_metrics_histograms_send_aggregation_metrics: bool, pub otlp_config_metrics_sums_cumulative_monotonic_mode: Option, pub otlp_config_metrics_sums_initial_cumulativ_monotonic_value: Option, @@ -104,6 +123,7 @@ pub struct Config { // - Traces pub otlp_config_traces_probabilistic_sampler_sampling_percentage: Option, // - Logs + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub otlp_config_logs_enabled: bool, } diff --git a/mod.rs b/mod.rs index 0d6ed14c..13fea537 100644 --- a/mod.rs +++ b/mod.rs @@ -884,4 +884,21 @@ pub mod tests { Ok(()) }); } + + #[test] + fn test_parse_bool_from_anything() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); + jail.set_env("DD_ENHANCED_METRICS", "1"); + jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "TRUE"); + jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "0"); + let config = get_config(Path::new(""), MOCK_REGION).expect("should parse config"); + assert_eq!(config.serverless_logs_enabled, true); + assert_eq!(config.enhanced_metrics, true); + assert_eq!(config.logs_config_use_compression, true); + assert_eq!(config.capture_lambda_payload, false); + Ok(()) + }); + } } diff --git a/yaml.rs b/yaml.rs index 484e14a9..176b4dbb 100644 --- a/yaml.rs +++ b/yaml.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use crate::config::{deserialize_apm_replace_rules, deserialize_processing_rules, ProcessingRule}; use datadog_trace_obfuscation::replacer::ReplaceRule; use serde::Deserialize; +use serde_aux::field_attributes::deserialize_bool_from_anything; use serde_json::Value; /// `Config` is a struct that represents some of the fields in the `datadog.yaml` file. @@ -52,7 +53,9 @@ pub struct ApmObfuscation { #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct ApmHttpObfuscation { + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub remove_query_string: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub remove_paths_with_digits: bool, } @@ -110,9 +113,12 @@ pub struct OtlpReceiverHttpConfig { #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct OtlpTracesConfig { + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub enabled: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub span_name_as_resource_name: bool, pub span_name_remappings: HashMap, + #[serde(deserialize_with = "deserialize_bool_from_anything")] pub ignore_missing_datadog_fields: bool, // NOT SUPORTED From ac8a7df8a0e7d30978aaa1b64fac7a7d01527a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 24 Apr 2025 12:43:41 -0400 Subject: [PATCH 060/112] chore(config): create `aws` module (#659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What? Refactors methods related to AWS config into its own module # Motivation Just cleaning and removing stuff from main – [SVLS-6686](https://datadoghq.atlassian.net/browse/SVLS-6686) [SVLS-6686]: https://datadoghq.atlassian.net/browse/SVLS-6686?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ --- aws.rs | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ mod.rs | 24 +----------------------- 2 files changed, 56 insertions(+), 23 deletions(-) create mode 100644 aws.rs diff --git a/aws.rs b/aws.rs new file mode 100644 index 00000000..6cf7e618 --- /dev/null +++ b/aws.rs @@ -0,0 +1,55 @@ +use std::{env, time::Instant}; + +const AWS_DEFAULT_REGION: &str = "AWS_DEFAULT_REGION"; +const AWS_ACCESS_KEY_ID: &str = "AWS_ACCESS_KEY_ID"; +const AWS_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY"; +const AWS_SESSION_TOKEN: &str = "AWS_SESSION_TOKEN"; +const AWS_CONTAINER_CREDENTIALS_FULL_URI: &str = "AWS_CONTAINER_CREDENTIALS_FULL_URI"; +const AWS_CONTAINER_AUTHORIZATION_TOKEN: &str = "AWS_CONTAINER_AUTHORIZATION_TOKEN"; +const AWS_LAMBDA_FUNCTION_NAME: &str = "AWS_LAMBDA_FUNCTION_NAME"; + +#[allow(clippy::module_name_repetitions)] +#[derive(Debug, Clone)] +pub struct AwsConfig { + pub region: String, + pub aws_access_key_id: String, + pub aws_secret_access_key: String, + pub aws_session_token: String, + pub function_name: String, + pub sandbox_init_time: Instant, + pub aws_container_credentials_full_uri: String, + pub aws_container_authorization_token: String, +} + +impl AwsConfig { + #[must_use] + pub fn from_env(start_time: Instant) -> Self { + Self { + region: env::var(AWS_DEFAULT_REGION).unwrap_or("us-east-1".to_string()), + aws_access_key_id: env::var(AWS_ACCESS_KEY_ID).unwrap_or_default(), + aws_secret_access_key: env::var(AWS_SECRET_ACCESS_KEY).unwrap_or_default(), + aws_session_token: env::var(AWS_SESSION_TOKEN).unwrap_or_default(), + aws_container_credentials_full_uri: env::var(AWS_CONTAINER_CREDENTIALS_FULL_URI) + .unwrap_or_default(), + aws_container_authorization_token: env::var(AWS_CONTAINER_AUTHORIZATION_TOKEN) + .unwrap_or_default(), + function_name: env::var(AWS_LAMBDA_FUNCTION_NAME).unwrap_or_default(), + sandbox_init_time: start_time, + } + } +} + +#[must_use] +pub fn get_aws_partition_by_region(region: &str) -> String { + match region { + r if r.starts_with("us-gov-") => "aws-us-gov".to_string(), + r if r.starts_with("cn-") => "aws-cn".to_string(), + _ => "aws".to_string(), + } +} + +#[must_use] +pub fn build_lambda_function_arn(account_id: &str, region: &str, function_name: &str) -> String { + let aws_partition = get_aws_partition_by_region(region); + format!("arn:{aws_partition}:lambda:{region}:{account_id}:function:{function_name}") +} diff --git a/mod.rs b/mod.rs index 13fea537..a51a202a 100644 --- a/mod.rs +++ b/mod.rs @@ -1,4 +1,5 @@ pub mod apm_replace_rule; +pub mod aws; pub mod env; pub mod flush_strategy; pub mod log_level; @@ -9,7 +10,6 @@ pub mod yaml; use datadog_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; use std::path::Path; -use std::time::Instant; use figment::providers::{Format, Yaml}; use figment::{providers::Env, Figment}; @@ -271,28 +271,6 @@ fn build_fqdn_logs(site: String) -> String { format!("https://http-intake.logs.{site}") } -#[allow(clippy::module_name_repetitions)] -#[derive(Debug, Clone)] -pub struct AwsConfig { - pub region: String, - pub aws_access_key_id: String, - pub aws_secret_access_key: String, - pub aws_session_token: String, - pub function_name: String, - pub sandbox_init_time: Instant, - pub aws_container_credentials_full_uri: String, - pub aws_container_authorization_token: String, -} - -#[must_use] -pub fn get_aws_partition_by_region(region: &str) -> String { - match region { - r if r.starts_with("us-gov-") => "aws-us-gov".to_string(), - r if r.starts_with("cn-") => "aws-cn".to_string(), - _ => "aws".to_string(), - } -} - #[cfg(test)] pub mod tests { use datadog_trace_obfuscation::replacer::parse_rules_from_string; From c6fcb6210173439d8ada55ec41b673734d4d258b Mon Sep 17 00:00:00 2001 From: Aleksandr Pasechnik Date: Tue, 6 May 2025 11:04:24 -0400 Subject: [PATCH 061/112] feat: [SVLS-6242] bottlecap fips builds (#644) Building bottlecap with fips mode. This is entirely focused on removing `ring` (and other non-FIPS-compliant dependencies from our `fips`-featured builds.) --- mod.rs | 96 ++++++++++++++++++++++------------------------------------ 1 file changed, 36 insertions(+), 60 deletions(-) diff --git a/mod.rs b/mod.rs index a51a202a..c4f990d6 100644 --- a/mod.rs +++ b/mod.rs @@ -34,7 +34,7 @@ fn log_fallback_reason(reason: &str) { println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); } -fn fallback(config: &EnvConfig, yaml_config: &YamlConfig, region: &str) -> Result<(), ConfigError> { +fn fallback(config: &EnvConfig, yaml_config: &YamlConfig) -> Result<(), ConfigError> { // Customer explicitly opted out of the Next Gen extension let opted_out = match config.extension_version.as_deref() { Some("compatibility") => true, @@ -97,17 +97,11 @@ fn fallback(config: &EnvConfig, yaml_config: &YamlConfig, region: &str) -> Resul return Err(ConfigError::UnsupportedField("otel".to_string())); } - // Govcloud Regions - if region.starts_with("us-gov-") { - log_fallback_reason("gov_region"); - return Err(ConfigError::UnsupportedField("gov_region".to_string())); - } - Ok(()) } #[allow(clippy::module_name_repetitions)] -pub fn get_config(config_directory: &Path, region: &str) -> Result { +pub fn get_config(config_directory: &Path) -> Result { let path = config_directory.join("datadog.yaml"); // Get default config fields (and ENV specific ones) @@ -129,7 +123,7 @@ pub fn get_config(config_directory: &Path, region: &str) -> Result Date: Fri, 16 May 2025 11:44:38 -0400 Subject: [PATCH 062/112] fix(config): remove `apm_ignore_resources` check in OTEL (#676) # What? Removes usage of `DD_APM_IGNORE_RESOURCES` in the OTEL span transform. # Why? 1. The implementation was incorrect and shouldn't check for resources to ignore in the transformation step. 2. It was not properly used in the `apm_config` for YAML files. # Notes: - Follow up PR to implement `APM_IGNORE_RESOURCES` properly in the Trace Agent. # More Learn about ignoring resources: https://docs.datadoghq.com/tracing/guide/ignoring_apm_resources/?tab=datadogyaml#ignoring-based-on-resources `DD_APM_IGNORE_RESOURCES` is specified as: ``` A list of regular expressions can be provided to exclude certain traces based on their resource name. All entries must be surrounded by double quotes and separated by commas. ``` A correct usage would be: ```env DD_APM_IGNORE_RESOURCES="(GET|POST) /healthcheck,API::NotesController#index" ``` or in yaml ```yaml apm_config: ignore_resources: ["(GET|POST) /healthcheck","API::NotesController#index"] ``` --- env.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/env.rs b/env.rs index b3e4f159..04846178 100644 --- a/env.rs +++ b/env.rs @@ -72,7 +72,6 @@ pub struct Config { #[serde(deserialize_with = "deserialize_bool_from_anything")] pub apm_config_obfuscation_http_remove_paths_with_digits: bool, pub apm_features: Vec, - pub apm_ignore_resources: Vec, // Metrics overrides pub dd_url: String, pub url: String, @@ -171,7 +170,6 @@ impl Default for Config { apm_config_obfuscation_http_remove_query_string: false, apm_config_obfuscation_http_remove_paths_with_digits: false, apm_features: vec![], - apm_ignore_resources: vec![], dd_url: String::default(), url: String::default(), // OTLP From 8e879503af8b622b011ecfedc1541f1e04a46895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Mon, 19 May 2025 15:49:54 +0200 Subject: [PATCH 063/112] feat(proxy): abstract lambda runtime api proxy (#669) # What? Abstracts the concept of the `proxy` from the Lambda Web Adapter implementation. This will unlock the usage of ASM. # How? Using `axum` crate, we create a new server proxy with specific routes from the Lambda Runtime API which we are interested in proxying. # Motivation ASM and [SVLS-6760](https://datadoghq.atlassian.net/browse/SVLS-6760) [SVLS-6760]: https://datadoghq.atlassian.net/browse/SVLS-6760?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ --- aws.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/aws.rs b/aws.rs index 6cf7e618..49067dd2 100644 --- a/aws.rs +++ b/aws.rs @@ -7,6 +7,9 @@ const AWS_SESSION_TOKEN: &str = "AWS_SESSION_TOKEN"; const AWS_CONTAINER_CREDENTIALS_FULL_URI: &str = "AWS_CONTAINER_CREDENTIALS_FULL_URI"; const AWS_CONTAINER_AUTHORIZATION_TOKEN: &str = "AWS_CONTAINER_AUTHORIZATION_TOKEN"; const AWS_LAMBDA_FUNCTION_NAME: &str = "AWS_LAMBDA_FUNCTION_NAME"; +const AWS_LAMBDA_RUNTIME_API: &str = "AWS_LAMBDA_RUNTIME_API"; +const AWS_LWA_LAMBDA_RUNTIME_API_PROXY: &str = "AWS_LWA_LAMBDA_RUNTIME_API_PROXY"; +const AWS_LAMBDA_EXEC_WRAPPER: &str = "AWS_LAMBDA_EXEC_WRAPPER"; #[allow(clippy::module_name_repetitions)] #[derive(Debug, Clone)] @@ -15,10 +18,13 @@ pub struct AwsConfig { pub aws_access_key_id: String, pub aws_secret_access_key: String, pub aws_session_token: String, - pub function_name: String, - pub sandbox_init_time: Instant, pub aws_container_credentials_full_uri: String, pub aws_container_authorization_token: String, + pub aws_lwa_proxy_lambda_runtime_api: Option, + pub function_name: String, + pub runtime_api: String, + pub sandbox_init_time: Instant, + pub exec_wrapper: Option, } impl AwsConfig { @@ -33,8 +39,11 @@ impl AwsConfig { .unwrap_or_default(), aws_container_authorization_token: env::var(AWS_CONTAINER_AUTHORIZATION_TOKEN) .unwrap_or_default(), + aws_lwa_proxy_lambda_runtime_api: env::var(AWS_LWA_LAMBDA_RUNTIME_API_PROXY).ok(), function_name: env::var(AWS_LAMBDA_FUNCTION_NAME).unwrap_or_default(), + runtime_api: env::var(AWS_LAMBDA_RUNTIME_API).unwrap_or_default(), sandbox_init_time: start_time, + exec_wrapper: env::var(AWS_LAMBDA_EXEC_WRAPPER).ok(), } } } From 46f682bcd2af351aa7557350301d9ac1dd8c57fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 28 May 2025 11:58:37 -0400 Subject: [PATCH 064/112] fix(config): fix otlp trace agent to start when right configuration is set (#680) # What? Ensures that OTLP agent is only enabled when the `otlp_config_receiver_protocols_http_endpoint` is set, and when `otlp_config_traces_enabled` is `true` # Motivation #678 # Notes OTEL agent should only spin up when receiver protocols endpoint is set, so this was a miss on my side. --- mod.rs | 49 +++++++------------- yaml.rs | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 151 insertions(+), 39 deletions(-) diff --git a/mod.rs b/mod.rs index c4f990d6..541fa83a 100644 --- a/mod.rs +++ b/mod.rs @@ -84,13 +84,11 @@ fn fallback(config: &EnvConfig, yaml_config: &YamlConfig) -> Result<(), ConfigEr .is_some() || config.otlp_config_logs_enabled; - let has_otlp_yaml_config = yaml_config.otlp_config.receiver.protocols.grpc.is_some() + let has_otlp_yaml_config = yaml_config.otlp_config_receiver_protocols_grpc().is_some() || yaml_config - .otlp_config - .traces - .probabilistic_sampler + .otlp_config_traces_probabilistic_sampler() .is_some() - || yaml_config.otlp_config.logs.is_some(); + || yaml_config.otlp_config_logs().is_some(); if has_otlp_env_config || has_otlp_yaml_config { log_fallback_reason("otel"); @@ -208,54 +206,41 @@ fn merge_config(config: &mut EnvConfig, yaml_config: &YamlConfig) { // OTLP // // - Receiver / HTTP + let yaml_otlp_config_receiver_protocols_http_endpoint = + yaml_config.otlp_config_receiver_protocols_http_endpoint(); if config .otlp_config_receiver_protocols_http_endpoint .is_none() - && !yaml_config - .otlp_config - .receiver - .protocols - .http - .endpoint - .is_empty() + && yaml_otlp_config_receiver_protocols_http_endpoint.is_some() { - config.otlp_config_receiver_protocols_http_endpoint = Some( - yaml_config - .otlp_config - .receiver - .protocols - .http - .endpoint - .clone(), - ); - } - - if !config.otlp_config_traces_enabled && yaml_config.otlp_config.traces.enabled { + config.otlp_config_receiver_protocols_http_endpoint = + yaml_otlp_config_receiver_protocols_http_endpoint.map(std::string::ToString::to_string); + } + + if !config.otlp_config_traces_enabled && yaml_config.otlp_config_traces_enabled() { config.otlp_config_traces_enabled = true; } if !config.otlp_config_ignore_missing_datadog_fields - && yaml_config.otlp_config.traces.ignore_missing_datadog_fields + && yaml_config.otlp_config_traces_ignore_missing_datadog_fields() { config.otlp_config_ignore_missing_datadog_fields = true; } if !config.otlp_config_traces_span_name_as_resource_name - && yaml_config.otlp_config.traces.span_name_as_resource_name + && yaml_config.otlp_config_traces_span_name_as_resource_name() { config.otlp_config_traces_span_name_as_resource_name = true; } + let yaml_otlp_config_traces_span_name_remappings = + yaml_config.otlp_config_traces_span_name_remappings(); if config.otlp_config_traces_span_name_remappings.is_empty() - && !yaml_config - .otlp_config - .traces - .span_name_remappings - .is_empty() + && !yaml_otlp_config_traces_span_name_remappings.is_empty() { config .otlp_config_traces_span_name_remappings - .clone_from(&yaml_config.otlp_config.traces.span_name_remappings); + .clone_from(&yaml_otlp_config_traces_span_name_remappings); } } diff --git a/yaml.rs b/yaml.rs index 176b4dbb..22e3aa5f 100644 --- a/yaml.rs +++ b/yaml.rs @@ -16,7 +16,89 @@ pub struct Config { pub logs_config: LogsConfig, pub apm_config: ApmConfig, pub proxy: ProxyConfig, - pub otlp_config: OtlpConfig, + pub otlp_config: Option, +} + +impl Config { + #[must_use] + pub fn otlp_config_receiver_protocols_http_endpoint(&self) -> Option<&str> { + self.otlp_config + .as_ref()? + .receiver + .as_ref()? + .protocols + .as_ref()? + .http + .as_ref()? + .endpoint + .as_deref() + } + + #[must_use] + pub fn otlp_config_receiver_protocols_grpc(&self) -> Option<&Value> { + self.otlp_config + .as_ref()? + .receiver + .as_ref()? + .protocols + .as_ref()? + .grpc + .as_ref() + } + + #[must_use] + pub fn otlp_config_traces_enabled(&self) -> bool { + self.otlp_config.as_ref().is_some_and(|otlp_config| { + otlp_config + .traces + .as_ref() + .is_some_and(|traces| traces.enabled) + }) + } + + #[must_use] + pub fn otlp_config_traces_ignore_missing_datadog_fields(&self) -> bool { + self.otlp_config.as_ref().is_some_and(|otlp_config| { + otlp_config + .traces + .as_ref() + .is_some_and(|traces| traces.ignore_missing_datadog_fields) + }) + } + + #[must_use] + pub fn otlp_config_traces_span_name_as_resource_name(&self) -> bool { + self.otlp_config.as_ref().is_some_and(|otlp_config| { + otlp_config + .traces + .as_ref() + .is_some_and(|traces| traces.span_name_as_resource_name) + }) + } + + #[must_use] + pub fn otlp_config_traces_span_name_remappings(&self) -> HashMap { + self.otlp_config + .as_ref() + .and_then(|otlp_config| otlp_config.traces.as_ref()) + .map(|traces| traces.span_name_remappings.clone()) + .unwrap_or_default() + } + + #[must_use] + pub fn otlp_config_traces_probabilistic_sampler(&self) -> Option<&Value> { + self.otlp_config + .as_ref() + .and_then(|otlp_config| otlp_config.traces.as_ref()) + .and_then(|traces| traces.probabilistic_sampler.as_ref()) + } + + #[must_use] + pub fn otlp_config_logs(&self) -> Option<&Value> { + self.otlp_config + .as_ref() + .and_then(|otlp_config| otlp_config.logs.as_ref()) + } } /// Logs Config @@ -77,8 +159,8 @@ pub struct ProxyConfig { #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct OtlpConfig { - pub receiver: OtlpReceiverConfig, - pub traces: OtlpTracesConfig, + pub receiver: Option, + pub traces: Option, // NOT SUPPORTED pub metrics: Option, @@ -89,14 +171,14 @@ pub struct OtlpConfig { #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct OtlpReceiverConfig { - pub protocols: OtlpReceiverProtocolsConfig, + pub protocols: Option, } #[derive(Debug, PartialEq, Deserialize, Clone, Default)] #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct OtlpReceiverProtocolsConfig { - pub http: OtlpReceiverHttpConfig, + pub http: Option, // NOT SUPPORTED pub grpc: Option, @@ -106,10 +188,22 @@ pub struct OtlpReceiverProtocolsConfig { #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct OtlpReceiverHttpConfig { - pub endpoint: String, + pub endpoint: Option, } -#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +impl Default for OtlpTracesConfig { + fn default() -> Self { + Self { + enabled: true, // Default this to true + span_name_as_resource_name: false, + span_name_remappings: HashMap::new(), + ignore_missing_datadog_fields: false, + probabilistic_sampler: None, + } + } +} + +#[derive(Debug, PartialEq, Deserialize, Clone)] #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct OtlpTracesConfig { @@ -124,3 +218,36 @@ pub struct OtlpTracesConfig { // NOT SUPORTED pub probabilistic_sampler: Option, } + +#[cfg(test)] +mod tests { + use std::path::Path; + + use crate::config::get_config; + + #[test] + fn test_otlp_config_receiver_protocols_http_endpoint() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r" + otlp_config: + receiver: + protocols: + http: + endpoint: 0.0.0.0:4318 + ", + )?; + + let config = get_config(Path::new("")).expect("should parse config"); + + assert_eq!( + config.otlp_config_receiver_protocols_http_endpoint, + Some("0.0.0.0:4318".to_string()) + ); + + Ok(()) + }); + } +} From ff27ddfc5cc1dc31e2dccc74327b21f799fb807d Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 3 Jun 2025 06:46:05 -0400 Subject: [PATCH 065/112] feat: continuous flushing strategy for high throughput functions (#684) This is a heavy refactor and new feature. - Introduces FlushDecision and separates it from FlushStrategy - Cleans up FlushControl logic and methods It also adds the ability to flush telemetry across multiple serial lambda invocations. This is done using the `continuous` strategy. This is a huge win for busy functions as seen in our test fleet, where the p99/max drops precipitously, which also causes the average to plummet. This also helps reduce the number of cold starts encountered during scaleup events, which further reduces latency along with costs: ![image](https://github.com/user-attachments/assets/14851e22-327d-43b0-8246-5780cfbf6ef7) Technical implementation: We spawn the task and collect the flush handles, then in the two periodic strategies we check if there were any errors or unresolved futures in the next flush cycle. If so, we switch to the `periodic` strategy to ensure flushing completes successfully. We don't adapt to the periodic strategy unless the last 20 invocations occurred within the `config.flush_timeout` value, which has been increased by default. This is a naive implementation. A better one would be to calculate the first derivative of the invocation periodicity. If the rate is increasing, we can adapt to the continuous strategy. If the rate slows, we should fall back to the periodic strategy. image The existing implementation is cautious in that we could definitely adapt sooner but don't. Todo: add a feature flag for continuous flushing? --- env.rs | 2 +- flush_strategy.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/env.rs b/env.rs index 04846178..4ba1e59c 100644 --- a/env.rs +++ b/env.rs @@ -135,7 +135,7 @@ impl Default for Config { api_key_secret_arn: String::default(), kms_api_key: String::default(), serverless_flush_strategy: FlushStrategy::Default, - flush_timeout: 5, + flush_timeout: 10, // Unified Tagging env: None, service: None, diff --git a/flush_strategy.rs b/flush_strategy.rs index dd4fea2c..307c92f8 100644 --- a/flush_strategy.rs +++ b/flush_strategy.rs @@ -12,6 +12,7 @@ pub enum FlushStrategy { End, EndPeriodically(PeriodicStrategy), Periodically(PeriodicStrategy), + Continuously(PeriodicStrategy), } // Deserialize for FlushStrategy From b714380d026286f84063637e7331e6830186c158 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 4 Jun 2025 13:03:02 -0400 Subject: [PATCH 066/112] fix: bump flush_timeout default (#697) A little goofy because we use this to determine when/how to move over to continuous flushing, but the gist is that our invocation context tracks the start time of each invocation. Because it's all local to a single sandbox, this means that the time diff between invocations includes post runtime duration, so it's very common to have 20 invocations greater than 10s if there are even a couple of periodic/end flushes in there. This customizable with `DD_FLUSH_TIMEOUT` so if people want to set it to a very short timeout, they are able to. --- env.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env.rs b/env.rs index 4ba1e59c..edd702ed 100644 --- a/env.rs +++ b/env.rs @@ -135,7 +135,7 @@ impl Default for Config { api_key_secret_arn: String::default(), kms_api_key: String::default(), serverless_flush_strategy: FlushStrategy::Default, - flush_timeout: 10, + flush_timeout: 30, // Unified Tagging env: None, service: None, From b853eb9aa5d59c9a22614a4f48848dfbc6f539fa Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Mon, 9 Jun 2025 13:48:37 -0400 Subject: [PATCH 067/112] feat: Allow users to specify continuous strategy (#701) https://datadoghq.atlassian.net/browse/SVLS-6994 --- flush_strategy.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flush_strategy.rs b/flush_strategy.rs index 307c92f8..a72a2416 100644 --- a/flush_strategy.rs +++ b/flush_strategy.rs @@ -36,6 +36,9 @@ impl<'de> Deserialize<'de> for FlushStrategy { (Some("periodically"), Some(interval)) => { Ok(FlushStrategy::Periodically(PeriodicStrategy { interval })) } + (Some("continuously"), Some(interval)) => { + Ok(FlushStrategy::Continuously(PeriodicStrategy { interval })) + } (Some("end"), Some(interval)) => { Ok(FlushStrategy::EndPeriodically(PeriodicStrategy { interval, From 6c42125c8f4b1c9738fcef63ce07f9a7a3e43626 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Tue, 10 Jun 2025 13:34:23 -0400 Subject: [PATCH 068/112] feat: Use http2 unless overridden or using a proxy (#706) We rolled out HTTP/2 support for logs in v81, which seems to have broken logs for some users relying on proxies which may not support http2. This change introduces a new configuration option called `use_http1`. 1. If `DD_HTTP_PROTOCOL` is explicitly set to http1, we'll use it 2. If `DD_HTTP_PROTOCOL` is not set and the user is using a proxy, we'll use http1 unless overridden by the `DD_HTTP_PROTOCOL` flag being set to anything other than `http1`. fixes #705 --- env.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/env.rs b/env.rs index edd702ed..8da18f28 100644 --- a/env.rs +++ b/env.rs @@ -39,6 +39,7 @@ pub struct Config { pub logs_config_compression_level: i32, pub logs_config_logs_dd_url: String, pub serverless_flush_strategy: FlushStrategy, + pub http_protocol: Option, #[serde(deserialize_with = "deserialize_bool_from_anything")] pub enhanced_metrics: bool, pub lambda_proc_enhanced_metrics: bool, @@ -135,6 +136,7 @@ impl Default for Config { api_key_secret_arn: String::default(), kms_api_key: String::default(), serverless_flush_strategy: FlushStrategy::Default, + http_protocol: None, // None means: use HTTP/2 by default, but fall back to HTTP/1 if proxy is configured flush_timeout: 30, // Unified Tagging env: None, From bd9f6140fb1f41a01ae7c1b12d6dca0cbbe9c817 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 11 Jun 2025 10:20:09 -0400 Subject: [PATCH 069/112] Dual shipping metrics support (#704) Adds support for dual shipping metrics to endpoints configured using the `additional_endpoints` YAML or `DD_ADDITIONAL_ENDPOINTS` env var config. For each configured endpoint/API key combination, we now create a separate `MetricsFlusher` to flush the same batch of metrics to multiple endpoints in parallel. Also, updates the retry logic to retry flushing for the specific flusher that encountered an error. Tested dual shipping metrics to 2 additional orgs/endpoints including eu1. Depends on dogstatsd changes: https://github.com/DataDog/serverless-components/pull/20 --- additional_endpoints.rs | 118 ++++++++++++++++++++++++++++++++++++++++ env.rs | 4 ++ mod.rs | 10 ++++ yaml.rs | 35 ++++++++++++ 4 files changed, 167 insertions(+) create mode 100644 additional_endpoints.rs diff --git a/additional_endpoints.rs b/additional_endpoints.rs new file mode 100644 index 00000000..281229d9 --- /dev/null +++ b/additional_endpoints.rs @@ -0,0 +1,118 @@ +use serde::{Deserialize, Deserializer}; +use serde_json::Value; +use std::collections::HashMap; +use tracing::error; + +#[allow(clippy::module_name_repetitions)] +pub fn deserialize_additional_endpoints<'de, D>( + deserializer: D, +) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let value = Value::deserialize(deserializer)?; + + match value { + Value::Object(map) => { + // For YAML format (object) in datadog.yaml + let mut result = HashMap::new(); + for (key, value) in map { + match value { + Value::Array(arr) => { + let urls: Vec = arr + .into_iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect(); + result.insert(key, urls); + } + _ => { + error!("Failed to deserialize additional endpoints - Invalid YAML format: expected array for key {}", key); + } + } + } + Ok(result) + } + Value::String(s) if !s.is_empty() => { + // For JSON format (string) in DD_ADDITIONAL_ENDPOINTS + if let Ok(map) = serde_json::from_str(&s) { + Ok(map) + } else { + error!("Failed to deserialize additional endpoints - Invalid JSON format"); + Ok(HashMap::new()) + } + } + _ => Ok(HashMap::new()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_deserialize_additional_endpoints_yaml() { + // Test YAML format (object) + let input = json!({ + "https://app.datadoghq.com": ["key1", "key2"], + "https://app.datadoghq.eu": ["key3"] + }); + + let result = deserialize_additional_endpoints(input).unwrap(); + + let mut expected = HashMap::new(); + expected.insert( + "https://app.datadoghq.com".to_string(), + vec!["key1".to_string(), "key2".to_string()], + ); + expected.insert( + "https://app.datadoghq.eu".to_string(), + vec!["key3".to_string()], + ); + + assert_eq!(result, expected); + } + + #[test] + fn test_deserialize_additional_endpoints_json() { + // Test JSON string format + let input = json!("{\"https://app.datadoghq.com\":[\"key1\",\"key2\"],\"https://app.datadoghq.eu\":[\"key3\"]}"); + + let result = deserialize_additional_endpoints(input).unwrap(); + + let mut expected = HashMap::new(); + expected.insert( + "https://app.datadoghq.com".to_string(), + vec!["key1".to_string(), "key2".to_string()], + ); + expected.insert( + "https://app.datadoghq.eu".to_string(), + vec!["key3".to_string()], + ); + + assert_eq!(result, expected); + } + + #[test] + fn test_deserialize_additional_endpoints_invalid_or_empty() { + // Test empty YAML + let input = json!({}); + let result = deserialize_additional_endpoints(input).unwrap(); + assert!(result.is_empty()); + + // Test empty JSON + let input = json!(""); + let result = deserialize_additional_endpoints(input).unwrap(); + assert!(result.is_empty()); + + let input = json!({ + "https://app.datadoghq.com": "invalid-yaml" + }); + let result = deserialize_additional_endpoints(input).unwrap(); + assert!(result.is_empty()); + + let input = json!("invalid-json"); + let result = deserialize_additional_endpoints(input).unwrap(); + assert!(result.is_empty()); + } +} diff --git a/env.rs b/env.rs index 8da18f28..b0bc6cb6 100644 --- a/env.rs +++ b/env.rs @@ -1,3 +1,4 @@ +use crate::config::additional_endpoints::deserialize_additional_endpoints; use serde::{Deserialize, Deserializer}; use std::collections::HashMap; use std::vec; @@ -76,6 +77,8 @@ pub struct Config { // Metrics overrides pub dd_url: String, pub url: String, + #[serde(deserialize_with = "deserialize_additional_endpoints")] + pub additional_endpoints: HashMap>, // OTLP // // - Traces @@ -174,6 +177,7 @@ impl Default for Config { apm_features: vec![], dd_url: String::default(), url: String::default(), + additional_endpoints: HashMap::new(), // OTLP // // - Receiver diff --git a/mod.rs b/mod.rs index 541fa83a..6e784ea2 100644 --- a/mod.rs +++ b/mod.rs @@ -1,3 +1,4 @@ +pub mod additional_endpoints; pub mod apm_replace_rule; pub mod aws; pub mod env; @@ -242,6 +243,15 @@ fn merge_config(config: &mut EnvConfig, yaml_config: &YamlConfig) { .otlp_config_traces_span_name_remappings .clone_from(&yaml_otlp_config_traces_span_name_remappings); } + + // Dual Shipping + // + // - Metrics + if config.additional_endpoints.is_empty() { + config + .additional_endpoints + .clone_from(&yaml_config.additional_endpoints); + } } #[inline] diff --git a/yaml.rs b/yaml.rs index 22e3aa5f..e82b322c 100644 --- a/yaml.rs +++ b/yaml.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use crate::config::additional_endpoints::deserialize_additional_endpoints; use crate::config::{deserialize_apm_replace_rules, deserialize_processing_rules, ProcessingRule}; use datadog_trace_obfuscation::replacer::ReplaceRule; use serde::Deserialize; @@ -17,6 +18,8 @@ pub struct Config { pub apm_config: ApmConfig, pub proxy: ProxyConfig, pub otlp_config: Option, + #[serde(deserialize_with = "deserialize_additional_endpoints")] + pub additional_endpoints: HashMap>, } impl Config { @@ -221,6 +224,7 @@ pub struct OtlpTracesConfig { #[cfg(test)] mod tests { + use std::collections::HashMap; use std::path::Path; use crate::config::get_config; @@ -250,4 +254,35 @@ mod tests { Ok(()) }); } + + #[test] + fn test_parse_additional_endpoints_from_yaml() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r#" +additional_endpoints: + "https://app.datadoghq.com": + - apikey2 + - apikey3 + "https://app.datadoghq.eu": + - apikey4 +"#, + )?; + + let config = get_config(Path::new("")).expect("should parse config"); + let mut expected = HashMap::new(); + expected.insert( + "https://app.datadoghq.com".to_string(), + vec!["apikey2".to_string(), "apikey3".to_string()], + ); + expected.insert( + "https://app.datadoghq.eu".to_string(), + vec!["apikey4".to_string()], + ); + assert_eq!(config.additional_endpoints, expected); + Ok(()) + }); + } } From 61560744a868838bc512352851a74b04e14ee415 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 2 Jul 2025 17:11:15 -0400 Subject: [PATCH 070/112] chore: Separate AwsCredentials from AwsConfig (#716) # Problem Right now `AwsConfig` has a lot of fields, including the ones related to credential: ``` pub aws_access_key_id: String, pub aws_secret_access_key: String, pub aws_session_token: String, pub aws_container_credentials_full_uri: String, pub aws_container_authorization_token: String, ``` The next PR https://github.com/DataDog/datadog-lambda-extension/pull/717 wants to lazily load API key and the credentials. To do that, for the resolver function `resolve_secrets()`, I need to change the param `aws_config` from `&AwsConfig` to `Arc>`. Because `aws_config` is passed to many places, this change involves updating lots of functions, which is formidable. # This PR Separates these credential-related fields out from `AwsConfig` and creates a new struct `AwsCredentials` Thus, the next PR will only need to change the param `aws_credentials` from `&AwsCredentials` to `Arc>`. Because `aws_credentials` is not used in lots of places, the next PR becomes easier. https://datadoghq.atlassian.net/issues/SVLS-6996 https://datadoghq.atlassian.net/issues/SVLS-6998 --- aws.rs | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/aws.rs b/aws.rs index 49067dd2..b45fc008 100644 --- a/aws.rs +++ b/aws.rs @@ -15,11 +15,6 @@ const AWS_LAMBDA_EXEC_WRAPPER: &str = "AWS_LAMBDA_EXEC_WRAPPER"; #[derive(Debug, Clone)] pub struct AwsConfig { pub region: String, - pub aws_access_key_id: String, - pub aws_secret_access_key: String, - pub aws_session_token: String, - pub aws_container_credentials_full_uri: String, - pub aws_container_authorization_token: String, pub aws_lwa_proxy_lambda_runtime_api: Option, pub function_name: String, pub runtime_api: String, @@ -32,6 +27,29 @@ impl AwsConfig { pub fn from_env(start_time: Instant) -> Self { Self { region: env::var(AWS_DEFAULT_REGION).unwrap_or("us-east-1".to_string()), + aws_lwa_proxy_lambda_runtime_api: env::var(AWS_LWA_LAMBDA_RUNTIME_API_PROXY).ok(), + function_name: env::var(AWS_LAMBDA_FUNCTION_NAME).unwrap_or_default(), + runtime_api: env::var(AWS_LAMBDA_RUNTIME_API).unwrap_or_default(), + sandbox_init_time: start_time, + exec_wrapper: env::var(AWS_LAMBDA_EXEC_WRAPPER).ok(), + } + } +} + +#[allow(clippy::module_name_repetitions)] +#[derive(Debug, Clone)] +pub struct AwsCredentials { + pub aws_access_key_id: String, + pub aws_secret_access_key: String, + pub aws_session_token: String, + pub aws_container_credentials_full_uri: String, + pub aws_container_authorization_token: String, +} + +impl AwsCredentials { + #[must_use] + pub fn from_env() -> Self { + Self { aws_access_key_id: env::var(AWS_ACCESS_KEY_ID).unwrap_or_default(), aws_secret_access_key: env::var(AWS_SECRET_ACCESS_KEY).unwrap_or_default(), aws_session_token: env::var(AWS_SESSION_TOKEN).unwrap_or_default(), @@ -39,11 +57,6 @@ impl AwsConfig { .unwrap_or_default(), aws_container_authorization_token: env::var(AWS_CONTAINER_AUTHORIZATION_TOKEN) .unwrap_or_default(), - aws_lwa_proxy_lambda_runtime_api: env::var(AWS_LWA_LAMBDA_RUNTIME_API_PROXY).ok(), - function_name: env::var(AWS_LAMBDA_FUNCTION_NAME).unwrap_or_default(), - runtime_api: env::var(AWS_LAMBDA_RUNTIME_API).unwrap_or_default(), - sandbox_init_time: start_time, - exec_wrapper: env::var(AWS_LAMBDA_EXEC_WRAPPER).ok(), } } } From 33da8b8d2961ceb5a79ecfbfe0a1fc6d7b95b7a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 3 Jul 2025 14:41:32 -0400 Subject: [PATCH 071/112] chore(config): separate config from sources (#709) # What? Separates the configuration from sources, allowing it to be used in more use cases. # How? Creates new default configuration and separates the environment variables and YAML sources from the default. # Why? Make it easier to track changes in every source, as the field names might be different to what they are used at the configuration level. # Notes I expect to abstract this even more by providing it as a crate which can have features, that way customers can only use the sources and product specific fields they need. --------- Co-authored-by: Aleksandr Pasechnik Co-authored-by: Florentin Labelle --- env.rs | 852 ++++++++++++++++++++++++++++++++++++++----------- log_level.rs | 53 +++- mod.rs | 692 ++++++++++++++++++++++++++++++---------- yaml.rs | 876 +++++++++++++++++++++++++++++++++++++++++---------- 4 files changed, 1941 insertions(+), 532 deletions(-) diff --git a/env.rs b/env.rs index b0bc6cb6..5bec1eeb 100644 --- a/env.rs +++ b/env.rs @@ -1,238 +1,714 @@ -use crate::config::additional_endpoints::deserialize_additional_endpoints; -use serde::{Deserialize, Deserializer}; +use figment::{providers::Env, Figment}; +use serde::Deserialize; use std::collections::HashMap; -use std::vec; use datadog_trace_obfuscation::replacer::ReplaceRule; -use serde_aux::field_attributes::deserialize_bool_from_anything; -use serde_json::Value; - -use crate::config::{ - apm_replace_rule::deserialize_apm_replace_rules, - flush_strategy::FlushStrategy, - log_level::{deserialize_log_level, LogLevel}, - processing_rule::{deserialize_processing_rules, ProcessingRule}, - service_mapping::deserialize_service_mapping, - trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}, + +use crate::{ + config::{ + additional_endpoints::deserialize_additional_endpoints, + apm_replace_rule::deserialize_apm_replace_rules, + deserialize_array_from_comma_separated_string, deserialize_key_value_pairs, + deserialize_optional_bool_from_anything, deserialize_string_or_int, + flush_strategy::FlushStrategy, + log_level::LogLevel, + processing_rule::{deserialize_processing_rules, ProcessingRule}, + service_mapping::deserialize_service_mapping, + trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}, + Config, ConfigError, ConfigSource, + }, + merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, }; -#[derive(Debug, PartialEq, Deserialize, Clone)] +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] #[serde(default)] #[allow(clippy::struct_excessive_bools)] -pub struct Config { - pub site: String, - pub api_key: String, - pub api_key_secret_arn: String, - pub kms_api_key: String, +#[allow(clippy::module_name_repetitions)] +pub struct EnvConfig { + /// @env `DD_SITE` + /// + /// The Datadog site to send telemetry to + pub site: Option, + /// @env `DD_API_KEY` + /// + /// The Datadog API key used to submit telemetry to Datadog + pub api_key: Option, + /// @env `DD_LOG_LEVEL` + /// + /// Minimum log level of the Datadog Agent. + /// Valid log levels are: trace, debug, info, warn, and error. + pub log_level: Option, + + /// @env `DD_FLUSH_TIMEOUT` + /// + /// Flush timeout in seconds + /// todo(duncanista): find out where this comes from + /// todo(?): go agent adds jitter too + pub flush_timeout: Option, + + // Proxy + /// @env `DD_PROXY_HTTPS` + /// + /// Proxy endpoint for HTTPS connections (most Datadog traffic) + pub proxy_https: Option, + /// @env `DD_PROXY_NO_PROXY` + /// + /// Specify hosts the Agent should connect to directly, bypassing the proxy. + #[serde(deserialize_with = "deserialize_array_from_comma_separated_string")] + pub proxy_no_proxy: Vec, + /// @env `DD_HTTP_PROTOCOL` + /// + /// The HTTP protocol to use for the Datadog Agent. + /// The transport type to use for sending logs. Possible values are "auto" or "http1". + pub http_protocol: Option, + + // Metrics + /// @env `DD_DD_URL` + /// + /// @default `https://app.datadoghq.com` + /// + /// The host of the Datadog intake server to send **metrics** to, only set this option + /// if you need the Agent to send **metrics** to a custom URL, it overrides the site + /// setting defined in "site". It does not affect APM, Logs, Remote Configuration, + /// or Live Process intake which have their own "*_`dd_url`" settings. + /// + /// If `DD_DD_URL` and `DD_URL` are both set, `DD_DD_URL` is used in priority. + pub dd_url: Option, + /// @env `DD_URL` + /// + /// @default `https://app.datadoghq.com` + pub url: Option, + /// @env `DD_ADDITIONAL_ENDPOINTS` + /// + /// Additional endpoints to send metrics to. + /// + #[serde(deserialize_with = "deserialize_additional_endpoints")] + pub additional_endpoints: HashMap>, + + // Unified Service Tagging + /// @env `DD_ENV` + /// + /// The environment name where the agent is running. Attached in-app to every + /// metric, event, log, trace, and service check emitted by this Agent. #[serde(deserialize_with = "deserialize_string_or_int")] pub env: Option, + /// @env `DD_SERVICE` #[serde(deserialize_with = "deserialize_string_or_int")] pub service: Option, + /// @env `DD_VERSION` #[serde(deserialize_with = "deserialize_string_or_int")] pub version: Option, - pub tags: Option, - #[serde(deserialize_with = "deserialize_log_level")] - pub log_level: LogLevel, + /// @env `DD_TAGS` + #[serde(deserialize_with = "deserialize_key_value_pairs")] + pub tags: HashMap, + + // Logs + /// @env `DD_LOGS_CONFIG_LOGS_DD_URL` + /// + /// Define the endpoint and port to hit when using a proxy for logs. + pub logs_config_logs_dd_url: Option, + /// @env `DD_LOGS_CONFIG_PROCESSING_RULES` + /// + /// Global processing rules that are applied to all logs. The available rules are + /// "`exclude_at_match`", "`include_at_match`" and "`mask_sequences`". More information in Datadog documentation: + /// #[serde(deserialize_with = "deserialize_processing_rules")] pub logs_config_processing_rules: Option>, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub logs_config_use_compression: bool, - pub logs_config_compression_level: i32, - pub logs_config_logs_dd_url: String, - pub serverless_flush_strategy: FlushStrategy, - pub http_protocol: Option, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub enhanced_metrics: bool, - pub lambda_proc_enhanced_metrics: bool, - /// Flush timeout in seconds - pub flush_timeout: u64, //TODO go agent adds jitter too - pub https_proxy: Option, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub capture_lambda_payload: bool, - pub capture_lambda_payload_max_depth: u32, + /// @env `DD_LOGS_CONFIG_USE_COMPRESSION` + /// + /// If enabled, the Agent compresses logs before sending them. + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub logs_config_use_compression: Option, + /// @env `DD_LOGS_CONFIG_COMPRESSION_LEVEL` + /// + /// The `compression_level` parameter accepts values from 0 (no compression) + /// to 9 (maximum compression but higher resource usage). Only takes effect if + /// `use_compression` is set to `true`. + pub logs_config_compression_level: Option, + + // APM + // + /// @env `DD_SERVICE_MAPPING` #[serde(deserialize_with = "deserialize_service_mapping")] pub service_mapping: HashMap, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub serverless_logs_enabled: bool, + // + /// @env `DD_APM_DD_URL` + /// + /// Define the endpoint and port to hit when using a proxy for APM. + pub apm_dd_url: Option, + /// @env `DD_APM_REPLACE_TAGS` + /// + /// Defines a set of rules to replace or remove certain resources, tags containing + /// potentially sensitive information. + /// Each rule has to contain: + /// * name - string - The tag name to replace, for resources use "resource.name". + /// * pattern - string - The pattern to match the desired content to replace + /// * repl - string - what to inline if the pattern is matched + /// + /// + #[serde(deserialize_with = "deserialize_apm_replace_rules")] + pub apm_replace_tags: Option>, + /// @env `DD_APM_CONFIG_OBFUSCATION_HTTP_REMOVE_QUERY_STRING` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub apm_config_obfuscation_http_remove_query_string: Option, + /// @env `DD_APM_CONFIG_OBFUSCATION_HTTP_REMOVE_PATHS_WITH_DIGITS` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub apm_config_obfuscation_http_remove_paths_with_digits: Option, + /// @env `DD_APM_FEATURES` + #[serde(deserialize_with = "deserialize_array_from_comma_separated_string")] + pub apm_features: Vec, + // // Trace Propagation + /// @env `DD_TRACE_PROPAGATION_STYLE` #[serde(deserialize_with = "deserialize_trace_propagation_style")] pub trace_propagation_style: Vec, + /// @env `DD_TRACE_PROPAGATION_STYLE_EXTRACT` #[serde(deserialize_with = "deserialize_trace_propagation_style")] pub trace_propagation_style_extract: Vec, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub trace_propagation_extract_first: bool, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub trace_propagation_http_baggage_enabled: bool, - // APM - pub apm_config_apm_dd_url: String, - #[serde(deserialize_with = "deserialize_apm_replace_rules")] - pub apm_replace_tags: Option>, - #[serde(deserialize_with = "deserialize_apm_replace_rules")] - pub apm_config_replace_tags: Option>, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub apm_config_obfuscation_http_remove_query_string: bool, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub apm_config_obfuscation_http_remove_paths_with_digits: bool, - pub apm_features: Vec, - // Metrics overrides - pub dd_url: String, - pub url: String, - #[serde(deserialize_with = "deserialize_additional_endpoints")] - pub additional_endpoints: HashMap>, + /// @env `DD_TRACE_PROPAGATION_EXTRACT_FIRST` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub trace_propagation_extract_first: Option, + /// @env `DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub trace_propagation_http_baggage_enabled: Option, + // OTLP // - // - Traces - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub otlp_config_traces_enabled: bool, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub otlp_config_traces_span_name_as_resource_name: bool, + // - APM / Traces + /// @env `DD_OTLP_CONFIG_TRACES_ENABLED` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub otlp_config_traces_enabled: Option, + /// @env `DD_OTLP_CONFIG_TRACES_SPAN_NAME_AS_RESOURCE_NAME` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub otlp_config_traces_span_name_as_resource_name: Option, + /// @env `DD_OTLP_CONFIG_TRACES_SPAN_NAME_REMAPPINGS` + #[serde(deserialize_with = "deserialize_key_value_pairs")] pub otlp_config_traces_span_name_remappings: HashMap, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub otlp_config_ignore_missing_datadog_fields: bool, + /// @env `DD_OTLP_CONFIG_IGNORE_MISSING_DATADOG_FIELDS` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub otlp_config_ignore_missing_datadog_fields: Option, + // // - Receiver / HTTP + /// @env `DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT` pub otlp_config_receiver_protocols_http_endpoint: Option, - // - // - // Fallback Config - pub extension_version: Option, - // AppSec - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub serverless_appsec_enabled: bool, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub appsec_enabled: bool, - // OTLP + // - Unsupported Configuration // // - Receiver / GRPC + /// @env `DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT` pub otlp_config_receiver_protocols_grpc_endpoint: Option, + /// @env `DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_TRANSPORT` pub otlp_config_receiver_protocols_grpc_transport: Option, + /// @env `DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_MAX_RECV_MSG_SIZE_MIB` pub otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib: Option, // - Metrics - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub otlp_config_metrics_enabled: bool, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub otlp_config_metrics_resource_attributes_as_tags: bool, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub otlp_config_metrics_instrumentation_scope_metadata_as_tags: bool, + /// @env `DD_OTLP_CONFIG_METRICS_ENABLED` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub otlp_config_metrics_enabled: Option, + /// @env `DD_OTLP_CONFIG_METRICS_RESOURCE_ATTRIBUTES_AS_TAGS` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub otlp_config_metrics_resource_attributes_as_tags: Option, + /// @env `DD_OTLP_CONFIG_METRICS_INSTRUMENTATION_SCOPE_METADATA_AS_TAGS` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub otlp_config_metrics_instrumentation_scope_metadata_as_tags: Option, + /// @env `DD_OTLP_CONFIG_METRICS_TAG_CARDINALITY` pub otlp_config_metrics_tag_cardinality: Option, + /// @env `DD_OTLP_CONFIG_METRICS_DELTA_TTL` pub otlp_config_metrics_delta_ttl: Option, + /// @env `DD_OTLP_CONFIG_METRICS_HISTOGRAMS_MODE` pub otlp_config_metrics_histograms_mode: Option, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub otlp_config_metrics_histograms_send_count_sum_metrics: bool, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub otlp_config_metrics_histograms_send_aggregation_metrics: bool, + /// @env `DD_OTLP_CONFIG_METRICS_HISTOGRAMS_SEND_COUNT_SUM_METRICS` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub otlp_config_metrics_histograms_send_count_sum_metrics: Option, + /// @env `DD_OTLP_CONFIG_METRICS_HISTOGRAMS_SEND_AGGREGATION_METRICS` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub otlp_config_metrics_histograms_send_aggregation_metrics: Option, pub otlp_config_metrics_sums_cumulative_monotonic_mode: Option, + /// @env `DD_OTLP_CONFIG_METRICS_SUMS_INITIAL_CUMULATIVE_MONOTONIC_VALUE` pub otlp_config_metrics_sums_initial_cumulativ_monotonic_value: Option, + /// @env `DD_OTLP_CONFIG_METRICS_SUMMARIES_MODE` pub otlp_config_metrics_summaries_mode: Option, // - Traces + /// @env `DD_OTLP_CONFIG_TRACES_PROBABILISTIC_SAMPLER_SAMPLING_PERCENTAGE` pub otlp_config_traces_probabilistic_sampler_sampling_percentage: Option, // - Logs - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub otlp_config_logs_enabled: bool, + /// @env `DD_OTLP_CONFIG_LOGS_ENABLED` + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub otlp_config_logs_enabled: Option, + + // AWS Lambda + /// @env `DD_API_KEY_SECRET_ARN` + /// + /// The AWS ARN of the secret containing the Datadog API key. + pub api_key_secret_arn: Option, + /// @env `DD_KMS_API_KEY` + /// + /// The AWS KMS API key to use for the Datadog Agent. + pub kms_api_key: Option, + /// @env `DD_SERVERLESS_LOGS_ENABLED` + /// + /// Enable logs for AWS Lambda. Default is `true`. + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub serverless_logs_enabled: Option, + /// @env `DD_SERVERLESS_FLUSH_STRATEGY` + /// + /// The flush strategy to use for AWS Lambda. + pub serverless_flush_strategy: Option, + /// @env `DD_ENHANCED_METRICS` + /// + /// Enable enhanced metrics for AWS Lambda. Default is `true`. + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub enhanced_metrics: Option, + /// @env `DD_LAMBDA_PROC_ENHANCED_METRICS` + /// + /// Enable Lambda process metrics for AWS Lambda. Default is `true`. + /// + /// This is for metrics like: + /// - CPU usage + /// - Network usage + /// - File descriptor count + /// - Thread count + /// - Temp directory usage + pub lambda_proc_enhanced_metrics: Option, + /// @env `DD_CAPTURE_LAMBDA_PAYLOAD` + /// + /// Enable capture of the Lambda request and response payloads. + /// Default is `false`. + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub capture_lambda_payload: Option, + /// @env `DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH` + /// + /// The maximum depth of the Lambda payload to capture. + /// Default is `10`. Requires `capture_lambda_payload` to be `true`. + pub capture_lambda_payload_max_depth: Option, + /// @env `DD_SERVERLESS_APPSEC_ENABLED` + /// + /// Enable Application and API Protection (AAP), previously known as AppSec/ASM, for AWS Lambda. + /// Default is `false`. + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub serverless_appsec_enabled: Option, + /// @env `DD_EXTENSION_VERSION` + /// + /// Used to decide which version of the Datadog Lambda Extension to use. + /// When set to `compatibility`, the extension will boot up in legacy mode. + pub extension_version: Option, } -impl Default for Config { - fn default() -> Self { - Config { - // General - site: String::default(), - api_key: String::default(), - api_key_secret_arn: String::default(), - kms_api_key: String::default(), - serverless_flush_strategy: FlushStrategy::Default, - http_protocol: None, // None means: use HTTP/2 by default, but fall back to HTTP/1 if proxy is configured - flush_timeout: 30, - // Unified Tagging - env: None, - service: None, - version: None, - tags: None, - // Logs - log_level: LogLevel::default(), - logs_config_processing_rules: None, - logs_config_use_compression: true, - logs_config_compression_level: 6, - logs_config_logs_dd_url: String::default(), - // Metrics - enhanced_metrics: true, - lambda_proc_enhanced_metrics: true, - https_proxy: None, - capture_lambda_payload: false, - capture_lambda_payload_max_depth: 10, - service_mapping: HashMap::new(), - serverless_logs_enabled: true, - // Trace Propagation - trace_propagation_style: vec![ - TracePropagationStyle::Datadog, - TracePropagationStyle::TraceContext, - ], - trace_propagation_style_extract: vec![], - trace_propagation_extract_first: false, - trace_propagation_http_baggage_enabled: false, - // APM - apm_config_apm_dd_url: String::default(), - apm_replace_tags: None, - apm_config_replace_tags: None, - apm_config_obfuscation_http_remove_query_string: false, - apm_config_obfuscation_http_remove_paths_with_digits: false, - apm_features: vec![], - dd_url: String::default(), - url: String::default(), - additional_endpoints: HashMap::new(), - // OTLP - // - // - Receiver - otlp_config_receiver_protocols_http_endpoint: None, - // - Traces - otlp_config_traces_enabled: true, - otlp_config_ignore_missing_datadog_fields: false, - otlp_config_traces_span_name_as_resource_name: false, - otlp_config_traces_span_name_remappings: HashMap::new(), - // - // Fallback Config (NOT SUPPORTED yet) - extension_version: None, - // AppSec - serverless_appsec_enabled: false, - appsec_enabled: false, - // OTLP - // - // - Receiver - otlp_config_receiver_protocols_grpc_endpoint: None, - otlp_config_receiver_protocols_grpc_transport: None, - otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib: None, - // - Metrics - otlp_config_metrics_enabled: false, // TODO(duncanista): Go Agent default is to true - otlp_config_metrics_resource_attributes_as_tags: false, - otlp_config_metrics_instrumentation_scope_metadata_as_tags: false, - otlp_config_metrics_tag_cardinality: None, - otlp_config_metrics_delta_ttl: None, - otlp_config_metrics_histograms_mode: None, - otlp_config_metrics_histograms_send_count_sum_metrics: false, - otlp_config_metrics_histograms_send_aggregation_metrics: false, - otlp_config_metrics_sums_cumulative_monotonic_mode: None, - otlp_config_metrics_sums_initial_cumulativ_monotonic_value: None, - otlp_config_metrics_summaries_mode: None, - // - Traces - otlp_config_traces_probabilistic_sampler_sampling_percentage: None, - // - Logs - otlp_config_logs_enabled: false, - } - } +#[allow(clippy::too_many_lines)] +fn merge_config(config: &mut Config, env_config: &EnvConfig) { + // Basic fields + merge_string!(config, env_config, site); + merge_string!(config, env_config, api_key); + merge_option_to_value!(config, env_config, log_level); + merge_option_to_value!(config, env_config, flush_timeout); + + // Unified Service Tagging + merge_option!(config, env_config, env); + merge_option!(config, env_config, service); + merge_option!(config, env_config, version); + merge_hashmap!(config, env_config, tags); + + // Proxy + merge_option!(config, env_config, proxy_https); + merge_vec!(config, env_config, proxy_no_proxy); + merge_option!(config, env_config, http_protocol); + + // Endpoints + merge_string!(config, env_config, dd_url); + merge_string!(config, env_config, url); + merge_hashmap!(config, env_config, additional_endpoints); + + // Logs + merge_string!(config, env_config, logs_config_logs_dd_url); + merge_option!(config, env_config, logs_config_processing_rules); + merge_option_to_value!(config, env_config, logs_config_use_compression); + merge_option_to_value!(config, env_config, logs_config_compression_level); + + // APM + merge_hashmap!(config, env_config, service_mapping); + merge_string!(config, env_config, apm_dd_url); + merge_option!(config, env_config, apm_replace_tags); + merge_option_to_value!( + config, + env_config, + apm_config_obfuscation_http_remove_query_string + ); + merge_option_to_value!( + config, + env_config, + apm_config_obfuscation_http_remove_paths_with_digits + ); + merge_vec!(config, env_config, apm_features); + + // Trace Propagation + merge_vec!(config, env_config, trace_propagation_style); + merge_vec!(config, env_config, trace_propagation_style_extract); + merge_option_to_value!(config, env_config, trace_propagation_extract_first); + merge_option_to_value!(config, env_config, trace_propagation_http_baggage_enabled); + + // OTLP + merge_option_to_value!(config, env_config, otlp_config_traces_enabled); + merge_option_to_value!( + config, + env_config, + otlp_config_traces_span_name_as_resource_name + ); + merge_hashmap!(config, env_config, otlp_config_traces_span_name_remappings); + merge_option_to_value!( + config, + env_config, + otlp_config_ignore_missing_datadog_fields + ); + merge_option!( + config, + env_config, + otlp_config_receiver_protocols_http_endpoint + ); + merge_option!( + config, + env_config, + otlp_config_receiver_protocols_grpc_endpoint + ); + merge_option!( + config, + env_config, + otlp_config_receiver_protocols_grpc_transport + ); + merge_option!( + config, + env_config, + otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib + ); + merge_option_to_value!(config, env_config, otlp_config_metrics_enabled); + merge_option_to_value!( + config, + env_config, + otlp_config_metrics_resource_attributes_as_tags + ); + merge_option_to_value!( + config, + env_config, + otlp_config_metrics_instrumentation_scope_metadata_as_tags + ); + merge_option!(config, env_config, otlp_config_metrics_tag_cardinality); + merge_option!(config, env_config, otlp_config_metrics_delta_ttl); + merge_option!(config, env_config, otlp_config_metrics_histograms_mode); + merge_option_to_value!( + config, + env_config, + otlp_config_metrics_histograms_send_count_sum_metrics + ); + merge_option_to_value!( + config, + env_config, + otlp_config_metrics_histograms_send_aggregation_metrics + ); + merge_option!( + config, + env_config, + otlp_config_metrics_sums_cumulative_monotonic_mode + ); + merge_option!( + config, + env_config, + otlp_config_metrics_sums_initial_cumulativ_monotonic_value + ); + merge_option!(config, env_config, otlp_config_metrics_summaries_mode); + merge_option!( + config, + env_config, + otlp_config_traces_probabilistic_sampler_sampling_percentage + ); + merge_option_to_value!(config, env_config, otlp_config_logs_enabled); + + // AWS Lambda + merge_string!(config, env_config, api_key_secret_arn); + merge_string!(config, env_config, kms_api_key); + merge_option_to_value!(config, env_config, serverless_logs_enabled); + merge_option_to_value!(config, env_config, serverless_flush_strategy); + merge_option_to_value!(config, env_config, enhanced_metrics); + merge_option_to_value!(config, env_config, lambda_proc_enhanced_metrics); + merge_option_to_value!(config, env_config, capture_lambda_payload); + merge_option_to_value!(config, env_config, capture_lambda_payload_max_depth); + merge_option_to_value!(config, env_config, serverless_appsec_enabled); + merge_option!(config, env_config, extension_version); } -fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let value = Value::deserialize(deserializer)?; - match value { - Value::String(s) => { - if s.trim().is_empty() { - Ok(None) - } else { - Ok(Some(s)) +#[derive(Debug, PartialEq, Clone, Copy)] +#[allow(clippy::module_name_repetitions)] +pub struct EnvConfigSource; + +impl ConfigSource for EnvConfigSource { + fn load(&self, config: &mut Config) -> Result<(), ConfigError> { + let figment = Figment::new() + .merge(Env::prefixed("DATADOG_")) + .merge(Env::prefixed("DD_")); + + match figment.extract::() { + Ok(env_config) => merge_config(config, &env_config), + Err(e) => { + return Err(ConfigError::ParseError(format!( + "Failed to parse config from environment variables: {e}, using default config.", + ))); } } - Value::Number(n) => Ok(Some(n.to_string())), - _ => Err(serde::de::Error::custom("expected a string or an integer")), + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{ + flush_strategy::{FlushStrategy, PeriodicStrategy}, + log_level::LogLevel, + processing_rule::{Kind, ProcessingRule}, + trace_propagation_style::TracePropagationStyle, + Config, + }; + + #[test] + fn test_merge_config_overrides_with_environment_variables() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + + // Set environment variables here + jail.set_env("DD_SITE", "test-site"); + jail.set_env("DD_API_KEY", "test-api-key"); + jail.set_env("DD_LOG_LEVEL", "debug"); + jail.set_env("DD_FLUSH_TIMEOUT", "42"); + + // Proxy + jail.set_env("DD_PROXY_HTTPS", "https://proxy.example.com"); + jail.set_env("DD_PROXY_NO_PROXY", "localhost,127.0.0.1"); + jail.set_env("DD_HTTP_PROTOCOL", "http1"); + + // Metrics + jail.set_env("DD_DD_URL", "https://metrics.datadoghq.com"); + jail.set_env("DD_URL", "https://app.datadoghq.com"); + jail.set_env( + "DD_ADDITIONAL_ENDPOINTS", + "{\"https://app.datadoghq.com\": [\"apikey2\", \"apikey3\"], \"https://app.datadoghq.eu\": [\"apikey4\"]}", + ); + + // Unified Service Tagging + jail.set_env("DD_ENV", "test-env"); + jail.set_env("DD_SERVICE", "test-service"); + jail.set_env("DD_VERSION", "1.0.0"); + jail.set_env("DD_TAGS", "team:test-team,project:test-project"); + + // Logs + jail.set_env("DD_LOGS_CONFIG_LOGS_DD_URL", "https://logs.datadoghq.com"); + jail.set_env( + "DD_LOGS_CONFIG_PROCESSING_RULES", + r#"[{"type":"exclude_at_match","name":"exclude","pattern":"exclude"}]"#, + ); + jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "false"); + jail.set_env("DD_LOGS_CONFIG_COMPRESSION_LEVEL", "3"); + + // APM + jail.set_env("DD_SERVICE_MAPPING", "old-service:new-service"); + jail.set_env("DD_APPSEC_ENABLED", "true"); + jail.set_env("DD_APM_DD_URL", "https://apm.datadoghq.com"); + jail.set_env( + "DD_APM_REPLACE_TAGS", + r#"[{"name":"test-tag","pattern":"test-pattern","repl":"replacement"}]"#, + ); + jail.set_env("DD_APM_CONFIG_OBFUSCATION_HTTP_REMOVE_QUERY_STRING", "true"); + jail.set_env( + "DD_APM_CONFIG_OBFUSCATION_HTTP_REMOVE_PATHS_WITH_DIGITS", + "true", + ); + jail.set_env( + "DD_APM_FEATURES", + "enable_otlp_compute_top_level_by_span_kind,enable_stats_by_span_kind", + ); + + // Trace Propagation + jail.set_env("DD_TRACE_PROPAGATION_STYLE", "datadog"); + jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "b3"); + jail.set_env("DD_TRACE_PROPAGATION_EXTRACT_FIRST", "true"); + jail.set_env("DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED", "true"); + + // OTLP + jail.set_env("DD_OTLP_CONFIG_TRACES_ENABLED", "false"); + jail.set_env("DD_OTLP_CONFIG_TRACES_SPAN_NAME_AS_RESOURCE_NAME", "true"); + jail.set_env( + "DD_OTLP_CONFIG_TRACES_SPAN_NAME_REMAPPINGS", + "old-span:new-span", + ); + jail.set_env("DD_OTLP_CONFIG_IGNORE_MISSING_DATADOG_FIELDS", "true"); + jail.set_env( + "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT", + "http://localhost:4318", + ); + jail.set_env( + "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", + "http://localhost:4317", + ); + jail.set_env("DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_TRANSPORT", "tcp"); + jail.set_env( + "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_MAX_RECV_MSG_SIZE_MIB", + "4", + ); + jail.set_env("DD_OTLP_CONFIG_METRICS_ENABLED", "true"); + jail.set_env("DD_OTLP_CONFIG_METRICS_RESOURCE_ATTRIBUTES_AS_TAGS", "true"); + jail.set_env( + "DD_OTLP_CONFIG_METRICS_INSTRUMENTATION_SCOPE_METADATA_AS_TAGS", + "true", + ); + jail.set_env("DD_OTLP_CONFIG_METRICS_TAG_CARDINALITY", "low"); + jail.set_env("DD_OTLP_CONFIG_METRICS_DELTA_TTL", "3600"); + jail.set_env("DD_OTLP_CONFIG_METRICS_HISTOGRAMS_MODE", "counters"); + jail.set_env( + "DD_OTLP_CONFIG_METRICS_HISTOGRAMS_SEND_COUNT_SUM_METRICS", + "true", + ); + jail.set_env( + "DD_OTLP_CONFIG_METRICS_HISTOGRAMS_SEND_AGGREGATION_METRICS", + "true", + ); + jail.set_env( + "DD_OTLP_CONFIG_METRICS_SUMS_CUMULATIVE_MONOTONIC_MODE", + "to_delta", + ); + jail.set_env( + "DD_OTLP_CONFIG_METRICS_SUMS_INITIAL_CUMULATIV_MONOTONIC_VALUE", + "auto", + ); + jail.set_env("DD_OTLP_CONFIG_METRICS_SUMMARIES_MODE", "quantiles"); + jail.set_env( + "DD_OTLP_CONFIG_TRACES_PROBABILISTIC_SAMPLER_SAMPLING_PERCENTAGE", + "50", + ); + jail.set_env("DD_OTLP_CONFIG_LOGS_ENABLED", "true"); + + // AWS Lambda + jail.set_env( + "DD_API_KEY_SECRET_ARN", + "arn:aws:secretsmanager:region:account:secret:datadog-api-key", + ); + jail.set_env("DD_KMS_API_KEY", "test-kms-key"); + jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); + jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,60000"); + jail.set_env("DD_ENHANCED_METRICS", "false"); + jail.set_env("DD_LAMBDA_PROC_ENHANCED_METRICS", "false"); + jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "true"); + jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "5"); + jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); + jail.set_env("DD_EXTENSION_VERSION", "compatibility"); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + let expected_config = Config { + site: "test-site".to_string(), + api_key: "test-api-key".to_string(), + log_level: LogLevel::Debug, + flush_timeout: 42, + proxy_https: Some("https://proxy.example.com".to_string()), + proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], + http_protocol: Some("http1".to_string()), + dd_url: "https://metrics.datadoghq.com".to_string(), + url: "https://app.datadoghq.com".to_string(), + additional_endpoints: HashMap::from([ + ( + "https://app.datadoghq.com".to_string(), + vec!["apikey2".to_string(), "apikey3".to_string()], + ), + ( + "https://app.datadoghq.eu".to_string(), + vec!["apikey4".to_string()], + ), + ]), + env: Some("test-env".to_string()), + service: Some("test-service".to_string()), + version: Some("1.0.0".to_string()), + tags: HashMap::from([ + ("team".to_string(), "test-team".to_string()), + ("project".to_string(), "test-project".to_string()), + ]), + logs_config_logs_dd_url: "https://logs.datadoghq.com".to_string(), + logs_config_processing_rules: Some(vec![ProcessingRule { + kind: Kind::ExcludeAtMatch, + name: "exclude".to_string(), + pattern: "exclude".to_string(), + replace_placeholder: None, + }]), + logs_config_use_compression: false, + logs_config_compression_level: 3, + service_mapping: HashMap::from([( + "old-service".to_string(), + "new-service".to_string(), + )]), + apm_dd_url: "https://apm.datadoghq.com".to_string(), + apm_replace_tags: Some( + datadog_trace_obfuscation::replacer::parse_rules_from_string( + r#"[{"name":"test-tag","pattern":"test-pattern","repl":"replacement"}]"#, + ) + .expect("Failed to parse replace rules"), + ), + apm_config_obfuscation_http_remove_query_string: true, + apm_config_obfuscation_http_remove_paths_with_digits: true, + apm_features: vec![ + "enable_otlp_compute_top_level_by_span_kind".to_string(), + "enable_stats_by_span_kind".to_string(), + ], + trace_propagation_style: vec![TracePropagationStyle::Datadog], + trace_propagation_style_extract: vec![TracePropagationStyle::B3], + trace_propagation_extract_first: true, + trace_propagation_http_baggage_enabled: true, + otlp_config_traces_enabled: false, + otlp_config_traces_span_name_as_resource_name: true, + otlp_config_traces_span_name_remappings: HashMap::from([( + "old-span".to_string(), + "new-span".to_string(), + )]), + otlp_config_ignore_missing_datadog_fields: true, + otlp_config_receiver_protocols_http_endpoint: Some( + "http://localhost:4318".to_string(), + ), + otlp_config_receiver_protocols_grpc_endpoint: Some( + "http://localhost:4317".to_string(), + ), + otlp_config_receiver_protocols_grpc_transport: Some("tcp".to_string()), + otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib: Some(4), + otlp_config_metrics_enabled: true, + otlp_config_metrics_resource_attributes_as_tags: true, + otlp_config_metrics_instrumentation_scope_metadata_as_tags: true, + otlp_config_metrics_tag_cardinality: Some("low".to_string()), + otlp_config_metrics_delta_ttl: Some(3600), + otlp_config_metrics_histograms_mode: Some("counters".to_string()), + otlp_config_metrics_histograms_send_count_sum_metrics: true, + otlp_config_metrics_histograms_send_aggregation_metrics: true, + otlp_config_metrics_sums_cumulative_monotonic_mode: Some("to_delta".to_string()), + otlp_config_metrics_sums_initial_cumulativ_monotonic_value: Some( + "auto".to_string(), + ), + otlp_config_metrics_summaries_mode: Some("quantiles".to_string()), + otlp_config_traces_probabilistic_sampler_sampling_percentage: Some(50), + otlp_config_logs_enabled: true, + api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" + .to_string(), + kms_api_key: "test-kms-key".to_string(), + serverless_logs_enabled: false, + serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { + interval: 60000, + }), + enhanced_metrics: false, + lambda_proc_enhanced_metrics: false, + capture_lambda_payload: true, + capture_lambda_payload_max_depth: 5, + serverless_appsec_enabled: true, + extension_version: Some("compatibility".to_string()), + }; + + assert_eq!(config, expected_config); + + Ok(()) + }); } } diff --git a/log_level.rs b/log_level.rs index 86c255e1..7443f3ca 100644 --- a/log_level.rs +++ b/log_level.rs @@ -1,7 +1,10 @@ +use std::str::FromStr; + use serde::{Deserialize, Deserializer}; +use serde_json::Value; use tracing::error; -#[derive(Clone, Copy, Debug, PartialEq, Deserialize, Default)] +#[derive(Clone, Copy, Debug, PartialEq, Default)] pub enum LogLevel { /// Designates very serious errors. Error, @@ -42,20 +45,40 @@ impl LogLevel { } } -#[allow(clippy::module_name_repetitions)] -pub fn deserialize_log_level<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let s: String = Deserialize::deserialize(deserializer)?; - match s.to_lowercase().as_str() { - "error" => Ok(LogLevel::Error), - "warn" => Ok(LogLevel::Warn), - "info" => Ok(LogLevel::Info), - "debug" => Ok(LogLevel::Debug), - "trace" => Ok(LogLevel::Trace), - _ => { - error!("Unknown log level: {}, using warn", s); +impl FromStr for LogLevel { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "error" => Ok(LogLevel::Error), + "warn" => Ok(LogLevel::Warn), + "info" => Ok(LogLevel::Info), + "debug" => Ok(LogLevel::Debug), + "trace" => Ok(LogLevel::Trace), + _ => Err(format!( + "Invalid log level: '{s}'. Valid levels are: error, warn, info, debug, trace", + )), + } + } +} + +impl<'de> Deserialize<'de> for LogLevel { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + + if let Value::String(s) = value { + match LogLevel::from_str(&s) { + Ok(level) => Ok(level), + Err(e) => { + error!("{}", e); + Ok(LogLevel::Warn) + } + } + } else { + error!("Expected a string for log level, got {:?}", value); Ok(LogLevel::Warn) } } diff --git a/mod.rs b/mod.rs index 6e784ea2..157b8916 100644 --- a/mod.rs +++ b/mod.rs @@ -9,20 +9,131 @@ pub mod service_mapping; pub mod trace_propagation_style; pub mod yaml; +use datadog_trace_obfuscation::replacer::ReplaceRule; use datadog_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; -use std::path::Path; +use serde::{Deserialize, Deserializer}; +use serde_aux::prelude::deserialize_bool_from_anything; +use serde_json::Value; -use figment::providers::{Format, Yaml}; -use figment::{providers::Env, Figment}; +use std::path::Path; +use std::{collections::HashMap, fmt}; +use tracing::{debug, error}; use crate::config::{ apm_replace_rule::deserialize_apm_replace_rules, - env::Config as EnvConfig, + env::EnvConfigSource, + flush_strategy::FlushStrategy, + log_level::LogLevel, processing_rule::{deserialize_processing_rules, ProcessingRule}, - yaml::Config as YamlConfig, + trace_propagation_style::TracePropagationStyle, + yaml::YamlConfigSource, }; -pub type Config = EnvConfig; +/// Helper macro to merge Option fields to String fields +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_string { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if let Some(value) = &$source.$source_field { + $config.$config_field.clone_from(value); + } + }; + ($config:expr, $source:expr, $field:ident) => { + if let Some(value) = &$source.$field { + $config.$field.clone_from(value); + } + }; +} + +/// Helper macro to merge Option fields where T implements Clone +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_option { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if $source.$source_field.is_some() { + $config.$config_field.clone_from(&$source.$source_field); + } + }; + ($config:expr, $source:expr, $field:ident) => { + if $source.$field.is_some() { + $config.$field.clone_from(&$source.$field); + } + }; +} + +/// Helper macro to merge Option fields to T fields when Option is Some +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_option_to_value { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if let Some(value) = &$source.$source_field { + $config.$config_field = value.clone(); + } + }; + ($config:expr, $source:expr, $field:ident) => { + if let Some(value) = &$source.$field { + $config.$field = value.clone(); + } + }; +} + +/// Helper macro to merge `Vec` fields when `Vec` is not empty +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_vec { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if !$source.$source_field.is_empty() { + $config.$config_field.clone_from(&$source.$source_field); + } + }; + ($config:expr, $source:expr, $field:ident) => { + if !$source.$field.is_empty() { + $config.$field.clone_from(&$source.$field); + } + }; +} + +// nit: these will replace one map with the other, not merge the maps togehter, right? +/// Helper macro to merge `HashMap` fields when `HashMap` is not empty +/// +/// Providing one field argument will merge the value from the source config field into the config +/// field. +/// +/// Providing two field arguments will merge the value from the source config field into the config +/// field if the value is not empty. +#[macro_export] +macro_rules! merge_hashmap { + ($config:expr, $config_field:ident, $source:expr, $source_field:ident) => { + if !$source.$source_field.is_empty() { + $config.$config_field.clone_from(&$source.$source_field); + } + }; + ($config:expr, $source:expr, $field:ident) => { + if !$source.$field.is_empty() { + $config.$field.clone_from(&$source.$field); + } + }; +} #[derive(Debug, PartialEq)] #[allow(clippy::module_name_repetitions)] @@ -31,11 +142,276 @@ pub enum ConfigError { UnsupportedField(String), } +#[allow(clippy::module_name_repetitions)] +pub trait ConfigSource { + fn load(&self, config: &mut Config) -> Result<(), ConfigError>; +} + +#[derive(Default)] +#[allow(clippy::module_name_repetitions)] +pub struct ConfigBuilder { + sources: Vec>, + config: Config, +} + +#[allow(clippy::module_name_repetitions)] +impl ConfigBuilder { + #[must_use] + pub fn add_source(mut self, source: Box) -> Self { + self.sources.push(source); + self + } + + pub fn build(&mut self) -> Config { + let mut failed_sources = 0; + for source in &self.sources { + match source.load(&mut self.config) { + Ok(()) => (), + Err(e) => { + error!("Failed to load config: {:?}", e); + failed_sources += 1; + } + } + } + + if !self.sources.is_empty() && failed_sources == self.sources.len() { + debug!("All sources failed to load config, using default config."); + } + + if self.config.site.is_empty() { + self.config.site = "datadoghq.com".to_string(); + } + + // If `proxy_https` is not set, set it from `HTTPS_PROXY` environment variable + // if it exists + if let Ok(https_proxy) = std::env::var("HTTPS_PROXY") { + if self.config.proxy_https.is_none() { + self.config.proxy_https = Some(https_proxy); + } + } + + // If `proxy_https` is set, check if the site is in `NO_PROXY` environment variable + // or in the `proxy_no_proxy` config field. + if self.config.proxy_https.is_some() { + let site_in_no_proxy = std::env::var("NO_PROXY") + .map_or(false, |no_proxy| no_proxy.contains(&self.config.site)) + || self + .config + .proxy_no_proxy + .iter() + .any(|no_proxy| no_proxy.contains(&self.config.site)); + if site_in_no_proxy { + self.config.proxy_https = None; + } + } + + // If extraction is not set, set it to the same as the propagation style + if self.config.trace_propagation_style_extract.is_empty() { + self.config + .trace_propagation_style_extract + .clone_from(&self.config.trace_propagation_style); + } + + // If Logs URL is not set, set it to the default + if self.config.logs_config_logs_dd_url.is_empty() { + self.config.logs_config_logs_dd_url = build_fqdn_logs(self.config.site.clone()); + } + + // If APM URL is not set, set it to the default + if self.config.apm_dd_url.is_empty() { + self.config.apm_dd_url = trace_intake_url(self.config.site.clone().as_str()); + } else { + // If APM URL is set, add the site to the URL + self.config.apm_dd_url = trace_intake_url_prefixed(self.config.apm_dd_url.as_str()); + } + + self.config.clone() + } +} + +#[derive(Debug, PartialEq, Clone)] +#[allow(clippy::module_name_repetitions)] +#[allow(clippy::struct_excessive_bools)] +pub struct Config { + pub site: String, + pub api_key: String, + pub log_level: LogLevel, + + pub flush_timeout: u64, + + // Proxy + pub proxy_https: Option, + pub proxy_no_proxy: Vec, + pub http_protocol: Option, + + // Endpoints + pub dd_url: String, + pub url: String, + pub additional_endpoints: HashMap>, + + // Unified Service Tagging + pub env: Option, + pub service: Option, + pub version: Option, + pub tags: HashMap, + + // Logs + pub logs_config_logs_dd_url: String, + pub logs_config_processing_rules: Option>, + pub logs_config_use_compression: bool, + pub logs_config_compression_level: i32, + + // APM + // + pub service_mapping: HashMap, + // + pub apm_dd_url: String, + pub apm_replace_tags: Option>, + pub apm_config_obfuscation_http_remove_query_string: bool, + pub apm_config_obfuscation_http_remove_paths_with_digits: bool, + pub apm_features: Vec, + // + // Trace Propagation + pub trace_propagation_style: Vec, + pub trace_propagation_style_extract: Vec, + pub trace_propagation_extract_first: bool, + pub trace_propagation_http_baggage_enabled: bool, + + // OTLP + // + // - APM / Traces + pub otlp_config_traces_enabled: bool, + pub otlp_config_traces_span_name_as_resource_name: bool, + pub otlp_config_traces_span_name_remappings: HashMap, + pub otlp_config_ignore_missing_datadog_fields: bool, + // + // - Receiver / HTTP + pub otlp_config_receiver_protocols_http_endpoint: Option, + // - Unsupported Configuration + // + // - Receiver / GRPC + pub otlp_config_receiver_protocols_grpc_endpoint: Option, + pub otlp_config_receiver_protocols_grpc_transport: Option, + pub otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib: Option, + // - Metrics + pub otlp_config_metrics_enabled: bool, + pub otlp_config_metrics_resource_attributes_as_tags: bool, + pub otlp_config_metrics_instrumentation_scope_metadata_as_tags: bool, + pub otlp_config_metrics_tag_cardinality: Option, + pub otlp_config_metrics_delta_ttl: Option, + pub otlp_config_metrics_histograms_mode: Option, + pub otlp_config_metrics_histograms_send_count_sum_metrics: bool, + pub otlp_config_metrics_histograms_send_aggregation_metrics: bool, + pub otlp_config_metrics_sums_cumulative_monotonic_mode: Option, + // nit: is the e in cumulative missing intentionally? + pub otlp_config_metrics_sums_initial_cumulativ_monotonic_value: Option, + pub otlp_config_metrics_summaries_mode: Option, + // - Traces + pub otlp_config_traces_probabilistic_sampler_sampling_percentage: Option, + // - Logs + pub otlp_config_logs_enabled: bool, + + // AWS Lambda + pub api_key_secret_arn: String, + pub kms_api_key: String, + pub serverless_logs_enabled: bool, + pub serverless_flush_strategy: FlushStrategy, + pub enhanced_metrics: bool, + pub lambda_proc_enhanced_metrics: bool, + pub capture_lambda_payload: bool, + pub capture_lambda_payload_max_depth: u32, + pub serverless_appsec_enabled: bool, + pub extension_version: Option, +} + +impl Default for Config { + fn default() -> Self { + Self { + site: String::default(), + api_key: String::default(), + log_level: LogLevel::default(), + flush_timeout: 30, + + // Proxy + proxy_https: None, + proxy_no_proxy: vec![], + http_protocol: None, + + // Endpoints + dd_url: String::default(), + url: String::default(), + additional_endpoints: HashMap::new(), + + // Unified Service Tagging + env: None, + service: None, + version: None, + tags: HashMap::new(), + + // Logs + logs_config_logs_dd_url: String::default(), + logs_config_processing_rules: None, + logs_config_use_compression: true, + logs_config_compression_level: 6, + + // APM + service_mapping: HashMap::new(), + apm_dd_url: String::default(), + apm_replace_tags: None, + apm_config_obfuscation_http_remove_query_string: false, + apm_config_obfuscation_http_remove_paths_with_digits: false, + apm_features: vec![], + trace_propagation_style: vec![ + TracePropagationStyle::Datadog, + TracePropagationStyle::TraceContext, + ], + trace_propagation_style_extract: vec![], + trace_propagation_extract_first: false, + trace_propagation_http_baggage_enabled: false, + + // OTLP + otlp_config_traces_enabled: true, + otlp_config_traces_span_name_as_resource_name: false, + otlp_config_traces_span_name_remappings: HashMap::new(), + otlp_config_ignore_missing_datadog_fields: false, + otlp_config_receiver_protocols_http_endpoint: None, + otlp_config_receiver_protocols_grpc_endpoint: None, + otlp_config_receiver_protocols_grpc_transport: None, + otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib: None, + otlp_config_metrics_enabled: false, // TODO(duncanista): Go Agent default is to true + otlp_config_metrics_resource_attributes_as_tags: false, + otlp_config_metrics_instrumentation_scope_metadata_as_tags: false, + otlp_config_metrics_tag_cardinality: None, + otlp_config_metrics_delta_ttl: None, + otlp_config_metrics_histograms_mode: None, + otlp_config_metrics_histograms_send_count_sum_metrics: false, + otlp_config_metrics_histograms_send_aggregation_metrics: false, + otlp_config_metrics_sums_cumulative_monotonic_mode: None, + otlp_config_metrics_sums_initial_cumulativ_monotonic_value: None, + otlp_config_metrics_summaries_mode: None, + otlp_config_traces_probabilistic_sampler_sampling_percentage: None, + otlp_config_logs_enabled: false, + + // AWS Lambda + api_key_secret_arn: String::default(), + kms_api_key: String::default(), + serverless_logs_enabled: true, + serverless_flush_strategy: FlushStrategy::Default, + enhanced_metrics: true, + lambda_proc_enhanced_metrics: true, + capture_lambda_payload: false, + capture_lambda_payload_max_depth: 10, + serverless_appsec_enabled: false, + extension_version: None, + } + } +} + fn log_fallback_reason(reason: &str) { println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); } -fn fallback(config: &EnvConfig, yaml_config: &YamlConfig) -> Result<(), ConfigError> { +fn fallback(config: &Config) -> Result<(), ConfigError> { // Customer explicitly opted out of the Next Gen extension let opted_out = match config.extension_version.as_deref() { Some("compatibility") => true, @@ -50,13 +426,15 @@ fn fallback(config: &EnvConfig, yaml_config: &YamlConfig) -> Result<(), ConfigEr )); } - if config.serverless_appsec_enabled || config.appsec_enabled { + if config.serverless_appsec_enabled { log_fallback_reason("appsec_enabled"); - return Err(ConfigError::UnsupportedField("appsec_enabled".to_string())); + return Err(ConfigError::UnsupportedField( + "serverless_appsec_enabled".to_string(), + )); } // OTLP - let has_otlp_env_config = config + let has_otlp_config = config .otlp_config_receiver_protocols_grpc_endpoint .is_some() || config @@ -85,13 +463,7 @@ fn fallback(config: &EnvConfig, yaml_config: &YamlConfig) -> Result<(), ConfigEr .is_some() || config.otlp_config_logs_enabled; - let has_otlp_yaml_config = yaml_config.otlp_config_receiver_protocols_grpc().is_some() - || yaml_config - .otlp_config_traces_probabilistic_sampler() - .is_some() - || yaml_config.otlp_config_logs().is_some(); - - if has_otlp_env_config || has_otlp_yaml_config { + if has_otlp_config { log_fallback_reason("otel"); return Err(ConfigError::UnsupportedField("otel".to_string())); } @@ -100,164 +472,127 @@ fn fallback(config: &EnvConfig, yaml_config: &YamlConfig) -> Result<(), ConfigEr } #[allow(clippy::module_name_repetitions)] -pub fn get_config(config_directory: &Path) -> Result { - let path = config_directory.join("datadog.yaml"); - - // Get default config fields (and ENV specific ones) - let figment = Figment::new() - .merge(Yaml::file(&path)) - .merge(Env::prefixed("DATADOG_")) - .merge(Env::prefixed("DD_")) - .merge(Env::raw().only(&["HTTPS_PROXY"])); - - // Get YAML nested fields - let yaml_figment = Figment::from(Yaml::file(&path)); - - let (mut config, yaml_config): (EnvConfig, YamlConfig) = - match (figment.extract(), yaml_figment.extract()) { - (Ok(env_config), Ok(yaml_config)) => (env_config, yaml_config), - (_, Err(err)) | (Err(err), _) => { - println!("Failed to parse Datadog config: {err}"); - return Err(ConfigError::ParseError(err.to_string())); - } - }; - - fallback(&config, &yaml_config)?; - - // Set site if empty - if config.site.is_empty() { - config.site = "datadoghq.com".to_string(); - } - - // NOTE: Must happen after config.site is set - // Prefer DD_PROXY_HTTPS over HTTPS_PROXY - // No else needed as HTTPS_PROXY is handled by reqwest and built into trace client - if let Ok(https_proxy) = std::env::var("DD_PROXY_HTTPS").or_else(|_| { - yaml_config - .proxy - .https - .clone() - .ok_or(std::env::VarError::NotPresent) - }) { - let no_proxy = yaml_config.proxy.no_proxy.clone(); - if std::env::var("NO_PROXY").map_or(false, |no_proxy| no_proxy.contains(&config.site)) - || no_proxy.map_or(false, |no_proxy| no_proxy.contains(&config.site)) - { - config.https_proxy = None; - } else { - config.https_proxy = Some(https_proxy); - } - } +pub fn get_config(config_directory: &Path) -> Result { + let path: std::path::PathBuf = config_directory.join("datadog.yaml"); + let mut config_builder = ConfigBuilder::default() + .add_source(Box::new(YamlConfigSource { path })) + .add_source(Box::new(EnvConfigSource)); + + let config = config_builder.build(); - merge_config(&mut config, &yaml_config); + fallback(&config)?; - // Metrics are handled by dogstatsd in Main Ok(config) } -/// Merge YAML nested fields into `EnvConfig` -/// -fn merge_config(config: &mut EnvConfig, yaml_config: &YamlConfig) { - // Set logs_config_processing_rules if not defined in env - if config.logs_config_processing_rules.is_none() { - if let Some(processing_rules) = yaml_config.logs_config.processing_rules.as_ref() { - config.logs_config_processing_rules = Some(processing_rules.clone()); - } - } - - // Trace Propagation - // - // If not set by the user, set defaults - if config.trace_propagation_style_extract.is_empty() { - config - .trace_propagation_style_extract - .clone_from(&config.trace_propagation_style); - } - if config.logs_config_logs_dd_url.is_empty() { - config.logs_config_logs_dd_url = build_fqdn_logs(config.site.clone()); - } - - if config.apm_config_apm_dd_url.is_empty() { - config.apm_config_apm_dd_url = trace_intake_url(config.site.clone().as_str()); - } else { - config.apm_config_apm_dd_url = - trace_intake_url_prefixed(config.apm_config_apm_dd_url.as_str()); - } +#[inline] +#[must_use] +fn build_fqdn_logs(site: String) -> String { + format!("https://http-intake.logs.{site}") +} - if config.apm_replace_tags.is_none() { - if let Some(rules) = yaml_config.apm_config.replace_tags.as_ref() { - config.apm_replace_tags = Some(rules.clone()); +pub fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Value::deserialize(deserializer)?; + match value { + Value::String(s) => { + if s.trim().is_empty() { + Ok(None) + } else { + Ok(Some(s)) + } } + Value::Number(n) => Ok(Some(n.to_string())), + _ => Err(serde::de::Error::custom("expected a string or an integer")), } +} - if !config.apm_config_obfuscation_http_remove_paths_with_digits { - if let Some(obfuscation) = yaml_config.apm_config.obfuscation { - config.apm_config_obfuscation_http_remove_paths_with_digits = - obfuscation.http.remove_paths_with_digits; +pub fn deserialize_optional_bool_from_anything<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + // First try to deserialize as Option<_> to handle null/missing values + let opt: Option = Option::deserialize(deserializer)?; + + match opt { + None => Ok(None), + Some(value) => { + // Use your existing method by deserializing the value + let bool_result = deserialize_bool_from_anything(value).map_err(|e| { + serde::de::Error::custom(format!("Failed to deserialize bool: {e}")) + })?; + Ok(Some(bool_result)) } } +} - if !config.apm_config_obfuscation_http_remove_query_string { - if let Some(obfuscation) = yaml_config.apm_config.obfuscation { - config.apm_config_obfuscation_http_remove_query_string = - obfuscation.http.remove_query_string; - } - } +pub fn deserialize_key_value_pairs<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct KeyValueVisitor; - // OTLP - // - // - Receiver / HTTP - let yaml_otlp_config_receiver_protocols_http_endpoint = - yaml_config.otlp_config_receiver_protocols_http_endpoint(); - if config - .otlp_config_receiver_protocols_http_endpoint - .is_none() - && yaml_otlp_config_receiver_protocols_http_endpoint.is_some() - { - config.otlp_config_receiver_protocols_http_endpoint = - yaml_otlp_config_receiver_protocols_http_endpoint.map(std::string::ToString::to_string); - } + impl<'de> serde::de::Visitor<'de> for KeyValueVisitor { + type Value = HashMap; - if !config.otlp_config_traces_enabled && yaml_config.otlp_config_traces_enabled() { - config.otlp_config_traces_enabled = true; - } + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a string in format 'key1:value1,key2:value2' or 'key1:value1'") + } - if !config.otlp_config_ignore_missing_datadog_fields - && yaml_config.otlp_config_traces_ignore_missing_datadog_fields() - { - config.otlp_config_ignore_missing_datadog_fields = true; - } + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + let mut map = HashMap::new(); - if !config.otlp_config_traces_span_name_as_resource_name - && yaml_config.otlp_config_traces_span_name_as_resource_name() - { - config.otlp_config_traces_span_name_as_resource_name = true; - } + for tag in value.split(',') { + let parts = tag.split(':').collect::>(); + if parts.len() == 2 { + map.insert(parts[0].to_string(), parts[1].to_string()); + } + } - let yaml_otlp_config_traces_span_name_remappings = - yaml_config.otlp_config_traces_span_name_remappings(); - if config.otlp_config_traces_span_name_remappings.is_empty() - && !yaml_otlp_config_traces_span_name_remappings.is_empty() - { - config - .otlp_config_traces_span_name_remappings - .clone_from(&yaml_otlp_config_traces_span_name_remappings); + Ok(map) + } } - // Dual Shipping - // - // - Metrics - if config.additional_endpoints.is_empty() { - config - .additional_endpoints - .clone_from(&yaml_config.additional_endpoints); - } + deserializer.deserialize_str(KeyValueVisitor) } -#[inline] -#[must_use] -fn build_fqdn_logs(site: String) -> String { - format!("https://http-intake.logs.{site}") +pub fn deserialize_array_from_comma_separated_string<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s: String = String::deserialize(deserializer)?; + Ok(s.split(',') + .map(|feature| feature.trim().to_string()) + .filter(|feature| !feature.is_empty()) + .collect()) +} + +pub fn deserialize_key_value_pair_array_to_hashmap<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let array: Vec = Vec::deserialize(deserializer)?; + let mut map = HashMap::new(); + for s in array { + let parts = s.split(':').collect::>(); + if parts.len() == 2 { + map.insert(parts[0].to_string(), parts[1].to_string()); + } + } + Ok(map) } #[cfg(test)] @@ -357,14 +692,11 @@ pub mod tests { fn test_support_pci_traces_intake_url() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env( - "DD_APM_CONFIG_APM_DD_URL", - "https://trace-pci.agent.datadoghq.com", - ); + jail.set_env("DD_APM_DD_URL", "https://trace-pci.agent.datadoghq.com"); let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( - config.apm_config_apm_dd_url, + config.apm_dd_url, "https://trace-pci.agent.datadoghq.com/api/v0.2/traces".to_string() ); Ok(()) @@ -415,7 +747,7 @@ pub mod tests { let config = get_config(Path::new("")).expect_err("should reject unknown fields"); assert_eq!( config, - ConfigError::UnsupportedField("appsec_enabled".to_string()) + ConfigError::UnsupportedField("serverless_appsec_enabled".to_string()) ); Ok(()) }); @@ -442,6 +774,7 @@ pub mod tests { fn test_parse_config_file() { figment::Jail::expect_with(|jail| { jail.clear_env(); + // nit: does parsing an empty file actually test "parse config file"? jail.create_file( "datadog.yaml", r" @@ -482,16 +815,16 @@ pub mod tests { let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, - EnvConfig { + Config { site: "datadoghq.com".to_string(), trace_propagation_style_extract: vec![ TracePropagationStyle::Datadog, TracePropagationStyle::TraceContext ], logs_config_logs_dd_url: "https://http-intake.logs.datadoghq.com".to_string(), - apm_config_apm_dd_url: trace_intake_url("datadoghq.com").to_string(), + apm_dd_url: trace_intake_url("datadoghq.com").to_string(), dd_url: String::new(), // We add the prefix in main.rs - ..EnvConfig::default() + ..Config::default() } ); Ok(()) @@ -504,7 +837,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_PROXY_HTTPS", "my-proxy:3128"); let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!(config.https_proxy, Some("my-proxy:3128".to_string())); + assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); } @@ -520,7 +853,7 @@ pub mod tests { "127.0.0.1,localhost,172.16.0.0/12,us-east-1.amazonaws.com,datadoghq.eu", ); let config = get_config(Path::new("")).expect("should parse noproxy"); - assert_eq!(config.https_proxy, None); + assert_eq!(config.proxy_https, None); Ok(()) }); } @@ -538,7 +871,7 @@ pub mod tests { )?; let config = get_config(Path::new("")).expect("should parse weird proxy config"); - assert_eq!(config.https_proxy, Some("my-proxy:3128".to_string())); + assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); } @@ -558,7 +891,7 @@ pub mod tests { )?; let config = get_config(Path::new("")).expect("should parse weird proxy config"); - assert_eq!(config.https_proxy, None); + assert_eq!(config.proxy_https, None); // Assertion to ensure config.site runs before proxy // because we chenck that noproxy contains the site assert_eq!(config.site, "datadoghq.com"); @@ -850,4 +1183,29 @@ pub mod tests { Ok(()) }); } + + #[test] + fn test_overrides_config_based_on_priority() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r#" + site: us3.datadoghq.com + api_key: "yaml-api-key" + log_level: "debug" + "#, + )?; + jail.set_env("DD_SITE", "us5.datadoghq.com"); + jail.set_env("DD_API_KEY", "env-api-key"); + jail.set_env("DD_FLUSH_TIMEOUT", "10"); + let config = get_config(Path::new("")).expect("should parse config"); + + assert_eq!(config.site, "us5.datadoghq.com"); + assert_eq!(config.api_key, "env-api-key"); + assert_eq!(config.log_level, LogLevel::Debug); + assert_eq!(config.flush_timeout, 10); + Ok(()) + }); + } } diff --git a/yaml.rs b/yaml.rs index e82b322c..2e1c6286 100644 --- a/yaml.rs +++ b/yaml.rs @@ -1,107 +1,107 @@ -use std::collections::HashMap; - -use crate::config::additional_endpoints::deserialize_additional_endpoints; -use crate::config::{deserialize_apm_replace_rules, deserialize_processing_rules, ProcessingRule}; +use std::{collections::HashMap, path::PathBuf}; + +use crate::{ + config::{ + additional_endpoints::deserialize_additional_endpoints, + deserialize_apm_replace_rules, deserialize_key_value_pair_array_to_hashmap, + deserialize_optional_bool_from_anything, deserialize_processing_rules, + deserialize_string_or_int, + flush_strategy::FlushStrategy, + log_level::LogLevel, + service_mapping::deserialize_service_mapping, + trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}, + Config, ConfigError, ConfigSource, ProcessingRule, + }, + merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, +}; use datadog_trace_obfuscation::replacer::ReplaceRule; +use figment::{ + providers::{Format, Yaml}, + Figment, +}; use serde::Deserialize; -use serde_aux::field_attributes::deserialize_bool_from_anything; -use serde_json::Value; -/// `Config` is a struct that represents some of the fields in the `datadog.yaml` file. +/// `YamlConfig` is a struct that represents some of the fields in the `datadog.yaml` file. /// -/// It is used to deserialize the `datadog.yaml` file into a struct that can be merged with the `Config` struct. +/// It is used to deserialize the `datadog.yaml` file into a struct that can be merged +/// with the `Config` struct. #[derive(Debug, PartialEq, Deserialize, Clone, Default)] #[serde(default)] #[allow(clippy::module_name_repetitions)] -pub struct Config { - pub logs_config: LogsConfig, - pub apm_config: ApmConfig, - pub proxy: ProxyConfig, - pub otlp_config: Option, - #[serde(deserialize_with = "deserialize_additional_endpoints")] - pub additional_endpoints: HashMap>, -} +pub struct YamlConfig { + pub site: Option, + pub api_key: Option, + pub log_level: Option, -impl Config { - #[must_use] - pub fn otlp_config_receiver_protocols_http_endpoint(&self) -> Option<&str> { - self.otlp_config - .as_ref()? - .receiver - .as_ref()? - .protocols - .as_ref()? - .http - .as_ref()? - .endpoint - .as_deref() - } + pub flush_timeout: Option, - #[must_use] - pub fn otlp_config_receiver_protocols_grpc(&self) -> Option<&Value> { - self.otlp_config - .as_ref()? - .receiver - .as_ref()? - .protocols - .as_ref()? - .grpc - .as_ref() - } + // Proxy + pub proxy: ProxyConfig, + // nit: this should probably be in the endpoints section + pub dd_url: Option, + pub http_protocol: Option, - #[must_use] - pub fn otlp_config_traces_enabled(&self) -> bool { - self.otlp_config.as_ref().is_some_and(|otlp_config| { - otlp_config - .traces - .as_ref() - .is_some_and(|traces| traces.enabled) - }) - } + // Endpoints + #[serde(deserialize_with = "deserialize_additional_endpoints")] + /// Field used for Dual Shipping for Metrics + pub additional_endpoints: HashMap>, - #[must_use] - pub fn otlp_config_traces_ignore_missing_datadog_fields(&self) -> bool { - self.otlp_config.as_ref().is_some_and(|otlp_config| { - otlp_config - .traces - .as_ref() - .is_some_and(|traces| traces.ignore_missing_datadog_fields) - }) - } + // Unified Service Tagging + #[serde(deserialize_with = "deserialize_string_or_int")] + pub env: Option, + #[serde(deserialize_with = "deserialize_string_or_int")] + pub service: Option, + #[serde(deserialize_with = "deserialize_string_or_int")] + pub version: Option, + #[serde(deserialize_with = "deserialize_key_value_pair_array_to_hashmap")] + pub tags: HashMap, + + // Logs + pub logs_config: LogsConfig, - #[must_use] - pub fn otlp_config_traces_span_name_as_resource_name(&self) -> bool { - self.otlp_config.as_ref().is_some_and(|otlp_config| { - otlp_config - .traces - .as_ref() - .is_some_and(|traces| traces.span_name_as_resource_name) - }) - } + // APM + pub apm_config: ApmConfig, + #[serde(deserialize_with = "deserialize_service_mapping")] + pub service_mapping: HashMap, + // Trace Propagation + #[serde(deserialize_with = "deserialize_trace_propagation_style")] + pub trace_propagation_style: Vec, + #[serde(deserialize_with = "deserialize_trace_propagation_style")] + pub trace_propagation_style_extract: Vec, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub trace_propagation_extract_first: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub trace_propagation_http_baggage_enabled: Option, + + // OTLP + pub otlp_config: Option, - #[must_use] - pub fn otlp_config_traces_span_name_remappings(&self) -> HashMap { - self.otlp_config - .as_ref() - .and_then(|otlp_config| otlp_config.traces.as_ref()) - .map(|traces| traces.span_name_remappings.clone()) - .unwrap_or_default() - } + // AWS Lambda + pub api_key_secret_arn: Option, + pub kms_api_key: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub serverless_logs_enabled: Option, + pub serverless_flush_strategy: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub enhanced_metrics: Option, + pub lambda_proc_enhanced_metrics: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub capture_lambda_payload: Option, + pub capture_lambda_payload_max_depth: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub serverless_appsec_enabled: Option, + pub extension_version: Option, +} - #[must_use] - pub fn otlp_config_traces_probabilistic_sampler(&self) -> Option<&Value> { - self.otlp_config - .as_ref() - .and_then(|otlp_config| otlp_config.traces.as_ref()) - .and_then(|traces| traces.probabilistic_sampler.as_ref()) - } +/// Proxy Config +/// - #[must_use] - pub fn otlp_config_logs(&self) -> Option<&Value> { - self.otlp_config - .as_ref() - .and_then(|otlp_config| otlp_config.logs.as_ref()) - } +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct ProxyConfig { + pub https: Option, + pub no_proxy: Option>, } /// Logs Config @@ -111,8 +111,12 @@ impl Config { #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct LogsConfig { + pub logs_dd_url: Option, #[serde(deserialize_with = "deserialize_processing_rules")] pub processing_rules: Option>, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub use_compression: Option, + pub compression_level: Option, } /// APM Config @@ -122,9 +126,27 @@ pub struct LogsConfig { #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct ApmConfig { + pub apm_dd_url: Option, #[serde(deserialize_with = "deserialize_apm_replace_rules")] pub replace_tags: Option>, pub obfuscation: Option, + pub features: Vec, +} + +impl ApmConfig { + #[must_use] + pub fn obfuscation_http_remove_query_string(&self) -> Option { + self.obfuscation + .as_ref() + .and_then(|obfuscation| obfuscation.http.remove_query_string) + } + + #[must_use] + pub fn obfuscation_http_remove_paths_with_digits(&self) -> Option { + self.obfuscation + .as_ref() + .and_then(|obfuscation| obfuscation.http.remove_paths_with_digits) + } } #[derive(Debug, PartialEq, Deserialize, Clone, Copy, Default)] @@ -138,21 +160,10 @@ pub struct ApmObfuscation { #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct ApmHttpObfuscation { - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub remove_query_string: bool, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub remove_paths_with_digits: bool, -} - -/// Proxy Config -/// - -#[derive(Debug, PartialEq, Deserialize, Clone, Default)] -#[serde(default)] -#[allow(clippy::module_name_repetitions)] -pub struct ProxyConfig { - pub https: Option, - pub no_proxy: Option>, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub remove_query_string: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub remove_paths_with_digits: Option, } /// OTLP Config @@ -166,8 +177,8 @@ pub struct OtlpConfig { pub traces: Option, // NOT SUPPORTED - pub metrics: Option, - pub logs: Option, + pub metrics: Option, + pub logs: Option, } #[derive(Debug, PartialEq, Deserialize, Clone, Default)] @@ -184,7 +195,7 @@ pub struct OtlpReceiverProtocolsConfig { pub http: Option, // NOT SUPPORTED - pub grpc: Option, + pub grpc: Option, } #[derive(Debug, PartialEq, Deserialize, Clone, Default)] @@ -194,94 +205,635 @@ pub struct OtlpReceiverHttpConfig { pub endpoint: Option, } -impl Default for OtlpTracesConfig { - fn default() -> Self { - Self { - enabled: true, // Default this to true - span_name_as_resource_name: false, - span_name_remappings: HashMap::new(), - ignore_missing_datadog_fields: false, - probabilistic_sampler: None, - } - } +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct OtlpReceiverGrpcConfig { + pub endpoint: Option, + pub transport: Option, + pub max_recv_msg_size_mib: Option, } -#[derive(Debug, PartialEq, Deserialize, Clone)] +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct OtlpTracesConfig { - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub enabled: bool, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub span_name_as_resource_name: bool, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub enabled: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub span_name_as_resource_name: Option, pub span_name_remappings: HashMap, - #[serde(deserialize_with = "deserialize_bool_from_anything")] - pub ignore_missing_datadog_fields: bool, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub ignore_missing_datadog_fields: Option, // NOT SUPORTED - pub probabilistic_sampler: Option, + pub probabilistic_sampler: Option, } -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::path::Path; +#[derive(Debug, PartialEq, Clone, Deserialize, Default, Copy)] +pub struct OtlpTracesProbabilisticSampler { + pub sampling_percentage: Option, +} - use crate::config::get_config; +#[derive(Debug, PartialEq, Deserialize, Clone, Default)] +#[serde(default)] +pub struct OtlpMetricsConfig { + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub enabled: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub resource_attributes_as_tags: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub instrumentation_scope_metadata_as_tags: Option, + pub tag_cardinality: Option, + pub delta_ttl: Option, + pub histograms: Option, + pub sums: Option, + pub summaries: Option, +} - #[test] - fn test_otlp_config_receiver_protocols_http_endpoint() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.create_file( - "datadog.yaml", - r" - otlp_config: - receiver: - protocols: - http: - endpoint: 0.0.0.0:4318 - ", - )?; +#[derive(Debug, PartialEq, Clone, Deserialize, Default)] +#[serde(default)] +pub struct OtlpMetricsHistograms { + pub mode: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub send_count_sum_metrics: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub send_aggregation_metrics: Option, +} + +#[derive(Debug, PartialEq, Clone, Deserialize, Default)] +#[serde(default)] +pub struct OtlpMetricsSums { + pub cumulative_monotonic_mode: Option, + pub initial_cumulative_monotonic_value: Option, +} + +#[derive(Debug, PartialEq, Clone, Deserialize, Default)] +#[serde(default)] +pub struct OtlpMetricsSummaries { + pub mode: Option, +} + +#[derive(Debug, PartialEq, Clone, Deserialize, Default, Copy)] +#[serde(default)] +pub struct OtlpLogsConfig { + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub enabled: Option, +} + +impl OtlpConfig { + #[must_use] + pub fn receiver_protocols_http_endpoint(&self) -> Option { + self.receiver.as_ref().and_then(|receiver| { + receiver.protocols.as_ref().and_then(|protocols| { + protocols + .http + .as_ref() + .and_then(|http| http.endpoint.clone()) + }) + }) + } + + #[must_use] + pub fn receiver_protocols_grpc(&self) -> Option<&OtlpReceiverGrpcConfig> { + self.receiver.as_ref().and_then(|receiver| { + receiver + .protocols + .as_ref() + .and_then(|protocols| protocols.grpc.as_ref()) + }) + } + + #[must_use] + pub fn traces_enabled(&self) -> Option { + self.traces.as_ref().and_then(|traces| traces.enabled) + } + + #[must_use] + pub fn traces_ignore_missing_datadog_fields(&self) -> Option { + self.traces + .as_ref() + .and_then(|traces| traces.ignore_missing_datadog_fields) + } + + #[must_use] + pub fn traces_span_name_as_resource_name(&self) -> Option { + self.traces + .as_ref() + .and_then(|traces| traces.span_name_as_resource_name) + } + + #[must_use] + pub fn traces_span_name_remappings(&self) -> HashMap { + self.traces + .as_ref() + .map(|traces| traces.span_name_remappings.clone()) + .unwrap_or_default() + } + + #[must_use] + pub fn traces_probabilistic_sampler(&self) -> Option<&OtlpTracesProbabilisticSampler> { + self.traces + .as_ref() + .and_then(|traces| traces.probabilistic_sampler.as_ref()) + } + + #[must_use] + pub fn logs(&self) -> Option<&OtlpLogsConfig> { + self.logs.as_ref() + } +} + +#[allow(clippy::too_many_lines)] +fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { + // Basic fields + merge_string!(config, yaml_config, site); + merge_string!(config, yaml_config, api_key); + merge_option_to_value!(config, yaml_config, log_level); + merge_option_to_value!(config, yaml_config, flush_timeout); + + // Unified Service Tagging + merge_option!(config, yaml_config, env); + merge_option!(config, yaml_config, service); + merge_option!(config, yaml_config, version); + merge_hashmap!(config, yaml_config, tags); + + // Proxy + merge_option!(config, proxy_https, yaml_config.proxy, https); + merge_option_to_value!(config, proxy_no_proxy, yaml_config.proxy, no_proxy); + merge_option!(config, yaml_config, http_protocol); + + // Endpoints + merge_hashmap!(config, yaml_config, additional_endpoints); + merge_string!(config, yaml_config, dd_url); + + // Logs + merge_string!( + config, + logs_config_logs_dd_url, + yaml_config.logs_config, + logs_dd_url + ); + merge_option!( + config, + logs_config_processing_rules, + yaml_config.logs_config, + processing_rules + ); + merge_option_to_value!( + config, + logs_config_use_compression, + yaml_config.logs_config, + use_compression + ); + merge_option_to_value!( + config, + logs_config_compression_level, + yaml_config.logs_config, + compression_level + ); + + // APM + merge_hashmap!(config, yaml_config, service_mapping); + merge_string!(config, apm_dd_url, yaml_config.apm_config, apm_dd_url); + merge_option!( + config, + apm_replace_tags, + yaml_config.apm_config, + replace_tags + ); + + // Not using the macro here because we need to call a method on the struct + if let Some(remove_query_string) = yaml_config + .apm_config + .obfuscation_http_remove_query_string() + { + config + .apm_config_obfuscation_http_remove_query_string + .clone_from(&remove_query_string); + } + if let Some(remove_paths_with_digits) = yaml_config + .apm_config + .obfuscation_http_remove_paths_with_digits() + { + config + .apm_config_obfuscation_http_remove_paths_with_digits + .clone_from(&remove_paths_with_digits); + } + + merge_vec!(config, apm_features, yaml_config.apm_config, features); + + // Trace Propagation + merge_vec!(config, yaml_config, trace_propagation_style); + merge_vec!(config, yaml_config, trace_propagation_style_extract); + merge_option_to_value!(config, yaml_config, trace_propagation_extract_first); + merge_option_to_value!(config, yaml_config, trace_propagation_http_baggage_enabled); + + // OTLP + if let Some(otlp_config) = &yaml_config.otlp_config { + // Traces + + // Not using macros in some cases because we need to call a method on the struct + if let Some(traces_enabled) = otlp_config.traces_enabled() { + config + .otlp_config_traces_enabled + .clone_from(&traces_enabled); + } + if let Some(traces_span_name_as_resource_name) = + otlp_config.traces_span_name_as_resource_name() + { + config + .otlp_config_traces_span_name_as_resource_name + .clone_from(&traces_span_name_as_resource_name); + } - let config = get_config(Path::new("")).expect("should parse config"); + let traces_span_name_remappings = otlp_config.traces_span_name_remappings(); + if !traces_span_name_remappings.is_empty() { + config + .otlp_config_traces_span_name_remappings + .clone_from(&traces_span_name_remappings); + } + if let Some(traces_ignore_missing_datadog_fields) = + otlp_config.traces_ignore_missing_datadog_fields() + { + config + .otlp_config_ignore_missing_datadog_fields + .clone_from(&traces_ignore_missing_datadog_fields); + } - assert_eq!( - config.otlp_config_receiver_protocols_http_endpoint, - Some("0.0.0.0:4318".to_string()) + if let Some(probabilistic_sampler) = otlp_config.traces_probabilistic_sampler() { + merge_option!( + config, + otlp_config_traces_probabilistic_sampler_sampling_percentage, + probabilistic_sampler, + sampling_percentage ); + } - Ok(()) - }); + // Receiver + let receiver_protocols_http_endpoint = otlp_config.receiver_protocols_http_endpoint(); + if receiver_protocols_http_endpoint.is_some() { + config + .otlp_config_receiver_protocols_http_endpoint + .clone_from(&receiver_protocols_http_endpoint); + } + + if let Some(receiver_protocols_grpc) = otlp_config.receiver_protocols_grpc() { + merge_option!( + config, + otlp_config_receiver_protocols_grpc_endpoint, + receiver_protocols_grpc, + endpoint + ); + merge_option!( + config, + otlp_config_receiver_protocols_grpc_transport, + receiver_protocols_grpc, + transport + ); + merge_option!( + config, + otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib, + receiver_protocols_grpc, + max_recv_msg_size_mib + ); + } + + // Metrics + if let Some(metrics) = &otlp_config.metrics { + merge_option_to_value!(config, otlp_config_metrics_enabled, metrics, enabled); + merge_option_to_value!( + config, + otlp_config_metrics_resource_attributes_as_tags, + metrics, + resource_attributes_as_tags + ); + merge_option_to_value!( + config, + otlp_config_metrics_instrumentation_scope_metadata_as_tags, + metrics, + instrumentation_scope_metadata_as_tags + ); + merge_option!( + config, + otlp_config_metrics_tag_cardinality, + metrics, + tag_cardinality + ); + merge_option!(config, otlp_config_metrics_delta_ttl, metrics, delta_ttl); + if let Some(histograms) = &metrics.histograms { + merge_option_to_value!( + config, + otlp_config_metrics_histograms_send_count_sum_metrics, + histograms, + send_count_sum_metrics + ); + merge_option_to_value!( + config, + otlp_config_metrics_histograms_send_aggregation_metrics, + histograms, + send_aggregation_metrics + ); + merge_option!( + config, + otlp_config_metrics_histograms_mode, + histograms, + mode + ); + } + if let Some(sums) = &metrics.sums { + merge_option!( + config, + otlp_config_metrics_sums_cumulative_monotonic_mode, + sums, + cumulative_monotonic_mode + ); + merge_option!( + config, + otlp_config_metrics_sums_initial_cumulativ_monotonic_value, + sums, + initial_cumulative_monotonic_value + ); + } + if let Some(summaries) = &metrics.summaries { + merge_option!(config, otlp_config_metrics_summaries_mode, summaries, mode); + } + } + + // Logs + if let Some(logs) = &otlp_config.logs { + merge_option_to_value!(config, otlp_config_logs_enabled, logs, enabled); + } } + // AWS Lambda + merge_string!(config, yaml_config, api_key_secret_arn); + merge_string!(config, yaml_config, kms_api_key); + merge_option_to_value!(config, yaml_config, serverless_logs_enabled); + merge_option_to_value!(config, yaml_config, serverless_flush_strategy); + merge_option_to_value!(config, yaml_config, enhanced_metrics); + merge_option_to_value!(config, yaml_config, lambda_proc_enhanced_metrics); + merge_option_to_value!(config, yaml_config, capture_lambda_payload); + merge_option_to_value!(config, yaml_config, capture_lambda_payload_max_depth); + merge_option_to_value!(config, yaml_config, serverless_appsec_enabled); + merge_option!(config, yaml_config, extension_version); +} + +#[derive(Debug, PartialEq, Clone)] +#[allow(clippy::module_name_repetitions)] +pub struct YamlConfigSource { + pub path: PathBuf, +} + +impl ConfigSource for YamlConfigSource { + fn load(&self, config: &mut Config) -> Result<(), ConfigError> { + let figment = Figment::new().merge(Yaml::file(self.path.clone())); + + match figment.extract::() { + Ok(yaml_config) => merge_config(config, &yaml_config), + Err(e) => { + return Err(ConfigError::ParseError(format!( + "Failed to parse config from yaml file: {e}, using default config." + ))); + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use crate::config::{flush_strategy::PeriodicStrategy, processing_rule::Kind}; + + use super::*; + #[test] - fn test_parse_additional_endpoints_from_yaml() { + fn test_merge_config_overrides_with_yaml_file() { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.create_file( "datadog.yaml", r#" +# Basic fields +site: "test-site" +api_key: "test-api-key" +log_level: "debug" +flush_timeout: 42 + +# Proxy +proxy: + https: "https://proxy.example.com" + no_proxy: ["localhost", "127.0.0.1"] +dd_url: "https://metrics.datadoghq.com" +http_protocol: "http1" + +# Endpoints additional_endpoints: "https://app.datadoghq.com": - apikey2 - apikey3 "https://app.datadoghq.eu": - apikey4 + +# Unified Service Tagging +env: "test-env" +service: "test-service" +version: "1.0.0" +tags: + - "team:test-team" + - "project:test-project" + +# Logs +logs_config: + logs_dd_url: "https://logs.datadoghq.com" + processing_rules: + - name: "test-exclude" + type: "exclude_at_match" + pattern: "test-pattern" + use_compression: false + compression_level: 3 + +# APM +apm_config: + apm_dd_url: "https://apm.datadoghq.com" + replace_tags: [] + obfuscation: + http: + remove_query_string: true + remove_paths_with_digits: true + features: + - "enable_otlp_compute_top_level_by_span_kind" + - "enable_stats_by_span_kind" + +service_mapping: old-service:new-service + +# Trace Propagation +trace_propagation_style: "datadog" +trace_propagation_style_extract: "b3" +trace_propagation_extract_first: true +trace_propagation_http_baggage_enabled: true + +# OTLP +otlp_config: + receiver: + protocols: + http: + endpoint: "http://localhost:4318" + grpc: + endpoint: "http://localhost:4317" + transport: "tcp" + max_recv_msg_size_mib: 4 + traces: + enabled: false + span_name_as_resource_name: true + span_name_remappings: + "old-span": "new-span" + ignore_missing_datadog_fields: true + probabilistic_sampler: + sampling_percentage: 50 + metrics: + enabled: true + resource_attributes_as_tags: true + instrumentation_scope_metadata_as_tags: true + tag_cardinality: "low" + delta_ttl: 3600 + histograms: + mode: "counters" + send_count_sum_metrics: true + send_aggregation_metrics: true + sums: + cumulative_monotonic_mode: "to_delta" + initial_cumulative_monotonic_value: "auto" + summaries: + mode: "quantiles" + logs: + enabled: true + +# AWS Lambda +api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" +kms_api_key: "test-kms-key" +serverless_logs_enabled: false +serverless_flush_strategy: "periodically,60000" +enhanced_metrics: false +lambda_proc_enhanced_metrics: false +capture_lambda_payload: true +capture_lambda_payload_max_depth: 5 +serverless_appsec_enabled: true +extension_version: "compatibility" "#, )?; - let config = get_config(Path::new("")).expect("should parse config"); - let mut expected = HashMap::new(); - expected.insert( - "https://app.datadoghq.com".to_string(), - vec!["apikey2".to_string(), "apikey3".to_string()], - ); - expected.insert( - "https://app.datadoghq.eu".to_string(), - vec!["apikey4".to_string()], - ); - assert_eq!(config.additional_endpoints, expected); + let mut config = Config::default(); + let yaml_config_source = YamlConfigSource { + path: Path::new("datadog.yaml").to_path_buf(), + }; + yaml_config_source + .load(&mut config) + .expect("Failed to load config"); + + let expected_config = Config { + site: "test-site".to_string(), + api_key: "test-api-key".to_string(), + log_level: LogLevel::Debug, + flush_timeout: 42, + proxy_https: Some("https://proxy.example.com".to_string()), + proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], + http_protocol: Some("http1".to_string()), + dd_url: "https://metrics.datadoghq.com".to_string(), + url: "".to_string(), // doesnt exist in yaml + additional_endpoints: HashMap::from([ + ( + "https://app.datadoghq.com".to_string(), + vec!["apikey2".to_string(), "apikey3".to_string()], + ), + ( + "https://app.datadoghq.eu".to_string(), + vec!["apikey4".to_string()], + ), + ]), + env: Some("test-env".to_string()), + service: Some("test-service".to_string()), + version: Some("1.0.0".to_string()), + tags: HashMap::from([ + ("team".to_string(), "test-team".to_string()), + ("project".to_string(), "test-project".to_string()), + ]), + logs_config_logs_dd_url: "https://logs.datadoghq.com".to_string(), + logs_config_processing_rules: Some(vec![ProcessingRule { + name: "test-exclude".to_string(), + pattern: "test-pattern".to_string(), + kind: Kind::ExcludeAtMatch, + replace_placeholder: None, + }]), + logs_config_use_compression: false, + logs_config_compression_level: 3, + service_mapping: HashMap::from([( + "old-service".to_string(), + "new-service".to_string(), + )]), + apm_dd_url: "https://apm.datadoghq.com".to_string(), + apm_replace_tags: Some(vec![]), + apm_config_obfuscation_http_remove_query_string: true, + apm_config_obfuscation_http_remove_paths_with_digits: true, + apm_features: vec![ + "enable_otlp_compute_top_level_by_span_kind".to_string(), + "enable_stats_by_span_kind".to_string(), + ], + trace_propagation_style: vec![TracePropagationStyle::Datadog], + trace_propagation_style_extract: vec![TracePropagationStyle::B3], + trace_propagation_extract_first: true, + trace_propagation_http_baggage_enabled: true, + otlp_config_traces_enabled: false, + otlp_config_traces_span_name_as_resource_name: true, + otlp_config_traces_span_name_remappings: HashMap::from([( + "old-span".to_string(), + "new-span".to_string(), + )]), + otlp_config_ignore_missing_datadog_fields: true, + otlp_config_receiver_protocols_http_endpoint: Some( + "http://localhost:4318".to_string(), + ), + otlp_config_receiver_protocols_grpc_endpoint: Some( + "http://localhost:4317".to_string(), + ), + otlp_config_receiver_protocols_grpc_transport: Some("tcp".to_string()), + otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib: Some(4), + otlp_config_metrics_enabled: true, + otlp_config_metrics_resource_attributes_as_tags: true, + otlp_config_metrics_instrumentation_scope_metadata_as_tags: true, + otlp_config_metrics_tag_cardinality: Some("low".to_string()), + otlp_config_metrics_delta_ttl: Some(3600), + otlp_config_metrics_histograms_mode: Some("counters".to_string()), + otlp_config_metrics_histograms_send_count_sum_metrics: true, + otlp_config_metrics_histograms_send_aggregation_metrics: true, + otlp_config_metrics_sums_cumulative_monotonic_mode: Some("to_delta".to_string()), + otlp_config_metrics_sums_initial_cumulativ_monotonic_value: Some( + "auto".to_string(), + ), + otlp_config_metrics_summaries_mode: Some("quantiles".to_string()), + otlp_config_traces_probabilistic_sampler_sampling_percentage: Some(50), + otlp_config_logs_enabled: true, + api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" + .to_string(), + kms_api_key: "test-kms-key".to_string(), + serverless_logs_enabled: false, + serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { + interval: 60000, + }), + enhanced_metrics: false, + lambda_proc_enhanced_metrics: false, + capture_lambda_payload: true, + capture_lambda_payload_max_depth: 5, + serverless_appsec_enabled: true, + extension_version: Some("compatibility".to_string()), + }; + + // Assert that + assert_eq!(config, expected_config); + Ok(()) }); } From 35009950814dc752389fa962a6a75103ff549b08 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Thu, 3 Jul 2025 18:57:47 -0400 Subject: [PATCH 072/112] Dual Shipping Logs Support (#718) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds support for dual shipping metrics to endpoints configured using the `logs_config` YAML or `DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS` env var config. Implemented a `LogsFlusher` as a wrapper to all the `Flusher` instances to manages flushing to all configured endpoints. Moved retry logic to `LogsFlusher`, as the retry request contains the endpoint details and does not have to be tied to a particular flusher. --------- Co-authored-by: jordan gonzález <30836115+duncanista@users.noreply.github.com> --- env.rs | 20 +++++++++++ logs_additional_endpoints.rs | 67 ++++++++++++++++++++++++++++++++++++ mod.rs | 5 +++ yaml.rs | 23 +++++++++++-- 4 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 logs_additional_endpoints.rs diff --git a/env.rs b/env.rs index 5bec1eeb..adbb5c74 100644 --- a/env.rs +++ b/env.rs @@ -12,6 +12,9 @@ use crate::{ deserialize_optional_bool_from_anything, deserialize_string_or_int, flush_strategy::FlushStrategy, log_level::LogLevel, + logs_additional_endpoints::{ + deserialize_logs_additional_endpoints, LogsAdditionalEndpoint, + }, processing_rule::{deserialize_processing_rules, ProcessingRule}, service_mapping::deserialize_service_mapping, trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}, @@ -125,6 +128,12 @@ pub struct EnvConfig { /// to 9 (maximum compression but higher resource usage). Only takes effect if /// `use_compression` is set to `true`. pub logs_config_compression_level: Option, + /// @env `DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS` + /// + /// Additional endpoints to send logs to. + /// + #[serde(deserialize_with = "deserialize_logs_additional_endpoints")] + pub logs_config_additional_endpoints: Vec, // APM // @@ -322,6 +331,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option!(config, env_config, logs_config_processing_rules); merge_option_to_value!(config, env_config, logs_config_use_compression); merge_option_to_value!(config, env_config, logs_config_compression_level); + merge_vec!(config, env_config, logs_config_additional_endpoints); // APM merge_hashmap!(config, env_config, service_mapping); @@ -505,6 +515,10 @@ mod tests { ); jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "false"); jail.set_env("DD_LOGS_CONFIG_COMPRESSION_LEVEL", "3"); + jail.set_env( + "DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS", + "[{\"api_key\": \"apikey2\", \"Host\": \"agent-http-intake.logs.datadoghq.com\", \"Port\": 443, \"is_reliable\": true}]", + ); // APM jail.set_env("DD_SERVICE_MAPPING", "old-service:new-service"); @@ -640,6 +654,12 @@ mod tests { }]), logs_config_use_compression: false, logs_config_compression_level: 3, + logs_config_additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "apikey2".to_string(), + host: "agent-http-intake.logs.datadoghq.com".to_string(), + port: 443, + is_reliable: true, + }], service_mapping: HashMap::from([( "old-service".to_string(), "new-service".to_string(), diff --git a/logs_additional_endpoints.rs b/logs_additional_endpoints.rs new file mode 100644 index 00000000..a6a5d94b --- /dev/null +++ b/logs_additional_endpoints.rs @@ -0,0 +1,67 @@ +use serde::{Deserialize, Deserializer}; +use serde_json::Value; +use tracing::error; + +#[derive(Debug, PartialEq, Clone, Deserialize)] +pub struct LogsAdditionalEndpoint { + pub api_key: String, + #[serde(rename = "Host")] + pub host: String, + #[serde(rename = "Port")] + pub port: u32, + pub is_reliable: bool, +} + +#[allow(clippy::module_name_repetitions)] +pub fn deserialize_logs_additional_endpoints<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Value::deserialize(deserializer)?; + + match value { + Value::String(s) if !s.is_empty() => { + // For JSON format (string) in DD_ADDITIONAL_ENDPOINTS + Ok(serde_json::from_str(&s).unwrap_or_else(|err| { + error!("Failed to deserialize DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS: {err}"); + vec![] + })) + } + _ => Ok(Vec::new()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_deserialize_logs_additional_endpoints_valid() { + let input = json!("[{\"api_key\": \"apiKey2\", \"Host\": \"agent-http-intake.logs.datadoghq.com\", \"Port\": 443, \"is_reliable\": true}]"); + + let result = deserialize_logs_additional_endpoints(input).unwrap(); + let mut expected = Vec::new(); + expected.push(LogsAdditionalEndpoint { + api_key: "apiKey2".to_string(), + host: "agent-http-intake.logs.datadoghq.com".to_string(), + port: 443, + is_reliable: true, + }); + + assert_eq!(result, expected); + } + + #[test] + fn test_deserialize_logs_additional_endpoints_invalid() { + // input missing "Port" field + let input = json!("[{\"api_key\": \"apiKey2\", \"Host\": \"agent-http-intake.logs.datadoghq.com\", \"is_reliable\": true}]"); + + let result = deserialize_logs_additional_endpoints(input).unwrap(); + let expected = Vec::new(); // expect empty list due to invalid input + + assert_eq!(result, expected); + } +} diff --git a/mod.rs b/mod.rs index 157b8916..95d00c70 100644 --- a/mod.rs +++ b/mod.rs @@ -4,6 +4,7 @@ pub mod aws; pub mod env; pub mod flush_strategy; pub mod log_level; +pub mod logs_additional_endpoints; pub mod processing_rule; pub mod service_mapping; pub mod trace_propagation_style; @@ -11,6 +12,7 @@ pub mod yaml; use datadog_trace_obfuscation::replacer::ReplaceRule; use datadog_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; + use serde::{Deserialize, Deserializer}; use serde_aux::prelude::deserialize_bool_from_anything; use serde_json::Value; @@ -24,6 +26,7 @@ use crate::config::{ env::EnvConfigSource, flush_strategy::FlushStrategy, log_level::LogLevel, + logs_additional_endpoints::LogsAdditionalEndpoint, processing_rule::{deserialize_processing_rules, ProcessingRule}, trace_propagation_style::TracePropagationStyle, yaml::YamlConfigSource, @@ -260,6 +263,7 @@ pub struct Config { pub logs_config_processing_rules: Option>, pub logs_config_use_compression: bool, pub logs_config_compression_level: i32, + pub logs_config_additional_endpoints: Vec, // APM // @@ -353,6 +357,7 @@ impl Default for Config { logs_config_processing_rules: None, logs_config_use_compression: true, logs_config_compression_level: 6, + logs_config_additional_endpoints: Vec::new(), // APM service_mapping: HashMap::new(), diff --git a/yaml.rs b/yaml.rs index 2e1c6286..f35a54bf 100644 --- a/yaml.rs +++ b/yaml.rs @@ -8,6 +8,7 @@ use crate::{ deserialize_string_or_int, flush_strategy::FlushStrategy, log_level::LogLevel, + logs_additional_endpoints::LogsAdditionalEndpoint, service_mapping::deserialize_service_mapping, trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}, Config, ConfigError, ConfigSource, ProcessingRule, @@ -117,6 +118,7 @@ pub struct LogsConfig { #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub use_compression: Option, pub compression_level: Option, + pub additional_endpoints: Vec, } /// APM Config @@ -392,6 +394,12 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { yaml_config.logs_config, compression_level ); + merge_vec!( + config, + logs_config_additional_endpoints, + yaml_config.logs_config, + additional_endpoints + ); // APM merge_hashmap!(config, yaml_config, service_mapping); @@ -633,8 +641,8 @@ http_protocol: "http1" # Endpoints additional_endpoints: "https://app.datadoghq.com": - - apikey2 - - apikey3 + - "apikey2" + - "apikey3" "https://app.datadoghq.eu": - apikey4 @@ -655,6 +663,11 @@ logs_config: pattern: "test-pattern" use_compression: false compression_level: 3 + additional_endpoints: + - api_key: "apikey2" + Host: "agent-http-intake.logs.datadoghq.com" + Port: 443 + is_reliable: true # APM apm_config: @@ -770,6 +783,12 @@ extension_version: "compatibility" }]), logs_config_use_compression: false, logs_config_compression_level: 3, + logs_config_additional_endpoints: vec![LogsAdditionalEndpoint { + api_key: "apikey2".to_string(), + host: "agent-http-intake.logs.datadoghq.com".to_string(), + port: 443, + is_reliable: true, + }], service_mapping: HashMap::from([( "old-service".to_string(), "new-service".to_string(), From 076d14f759db9a545a417cf0cc9325d598a442f4 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Mon, 14 Jul 2025 19:03:43 -0400 Subject: [PATCH 073/112] chore: upgrade rust version for toolchain to 1.84.1 (#743) # This PR 1. In `rust-toolchain.toml`, upgrade Rust version from `1.81.0` to `1.84.1`. 2. Fix/mute clippy errors caused by the upgrade - some errors require non-trivial code changes, so I muted them for now and added a TODO to fix them in separate PRs. # Motivation `libdatadog` now uses `1.84.1` https://github.com/DataDog/libdatadog/blame/main/Cargo.toml#L62 To test changes on `libdatadog`, I need to change the Rust version in `datadog-lambda-extension` to 1.84.1 as well. Making this a separate PR: 1. so it's easier to test multiple PRs that depend on changes on `libdatadog` in parallel after I merge this PR to main. 4. because this PR also involves lots of code changes needed to make clippy happy --- mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mod.rs b/mod.rs index 95d00c70..8b790903 100644 --- a/mod.rs +++ b/mod.rs @@ -197,7 +197,7 @@ impl ConfigBuilder { // or in the `proxy_no_proxy` config field. if self.config.proxy_https.is_some() { let site_in_no_proxy = std::env::var("NO_PROXY") - .map_or(false, |no_proxy| no_proxy.contains(&self.config.site)) + .is_ok_and(|no_proxy| no_proxy.contains(&self.config.site)) || self .config .proxy_no_proxy @@ -543,7 +543,7 @@ where { struct KeyValueVisitor; - impl<'de> serde::de::Visitor<'de> for KeyValueVisitor { + impl serde::de::Visitor<'_> for KeyValueVisitor { type Value = HashMap; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { From 61453da9c11ce688ef7824f6015d67fa3b42932a Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Mon, 21 Jul 2025 10:01:38 -0400 Subject: [PATCH 074/112] feat: dual shipping APM support (#735) Adds support for dual shipping traces to endpoints configured using the `apm_config` YAML or `DD_APM_CONFIG_ADDITIONAL_ENDPOINTS` env var config. #### Additional Notes: - Bumped libdatadog (and serverless-components) to include https://github.com/DataDog/libdatadog/pull/1139 - Adds configuration option to set compression level for trace payloads --- env.rs | 27 +++++++++++++++++++++++++++ mod.rs | 4 ++++ yaml.rs | 37 +++++++++++++++++++++++++++++++++++-- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/env.rs b/env.rs index adbb5c74..7509a1da 100644 --- a/env.rs +++ b/env.rs @@ -163,9 +163,21 @@ pub struct EnvConfig { /// @env `DD_APM_CONFIG_OBFUSCATION_HTTP_REMOVE_PATHS_WITH_DIGITS` #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub apm_config_obfuscation_http_remove_paths_with_digits: Option, + /// @env `DD_APM_CONFIG_COMPRESSION_LEVEL` + /// + /// The Agent compresses traces before sending them. The `compression_level` parameter + /// accepts values from 0 (no compression) to 9 (maximum compression but + /// higher resource usage). + pub apm_config_compression_level: Option, /// @env `DD_APM_FEATURES` #[serde(deserialize_with = "deserialize_array_from_comma_separated_string")] pub apm_features: Vec, + /// @env `DD_APM_ADDITIONAL_ENDPOINTS` + /// + /// Additional endpoints to send traces to. + /// + #[serde(deserialize_with = "deserialize_additional_endpoints")] + pub apm_additional_endpoints: HashMap>, // // Trace Propagation /// @env `DD_TRACE_PROPAGATION_STYLE` @@ -347,7 +359,9 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { env_config, apm_config_obfuscation_http_remove_paths_with_digits ); + merge_option_to_value!(config, env_config, apm_config_compression_level); merge_vec!(config, env_config, apm_features); + merge_hashmap!(config, env_config, apm_additional_endpoints); // Trace Propagation merge_vec!(config, env_config, trace_propagation_style); @@ -533,10 +547,12 @@ mod tests { "DD_APM_CONFIG_OBFUSCATION_HTTP_REMOVE_PATHS_WITH_DIGITS", "true", ); + jail.set_env("DD_APM_CONFIG_COMPRESSION_LEVEL", "3"); jail.set_env( "DD_APM_FEATURES", "enable_otlp_compute_top_level_by_span_kind,enable_stats_by_span_kind", ); + jail.set_env("DD_APM_ADDITIONAL_ENDPOINTS", "{\"https://trace.agent.datadoghq.com\": [\"apikey2\", \"apikey3\"], \"https://trace.agent.datadoghq.eu\": [\"apikey4\"]}"); // Trace Propagation jail.set_env("DD_TRACE_PROPAGATION_STYLE", "datadog"); @@ -673,10 +689,21 @@ mod tests { ), apm_config_obfuscation_http_remove_query_string: true, apm_config_obfuscation_http_remove_paths_with_digits: true, + apm_config_compression_level: 3, apm_features: vec![ "enable_otlp_compute_top_level_by_span_kind".to_string(), "enable_stats_by_span_kind".to_string(), ], + apm_additional_endpoints: HashMap::from([ + ( + "https://trace.agent.datadoghq.com".to_string(), + vec!["apikey2".to_string(), "apikey3".to_string()], + ), + ( + "https://trace.agent.datadoghq.eu".to_string(), + vec!["apikey4".to_string()], + ), + ]), trace_propagation_style: vec![TracePropagationStyle::Datadog], trace_propagation_style_extract: vec![TracePropagationStyle::B3], trace_propagation_extract_first: true, diff --git a/mod.rs b/mod.rs index 8b790903..e1e2f7f1 100644 --- a/mod.rs +++ b/mod.rs @@ -273,7 +273,9 @@ pub struct Config { pub apm_replace_tags: Option>, pub apm_config_obfuscation_http_remove_query_string: bool, pub apm_config_obfuscation_http_remove_paths_with_digits: bool, + pub apm_config_compression_level: i32, pub apm_features: Vec, + pub apm_additional_endpoints: HashMap>, // // Trace Propagation pub trace_propagation_style: Vec, @@ -365,7 +367,9 @@ impl Default for Config { apm_replace_tags: None, apm_config_obfuscation_http_remove_query_string: false, apm_config_obfuscation_http_remove_paths_with_digits: false, + apm_config_compression_level: 6, apm_features: vec![], + apm_additional_endpoints: HashMap::new(), trace_propagation_style: vec![ TracePropagationStyle::Datadog, TracePropagationStyle::TraceContext, diff --git a/yaml.rs b/yaml.rs index f35a54bf..7b944ea4 100644 --- a/yaml.rs +++ b/yaml.rs @@ -132,7 +132,10 @@ pub struct ApmConfig { #[serde(deserialize_with = "deserialize_apm_replace_rules")] pub replace_tags: Option>, pub obfuscation: Option, + pub compression_level: Option, pub features: Vec, + #[serde(deserialize_with = "deserialize_additional_endpoints")] + pub additional_endpoints: HashMap>, } impl ApmConfig { @@ -410,6 +413,18 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { yaml_config.apm_config, replace_tags ); + merge_option_to_value!( + config, + apm_config_compression_level, + yaml_config.apm_config, + compression_level + ); + merge_hashmap!( + config, + apm_additional_endpoints, + yaml_config.apm_config, + additional_endpoints + ); // Not using the macro here because we need to call a method on the struct if let Some(remove_query_string) = yaml_config @@ -641,8 +656,8 @@ http_protocol: "http1" # Endpoints additional_endpoints: "https://app.datadoghq.com": - - "apikey2" - - "apikey3" + - apikey2 + - apikey3 "https://app.datadoghq.eu": - apikey4 @@ -677,9 +692,16 @@ apm_config: http: remove_query_string: true remove_paths_with_digits: true + compression_level: 3 features: - "enable_otlp_compute_top_level_by_span_kind" - "enable_stats_by_span_kind" + additional_endpoints: + "https://trace.agent.datadoghq.com": + - apikey2 + - apikey3 + "https://trace.agent.datadoghq.eu": + - apikey4 service_mapping: old-service:new-service @@ -797,10 +819,21 @@ extension_version: "compatibility" apm_replace_tags: Some(vec![]), apm_config_obfuscation_http_remove_query_string: true, apm_config_obfuscation_http_remove_paths_with_digits: true, + apm_config_compression_level: 3, apm_features: vec![ "enable_otlp_compute_top_level_by_span_kind".to_string(), "enable_stats_by_span_kind".to_string(), ], + apm_additional_endpoints: HashMap::from([ + ( + "https://trace.agent.datadoghq.com".to_string(), + vec!["apikey2".to_string(), "apikey3".to_string()], + ), + ( + "https://trace.agent.datadoghq.eu".to_string(), + vec!["apikey4".to_string()], + ), + ]), trace_propagation_style: vec![TracePropagationStyle::Datadog], trace_propagation_style_extract: vec![TracePropagationStyle::B3], trace_propagation_extract_first: true, From df090c05d33c769e5edf707d86c2abb5dae78a82 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Mon, 21 Jul 2025 11:45:09 -0400 Subject: [PATCH 075/112] chore: Add doc and rename function for flushing strategy (#740) # Motivation It took me quite some effort to understand flushing strategies. I want to make it easier to understand for me and future developers. # This PR Tries to make flushing strategy code more readable: 1. Add/move comments 2. Create an enum `ConcreteFlushStrategy`, which doesn't contain `Default` because it is required to be resolved to a concrete strategy 3. Rename `should_adapt` to `evaluate_concrete_strategy()` # To reviewers There are still a few things I don't understand, which are marked with `TODO`. Appreciate explanation! Also correct me if any comment I added is wrong. --- flush_strategy.rs | 17 +++++++++++++++++ mod.rs | 1 + 2 files changed, 18 insertions(+) diff --git a/flush_strategy.rs b/flush_strategy.rs index a72a2416..51e9710e 100644 --- a/flush_strategy.rs +++ b/flush_strategy.rs @@ -8,10 +8,27 @@ pub struct PeriodicStrategy { #[derive(Clone, Copy, Debug, PartialEq)] pub enum FlushStrategy { + // Flush every 1s and at the end of the invocation Default, + // User specifies the interval in milliseconds, will not block on the runtimeDone event + Periodically(PeriodicStrategy), + // Always flush at the end of the invocation End, + // Flush both (1) at the end of the invocation and (2) periodically with the specified interval EndPeriodically(PeriodicStrategy), + // Flush in a non-blocking, asynchronous manner, so the next invocation can start without waiting + // for the flush to complete + Continuously(PeriodicStrategy), +} + +// A restricted subset of `FlushStrategy`. The Default strategy is now allowed, which is required to be +// translated into a concrete strategy. +#[allow(clippy::module_name_repetitions)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ConcreteFlushStrategy { Periodically(PeriodicStrategy), + End, + EndPeriodically(PeriodicStrategy), Continuously(PeriodicStrategy), } diff --git a/mod.rs b/mod.rs index e1e2f7f1..8cbc3284 100644 --- a/mod.rs +++ b/mod.rs @@ -240,6 +240,7 @@ pub struct Config { pub api_key: String, pub log_level: LogLevel, + // Timeout for the request to flush data to Datadog endpoint pub flush_timeout: u64, // Proxy From b7d25abebf93f351a1e9fef6714f68a992f19a8e Mon Sep 17 00:00:00 2001 From: Romain Marcadier Date: Thu, 24 Jul 2025 19:56:26 +0200 Subject: [PATCH 076/112] chore: upgrade to edition 2024 and fix all linter warnings (#754) Also updates CI to run `clippy` on `--all-targets` so that linter errors aren't ignored on side targets such as tests. --- additional_endpoints.rs | 27 +++++++++++++++++++-------- apm_replace_rule.rs | 2 +- env.rs | 13 +++++++------ logs_additional_endpoints.rs | 19 ++++++++++++------- mod.rs | 10 +++++----- yaml.rs | 9 +++++---- 6 files changed, 49 insertions(+), 31 deletions(-) diff --git a/additional_endpoints.rs b/additional_endpoints.rs index 281229d9..16611833 100644 --- a/additional_endpoints.rs +++ b/additional_endpoints.rs @@ -26,7 +26,10 @@ where result.insert(key, urls); } _ => { - error!("Failed to deserialize additional endpoints - Invalid YAML format: expected array for key {}", key); + error!( + "Failed to deserialize additional endpoints - Invalid YAML format: expected array for key {}", + key + ); } } } @@ -58,7 +61,8 @@ mod tests { "https://app.datadoghq.eu": ["key3"] }); - let result = deserialize_additional_endpoints(input).unwrap(); + let result = deserialize_additional_endpoints(input) + .expect("Failed to deserialize additional endpoints"); let mut expected = HashMap::new(); expected.insert( @@ -76,9 +80,12 @@ mod tests { #[test] fn test_deserialize_additional_endpoints_json() { // Test JSON string format - let input = json!("{\"https://app.datadoghq.com\":[\"key1\",\"key2\"],\"https://app.datadoghq.eu\":[\"key3\"]}"); + let input = json!( + "{\"https://app.datadoghq.com\":[\"key1\",\"key2\"],\"https://app.datadoghq.eu\":[\"key3\"]}" + ); - let result = deserialize_additional_endpoints(input).unwrap(); + let result = deserialize_additional_endpoints(input) + .expect("Failed to deserialize additional endpoints"); let mut expected = HashMap::new(); expected.insert( @@ -97,22 +104,26 @@ mod tests { fn test_deserialize_additional_endpoints_invalid_or_empty() { // Test empty YAML let input = json!({}); - let result = deserialize_additional_endpoints(input).unwrap(); + let result = deserialize_additional_endpoints(input) + .expect("Failed to deserialize additional endpoints"); assert!(result.is_empty()); // Test empty JSON let input = json!(""); - let result = deserialize_additional_endpoints(input).unwrap(); + let result = deserialize_additional_endpoints(input) + .expect("Failed to deserialize additional endpoints"); assert!(result.is_empty()); let input = json!({ "https://app.datadoghq.com": "invalid-yaml" }); - let result = deserialize_additional_endpoints(input).unwrap(); + let result = deserialize_additional_endpoints(input) + .expect("Failed to deserialize additional endpoints"); assert!(result.is_empty()); let input = json!("invalid-json"); - let result = deserialize_additional_endpoints(input).unwrap(); + let result = deserialize_additional_endpoints(input) + .expect("Failed to deserialize additional endpoints"); assert!(result.is_empty()); } } diff --git a/apm_replace_rule.rs b/apm_replace_rule.rs index 843fd07a..e63d94bc 100644 --- a/apm_replace_rule.rs +++ b/apm_replace_rule.rs @@ -1,4 +1,4 @@ -use datadog_trace_obfuscation::replacer::{parse_rules_from_string, ReplaceRule}; +use datadog_trace_obfuscation::replacer::{ReplaceRule, parse_rules_from_string}; use serde::de::{Deserializer, SeqAccess, Visitor}; use serde::{Deserialize, Serialize}; use serde_json; diff --git a/env.rs b/env.rs index 7509a1da..ad91db2f 100644 --- a/env.rs +++ b/env.rs @@ -1,4 +1,4 @@ -use figment::{providers::Env, Figment}; +use figment::{Figment, providers::Env}; use serde::Deserialize; use std::collections::HashMap; @@ -6,6 +6,7 @@ use datadog_trace_obfuscation::replacer::ReplaceRule; use crate::{ config::{ + Config, ConfigError, ConfigSource, additional_endpoints::deserialize_additional_endpoints, apm_replace_rule::deserialize_apm_replace_rules, deserialize_array_from_comma_separated_string, deserialize_key_value_pairs, @@ -13,12 +14,11 @@ use crate::{ flush_strategy::FlushStrategy, log_level::LogLevel, logs_additional_endpoints::{ - deserialize_logs_additional_endpoints, LogsAdditionalEndpoint, + LogsAdditionalEndpoint, deserialize_logs_additional_endpoints, }, - processing_rule::{deserialize_processing_rules, ProcessingRule}, + processing_rule::{ProcessingRule, deserialize_processing_rules}, service_mapping::deserialize_service_mapping, - trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}, - Config, ConfigError, ConfigSource, + trace_propagation_style::{TracePropagationStyle, deserialize_trace_propagation_style}, }, merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, }; @@ -484,14 +484,15 @@ impl ConfigSource for EnvConfigSource { mod tests { use super::*; use crate::config::{ + Config, flush_strategy::{FlushStrategy, PeriodicStrategy}, log_level::LogLevel, processing_rule::{Kind, ProcessingRule}, trace_propagation_style::TracePropagationStyle, - Config, }; #[test] + #[allow(clippy::too_many_lines)] fn test_merge_config_overrides_with_environment_variables() { figment::Jail::expect_with(|jail| { jail.clear_env(); diff --git a/logs_additional_endpoints.rs b/logs_additional_endpoints.rs index a6a5d94b..f3d18c15 100644 --- a/logs_additional_endpoints.rs +++ b/logs_additional_endpoints.rs @@ -40,16 +40,18 @@ mod tests { #[test] fn test_deserialize_logs_additional_endpoints_valid() { - let input = json!("[{\"api_key\": \"apiKey2\", \"Host\": \"agent-http-intake.logs.datadoghq.com\", \"Port\": 443, \"is_reliable\": true}]"); + let input = json!( + "[{\"api_key\": \"apiKey2\", \"Host\": \"agent-http-intake.logs.datadoghq.com\", \"Port\": 443, \"is_reliable\": true}]" + ); - let result = deserialize_logs_additional_endpoints(input).unwrap(); - let mut expected = Vec::new(); - expected.push(LogsAdditionalEndpoint { + let result = deserialize_logs_additional_endpoints(input) + .expect("Failed to deserialize logs additional endpoints"); + let expected = vec![LogsAdditionalEndpoint { api_key: "apiKey2".to_string(), host: "agent-http-intake.logs.datadoghq.com".to_string(), port: 443, is_reliable: true, - }); + }]; assert_eq!(result, expected); } @@ -57,9 +59,12 @@ mod tests { #[test] fn test_deserialize_logs_additional_endpoints_invalid() { // input missing "Port" field - let input = json!("[{\"api_key\": \"apiKey2\", \"Host\": \"agent-http-intake.logs.datadoghq.com\", \"is_reliable\": true}]"); + let input = json!( + "[{\"api_key\": \"apiKey2\", \"Host\": \"agent-http-intake.logs.datadoghq.com\", \"is_reliable\": true}]" + ); - let result = deserialize_logs_additional_endpoints(input).unwrap(); + let result = deserialize_logs_additional_endpoints(input) + .expect("Failed to deserialize logs additional endpoints"); let expected = Vec::new(); // expect empty list due to invalid input assert_eq!(result, expected); diff --git a/mod.rs b/mod.rs index 8cbc3284..2f7d4308 100644 --- a/mod.rs +++ b/mod.rs @@ -27,7 +27,7 @@ use crate::config::{ flush_strategy::FlushStrategy, log_level::LogLevel, logs_additional_endpoints::LogsAdditionalEndpoint, - processing_rule::{deserialize_processing_rules, ProcessingRule}, + processing_rule::{ProcessingRule, deserialize_processing_rules}, trace_propagation_style::TracePropagationStyle, yaml::YamlConfigSource, }; @@ -1186,10 +1186,10 @@ pub mod tests { jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "TRUE"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "0"); let config = get_config(Path::new("")).expect("should parse config"); - assert_eq!(config.serverless_logs_enabled, true); - assert_eq!(config.enhanced_metrics, true); - assert_eq!(config.logs_config_use_compression, true); - assert_eq!(config.capture_lambda_payload, false); + assert!(config.serverless_logs_enabled); + assert!(config.enhanced_metrics); + assert!(config.logs_config_use_compression); + assert!(!config.capture_lambda_payload); Ok(()) }); } diff --git a/yaml.rs b/yaml.rs index 7b944ea4..1045fd66 100644 --- a/yaml.rs +++ b/yaml.rs @@ -2,6 +2,7 @@ use std::{collections::HashMap, path::PathBuf}; use crate::{ config::{ + Config, ConfigError, ConfigSource, ProcessingRule, additional_endpoints::deserialize_additional_endpoints, deserialize_apm_replace_rules, deserialize_key_value_pair_array_to_hashmap, deserialize_optional_bool_from_anything, deserialize_processing_rules, @@ -10,15 +11,14 @@ use crate::{ log_level::LogLevel, logs_additional_endpoints::LogsAdditionalEndpoint, service_mapping::deserialize_service_mapping, - trace_propagation_style::{deserialize_trace_propagation_style, TracePropagationStyle}, - Config, ConfigError, ConfigSource, ProcessingRule, + trace_propagation_style::{TracePropagationStyle, deserialize_trace_propagation_style}, }, merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, }; use datadog_trace_obfuscation::replacer::ReplaceRule; use figment::{ - providers::{Format, Yaml}, Figment, + providers::{Format, Yaml}, }; use serde::Deserialize; @@ -634,6 +634,7 @@ mod tests { use super::*; #[test] + #[allow(clippy::too_many_lines)] fn test_merge_config_overrides_with_yaml_file() { figment::Jail::expect_with(|jail| { jail.clear_env(); @@ -778,7 +779,7 @@ extension_version: "compatibility" proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), dd_url: "https://metrics.datadoghq.com".to_string(), - url: "".to_string(), // doesnt exist in yaml + url: String::new(), // doesnt exist in yaml additional_endpoints: HashMap::from([ ( "https://app.datadoghq.com".to_string(), From ab26905a7b7df69bfa04f7b0c26295fe7572cb7d Mon Sep 17 00:00:00 2001 From: Zarir Hamza Date: Thu, 7 Aug 2025 15:23:42 -0400 Subject: [PATCH 077/112] fix(apm): Enhance Synthetic Span Service Representation (#751) ### What does this PR do? Rollout of span naming changes to align serverless product with tracer to create streamlined Service Representation for Serverless Key Changes: - Change service name to match instance name for all managed services (aws.lambda -> lambda name, etc) (breaking) - Opt out via `DD_TRACE_AWS_SERVICE_REPRESENTATION_ENABLED` - Add `span.kind:server` on synthetic spans made via span-inferrer, cold start and lambda invocation spans - Remove `_dd.base_service` tags on synthetic spans to avoid unintentional service override ### Motivation Improve Service Map for Serverless. This allows for synthetic spans to have their own service on the map which connects with the inferred spans from the tracer side. --- env.rs | 8 ++++++++ mod.rs | 2 ++ yaml.rs | 9 +++++++++ 3 files changed, 19 insertions(+) diff --git a/env.rs b/env.rs index ad91db2f..46b2717f 100644 --- a/env.rs +++ b/env.rs @@ -178,6 +178,11 @@ pub struct EnvConfig { /// #[serde(deserialize_with = "deserialize_additional_endpoints")] pub apm_additional_endpoints: HashMap>, + /// @env `DD_TRACE_AWS_SERVICE_REPRESENTATION_ENABLED` + /// + /// Enable the new AWS-resource naming logic in the tracer. + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub trace_aws_service_representation_enabled: Option, // // Trace Propagation /// @env `DD_TRACE_PROPAGATION_STYLE` @@ -362,6 +367,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option_to_value!(config, env_config, apm_config_compression_level); merge_vec!(config, env_config, apm_features); merge_hashmap!(config, env_config, apm_additional_endpoints); + merge_option_to_value!(config, env_config, trace_aws_service_representation_enabled); // Trace Propagation merge_vec!(config, env_config, trace_propagation_style); @@ -560,6 +566,7 @@ mod tests { jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "b3"); jail.set_env("DD_TRACE_PROPAGATION_EXTRACT_FIRST", "true"); jail.set_env("DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED", "true"); + jail.set_env("DD_TRACE_AWS_SERVICE_REPRESENTATION_ENABLED", "true"); // OTLP jail.set_env("DD_OTLP_CONFIG_TRACES_ENABLED", "false"); @@ -709,6 +716,7 @@ mod tests { trace_propagation_style_extract: vec![TracePropagationStyle::B3], trace_propagation_extract_first: true, trace_propagation_http_baggage_enabled: true, + trace_aws_service_representation_enabled: true, otlp_config_traces_enabled: false, otlp_config_traces_span_name_as_resource_name: true, otlp_config_traces_span_name_remappings: HashMap::from([( diff --git a/mod.rs b/mod.rs index 2f7d4308..3a5f2f84 100644 --- a/mod.rs +++ b/mod.rs @@ -283,6 +283,7 @@ pub struct Config { pub trace_propagation_style_extract: Vec, pub trace_propagation_extract_first: bool, pub trace_propagation_http_baggage_enabled: bool, + pub trace_aws_service_representation_enabled: bool, // OTLP // @@ -371,6 +372,7 @@ impl Default for Config { apm_config_compression_level: 6, apm_features: vec![], apm_additional_endpoints: HashMap::new(), + trace_aws_service_representation_enabled: true, trace_propagation_style: vec![ TracePropagationStyle::Datadog, TracePropagationStyle::TraceContext, diff --git a/yaml.rs b/yaml.rs index 1045fd66..7620f5ab 100644 --- a/yaml.rs +++ b/yaml.rs @@ -64,6 +64,8 @@ pub struct YamlConfig { pub apm_config: ApmConfig, #[serde(deserialize_with = "deserialize_service_mapping")] pub service_mapping: HashMap, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub trace_aws_service_representation_enabled: Option, // Trace Propagation #[serde(deserialize_with = "deserialize_trace_propagation_style")] pub trace_propagation_style: Vec, @@ -451,6 +453,11 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_vec!(config, yaml_config, trace_propagation_style_extract); merge_option_to_value!(config, yaml_config, trace_propagation_extract_first); merge_option_to_value!(config, yaml_config, trace_propagation_http_baggage_enabled); + merge_option_to_value!( + config, + yaml_config, + trace_aws_service_representation_enabled + ); // OTLP if let Some(otlp_config) = &yaml_config.otlp_config { @@ -711,6 +718,7 @@ trace_propagation_style: "datadog" trace_propagation_style_extract: "b3" trace_propagation_extract_first: true trace_propagation_http_baggage_enabled: true +trace_aws_service_representation_enabled: true # OTLP otlp_config: @@ -839,6 +847,7 @@ extension_version: "compatibility" trace_propagation_style_extract: vec![TracePropagationStyle::B3], trace_propagation_extract_first: true, trace_propagation_http_baggage_enabled: true, + trace_aws_service_representation_enabled: true, otlp_config_traces_enabled: false, otlp_config_traces_span_name_as_resource_name: true, otlp_config_traces_span_name_remappings: HashMap::from([( From e8ce82eec50c92a51ac74a07eb287fdcb0ddd1f1 Mon Sep 17 00:00:00 2001 From: Romain Marcadier Date: Wed, 27 Aug 2025 20:19:26 +0200 Subject: [PATCH 078/112] feat: port of Serverless AAP from Go to Rust (#755) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What? Ports the Serverless App & API Protection feature (AAP, also known as Serverless AppSec) from the Go extension to Rust. This is using https://github.com/DataDog/libddwaf-rust to provide bindings to the in-app WAF. This provides enhanced support for API Protection (notably, response schema collection) compared to the Go version. Tradeoff is that XML request and response security processing is not currently supported in this version (it was in Go, but likely seldom used). This introduces a `bottlecap::appsec::processor::Processor` that is integrated in the `bottlecap::proxy::Interceptor` (for request & response acquisition) as well as in the `bottlecap::trace_processor::TraceProcessor` (to decorate the `aws.lambda` span with security data). # Why? We plan on decommissioning the Go version of the agent and a tracer-side version of the Serverless AAP feature will not be available across all supported language runtimes before several weeks/months. Also [SVLS-5286](https://datadoghq.atlassian.net/browse/SVLS-5286) # Notes [SVLS-5286]: https://datadoghq.atlassian.net/browse/SVLS-5286?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ --------- Co-authored-by: jordan gonzález <30836115+duncanista@users.noreply.github.com> --- env.rs | 38 +++++++++++++++- mod.rs | 135 +++++++++++++++++++++++++++++++++++++++++++++++++++----- yaml.rs | 26 ++++++++++- 3 files changed, 185 insertions(+), 14 deletions(-) diff --git a/env.rs b/env.rs index 46b2717f..9f71c1d6 100644 --- a/env.rs +++ b/env.rs @@ -1,6 +1,7 @@ use figment::{Figment, providers::Env}; use serde::Deserialize; use std::collections::HashMap; +use std::time::Duration; use datadog_trace_obfuscation::replacer::ReplaceRule; @@ -10,7 +11,8 @@ use crate::{ additional_endpoints::deserialize_additional_endpoints, apm_replace_rule::deserialize_apm_replace_rules, deserialize_array_from_comma_separated_string, deserialize_key_value_pairs, - deserialize_optional_bool_from_anything, deserialize_string_or_int, + deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, + deserialize_optional_duration_from_seconds, deserialize_string_or_int, flush_strategy::FlushStrategy, log_level::LogLevel, logs_additional_endpoints::{ @@ -312,6 +314,25 @@ pub struct EnvConfig { /// Default is `false`. #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub serverless_appsec_enabled: Option, + /// @env `DD_APPSEC_RULES` + /// + /// The path to a user-configured App & API Protection ruleset (in JSON format). + pub appsec_rules: Option, + /// @env `DD_APPSEC_WAF_TIMEOUT` + /// + /// The timeout for the WAF to process a request, in microseconds. + #[serde(deserialize_with = "deserialize_optional_duration_from_microseconds")] + pub appsec_waf_timeout: Option, + /// @env `DD_API_SECURITY_ENABLED` + /// + /// Enable API Security for AWS Lambda. + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub api_security_enabled: Option, + /// @env `DD_API_SECURITY_SAMPLE_DELAY` + /// + /// The delay between two samples of the API Security schema collection, in seconds. + #[serde(deserialize_with = "deserialize_optional_duration_from_seconds")] + pub api_security_sample_delay: Option, /// @env `DD_EXTENSION_VERSION` /// /// Used to decide which version of the Datadog Lambda Extension to use. @@ -460,6 +481,10 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option_to_value!(config, env_config, capture_lambda_payload); merge_option_to_value!(config, env_config, capture_lambda_payload_max_depth); merge_option_to_value!(config, env_config, serverless_appsec_enabled); + merge_option!(config, env_config, appsec_rules); + merge_option_to_value!(config, env_config, appsec_waf_timeout); + merge_option_to_value!(config, env_config, api_security_enabled); + merge_option_to_value!(config, env_config, api_security_sample_delay); merge_option!(config, env_config, extension_version); } @@ -486,8 +511,11 @@ impl ConfigSource for EnvConfigSource { } } +#[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] mod tests { + use std::time::Duration; + use super::*; use crate::config::{ Config, @@ -634,6 +662,10 @@ mod tests { jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "true"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "5"); jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); + jail.set_env("DD_APPSEC_RULES", "/path/to/rules.json"); + jail.set_env("DD_APPSEC_WAF_TIMEOUT", "1000000"); // Microseconds + jail.set_env("DD_API_SECURITY_ENABLED", "0"); // Seconds + jail.set_env("DD_API_SECURITY_SAMPLE_DELAY", "60"); // Seconds jail.set_env("DD_EXTENSION_VERSION", "compatibility"); let mut config = Config::default(); @@ -759,6 +791,10 @@ mod tests { capture_lambda_payload: true, capture_lambda_payload_max_depth: 5, serverless_appsec_enabled: true, + appsec_rules: Some("/path/to/rules.json".to_string()), + appsec_waf_timeout: Duration::from_secs(1), + api_security_enabled: false, + api_security_sample_delay: Duration::from_secs(60), extension_version: Some("compatibility".to_string()), }; diff --git a/mod.rs b/mod.rs index 3a5f2f84..faeeca2a 100644 --- a/mod.rs +++ b/mod.rs @@ -18,6 +18,7 @@ use serde_aux::prelude::deserialize_bool_from_anything; use serde_json::Value; use std::path::Path; +use std::time::Duration; use std::{collections::HashMap, fmt}; use tracing::{debug, error}; @@ -31,6 +32,7 @@ use crate::config::{ trace_propagation_style::TracePropagationStyle, yaml::YamlConfigSource, }; +use crate::proc::has_dotnet_binary; /// Helper macro to merge Option fields to String fields /// @@ -328,7 +330,13 @@ pub struct Config { pub lambda_proc_enhanced_metrics: bool, pub capture_lambda_payload: bool, pub capture_lambda_payload_max_depth: u32, + pub serverless_appsec_enabled: bool, + pub appsec_rules: Option, + pub appsec_waf_timeout: Duration, + pub api_security_enabled: bool, + pub api_security_sample_delay: Duration, + pub extension_version: Option, } @@ -413,7 +421,13 @@ impl Default for Config { lambda_proc_enhanced_metrics: true, capture_lambda_payload: false, capture_lambda_payload_max_depth: 10, + serverless_appsec_enabled: false, + appsec_rules: None, + appsec_waf_timeout: Duration::from_millis(5), + api_security_enabled: true, + api_security_sample_delay: Duration::from_secs(30), + extension_version: None, } } @@ -438,10 +452,12 @@ fn fallback(config: &Config) -> Result<(), ConfigError> { )); } - if config.serverless_appsec_enabled { - log_fallback_reason("appsec_enabled"); + // ASM / .NET + // todo(duncanista): Remove once the .NET runtime is fixed + if config.serverless_appsec_enabled && has_dotnet_binary() { + log_fallback_reason("serverless_appsec_enabled_dotnet"); return Err(ConfigError::UnsupportedField( - "serverless_appsec_enabled".to_string(), + "serverless_appsec_enabled_dotnet".to_string(), )); } @@ -607,16 +623,53 @@ where Ok(map) } +pub fn deserialize_optional_duration_from_microseconds<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + Ok(Option::::deserialize(deserializer)?.map(Duration::from_micros)) +} + +pub fn deserialize_optional_duration_from_seconds<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + struct DurationVisitor; + impl serde::de::Visitor<'_> for DurationVisitor { + type Value = Option; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "a duration in seconds (integer or float)") + } + fn visit_u64(self, v: u64) -> Result { + Ok(Some(Duration::from_secs(v))) + } + fn visit_i64(self, v: i64) -> Result { + if v < 0 { + return Err(E::custom("negative durations are not allowed")); + } + self.visit_u64(u64::try_from(v).expect("positive i64 to u64 conversion never fails")) + } + fn visit_f64(self, v: f64) -> Result { + if v < 0f64 { + return Err(E::custom("negative durations are not allowed")); + } + Ok(Some(Duration::from_secs_f64(v))) + } + } + deserializer.deserialize_any(DurationVisitor) +} + +#[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] pub mod tests { use datadog_trace_obfuscation::replacer::parse_rules_from_string; use super::*; - use crate::config::flush_strategy::{FlushStrategy, PeriodicStrategy}; - use crate::config::log_level::LogLevel; - use crate::config::processing_rule; - use crate::config::trace_propagation_style::TracePropagationStyle; + use crate::config::{ + flush_strategy::{FlushStrategy, PeriodicStrategy}, + log_level::LogLevel, + processing_rule::ProcessingRule, + trace_propagation_style::TracePropagationStyle, + }; #[test] fn test_reject_on_opted_out() { @@ -754,13 +807,13 @@ pub mod tests { fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); + jail.set_env( + "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", + "localhost:4138", + ); let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!( - config, - ConfigError::UnsupportedField("serverless_appsec_enabled".to_string()) - ); + assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); Ok(()) }); } @@ -1220,4 +1273,62 @@ pub mod tests { Ok(()) }); } + + #[test] + fn test_parse_duration_from_microseconds() { + #[derive(Deserialize, Debug, PartialEq, Eq)] + struct Value { + #[serde(default)] + #[serde(deserialize_with = "deserialize_optional_duration_from_microseconds")] + duration: Option, + } + + assert_eq!( + serde_json::from_str::("{}").expect("failed to parse JSON"), + Value { duration: None } + ); + serde_json::from_str::(r#"{"duration":-1}"#) + .expect_err("should have failed parsing"); + assert_eq!( + serde_json::from_str::(r#"{"duration":1000000}"#).expect("failed to parse JSON"), + Value { + duration: Some(Duration::from_secs(1)) + } + ); + serde_json::from_str::(r#"{"duration":-1.5}"#) + .expect_err("should have failed parsing"); + serde_json::from_str::(r#"{"duration":1.5}"#) + .expect_err("should have failed parsing"); + } + + #[test] + fn test_parse_duration_from_seconds() { + #[derive(Deserialize, Debug, PartialEq, Eq)] + struct Value { + #[serde(default)] + #[serde(deserialize_with = "deserialize_optional_duration_from_seconds")] + duration: Option, + } + + assert_eq!( + serde_json::from_str::("{}").expect("failed to parse JSON"), + Value { duration: None } + ); + serde_json::from_str::(r#"{"duration":-1}"#) + .expect_err("should have failed parsing"); + assert_eq!( + serde_json::from_str::(r#"{"duration":1}"#).expect("failed to parse JSON"), + Value { + duration: Some(Duration::from_secs(1)) + } + ); + serde_json::from_str::(r#"{"duration":-1.5}"#) + .expect_err("should have failed parsing"); + assert_eq!( + serde_json::from_str::(r#"{"duration":1.5}"#).expect("failed to parse JSON"), + Value { + duration: Some(Duration::from_millis(1500)) + } + ); + } } diff --git a/yaml.rs b/yaml.rs index 7620f5ab..e3f19984 100644 --- a/yaml.rs +++ b/yaml.rs @@ -1,3 +1,4 @@ +use std::time::Duration; use std::{collections::HashMap, path::PathBuf}; use crate::{ @@ -5,7 +6,8 @@ use crate::{ Config, ConfigError, ConfigSource, ProcessingRule, additional_endpoints::deserialize_additional_endpoints, deserialize_apm_replace_rules, deserialize_key_value_pair_array_to_hashmap, - deserialize_optional_bool_from_anything, deserialize_processing_rules, + deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, + deserialize_optional_duration_from_seconds, deserialize_processing_rules, deserialize_string_or_int, flush_strategy::FlushStrategy, log_level::LogLevel, @@ -93,6 +95,12 @@ pub struct YamlConfig { pub capture_lambda_payload_max_depth: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub serverless_appsec_enabled: Option, + pub appsec_rules: Option, + #[serde(deserialize_with = "deserialize_optional_duration_from_microseconds")] + pub appsec_waf_timeout: Option, + pub api_security_enabled: Option, + #[serde(deserialize_with = "deserialize_optional_duration_from_seconds")] + pub api_security_sample_delay: Option, pub extension_version: Option, } @@ -606,6 +614,10 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option_to_value!(config, yaml_config, capture_lambda_payload); merge_option_to_value!(config, yaml_config, capture_lambda_payload_max_depth); merge_option_to_value!(config, yaml_config, serverless_appsec_enabled); + merge_option!(config, yaml_config, appsec_rules); + merge_option_to_value!(config, yaml_config, appsec_waf_timeout); + merge_option_to_value!(config, yaml_config, api_security_enabled); + merge_option_to_value!(config, yaml_config, api_security_sample_delay); merge_option!(config, yaml_config, extension_version); } @@ -632,9 +644,11 @@ impl ConfigSource for YamlConfigSource { } } +#[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] mod tests { use std::path::Path; + use std::time::Duration; use crate::config::{flush_strategy::PeriodicStrategy, processing_rule::Kind}; @@ -766,6 +780,10 @@ lambda_proc_enhanced_metrics: false capture_lambda_payload: true capture_lambda_payload_max_depth: 5 serverless_appsec_enabled: true +appsec_rules: "/path/to/rules.json" +appsec_waf_timeout: 1000000 # Microseconds +api_security_enabled: false +api_security_sample_delay: 60 # Seconds extension_version: "compatibility" "#, )?; @@ -889,7 +907,13 @@ extension_version: "compatibility" lambda_proc_enhanced_metrics: false, capture_lambda_payload: true, capture_lambda_payload_max_depth: 5, + serverless_appsec_enabled: true, + appsec_rules: Some("/path/to/rules.json".to_string()), + appsec_waf_timeout: Duration::from_secs(1), + api_security_enabled: false, + api_security_sample_delay: Duration::from_secs(60), + extension_version: Some("compatibility".to_string()), }; From caa87502939a53d686e8656d4eccecafdf0d5d42 Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Thu, 28 Aug 2025 14:59:23 -0400 Subject: [PATCH 079/112] feat: No longer launch Go-based agent for compatibility/OTLP/AAP config (#788) https://datadoghq.atlassian.net/browse/SVLS-7398 - As part of coming release, bottlecap agent no longer launches Go-based agent when compatibility/AAP/OTLP features are active - Emit the same metric when detecting any of above configuration - Update corresponding unit tests Manifests: - [Test lambda function](https://us-east-1.console.aws.amazon.com/lambda/home?region=us-east-1#/functions/ltn1-fullinstrument-bn-cold-python310-lambda?code=&subtab=envVars&tab=testing) with [logs](https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups/log-group/$252Faws$252Flambda$252Fltn1-fullinstrument-bn-cold-python310-lambda/log-events/2025$252F08$252F21$252F$255B$2524LATEST$255Df3788d359677452dad162488ff15456f$3FfilterPattern$3Dotel) showing compatibility/AAP/OTPL are enabled image - [Logging](https://app.datadoghq.com/logs/livetail?query=functionname%3Altn1-fullinstrument-bn-cold-python310-lambda%20Metric&agg_m=count&agg_m_source=base&agg_t=count&cols=host%2Cservice&fromUser=true&messageDisplay=inline&refresh_mode=paused&storage=driveline&stream_sort=desc&viz=stream&from_ts=1755787655569&to_ts=1755787689060&live=false) image - [Metric](https://app.datadoghq.com/screen/integration/aws_lambda_enhanced_metrics?fromUser=false&fullscreen_end_ts=1755788220000&fullscreen_paused=true&fullscreen_refresh_mode=paused&fullscreen_section=overview&fullscreen_start_ts=1755787200000&fullscreen_widget=2&graph-explorer__tile_def=N4IgbglgXiBcIBcD2AHANhAzgkAaEAxgK7ZIC2A%2BhgHYDWmcA2gLr4BOApgI5EfYOxGoTphRJqmDhQBmSNmQCGOeJgIK0CtnhA8ObCHyagAJkoUVMSImwIc4IMhwT6CDfNQWP7utgE8AjNo%2BvvaYRGSwpggKxkgA5gB0kmxgemh8mAkcAB4IHBIQ4gnSChBoSKlswAAkCgDumBQKBARW1Ai41ZxxhdSd0kTUBAi9AL4ABABGvuPAA0Mj4h6OowkKja2DCAAUAJTaCnFx3UpyoeEgo6wgsvJEGgJCN3Jk9wrevH6BV-iWbMqgTbtOAAJgADPg5MY9BRpkZEL4UHZ4LdXhptBBqNDsnAISAoXp7NDVJdmKMfiBsL50nBgOSgA&refresh_mode=sliding&from_ts=1755783890661&to_ts=1755787490661&live=true) image --- mod.rs | 120 +++++++++++++++++++++++++-------------------------------- 1 file changed, 53 insertions(+), 67 deletions(-) diff --git a/mod.rs b/mod.rs index faeeca2a..4f507326 100644 --- a/mod.rs +++ b/mod.rs @@ -434,10 +434,13 @@ impl Default for Config { } fn log_fallback_reason(reason: &str) { - println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); + error!("Fallback support for {reason} is no longer available."); } -fn fallback(config: &Config) -> Result<(), ConfigError> { +#[must_use = "fallback reasons should be processed to emit appropriate metrics"] +pub fn fallback(config: &Config) -> Vec { + let mut fallback_reasons = Vec::new(); + // Customer explicitly opted out of the Next Gen extension let opted_out = match config.extension_version.as_deref() { Some("compatibility") => true, @@ -446,21 +449,18 @@ fn fallback(config: &Config) -> Result<(), ConfigError> { }; if opted_out { - log_fallback_reason("extension_version"); - return Err(ConfigError::UnsupportedField( - "extension_version".to_string(), - )); + let reason = "extension_version"; + log_fallback_reason(reason); + fallback_reasons.push(reason.to_string()); } // ASM / .NET // todo(duncanista): Remove once the .NET runtime is fixed if config.serverless_appsec_enabled && has_dotnet_binary() { - log_fallback_reason("serverless_appsec_enabled_dotnet"); - return Err(ConfigError::UnsupportedField( - "serverless_appsec_enabled_dotnet".to_string(), - )); + let reason = "serverless_appsec_enabled_dotnet"; + log_fallback_reason(reason); + fallback_reasons.push(reason.to_string()); } - // OTLP let has_otlp_config = config .otlp_config_receiver_protocols_grpc_endpoint @@ -492,25 +492,22 @@ fn fallback(config: &Config) -> Result<(), ConfigError> { || config.otlp_config_logs_enabled; if has_otlp_config { - log_fallback_reason("otel"); - return Err(ConfigError::UnsupportedField("otel".to_string())); + let reason = "otel"; + log_fallback_reason(reason); + fallback_reasons.push(reason.to_string()); } - Ok(()) + fallback_reasons } #[allow(clippy::module_name_repetitions)] -pub fn get_config(config_directory: &Path) -> Result { +#[must_use = "configuration must be used to initialize the application"] +pub fn get_config(config_directory: &Path) -> Config { let path: std::path::PathBuf = config_directory.join("datadog.yaml"); let mut config_builder = ConfigBuilder::default() .add_source(Box::new(YamlConfigSource { path })) .add_source(Box::new(EnvConfigSource)); - - let config = config_builder.build(); - - fallback(&config)?; - - Ok(config) + config_builder.build() } #[inline] @@ -676,11 +673,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_EXTENSION_VERSION", "compatibility"); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!( - config, - ConfigError::UnsupportedField("extension_version".to_string()) - ); + let _config = get_config(Path::new("")); Ok(()) }); } @@ -694,8 +687,7 @@ pub mod tests { "localhost:4138", ); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); + let _config = get_config(Path::new("")); Ok(()) }); } @@ -715,8 +707,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); + let _config = get_config(Path::new("")); Ok(()) }); } @@ -726,7 +717,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_logs_dd_url, "https://http-intake.logs.datadoghq.com".to_string() @@ -744,7 +735,7 @@ pub mod tests { "agent-http-intake-pci.logs.datadoghq.com:443", ); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_logs_dd_url, "agent-http-intake-pci.logs.datadoghq.com:443".to_string() @@ -759,7 +750,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_APM_DD_URL", "https://trace-pci.agent.datadoghq.com"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.apm_dd_url, "https://trace-pci.agent.datadoghq.com/api/v0.2/traces".to_string() @@ -774,7 +765,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_DD_URL", "custom_proxy:3128"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.dd_url, "custom_proxy:3128".to_string()); Ok(()) }); @@ -786,7 +777,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_URL", "custom_proxy:3128"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.url, "custom_proxy:3128".to_string()); Ok(()) }); @@ -797,7 +788,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.dd_url, String::new()); Ok(()) }); @@ -807,13 +798,9 @@ pub mod tests { fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env( - "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", - "localhost:4138", - ); + jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); + let _config = get_config(Path::new("")); Ok(()) }); } @@ -829,7 +816,7 @@ pub mod tests { ", )?; jail.set_env("DD_SITE", "datad0g.com"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "datad0g.com"); Ok(()) }); @@ -845,7 +832,7 @@ pub mod tests { r" ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "datadoghq.com"); Ok(()) }); @@ -856,7 +843,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SITE", "datadoghq.eu"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "datadoghq.eu"); Ok(()) }); @@ -867,7 +854,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_LOG_LEVEL", "TRACE"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.log_level, LogLevel::Trace); Ok(()) }); @@ -877,7 +864,7 @@ pub mod tests { fn test_parse_default() { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config, Config { @@ -901,7 +888,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_PROXY_HTTPS", "my-proxy:3128"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -917,7 +904,7 @@ pub mod tests { "NO_PROXY", "127.0.0.1,localhost,172.16.0.0/12,us-east-1.amazonaws.com,datadoghq.eu", ); - let config = get_config(Path::new("")).expect("should parse noproxy"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, None); Ok(()) }); @@ -935,7 +922,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect("should parse weird proxy config"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -955,7 +942,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect("should parse weird proxy config"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, None); // Assertion to ensure config.site runs before proxy // because we chenck that noproxy contains the site @@ -969,7 +956,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.serverless_flush_strategy, FlushStrategy::End); Ok(()) }); @@ -980,7 +967,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.serverless_flush_strategy, FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }) @@ -994,7 +981,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -1008,7 +995,7 @@ pub mod tests { "DD_SERVERLESS_FLUSH_STRATEGY", "periodically,invalid_interval", ); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -1021,7 +1008,7 @@ pub mod tests { jail.set_env("DD_VERSION", "123"); jail.set_env("DD_ENV", "123456890"); jail.set_env("DD_SERVICE", "123456"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.version.expect("failed to parse DD_VERSION"), "123"); assert_eq!(config.env.expect("failed to parse DD_ENV"), "123456890"); assert_eq!( @@ -1051,7 +1038,7 @@ pub mod tests { pattern: exclude-me-yaml ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -1080,7 +1067,7 @@ pub mod tests { pattern: exclude ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -1109,7 +1096,7 @@ pub mod tests { repl: 'REDACTED' ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); let rule = parse_rules_from_string( r#"[ {"name": "*", "pattern": "foo", "repl": "REDACTED"} @@ -1140,7 +1127,7 @@ pub mod tests { repl: 'REDACTED-YAML' ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); let rule = parse_rules_from_string( r#"[ {"name": "*", "pattern": "foo", "repl": "REDACTED-ENV"} @@ -1167,7 +1154,7 @@ pub mod tests { remove_paths_with_digits: true ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert!(config.apm_config_obfuscation_http_remove_query_string,); assert!(config.apm_config_obfuscation_http_remove_paths_with_digits,); Ok(()) @@ -1182,7 +1169,7 @@ pub mod tests { "datadog,tracecontext,b3,b3multi", ); jail.set_env("DD_EXTENSION_VERSION", "next"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); let expected_styles = vec![ TracePropagationStyle::Datadog, @@ -1201,7 +1188,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "datadog"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.trace_propagation_style, @@ -1226,8 +1213,7 @@ pub mod tests { "DD_APM_REPLACE_TAGS", r#"[{"name":"resource.name","pattern":"(.*)/(foo[:%].+)","repl":"$1/{foo}"}]"#, ); - let config = get_config(Path::new("")); - assert!(config.is_ok()); + let _config = get_config(Path::new("")); Ok(()) }); } @@ -1240,7 +1226,7 @@ pub mod tests { jail.set_env("DD_ENHANCED_METRICS", "1"); jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "TRUE"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "0"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert!(config.serverless_logs_enabled); assert!(config.enhanced_metrics); assert!(config.logs_config_use_compression); @@ -1264,7 +1250,7 @@ pub mod tests { jail.set_env("DD_SITE", "us5.datadoghq.com"); jail.set_env("DD_API_KEY", "env-api-key"); jail.set_env("DD_FLUSH_TIMEOUT", "10"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "us5.datadoghq.com"); assert_eq!(config.api_key, "env-api-key"); From d9885bed22ff159bc382a8c64653f60767c59ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 28 Aug 2025 17:44:45 -0400 Subject: [PATCH 080/112] Revert "feat: No longer launch Go-based agent for compatibility/OTLP/AAP config (#788)" This reverts commit 0f5984571eb842e5ce1cbadbec0f92d73befcd08. --- mod.rs | 120 ++++++++++++++++++++++++++++++++------------------------- 1 file changed, 67 insertions(+), 53 deletions(-) diff --git a/mod.rs b/mod.rs index 4f507326..faeeca2a 100644 --- a/mod.rs +++ b/mod.rs @@ -434,13 +434,10 @@ impl Default for Config { } fn log_fallback_reason(reason: &str) { - error!("Fallback support for {reason} is no longer available."); + println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); } -#[must_use = "fallback reasons should be processed to emit appropriate metrics"] -pub fn fallback(config: &Config) -> Vec { - let mut fallback_reasons = Vec::new(); - +fn fallback(config: &Config) -> Result<(), ConfigError> { // Customer explicitly opted out of the Next Gen extension let opted_out = match config.extension_version.as_deref() { Some("compatibility") => true, @@ -449,18 +446,21 @@ pub fn fallback(config: &Config) -> Vec { }; if opted_out { - let reason = "extension_version"; - log_fallback_reason(reason); - fallback_reasons.push(reason.to_string()); + log_fallback_reason("extension_version"); + return Err(ConfigError::UnsupportedField( + "extension_version".to_string(), + )); } // ASM / .NET // todo(duncanista): Remove once the .NET runtime is fixed if config.serverless_appsec_enabled && has_dotnet_binary() { - let reason = "serverless_appsec_enabled_dotnet"; - log_fallback_reason(reason); - fallback_reasons.push(reason.to_string()); + log_fallback_reason("serverless_appsec_enabled_dotnet"); + return Err(ConfigError::UnsupportedField( + "serverless_appsec_enabled_dotnet".to_string(), + )); } + // OTLP let has_otlp_config = config .otlp_config_receiver_protocols_grpc_endpoint @@ -492,22 +492,25 @@ pub fn fallback(config: &Config) -> Vec { || config.otlp_config_logs_enabled; if has_otlp_config { - let reason = "otel"; - log_fallback_reason(reason); - fallback_reasons.push(reason.to_string()); + log_fallback_reason("otel"); + return Err(ConfigError::UnsupportedField("otel".to_string())); } - fallback_reasons + Ok(()) } #[allow(clippy::module_name_repetitions)] -#[must_use = "configuration must be used to initialize the application"] -pub fn get_config(config_directory: &Path) -> Config { +pub fn get_config(config_directory: &Path) -> Result { let path: std::path::PathBuf = config_directory.join("datadog.yaml"); let mut config_builder = ConfigBuilder::default() .add_source(Box::new(YamlConfigSource { path })) .add_source(Box::new(EnvConfigSource)); - config_builder.build() + + let config = config_builder.build(); + + fallback(&config)?; + + Ok(config) } #[inline] @@ -673,7 +676,11 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_EXTENSION_VERSION", "compatibility"); - let _config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("extension_version".to_string()) + ); Ok(()) }); } @@ -687,7 +694,8 @@ pub mod tests { "localhost:4138", ); - let _config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); Ok(()) }); } @@ -707,7 +715,8 @@ pub mod tests { ", )?; - let _config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); Ok(()) }); } @@ -717,7 +726,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.logs_config_logs_dd_url, "https://http-intake.logs.datadoghq.com".to_string() @@ -735,7 +744,7 @@ pub mod tests { "agent-http-intake-pci.logs.datadoghq.com:443", ); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.logs_config_logs_dd_url, "agent-http-intake-pci.logs.datadoghq.com:443".to_string() @@ -750,7 +759,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_APM_DD_URL", "https://trace-pci.agent.datadoghq.com"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.apm_dd_url, "https://trace-pci.agent.datadoghq.com/api/v0.2/traces".to_string() @@ -765,7 +774,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_DD_URL", "custom_proxy:3128"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.dd_url, "custom_proxy:3128".to_string()); Ok(()) }); @@ -777,7 +786,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_URL", "custom_proxy:3128"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.url, "custom_proxy:3128".to_string()); Ok(()) }); @@ -788,7 +797,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.dd_url, String::new()); Ok(()) }); @@ -798,9 +807,13 @@ pub mod tests { fn test_allowed_but_disabled() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); + jail.set_env( + "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", + "localhost:4138", + ); - let _config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); Ok(()) }); } @@ -816,7 +829,7 @@ pub mod tests { ", )?; jail.set_env("DD_SITE", "datad0g.com"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.site, "datad0g.com"); Ok(()) }); @@ -832,7 +845,7 @@ pub mod tests { r" ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.site, "datadoghq.com"); Ok(()) }); @@ -843,7 +856,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SITE", "datadoghq.eu"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.site, "datadoghq.eu"); Ok(()) }); @@ -854,7 +867,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_LOG_LEVEL", "TRACE"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.log_level, LogLevel::Trace); Ok(()) }); @@ -864,7 +877,7 @@ pub mod tests { fn test_parse_default() { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, Config { @@ -888,7 +901,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_PROXY_HTTPS", "my-proxy:3128"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -904,7 +917,7 @@ pub mod tests { "NO_PROXY", "127.0.0.1,localhost,172.16.0.0/12,us-east-1.amazonaws.com,datadoghq.eu", ); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse noproxy"); assert_eq!(config.proxy_https, None); Ok(()) }); @@ -922,7 +935,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse weird proxy config"); assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -942,7 +955,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse weird proxy config"); assert_eq!(config.proxy_https, None); // Assertion to ensure config.site runs before proxy // because we chenck that noproxy contains the site @@ -956,7 +969,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::End); Ok(()) }); @@ -967,7 +980,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.serverless_flush_strategy, FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }) @@ -981,7 +994,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -995,7 +1008,7 @@ pub mod tests { "DD_SERVERLESS_FLUSH_STRATEGY", "periodically,invalid_interval", ); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -1008,7 +1021,7 @@ pub mod tests { jail.set_env("DD_VERSION", "123"); jail.set_env("DD_ENV", "123456890"); jail.set_env("DD_SERVICE", "123456"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.version.expect("failed to parse DD_VERSION"), "123"); assert_eq!(config.env.expect("failed to parse DD_ENV"), "123456890"); assert_eq!( @@ -1038,7 +1051,7 @@ pub mod tests { pattern: exclude-me-yaml ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -1067,7 +1080,7 @@ pub mod tests { pattern: exclude ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -1096,7 +1109,7 @@ pub mod tests { repl: 'REDACTED' ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); let rule = parse_rules_from_string( r#"[ {"name": "*", "pattern": "foo", "repl": "REDACTED"} @@ -1127,7 +1140,7 @@ pub mod tests { repl: 'REDACTED-YAML' ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); let rule = parse_rules_from_string( r#"[ {"name": "*", "pattern": "foo", "repl": "REDACTED-ENV"} @@ -1154,7 +1167,7 @@ pub mod tests { remove_paths_with_digits: true ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert!(config.apm_config_obfuscation_http_remove_query_string,); assert!(config.apm_config_obfuscation_http_remove_paths_with_digits,); Ok(()) @@ -1169,7 +1182,7 @@ pub mod tests { "datadog,tracecontext,b3,b3multi", ); jail.set_env("DD_EXTENSION_VERSION", "next"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); let expected_styles = vec![ TracePropagationStyle::Datadog, @@ -1188,7 +1201,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "datadog"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.trace_propagation_style, @@ -1213,7 +1226,8 @@ pub mod tests { "DD_APM_REPLACE_TAGS", r#"[{"name":"resource.name","pattern":"(.*)/(foo[:%].+)","repl":"$1/{foo}"}]"#, ); - let _config = get_config(Path::new("")); + let config = get_config(Path::new("")); + assert!(config.is_ok()); Ok(()) }); } @@ -1226,7 +1240,7 @@ pub mod tests { jail.set_env("DD_ENHANCED_METRICS", "1"); jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "TRUE"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "0"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert!(config.serverless_logs_enabled); assert!(config.enhanced_metrics); assert!(config.logs_config_use_compression); @@ -1250,7 +1264,7 @@ pub mod tests { jail.set_env("DD_SITE", "us5.datadoghq.com"); jail.set_env("DD_API_KEY", "env-api-key"); jail.set_env("DD_FLUSH_TIMEOUT", "10"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.site, "us5.datadoghq.com"); assert_eq!(config.api_key, "env-api-key"); From 63d6422f432c20ced95b952ef4d61f4f2c9ee88f Mon Sep 17 00:00:00 2001 From: jchrostek-dd Date: Fri, 29 Aug 2025 13:14:21 -0400 Subject: [PATCH 081/112] Ignoring Unwanted Resources in APM (#794) ## Task https://datadoghq.atlassian.net/browse/SVLS-6846 ## Overview We want to allow users to set filter tags which drops traces with root spans that match specified span tags. Specifically, users can set `DD_APM_FILTER_TAGS_REQUIRE` or `DD_APM_FILTER_TAGS_REJECT`. More info [here](https://docs.datadoghq.com/tracing/guide/ignoring_apm_resources/?tab=datadogyaml#trace-agent-configuration-options). ## Testing Deployed changes to Lambda. Invoked Lambda directly and through API Gateway to check with different root spans. Set the tags to either be REQUIRE or REJECT with value `name:aws.lambda`. Confirmed in logs and UI that we were dropping spans. --- env.rs | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- mod.rs | 55 +++++++++++++++++++++++++++++++++++++++++++++++++ yaml.rs | 4 ++++ 3 files changed, 120 insertions(+), 2 deletions(-) diff --git a/env.rs b/env.rs index 9f71c1d6..b6d444fc 100644 --- a/env.rs +++ b/env.rs @@ -10,8 +10,9 @@ use crate::{ Config, ConfigError, ConfigSource, additional_endpoints::deserialize_additional_endpoints, apm_replace_rule::deserialize_apm_replace_rules, - deserialize_array_from_comma_separated_string, deserialize_key_value_pairs, - deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, + deserialize_apm_filter_tags, deserialize_array_from_comma_separated_string, + deserialize_key_value_pairs, deserialize_optional_bool_from_anything, + deserialize_optional_duration_from_microseconds, deserialize_optional_duration_from_seconds, deserialize_string_or_int, flush_strategy::FlushStrategy, log_level::LogLevel, @@ -180,6 +181,34 @@ pub struct EnvConfig { /// #[serde(deserialize_with = "deserialize_additional_endpoints")] pub apm_additional_endpoints: HashMap>, + /// @env `DD_APM_FILTER_TAGS_REQUIRE` + /// + /// Space-separated list of key:value tag pairs that spans must match to be kept. + /// Only spans matching at least one of these tags will be sent to Datadog. + /// Example: "env:production service:api-gateway" + #[serde(deserialize_with = "deserialize_apm_filter_tags")] + pub apm_filter_tags_require: Option>, + /// @env `DD_APM_FILTER_TAGS_REJECT` + /// + /// Space-separated list of key:value tag pairs that will cause spans to be filtered out. + /// Spans matching any of these tags will be dropped. + /// Example: "env:development debug:true name:health.check" + #[serde(deserialize_with = "deserialize_apm_filter_tags")] + pub apm_filter_tags_reject: Option>, + /// @env `DD_APM_FILTER_TAGS_REGEX_REQUIRE` + /// + /// Space-separated list of key:value tag pairs with regex values that spans must match to be kept. + /// Only spans matching at least one of these regex patterns will be sent to Datadog. + /// Example: "env:^prod.*$ service:^api-.*$" + #[serde(deserialize_with = "deserialize_apm_filter_tags")] + pub apm_filter_tags_regex_require: Option>, + /// @env `DD_APM_FILTER_TAGS_REGEX_REJECT` + /// + /// Space-separated list of key:value tag pairs with regex values that will cause spans to be filtered out. + /// Spans matching any of these regex patterns will be dropped. + /// Example: "env:^test.*$ debug:^true$" + #[serde(deserialize_with = "deserialize_apm_filter_tags")] + pub apm_filter_tags_regex_reject: Option>, /// @env `DD_TRACE_AWS_SERVICE_REPRESENTATION_ENABLED` /// /// Enable the new AWS-resource naming logic in the tracer. @@ -388,6 +417,10 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option_to_value!(config, env_config, apm_config_compression_level); merge_vec!(config, env_config, apm_features); merge_hashmap!(config, env_config, apm_additional_endpoints); + merge_option!(config, env_config, apm_filter_tags_require); + merge_option!(config, env_config, apm_filter_tags_reject); + merge_option!(config, env_config, apm_filter_tags_regex_require); + merge_option!(config, env_config, apm_filter_tags_regex_reject); merge_option_to_value!(config, env_config, trace_aws_service_representation_enabled); // Trace Propagation @@ -588,6 +621,16 @@ mod tests { "enable_otlp_compute_top_level_by_span_kind,enable_stats_by_span_kind", ); jail.set_env("DD_APM_ADDITIONAL_ENDPOINTS", "{\"https://trace.agent.datadoghq.com\": [\"apikey2\", \"apikey3\"], \"https://trace.agent.datadoghq.eu\": [\"apikey4\"]}"); + jail.set_env("DD_APM_FILTER_TAGS_REQUIRE", "env:production service:api"); + jail.set_env("DD_APM_FILTER_TAGS_REJECT", "debug:true env:test"); + jail.set_env( + "DD_APM_FILTER_TAGS_REGEX_REQUIRE", + "env:^test.*$ debug:^true$", + ); + jail.set_env( + "DD_APM_FILTER_TAGS_REGEX_REJECT", + "env:^test.*$ debug:^true$", + ); // Trace Propagation jail.set_env("DD_TRACE_PROPAGATION_STYLE", "datadog"); @@ -744,6 +787,22 @@ mod tests { vec!["apikey4".to_string()], ), ]), + apm_filter_tags_require: Some(vec![ + "env:production".to_string(), + "service:api".to_string(), + ]), + apm_filter_tags_reject: Some(vec![ + "debug:true".to_string(), + "env:test".to_string(), + ]), + apm_filter_tags_regex_require: Some(vec![ + "env:^test.*$".to_string(), + "debug:^true$".to_string(), + ]), + apm_filter_tags_regex_reject: Some(vec![ + "env:^test.*$".to_string(), + "debug:^true$".to_string(), + ]), trace_propagation_style: vec![TracePropagationStyle::Datadog], trace_propagation_style_extract: vec![TracePropagationStyle::B3], trace_propagation_extract_first: true, diff --git a/mod.rs b/mod.rs index faeeca2a..0dc7b734 100644 --- a/mod.rs +++ b/mod.rs @@ -279,6 +279,10 @@ pub struct Config { pub apm_config_compression_level: i32, pub apm_features: Vec, pub apm_additional_endpoints: HashMap>, + pub apm_filter_tags_require: Option>, + pub apm_filter_tags_reject: Option>, + pub apm_filter_tags_regex_require: Option>, + pub apm_filter_tags_regex_reject: Option>, // // Trace Propagation pub trace_propagation_style: Vec, @@ -380,6 +384,10 @@ impl Default for Config { apm_config_compression_level: 6, apm_features: vec![], apm_additional_endpoints: HashMap::new(), + apm_filter_tags_require: None, + apm_filter_tags_reject: None, + apm_filter_tags_regex_require: None, + apm_filter_tags_regex_reject: None, trace_aws_service_representation_enabled: true, trace_propagation_style: vec![ TracePropagationStyle::Datadog, @@ -623,6 +631,53 @@ where Ok(map) } +/// Deserialize APM filter tags from space-separated "key:value" pairs, also support key-only tags +pub fn deserialize_apm_filter_tags<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + let opt: Option = Option::deserialize(deserializer)?; + + match opt { + None => Ok(None), + Some(s) if s.trim().is_empty() => Ok(None), + Some(s) => { + let tags: Vec = s + .split_whitespace() + .filter_map(|pair| { + let parts: Vec<&str> = pair.splitn(2, ':').collect(); + if parts.len() == 2 { + let key = parts[0].trim(); + let value = parts[1].trim(); + if key.is_empty() { + None + } else if value.is_empty() { + Some(key.to_string()) + } else { + Some(format!("{key}:{value}")) + } + } else if parts.len() == 1 { + let key = parts[0].trim(); + if key.is_empty() { + None + } else { + Some(key.to_string()) + } + } else { + None + } + }) + .collect(); + + if tags.is_empty() { + Ok(None) + } else { + Ok(Some(tags)) + } + } + } +} + pub fn deserialize_optional_duration_from_microseconds<'de, D: Deserializer<'de>>( deserializer: D, ) -> Result, D::Error> { diff --git a/yaml.rs b/yaml.rs index e3f19984..b8dca19b 100644 --- a/yaml.rs +++ b/yaml.rs @@ -915,6 +915,10 @@ extension_version: "compatibility" api_security_sample_delay: Duration::from_secs(60), extension_version: Some("compatibility".to_string()), + apm_filter_tags_require: None, + apm_filter_tags_reject: None, + apm_filter_tags_regex_require: None, + apm_filter_tags_regex_reject: None, }; // Assert that From fabfbcb877ff9daf50be37f64a97b3156d34007d Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Tue, 9 Sep 2025 14:21:36 -0400 Subject: [PATCH 082/112] feat: eat: Add hierarchical configurable compression levels (#800) feat: Add hierarchical configurable compression levels - Add global compression_level config parameter (0-9, default: 6) with fallback hierarchy - Support 2-level compression configuration: global level first, then module-specific - This makes configuration more convenient - set once globally or override per module - Apply compression configuration to metrics flushers and trace processor - Add environment variable DD_COMPRESSION_LEVEL for global setting Test - Configuration: image - ([log](https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups/log-group/$252Faws$252Flambda$252Fltn1-fullinstrument-bn-cold-python310-lambda/log-events/2025$252F08$252F25$252F$255B$2524LATEST$255D9c19719435bc48839f6f005d2b58b552)) Configuration: image --- env.rs | 47 +++++++++++++++++++++++++++++++++++++++++++---- mod.rs | 12 ++++++++++++ yaml.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 106 insertions(+), 9 deletions(-) diff --git a/env.rs b/env.rs index b6d444fc..8e472639 100644 --- a/env.rs +++ b/env.rs @@ -107,6 +107,12 @@ pub struct EnvConfig { /// @env `DD_TAGS` #[serde(deserialize_with = "deserialize_key_value_pairs")] pub tags: HashMap, + /// @env `DD_COMPRESSION_LEVEL` + /// + /// Global level `compression_level` parameter accepts values from 0 (no compression) + /// to 9 (maximum compression but higher resource usage). This value is effective only if + /// the individual component doesn't specify its own. + pub compression_level: Option, // Logs /// @env `DD_LOGS_CONFIG_LOGS_DD_URL` @@ -229,6 +235,12 @@ pub struct EnvConfig { #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub trace_propagation_http_baggage_enabled: Option, + /// @env `DD_METRICS_CONFIG_COMPRESSION_LEVEL` + /// The metrics compresses traces before sending them. The `compression_level` parameter + /// accepts values from 0 (no compression) to 9 (maximum compression but + /// higher resource usage). + pub metrics_config_compression_level: Option, + // OTLP // // - APM / Traces @@ -393,10 +405,18 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_string!(config, env_config, url); merge_hashmap!(config, env_config, additional_endpoints); + merge_option_to_value!(config, env_config, compression_level); + // Logs merge_string!(config, env_config, logs_config_logs_dd_url); merge_option!(config, env_config, logs_config_processing_rules); merge_option_to_value!(config, env_config, logs_config_use_compression); + merge_option_to_value!( + config, + logs_config_compression_level, + env_config, + compression_level + ); merge_option_to_value!(config, env_config, logs_config_compression_level); merge_vec!(config, env_config, logs_config_additional_endpoints); @@ -414,6 +434,12 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { env_config, apm_config_obfuscation_http_remove_paths_with_digits ); + merge_option_to_value!( + config, + apm_config_compression_level, + env_config, + compression_level + ); merge_option_to_value!(config, env_config, apm_config_compression_level); merge_vec!(config, env_config, apm_features); merge_hashmap!(config, env_config, apm_additional_endpoints); @@ -429,6 +455,15 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option_to_value!(config, env_config, trace_propagation_extract_first); merge_option_to_value!(config, env_config, trace_propagation_http_baggage_enabled); + // Metrics + merge_option_to_value!( + config, + metrics_config_compression_level, + env_config, + compression_level + ); + merge_option_to_value!(config, env_config, metrics_config_compression_level); + // OTLP merge_option_to_value!(config, env_config, otlp_config_traces_enabled); merge_option_to_value!( @@ -588,6 +623,7 @@ mod tests { jail.set_env("DD_SERVICE", "test-service"); jail.set_env("DD_VERSION", "1.0.0"); jail.set_env("DD_TAGS", "team:test-team,project:test-project"); + jail.set_env("DD_COMPRESSION_LEVEL", "4"); // Logs jail.set_env("DD_LOGS_CONFIG_LOGS_DD_URL", "https://logs.datadoghq.com"); @@ -596,7 +632,7 @@ mod tests { r#"[{"type":"exclude_at_match","name":"exclude","pattern":"exclude"}]"#, ); jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "false"); - jail.set_env("DD_LOGS_CONFIG_COMPRESSION_LEVEL", "3"); + jail.set_env("DD_LOGS_CONFIG_COMPRESSION_LEVEL", "1"); jail.set_env( "DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS", "[{\"api_key\": \"apikey2\", \"Host\": \"agent-http-intake.logs.datadoghq.com\", \"Port\": 443, \"is_reliable\": true}]", @@ -615,7 +651,7 @@ mod tests { "DD_APM_CONFIG_OBFUSCATION_HTTP_REMOVE_PATHS_WITH_DIGITS", "true", ); - jail.set_env("DD_APM_CONFIG_COMPRESSION_LEVEL", "3"); + jail.set_env("DD_APM_CONFIG_COMPRESSION_LEVEL", "2"); jail.set_env( "DD_APM_FEATURES", "enable_otlp_compute_top_level_by_span_kind,enable_stats_by_span_kind", @@ -632,6 +668,7 @@ mod tests { "env:^test.*$ debug:^true$", ); + jail.set_env("DD_METRICS_CONFIG_COMPRESSION_LEVEL", "3"); // Trace Propagation jail.set_env("DD_TRACE_PROPAGATION_STYLE", "datadog"); jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "b3"); @@ -721,6 +758,7 @@ mod tests { site: "test-site".to_string(), api_key: "test-api-key".to_string(), log_level: LogLevel::Debug, + compression_level: 4, flush_timeout: 42, proxy_https: Some("https://proxy.example.com".to_string()), proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], @@ -752,7 +790,7 @@ mod tests { replace_placeholder: None, }]), logs_config_use_compression: false, - logs_config_compression_level: 3, + logs_config_compression_level: 1, logs_config_additional_endpoints: vec![LogsAdditionalEndpoint { api_key: "apikey2".to_string(), host: "agent-http-intake.logs.datadoghq.com".to_string(), @@ -772,7 +810,7 @@ mod tests { ), apm_config_obfuscation_http_remove_query_string: true, apm_config_obfuscation_http_remove_paths_with_digits: true, - apm_config_compression_level: 3, + apm_config_compression_level: 2, apm_features: vec![ "enable_otlp_compute_top_level_by_span_kind".to_string(), "enable_stats_by_span_kind".to_string(), @@ -808,6 +846,7 @@ mod tests { trace_propagation_extract_first: true, trace_propagation_http_baggage_enabled: true, trace_aws_service_representation_enabled: true, + metrics_config_compression_level: 3, otlp_config_traces_enabled: false, otlp_config_traces_span_name_as_resource_name: true, otlp_config_traces_span_name_remappings: HashMap::from([( diff --git a/mod.rs b/mod.rs index 0dc7b734..59a0e5be 100644 --- a/mod.rs +++ b/mod.rs @@ -245,6 +245,10 @@ pub struct Config { // Timeout for the request to flush data to Datadog endpoint pub flush_timeout: u64, + // Global config of compression levels. + // It would be overridden by the setup for the individual component + pub compression_level: i32, + // Proxy pub proxy_https: Option, pub proxy_no_proxy: Vec, @@ -291,6 +295,9 @@ pub struct Config { pub trace_propagation_http_baggage_enabled: bool, pub trace_aws_service_representation_enabled: bool, + // Metrics + pub metrics_config_compression_level: i32, + // OTLP // // - APM / Traces @@ -368,6 +375,8 @@ impl Default for Config { version: None, tags: HashMap::new(), + compression_level: 6, + // Logs logs_config_logs_dd_url: String::default(), logs_config_processing_rules: None, @@ -397,6 +406,9 @@ impl Default for Config { trace_propagation_extract_first: false, trace_propagation_http_baggage_enabled: false, + // Metrics + metrics_config_compression_level: 6, + // OTLP otlp_config_traces_enabled: true, otlp_config_traces_span_name_as_resource_name: false, diff --git a/yaml.rs b/yaml.rs index b8dca19b..8017873b 100644 --- a/yaml.rs +++ b/yaml.rs @@ -38,6 +38,8 @@ pub struct YamlConfig { pub flush_timeout: Option, + pub compression_level: Option, + // Proxy pub proxy: ProxyConfig, // nit: this should probably be in the endpoints section @@ -78,6 +80,9 @@ pub struct YamlConfig { #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub trace_propagation_http_baggage_enabled: Option, + // Metrics + pub metrics_config: MetricsConfig, + // OTLP pub otlp_config: Option, @@ -131,6 +136,15 @@ pub struct LogsConfig { pub additional_endpoints: Vec, } +/// Metrics specific config +/// +#[derive(Debug, PartialEq, Deserialize, Clone, Copy, Default)] +#[serde(default)] +#[allow(clippy::module_name_repetitions)] +pub struct MetricsConfig { + pub compression_level: Option, +} + /// APM Config /// @@ -373,6 +387,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option!(config, yaml_config, version); merge_hashmap!(config, yaml_config, tags); + merge_option_to_value!(config, yaml_config, compression_level); // Proxy merge_option!(config, proxy_https, yaml_config.proxy, https); merge_option_to_value!(config, proxy_no_proxy, yaml_config.proxy, no_proxy); @@ -401,6 +416,12 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { yaml_config.logs_config, use_compression ); + merge_option_to_value!( + config, + logs_config_compression_level, + yaml_config, + compression_level + ); merge_option_to_value!( config, logs_config_compression_level, @@ -414,6 +435,20 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { additional_endpoints ); + merge_option_to_value!( + config, + metrics_config_compression_level, + yaml_config, + compression_level + ); + + merge_option_to_value!( + config, + metrics_config_compression_level, + yaml_config.metrics_config, + compression_level + ); + // APM merge_hashmap!(config, yaml_config, service_mapping); merge_string!(config, apm_dd_url, yaml_config.apm_config, apm_dd_url); @@ -423,6 +458,12 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { yaml_config.apm_config, replace_tags ); + merge_option_to_value!( + config, + apm_config_compression_level, + yaml_config, + compression_level + ); merge_option_to_value!( config, apm_config_compression_level, @@ -667,7 +708,7 @@ site: "test-site" api_key: "test-api-key" log_level: "debug" flush_timeout: 42 - +compression_level: 4 # Proxy proxy: https: "https://proxy.example.com" @@ -699,7 +740,7 @@ logs_config: type: "exclude_at_match" pattern: "test-pattern" use_compression: false - compression_level: 3 + compression_level: 1 additional_endpoints: - api_key: "apikey2" Host: "agent-http-intake.logs.datadoghq.com" @@ -714,7 +755,7 @@ apm_config: http: remove_query_string: true remove_paths_with_digits: true - compression_level: 3 + compression_level: 2 features: - "enable_otlp_compute_top_level_by_span_kind" - "enable_stats_by_span_kind" @@ -734,6 +775,9 @@ trace_propagation_extract_first: true trace_propagation_http_baggage_enabled: true trace_aws_service_representation_enabled: true +metrics_config: + compression_level: 3 + # OTLP otlp_config: receiver: @@ -801,6 +845,7 @@ extension_version: "compatibility" api_key: "test-api-key".to_string(), log_level: LogLevel::Debug, flush_timeout: 42, + compression_level: 4, proxy_https: Some("https://proxy.example.com".to_string()), proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), @@ -831,7 +876,7 @@ extension_version: "compatibility" replace_placeholder: None, }]), logs_config_use_compression: false, - logs_config_compression_level: 3, + logs_config_compression_level: 1, logs_config_additional_endpoints: vec![LogsAdditionalEndpoint { api_key: "apikey2".to_string(), host: "agent-http-intake.logs.datadoghq.com".to_string(), @@ -846,7 +891,7 @@ extension_version: "compatibility" apm_replace_tags: Some(vec![]), apm_config_obfuscation_http_remove_query_string: true, apm_config_obfuscation_http_remove_paths_with_digits: true, - apm_config_compression_level: 3, + apm_config_compression_level: 2, apm_features: vec![ "enable_otlp_compute_top_level_by_span_kind".to_string(), "enable_stats_by_span_kind".to_string(), @@ -866,6 +911,7 @@ extension_version: "compatibility" trace_propagation_extract_first: true, trace_propagation_http_baggage_enabled: true, trace_aws_service_representation_enabled: true, + metrics_config_compression_level: 3, otlp_config_traces_enabled: false, otlp_config_traces_span_name_as_resource_name: true, otlp_config_traces_span_name_remappings: HashMap::from([( From 2a4a41fe4d84cd641f85474aaf587afd3d595bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 10 Sep 2025 21:27:18 -0400 Subject: [PATCH 083/112] cherry pick: No longer launch Go-based agent for compatibility/OTLP/AAP config (#817) Cherry pick of previously reverted #788 https://datadoghq.atlassian.net/browse/SVLS-7398 - As part of coming release, bottlecap agent no longer launches Go-based agent when compatibility/AAP/OTLP features are active - Emit the same metric when detecting any of above configuration - Update corresponding unit tests Attention: it is an known issue with .Net https://github.com/aws/aws-lambda-dotnet/issues/2093 Manifests: - [Test lambda function](https://us-east-1.console.aws.amazon.com/lambda/home?region=us-east-1#/functions/ltn1-fullinstrument-bn-cold-python310-lambda?code=&subtab=envVars&tab=testing) with [logs](https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups/log-group/$252Faws$252Flambda$252Fltn1-fullinstrument-bn-cold-python310-lambda/log-events/2025$252F08$252F21$252F$255B$2524LATEST$255Df3788d359677452dad162488ff15456f$3FfilterPattern$3Dotel) showing compatibility/AAP/OTPL are enabled image - [Logging](https://app.datadoghq.com/logs/livetail?query=functionname%3Altn1-fullinstrument-bn-cold-python310-lambda%20Metric&agg_m=count&agg_m_source=base&agg_t=count&cols=host%2Cservice&fromUser=true&messageDisplay=inline&refresh_mode=paused&storage=driveline&stream_sort=desc&viz=stream&from_ts=1755787655569&to_ts=1755787689060&live=false) image - [Metric](https://app.datadoghq.com/screen/integration/aws_lambda_enhanced_metrics?fromUser=false&fullscreen_end_ts=1755788220000&fullscreen_paused=true&fullscreen_refresh_mode=paused&fullscreen_section=overview&fullscreen_start_ts=1755787200000&fullscreen_widget=2&graph-explorer__tile_def=N4IgbglgXiBcIBcD2AHANhAzgkAaEAxgK7ZIC2A%2BhgHYDWmcA2gLr4BOApgI5EfYOxGoTphRJqmDhQBmSNmQCGOeJgIK0CtnhA8ObCHyagAJkoUVMSImwIc4IMhwT6CDfNQWP7utgE8AjNo%2BvvaYRGSwpggKxkgA5gB0kmxgemh8mAkcAB4IHBIQ4gnSChBoSKlswAAkCgDumBQKBARW1Ai41ZxxhdSd0kTUBAi9AL4ABABGvuPAA0Mj4h6OowkKja2DCAAUAJTaCnFx3UpyoeEgo6wgsvJEGgJCN3Jk9wrevH6BV-iWbMqgTbtOAAJgADPg5MY9BRpkZEL4UHZ4LdXhptBBqNDsnAISAoXp7NDVJdmKMfiBsL50nBgOSgA&refresh_mode=sliding&from_ts=1755783890661&to_ts=1755787490661&live=true) image ==== Another manifest for .Net: - [Lambda function](https://us-east-1.console.aws.amazon.com/lambda/home?region=us-east-1#/functions/ltn1-fullinstrument-bn-cold-dotnet6-lambda?code=&subtab=envVars&tab=testing) - [Log](https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups/log-group/$252Faws$252Flambda$252Fltn1-fullinstrument-bn-cold-dotnet6-lambda/log-events/2025$252F08$252F29$252F$255B$2524LATEST$255D15ca867ee94049129ed461283ae46f01$3FfilterPattern$3Dfailover) - Configuration image - Log shows the issue reasons image image image --------- Co-authored-by: Tianning Li --- mod.rs | 216 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 137 insertions(+), 79 deletions(-) diff --git a/mod.rs b/mod.rs index 59a0e5be..c000856c 100644 --- a/mod.rs +++ b/mod.rs @@ -453,11 +453,14 @@ impl Default for Config { } } -fn log_fallback_reason(reason: &str) { - println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); +fn log_unsupported_reason(reason: &str) { + error!("Found unsupported config: {reason} is no longer available."); } -fn fallback(config: &Config) -> Result<(), ConfigError> { +#[must_use = "Unsupported reasons should be processed to emit appropriate metrics"] +pub fn inspect_config(config: &Config) -> Vec { + let mut unsupported_reasons = Vec::new(); + // Customer explicitly opted out of the Next Gen extension let opted_out = match config.extension_version.as_deref() { Some("compatibility") => true, @@ -466,21 +469,18 @@ fn fallback(config: &Config) -> Result<(), ConfigError> { }; if opted_out { - log_fallback_reason("extension_version"); - return Err(ConfigError::UnsupportedField( - "extension_version".to_string(), - )); + let reason = "extension_version"; + log_unsupported_reason(reason); + unsupported_reasons.push(reason.to_string()); } // ASM / .NET // todo(duncanista): Remove once the .NET runtime is fixed if config.serverless_appsec_enabled && has_dotnet_binary() { - log_fallback_reason("serverless_appsec_enabled_dotnet"); - return Err(ConfigError::UnsupportedField( - "serverless_appsec_enabled_dotnet".to_string(), - )); + let reason = "serverless_appsec_enabled_dotnet"; + log_unsupported_reason(reason); + unsupported_reasons.push(reason.to_string()); } - // OTLP let has_otlp_config = config .otlp_config_receiver_protocols_grpc_endpoint @@ -512,25 +512,22 @@ fn fallback(config: &Config) -> Result<(), ConfigError> { || config.otlp_config_logs_enabled; if has_otlp_config { - log_fallback_reason("otel"); - return Err(ConfigError::UnsupportedField("otel".to_string())); + let reason = "otel"; + log_unsupported_reason(reason); + unsupported_reasons.push(reason.to_string()); } - Ok(()) + unsupported_reasons } #[allow(clippy::module_name_repetitions)] -pub fn get_config(config_directory: &Path) -> Result { +#[must_use = "configuration must be used to initialize the application"] +pub fn get_config(config_directory: &Path) -> Config { let path: std::path::PathBuf = config_directory.join("datadog.yaml"); let mut config_builder = ConfigBuilder::default() .add_source(Box::new(YamlConfigSource { path })) .add_source(Box::new(EnvConfigSource)); - - let config = config_builder.build(); - - fallback(&config)?; - - Ok(config) + config_builder.build() } #[inline] @@ -739,19 +736,98 @@ pub mod tests { }; #[test] - fn test_reject_on_opted_out() { + fn test_baseline_case() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env("DD_EXTENSION_VERSION", "compatibility"); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!( - config, - ConfigError::UnsupportedField("extension_version".to_string()) - ); + let _config = get_config(Path::new("")); Ok(()) }); } + #[test] + fn test_inspect_config() { + struct TestCase { + name: &'static str, + env_vars: Vec<(&'static str, &'static str)>, + yaml_content: Option<&'static str>, + expected_reasons: Vec<&'static str>, + } + + let test_cases = vec![ + TestCase { + name: "default config - no unsupported reasons", + env_vars: vec![], + yaml_content: None, + expected_reasons: vec![], + }, + TestCase { + name: "extension_version compatibility - should discover", + env_vars: vec![("DD_EXTENSION_VERSION", "compatibility")], + yaml_content: None, + expected_reasons: vec!["extension_version"], + }, + TestCase { + name: "otlp config enabled - should discover", + env_vars: vec![( + "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", + "localhost:4317", + )], + yaml_content: None, + expected_reasons: vec!["otel"], + }, + TestCase { + name: "multiple unsupported reasons", + env_vars: vec![ + ("DD_EXTENSION_VERSION", "compatibility"), + ("DD_OTLP_CONFIG_METRICS_ENABLED", "true"), + ], + yaml_content: None, + expected_reasons: vec!["extension_version", "otel"], + }, + TestCase { + name: "otlp config via yaml - should discover", + env_vars: vec![], + yaml_content: Some( + r" + otlp_config: + receiver: + protocols: + grpc: + endpoint: localhost:4317 + ", + ), + expected_reasons: vec!["otel"], + }, + ]; + + for test_case in test_cases { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + + // Set environment variables + for (key, value) in &test_case.env_vars { + jail.set_env(key, value); + } + + // Create YAML file if provided + if let Some(yaml_content) = test_case.yaml_content { + jail.create_file("datadog.yaml", yaml_content)?; + } + + let config = get_config(Path::new("")); + let unsupported_reasons = inspect_config(&config); + + assert_eq!( + unsupported_reasons, test_case.expected_reasons, + "Test case '{}' failed: expected {:?}, got {:?}", + test_case.name, test_case.expected_reasons, unsupported_reasons + ); + + Ok(()) + }); + } + } + #[test] fn test_fallback_on_otel() { figment::Jail::expect_with(|jail| { @@ -761,8 +837,7 @@ pub mod tests { "localhost:4138", ); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); + let _config = get_config(Path::new("")); Ok(()) }); } @@ -782,8 +857,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); + let _config = get_config(Path::new("")); Ok(()) }); } @@ -793,7 +867,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_logs_dd_url, "https://http-intake.logs.datadoghq.com".to_string() @@ -811,7 +885,7 @@ pub mod tests { "agent-http-intake-pci.logs.datadoghq.com:443", ); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_logs_dd_url, "agent-http-intake-pci.logs.datadoghq.com:443".to_string() @@ -826,7 +900,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_APM_DD_URL", "https://trace-pci.agent.datadoghq.com"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.apm_dd_url, "https://trace-pci.agent.datadoghq.com/api/v0.2/traces".to_string() @@ -841,7 +915,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_DD_URL", "custom_proxy:3128"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.dd_url, "custom_proxy:3128".to_string()); Ok(()) }); @@ -853,7 +927,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_URL", "custom_proxy:3128"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.url, "custom_proxy:3128".to_string()); Ok(()) }); @@ -864,27 +938,12 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.dd_url, String::new()); Ok(()) }); } - #[test] - fn test_allowed_but_disabled() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env( - "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", - "localhost:4138", - ); - - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); - Ok(()) - }); - } - #[test] fn test_precedence() { figment::Jail::expect_with(|jail| { @@ -896,7 +955,7 @@ pub mod tests { ", )?; jail.set_env("DD_SITE", "datad0g.com"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "datad0g.com"); Ok(()) }); @@ -912,7 +971,7 @@ pub mod tests { r" ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "datadoghq.com"); Ok(()) }); @@ -923,7 +982,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SITE", "datadoghq.eu"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "datadoghq.eu"); Ok(()) }); @@ -934,7 +993,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_LOG_LEVEL", "TRACE"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.log_level, LogLevel::Trace); Ok(()) }); @@ -944,7 +1003,7 @@ pub mod tests { fn test_parse_default() { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config, Config { @@ -968,7 +1027,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_PROXY_HTTPS", "my-proxy:3128"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -984,7 +1043,7 @@ pub mod tests { "NO_PROXY", "127.0.0.1,localhost,172.16.0.0/12,us-east-1.amazonaws.com,datadoghq.eu", ); - let config = get_config(Path::new("")).expect("should parse noproxy"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, None); Ok(()) }); @@ -1002,7 +1061,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect("should parse weird proxy config"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -1022,7 +1081,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect("should parse weird proxy config"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, None); // Assertion to ensure config.site runs before proxy // because we chenck that noproxy contains the site @@ -1036,7 +1095,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.serverless_flush_strategy, FlushStrategy::End); Ok(()) }); @@ -1047,7 +1106,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.serverless_flush_strategy, FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }) @@ -1061,7 +1120,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -1075,7 +1134,7 @@ pub mod tests { "DD_SERVERLESS_FLUSH_STRATEGY", "periodically,invalid_interval", ); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -1088,7 +1147,7 @@ pub mod tests { jail.set_env("DD_VERSION", "123"); jail.set_env("DD_ENV", "123456890"); jail.set_env("DD_SERVICE", "123456"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.version.expect("failed to parse DD_VERSION"), "123"); assert_eq!(config.env.expect("failed to parse DD_ENV"), "123456890"); assert_eq!( @@ -1118,7 +1177,7 @@ pub mod tests { pattern: exclude-me-yaml ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -1147,7 +1206,7 @@ pub mod tests { pattern: exclude ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -1176,7 +1235,7 @@ pub mod tests { repl: 'REDACTED' ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); let rule = parse_rules_from_string( r#"[ {"name": "*", "pattern": "foo", "repl": "REDACTED"} @@ -1207,7 +1266,7 @@ pub mod tests { repl: 'REDACTED-YAML' ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); let rule = parse_rules_from_string( r#"[ {"name": "*", "pattern": "foo", "repl": "REDACTED-ENV"} @@ -1234,7 +1293,7 @@ pub mod tests { remove_paths_with_digits: true ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert!(config.apm_config_obfuscation_http_remove_query_string,); assert!(config.apm_config_obfuscation_http_remove_paths_with_digits,); Ok(()) @@ -1249,7 +1308,7 @@ pub mod tests { "datadog,tracecontext,b3,b3multi", ); jail.set_env("DD_EXTENSION_VERSION", "next"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); let expected_styles = vec![ TracePropagationStyle::Datadog, @@ -1268,7 +1327,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "datadog"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.trace_propagation_style, @@ -1293,8 +1352,7 @@ pub mod tests { "DD_APM_REPLACE_TAGS", r#"[{"name":"resource.name","pattern":"(.*)/(foo[:%].+)","repl":"$1/{foo}"}]"#, ); - let config = get_config(Path::new("")); - assert!(config.is_ok()); + let _config = get_config(Path::new("")); Ok(()) }); } @@ -1307,7 +1365,7 @@ pub mod tests { jail.set_env("DD_ENHANCED_METRICS", "1"); jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "TRUE"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "0"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert!(config.serverless_logs_enabled); assert!(config.enhanced_metrics); assert!(config.logs_config_use_compression); @@ -1331,7 +1389,7 @@ pub mod tests { jail.set_env("DD_SITE", "us5.datadoghq.com"); jail.set_env("DD_API_KEY", "env-api-key"); jail.set_env("DD_FLUSH_TIMEOUT", "10"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "us5.datadoghq.com"); assert_eq!(config.api_key, "env-api-key"); From a5ab92fd3eb8874e2d73823f02acaa042785ae38 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 17 Sep 2025 12:29:19 -0400 Subject: [PATCH 084/112] feat: [Trace Stats] Add feature flag DD_COMPUTE_TRACE_STATS (#841) ## This PR Adds a feature flag `DD_COMPUTE_TRACE_STATS`. - If true, trace stats will be computed from the extension side. In this case, we set `_dd.compute_stats` to `0`, so trace stats won't be computed on the backend. - If false, trace stats will NOT be computed from the extension side. In this case, we set `_dd.compute_stats` to `1`, so trace stats will be computed on the backend. - Defaults to false for now, so `_dd.compute_stats` still defaults to `1`, i.e. default behavior is not changed. - After we fully support computing trace stats on extension side, I will change the default to true then delete the flag. Jira: https://datadoghq.atlassian.net/browse/SVLS-7593 --- env.rs | 11 +++++++++++ mod.rs | 2 ++ yaml.rs | 5 +++++ 3 files changed, 18 insertions(+) diff --git a/env.rs b/env.rs index 8e472639..889b1952 100644 --- a/env.rs +++ b/env.rs @@ -349,10 +349,18 @@ pub struct EnvConfig { /// The maximum depth of the Lambda payload to capture. /// Default is `10`. Requires `capture_lambda_payload` to be `true`. pub capture_lambda_payload_max_depth: Option, + /// @env `DD_COMPUTE_TRACE_STATS` + /// + /// If true, enable computation of trace stats on the extension side. + /// If false, trace stats will be computed on the backend side. + /// Default is `false`. + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub compute_trace_stats: Option, /// @env `DD_SERVERLESS_APPSEC_ENABLED` /// /// Enable Application and API Protection (AAP), previously known as AppSec/ASM, for AWS Lambda. /// Default is `false`. + /// #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub serverless_appsec_enabled: Option, /// @env `DD_APPSEC_RULES` @@ -548,6 +556,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option_to_value!(config, env_config, lambda_proc_enhanced_metrics); merge_option_to_value!(config, env_config, capture_lambda_payload); merge_option_to_value!(config, env_config, capture_lambda_payload_max_depth); + merge_option_to_value!(config, env_config, compute_trace_stats); merge_option_to_value!(config, env_config, serverless_appsec_enabled); merge_option!(config, env_config, appsec_rules); merge_option_to_value!(config, env_config, appsec_waf_timeout); @@ -741,6 +750,7 @@ mod tests { jail.set_env("DD_LAMBDA_PROC_ENHANCED_METRICS", "false"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "true"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "5"); + jail.set_env("DD_COMPUTE_TRACE_STATS", "true"); jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); jail.set_env("DD_APPSEC_RULES", "/path/to/rules.json"); jail.set_env("DD_APPSEC_WAF_TIMEOUT", "1000000"); // Microseconds @@ -888,6 +898,7 @@ mod tests { lambda_proc_enhanced_metrics: false, capture_lambda_payload: true, capture_lambda_payload_max_depth: 5, + compute_trace_stats: true, serverless_appsec_enabled: true, appsec_rules: Some("/path/to/rules.json".to_string()), appsec_waf_timeout: Duration::from_secs(1), diff --git a/mod.rs b/mod.rs index c000856c..f4b609f6 100644 --- a/mod.rs +++ b/mod.rs @@ -341,6 +341,7 @@ pub struct Config { pub lambda_proc_enhanced_metrics: bool, pub capture_lambda_payload: bool, pub capture_lambda_payload_max_depth: u32, + pub compute_trace_stats: bool, pub serverless_appsec_enabled: bool, pub appsec_rules: Option, @@ -441,6 +442,7 @@ impl Default for Config { lambda_proc_enhanced_metrics: true, capture_lambda_payload: false, capture_lambda_payload_max_depth: 10, + compute_trace_stats: false, serverless_appsec_enabled: false, appsec_rules: None, diff --git a/yaml.rs b/yaml.rs index 8017873b..84d22514 100644 --- a/yaml.rs +++ b/yaml.rs @@ -99,6 +99,8 @@ pub struct YamlConfig { pub capture_lambda_payload: Option, pub capture_lambda_payload_max_depth: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub compute_trace_stats: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub serverless_appsec_enabled: Option, pub appsec_rules: Option, #[serde(deserialize_with = "deserialize_optional_duration_from_microseconds")] @@ -654,6 +656,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option_to_value!(config, yaml_config, lambda_proc_enhanced_metrics); merge_option_to_value!(config, yaml_config, capture_lambda_payload); merge_option_to_value!(config, yaml_config, capture_lambda_payload_max_depth); + merge_option_to_value!(config, yaml_config, compute_trace_stats); merge_option_to_value!(config, yaml_config, serverless_appsec_enabled); merge_option!(config, yaml_config, appsec_rules); merge_option_to_value!(config, yaml_config, appsec_waf_timeout); @@ -823,6 +826,7 @@ enhanced_metrics: false lambda_proc_enhanced_metrics: false capture_lambda_payload: true capture_lambda_payload_max_depth: 5 +compute_trace_stats: true serverless_appsec_enabled: true appsec_rules: "/path/to/rules.json" appsec_waf_timeout: 1000000 # Microseconds @@ -953,6 +957,7 @@ extension_version: "compatibility" lambda_proc_enhanced_metrics: false, capture_lambda_payload: true, capture_lambda_payload_max_depth: 5, + compute_trace_stats: true, serverless_appsec_enabled: true, appsec_rules: Some("/path/to/rules.json".to_string()), From 9819806dc21ce1a4ed15c2ee783fdba2e2c38533 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 18 Sep 2025 21:41:00 +0530 Subject: [PATCH 085/112] fix: use tokio time instead of std time because tokio time can be frozen (#846) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tokio time allows us to pause or sleep without blocking the runtime. It also allows time to be paused (mainly for tests). I think we may need the sleep to force blocking code to yield --------- Co-authored-by: jordan gonzález <30836115+duncanista@users.noreply.github.com> --- aws.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aws.rs b/aws.rs index b45fc008..62100e47 100644 --- a/aws.rs +++ b/aws.rs @@ -1,4 +1,5 @@ -use std::{env, time::Instant}; +use std::env; +use tokio::time::Instant; const AWS_DEFAULT_REGION: &str = "AWS_DEFAULT_REGION"; const AWS_ACCESS_KEY_ID: &str = "AWS_ACCESS_KEY_ID"; From 08b7982adf8be3075c1f9009ac7dd9816dc0447a Mon Sep 17 00:00:00 2001 From: jchrostek-dd Date: Tue, 23 Sep 2025 06:44:06 -0400 Subject: [PATCH 086/112] add support for observability pipeline (#826) ## Task https://datadoghq.atlassian.net/jira/software/c/projects/SVLS/boards/5420?quickFilter=7573&selectedIssue=SVLS-7525 ## Overview * Add support for sending logs to an Observability Pipeline instead of directly to Datadog. * To enable, customers must set `DD_ENABLE_OBSERVABILITY_PIPELINE_FORWARDING` to true, and `DD_LOGS_CONFIG_LOGS_DD_URL` to their Observability Pipeline endpoint. Will fast follow and update docs to reflect this. * Initially, I was using setting up the observability pipeline with 'Datadog Agent' as the source. This required us to format the log message in a certain format. However, chatting with the Observability Pipeline Team, they actually recommend we use 'Http Server' as the source for our pipeline setup instead since this just accepts any json. ## Testing Created an [observability pipeline](https://ddserverless.datadoghq.com/observability-pipelines/b15e4a64-880d-11f0-b622-da7ad0900002/view) and deployed a lambda function with the changes. Triggered the lambda function and confirmed we see it in our [logs](https://ddserverless.datadoghq.com/logs?query=function_arn%3A%22arn%3Aaws%3Alambda%3Aus-east-1%3A425362996713%3Afunction%3Aobcdkstackv3-hellofunctionv3ec5a2fbe-l9qvtrowzb5q%22&agg_m=count&agg_m_source=base&agg_t=count&cols=host%2Cservice&messageDisplay=inline&refresh_mode=sliding&storage=hot&stream_sort=desc&viz=stream&from_ts=1758196420534&to_ts=1758369220534&live=true). We know it is going through the observability pipeline because we can see an attached 'http_server' attached as the source type. --- env.rs | 17 +++++++++++++++++ mod.rs | 4 ++++ yaml.rs | 2 ++ 3 files changed, 23 insertions(+) diff --git a/env.rs b/env.rs index 889b1952..2342aa9f 100644 --- a/env.rs +++ b/env.rs @@ -144,6 +144,15 @@ pub struct EnvConfig { #[serde(deserialize_with = "deserialize_logs_additional_endpoints")] pub logs_config_additional_endpoints: Vec, + /// @env `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_ENABLED` + /// When true, emit plain json suitable for Observability Pipelines + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub observability_pipelines_worker_logs_enabled: Option, + /// @env `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL` + /// + /// The URL endpoint for sending logs to Observability Pipelines Worker + pub observability_pipelines_worker_logs_url: Option, + // APM // /// @env `DD_SERVICE_MAPPING` @@ -427,6 +436,12 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { ); merge_option_to_value!(config, env_config, logs_config_compression_level); merge_vec!(config, env_config, logs_config_additional_endpoints); + merge_option_to_value!( + config, + env_config, + observability_pipelines_worker_logs_enabled + ); + merge_string!(config, env_config, observability_pipelines_worker_logs_url); // APM merge_hashmap!(config, env_config, service_mapping); @@ -807,6 +822,8 @@ mod tests { port: 443, is_reliable: true, }], + observability_pipelines_worker_logs_enabled: false, + observability_pipelines_worker_logs_url: String::default(), service_mapping: HashMap::from([( "old-service".to_string(), "new-service".to_string(), diff --git a/mod.rs b/mod.rs index f4b609f6..ac6747b4 100644 --- a/mod.rs +++ b/mod.rs @@ -271,6 +271,8 @@ pub struct Config { pub logs_config_use_compression: bool, pub logs_config_compression_level: i32, pub logs_config_additional_endpoints: Vec, + pub observability_pipelines_worker_logs_enabled: bool, + pub observability_pipelines_worker_logs_url: String, // APM // @@ -384,6 +386,8 @@ impl Default for Config { logs_config_use_compression: true, logs_config_compression_level: 6, logs_config_additional_endpoints: Vec::new(), + observability_pipelines_worker_logs_enabled: false, + observability_pipelines_worker_logs_url: String::default(), // APM service_mapping: HashMap::new(), diff --git a/yaml.rs b/yaml.rs index 84d22514..0cab454c 100644 --- a/yaml.rs +++ b/yaml.rs @@ -887,6 +887,8 @@ extension_version: "compatibility" port: 443, is_reliable: true, }], + observability_pipelines_worker_logs_enabled: false, + observability_pipelines_worker_logs_url: String::default(), service_mapping: HashMap::from([( "old-service".to_string(), "new-service".to_string(), From d17d6946c37612c87386184f20ae14dac82be906 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 24 Sep 2025 15:27:20 -0400 Subject: [PATCH 087/112] feat: lower zstd default compression (#867) A quick test run showed our max duration skews on smaller lambda sizes with lots of data setting the zstd compression level to 6. Looks like we start to block the CPU at around thi smark. Gonna default it to 3, as tested below with 3 500k runs. image --- mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mod.rs b/mod.rs index ac6747b4..adcb5458 100644 --- a/mod.rs +++ b/mod.rs @@ -378,13 +378,13 @@ impl Default for Config { version: None, tags: HashMap::new(), - compression_level: 6, + compression_level: 3, // Logs logs_config_logs_dd_url: String::default(), logs_config_processing_rules: None, logs_config_use_compression: true, - logs_config_compression_level: 6, + logs_config_compression_level: 3, logs_config_additional_endpoints: Vec::new(), observability_pipelines_worker_logs_enabled: false, observability_pipelines_worker_logs_url: String::default(), @@ -395,7 +395,7 @@ impl Default for Config { apm_replace_tags: None, apm_config_obfuscation_http_remove_query_string: false, apm_config_obfuscation_http_remove_paths_with_digits: false, - apm_config_compression_level: 6, + apm_config_compression_level: 3, apm_features: vec![], apm_additional_endpoints: HashMap::new(), apm_filter_tags_require: None, @@ -412,7 +412,7 @@ impl Default for Config { trace_propagation_http_baggage_enabled: false, // Metrics - metrics_config_compression_level: 6, + metrics_config_compression_level: 3, // OTLP otlp_config_traces_enabled: true, From 577ed69c09b6d67ce9db3cb85f046849ae18ca38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 25 Sep 2025 16:17:47 -0400 Subject: [PATCH 088/112] revert(#817): reverts fallback config (#871) # What? This reverts commit 2396c4fe102677179c834c2dd65cb5b2ea79ca8f from #817 # Why? Need a release # Notes We'll cherry pick and bring it back at some point --- mod.rs | 216 +++++++++++++++++++++------------------------------------ 1 file changed, 79 insertions(+), 137 deletions(-) diff --git a/mod.rs b/mod.rs index adcb5458..027830ee 100644 --- a/mod.rs +++ b/mod.rs @@ -459,14 +459,11 @@ impl Default for Config { } } -fn log_unsupported_reason(reason: &str) { - error!("Found unsupported config: {reason} is no longer available."); +fn log_fallback_reason(reason: &str) { + println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); } -#[must_use = "Unsupported reasons should be processed to emit appropriate metrics"] -pub fn inspect_config(config: &Config) -> Vec { - let mut unsupported_reasons = Vec::new(); - +fn fallback(config: &Config) -> Result<(), ConfigError> { // Customer explicitly opted out of the Next Gen extension let opted_out = match config.extension_version.as_deref() { Some("compatibility") => true, @@ -475,18 +472,21 @@ pub fn inspect_config(config: &Config) -> Vec { }; if opted_out { - let reason = "extension_version"; - log_unsupported_reason(reason); - unsupported_reasons.push(reason.to_string()); + log_fallback_reason("extension_version"); + return Err(ConfigError::UnsupportedField( + "extension_version".to_string(), + )); } // ASM / .NET // todo(duncanista): Remove once the .NET runtime is fixed if config.serverless_appsec_enabled && has_dotnet_binary() { - let reason = "serverless_appsec_enabled_dotnet"; - log_unsupported_reason(reason); - unsupported_reasons.push(reason.to_string()); + log_fallback_reason("serverless_appsec_enabled_dotnet"); + return Err(ConfigError::UnsupportedField( + "serverless_appsec_enabled_dotnet".to_string(), + )); } + // OTLP let has_otlp_config = config .otlp_config_receiver_protocols_grpc_endpoint @@ -518,22 +518,25 @@ pub fn inspect_config(config: &Config) -> Vec { || config.otlp_config_logs_enabled; if has_otlp_config { - let reason = "otel"; - log_unsupported_reason(reason); - unsupported_reasons.push(reason.to_string()); + log_fallback_reason("otel"); + return Err(ConfigError::UnsupportedField("otel".to_string())); } - unsupported_reasons + Ok(()) } #[allow(clippy::module_name_repetitions)] -#[must_use = "configuration must be used to initialize the application"] -pub fn get_config(config_directory: &Path) -> Config { +pub fn get_config(config_directory: &Path) -> Result { let path: std::path::PathBuf = config_directory.join("datadog.yaml"); let mut config_builder = ConfigBuilder::default() .add_source(Box::new(YamlConfigSource { path })) .add_source(Box::new(EnvConfigSource)); - config_builder.build() + + let config = config_builder.build(); + + fallback(&config)?; + + Ok(config) } #[inline] @@ -742,98 +745,19 @@ pub mod tests { }; #[test] - fn test_baseline_case() { + fn test_reject_on_opted_out() { figment::Jail::expect_with(|jail| { jail.clear_env(); - let _config = get_config(Path::new("")); + jail.set_env("DD_EXTENSION_VERSION", "compatibility"); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!( + config, + ConfigError::UnsupportedField("extension_version".to_string()) + ); Ok(()) }); } - #[test] - fn test_inspect_config() { - struct TestCase { - name: &'static str, - env_vars: Vec<(&'static str, &'static str)>, - yaml_content: Option<&'static str>, - expected_reasons: Vec<&'static str>, - } - - let test_cases = vec![ - TestCase { - name: "default config - no unsupported reasons", - env_vars: vec![], - yaml_content: None, - expected_reasons: vec![], - }, - TestCase { - name: "extension_version compatibility - should discover", - env_vars: vec![("DD_EXTENSION_VERSION", "compatibility")], - yaml_content: None, - expected_reasons: vec!["extension_version"], - }, - TestCase { - name: "otlp config enabled - should discover", - env_vars: vec![( - "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", - "localhost:4317", - )], - yaml_content: None, - expected_reasons: vec!["otel"], - }, - TestCase { - name: "multiple unsupported reasons", - env_vars: vec![ - ("DD_EXTENSION_VERSION", "compatibility"), - ("DD_OTLP_CONFIG_METRICS_ENABLED", "true"), - ], - yaml_content: None, - expected_reasons: vec!["extension_version", "otel"], - }, - TestCase { - name: "otlp config via yaml - should discover", - env_vars: vec![], - yaml_content: Some( - r" - otlp_config: - receiver: - protocols: - grpc: - endpoint: localhost:4317 - ", - ), - expected_reasons: vec!["otel"], - }, - ]; - - for test_case in test_cases { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - - // Set environment variables - for (key, value) in &test_case.env_vars { - jail.set_env(key, value); - } - - // Create YAML file if provided - if let Some(yaml_content) = test_case.yaml_content { - jail.create_file("datadog.yaml", yaml_content)?; - } - - let config = get_config(Path::new("")); - let unsupported_reasons = inspect_config(&config); - - assert_eq!( - unsupported_reasons, test_case.expected_reasons, - "Test case '{}' failed: expected {:?}, got {:?}", - test_case.name, test_case.expected_reasons, unsupported_reasons - ); - - Ok(()) - }); - } - } - #[test] fn test_fallback_on_otel() { figment::Jail::expect_with(|jail| { @@ -843,7 +767,8 @@ pub mod tests { "localhost:4138", ); - let _config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); Ok(()) }); } @@ -863,7 +788,8 @@ pub mod tests { ", )?; - let _config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); Ok(()) }); } @@ -873,7 +799,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.logs_config_logs_dd_url, "https://http-intake.logs.datadoghq.com".to_string() @@ -891,7 +817,7 @@ pub mod tests { "agent-http-intake-pci.logs.datadoghq.com:443", ); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.logs_config_logs_dd_url, "agent-http-intake-pci.logs.datadoghq.com:443".to_string() @@ -906,7 +832,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_APM_DD_URL", "https://trace-pci.agent.datadoghq.com"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.apm_dd_url, "https://trace-pci.agent.datadoghq.com/api/v0.2/traces".to_string() @@ -921,7 +847,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_DD_URL", "custom_proxy:3128"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.dd_url, "custom_proxy:3128".to_string()); Ok(()) }); @@ -933,7 +859,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_URL", "custom_proxy:3128"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.url, "custom_proxy:3128".to_string()); Ok(()) }); @@ -944,12 +870,27 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.dd_url, String::new()); Ok(()) }); } + #[test] + fn test_allowed_but_disabled() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env( + "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", + "localhost:4138", + ); + + let config = get_config(Path::new("")).expect_err("should reject unknown fields"); + assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); + Ok(()) + }); + } + #[test] fn test_precedence() { figment::Jail::expect_with(|jail| { @@ -961,7 +902,7 @@ pub mod tests { ", )?; jail.set_env("DD_SITE", "datad0g.com"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.site, "datad0g.com"); Ok(()) }); @@ -977,7 +918,7 @@ pub mod tests { r" ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.site, "datadoghq.com"); Ok(()) }); @@ -988,7 +929,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SITE", "datadoghq.eu"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.site, "datadoghq.eu"); Ok(()) }); @@ -999,7 +940,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_LOG_LEVEL", "TRACE"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.log_level, LogLevel::Trace); Ok(()) }); @@ -1009,7 +950,7 @@ pub mod tests { fn test_parse_default() { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config, Config { @@ -1033,7 +974,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_PROXY_HTTPS", "my-proxy:3128"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -1049,7 +990,7 @@ pub mod tests { "NO_PROXY", "127.0.0.1,localhost,172.16.0.0/12,us-east-1.amazonaws.com,datadoghq.eu", ); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse noproxy"); assert_eq!(config.proxy_https, None); Ok(()) }); @@ -1067,7 +1008,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse weird proxy config"); assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -1087,7 +1028,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse weird proxy config"); assert_eq!(config.proxy_https, None); // Assertion to ensure config.site runs before proxy // because we chenck that noproxy contains the site @@ -1101,7 +1042,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::End); Ok(()) }); @@ -1112,7 +1053,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.serverless_flush_strategy, FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }) @@ -1126,7 +1067,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -1140,7 +1081,7 @@ pub mod tests { "DD_SERVERLESS_FLUSH_STRATEGY", "periodically,invalid_interval", ); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -1153,7 +1094,7 @@ pub mod tests { jail.set_env("DD_VERSION", "123"); jail.set_env("DD_ENV", "123456890"); jail.set_env("DD_SERVICE", "123456"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.version.expect("failed to parse DD_VERSION"), "123"); assert_eq!(config.env.expect("failed to parse DD_ENV"), "123456890"); assert_eq!( @@ -1183,7 +1124,7 @@ pub mod tests { pattern: exclude-me-yaml ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -1212,7 +1153,7 @@ pub mod tests { pattern: exclude ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -1241,7 +1182,7 @@ pub mod tests { repl: 'REDACTED' ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); let rule = parse_rules_from_string( r#"[ {"name": "*", "pattern": "foo", "repl": "REDACTED"} @@ -1272,7 +1213,7 @@ pub mod tests { repl: 'REDACTED-YAML' ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); let rule = parse_rules_from_string( r#"[ {"name": "*", "pattern": "foo", "repl": "REDACTED-ENV"} @@ -1299,7 +1240,7 @@ pub mod tests { remove_paths_with_digits: true ", )?; - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert!(config.apm_config_obfuscation_http_remove_query_string,); assert!(config.apm_config_obfuscation_http_remove_paths_with_digits,); Ok(()) @@ -1314,7 +1255,7 @@ pub mod tests { "datadog,tracecontext,b3,b3multi", ); jail.set_env("DD_EXTENSION_VERSION", "next"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); let expected_styles = vec![ TracePropagationStyle::Datadog, @@ -1333,7 +1274,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "datadog"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!( config.trace_propagation_style, @@ -1358,7 +1299,8 @@ pub mod tests { "DD_APM_REPLACE_TAGS", r#"[{"name":"resource.name","pattern":"(.*)/(foo[:%].+)","repl":"$1/{foo}"}]"#, ); - let _config = get_config(Path::new("")); + let config = get_config(Path::new("")); + assert!(config.is_ok()); Ok(()) }); } @@ -1371,7 +1313,7 @@ pub mod tests { jail.set_env("DD_ENHANCED_METRICS", "1"); jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "TRUE"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "0"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert!(config.serverless_logs_enabled); assert!(config.enhanced_metrics); assert!(config.logs_config_use_compression); @@ -1395,7 +1337,7 @@ pub mod tests { jail.set_env("DD_SITE", "us5.datadoghq.com"); jail.set_env("DD_API_KEY", "env-api-key"); jail.set_env("DD_FLUSH_TIMEOUT", "10"); - let config = get_config(Path::new("")); + let config = get_config(Path::new("")).expect("should parse config"); assert_eq!(config.site, "us5.datadoghq.com"); assert_eq!(config.api_key, "env-api-key"); From b5631c7b4a00ab0f8184542044ab076b94fd5b93 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Mon, 29 Sep 2025 11:01:50 -0400 Subject: [PATCH 089/112] chore: [Trace Stats] Rename env var DD_COMPUTE_TRACE_STATS (#875) # This PR As @apiarian-datadog suggested in https://github.com/DataDog/datadog-lambda-extension/pull/841#discussion_r2376111825, rename the feature flag `DD_COMPUTE_TRACE_STATS` to `DD_COMPUTE_TRACE_STATS_ON_EXTENSION` for clarity. # Notes Jira: https://datadoghq.atlassian.net/browse/SVLS-7593 --- env.rs | 10 +++++----- mod.rs | 4 ++-- yaml.rs | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/env.rs b/env.rs index 2342aa9f..7d31424e 100644 --- a/env.rs +++ b/env.rs @@ -358,13 +358,13 @@ pub struct EnvConfig { /// The maximum depth of the Lambda payload to capture. /// Default is `10`. Requires `capture_lambda_payload` to be `true`. pub capture_lambda_payload_max_depth: Option, - /// @env `DD_COMPUTE_TRACE_STATS` + /// @env `DD_COMPUTE_TRACE_STATS_ON_EXTENSION` /// /// If true, enable computation of trace stats on the extension side. /// If false, trace stats will be computed on the backend side. /// Default is `false`. #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub compute_trace_stats: Option, + pub compute_trace_stats_on_extension: Option, /// @env `DD_SERVERLESS_APPSEC_ENABLED` /// /// Enable Application and API Protection (AAP), previously known as AppSec/ASM, for AWS Lambda. @@ -571,7 +571,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option_to_value!(config, env_config, lambda_proc_enhanced_metrics); merge_option_to_value!(config, env_config, capture_lambda_payload); merge_option_to_value!(config, env_config, capture_lambda_payload_max_depth); - merge_option_to_value!(config, env_config, compute_trace_stats); + merge_option_to_value!(config, env_config, compute_trace_stats_on_extension); merge_option_to_value!(config, env_config, serverless_appsec_enabled); merge_option!(config, env_config, appsec_rules); merge_option_to_value!(config, env_config, appsec_waf_timeout); @@ -765,7 +765,7 @@ mod tests { jail.set_env("DD_LAMBDA_PROC_ENHANCED_METRICS", "false"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "true"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "5"); - jail.set_env("DD_COMPUTE_TRACE_STATS", "true"); + jail.set_env("DD_COMPUTE_TRACE_STATS_ON_EXTENSION", "true"); jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); jail.set_env("DD_APPSEC_RULES", "/path/to/rules.json"); jail.set_env("DD_APPSEC_WAF_TIMEOUT", "1000000"); // Microseconds @@ -915,7 +915,7 @@ mod tests { lambda_proc_enhanced_metrics: false, capture_lambda_payload: true, capture_lambda_payload_max_depth: 5, - compute_trace_stats: true, + compute_trace_stats_on_extension: true, serverless_appsec_enabled: true, appsec_rules: Some("/path/to/rules.json".to_string()), appsec_waf_timeout: Duration::from_secs(1), diff --git a/mod.rs b/mod.rs index 027830ee..a940b6e0 100644 --- a/mod.rs +++ b/mod.rs @@ -343,7 +343,7 @@ pub struct Config { pub lambda_proc_enhanced_metrics: bool, pub capture_lambda_payload: bool, pub capture_lambda_payload_max_depth: u32, - pub compute_trace_stats: bool, + pub compute_trace_stats_on_extension: bool, pub serverless_appsec_enabled: bool, pub appsec_rules: Option, @@ -446,7 +446,7 @@ impl Default for Config { lambda_proc_enhanced_metrics: true, capture_lambda_payload: false, capture_lambda_payload_max_depth: 10, - compute_trace_stats: false, + compute_trace_stats_on_extension: false, serverless_appsec_enabled: false, appsec_rules: None, diff --git a/yaml.rs b/yaml.rs index 0cab454c..0c0e39b4 100644 --- a/yaml.rs +++ b/yaml.rs @@ -99,7 +99,7 @@ pub struct YamlConfig { pub capture_lambda_payload: Option, pub capture_lambda_payload_max_depth: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] - pub compute_trace_stats: Option, + pub compute_trace_stats_on_extension: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub serverless_appsec_enabled: Option, pub appsec_rules: Option, @@ -656,7 +656,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option_to_value!(config, yaml_config, lambda_proc_enhanced_metrics); merge_option_to_value!(config, yaml_config, capture_lambda_payload); merge_option_to_value!(config, yaml_config, capture_lambda_payload_max_depth); - merge_option_to_value!(config, yaml_config, compute_trace_stats); + merge_option_to_value!(config, yaml_config, compute_trace_stats_on_extension); merge_option_to_value!(config, yaml_config, serverless_appsec_enabled); merge_option!(config, yaml_config, appsec_rules); merge_option_to_value!(config, yaml_config, appsec_waf_timeout); @@ -826,7 +826,7 @@ enhanced_metrics: false lambda_proc_enhanced_metrics: false capture_lambda_payload: true capture_lambda_payload_max_depth: 5 -compute_trace_stats: true +compute_trace_stats_on_extension: true serverless_appsec_enabled: true appsec_rules: "/path/to/rules.json" appsec_waf_timeout: 1000000 # Microseconds @@ -959,7 +959,7 @@ extension_version: "compatibility" lambda_proc_enhanced_metrics: false, capture_lambda_payload: true, capture_lambda_payload_max_depth: 5, - compute_trace_stats: true, + compute_trace_stats_on_extension: true, serverless_appsec_enabled: true, appsec_rules: Some("/path/to/rules.json".to_string()), From 6b7884de4e2aa223a29d1cc3aa87733c3ae8d873 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 1 Oct 2025 15:37:44 -0400 Subject: [PATCH 090/112] feat: remove failover to go (#882) Removes the failover to Go. If we can't parse any of the config options we log the failing value and move on with the default specified. --- apm_replace_rule.rs | 33 ++-- env.rs | 45 +++-- mod.rs | 343 ++++++++++++++++--------------------- processing_rule.rs | 29 ++-- service_mapping.rs | 13 +- trace_propagation_style.rs | 18 +- yaml.rs | 29 +++- 7 files changed, 257 insertions(+), 253 deletions(-) diff --git a/apm_replace_rule.rs b/apm_replace_rule.rs index e63d94bc..0e49161c 100644 --- a/apm_replace_rule.rs +++ b/apm_replace_rule.rs @@ -25,10 +25,13 @@ impl<'de> Visitor<'de> for StringOrReplaceRulesVisitor { where E: serde::de::Error, { - // Validate it's at least valid JSON - let _: serde_json::Value = - serde_json::from_str(value).map_err(|_| E::custom("Expected valid JSON string"))?; - Ok(value.to_string()) + match serde_json::from_str::(value) { + Ok(_) => Ok(value.to_string()), + Err(e) => { + tracing::error!("Invalid JSON string for APM replace rules: {}", e); + Ok(String::new()) + } + } } // Convert YAML sequences to JSON strings @@ -40,10 +43,13 @@ impl<'de> Visitor<'de> for StringOrReplaceRulesVisitor { while let Some(rule) = seq.next_element::()? { rules.push(rule); } - // Serialize to JSON string for compatibility with parse_rules_from_string - serde_json::to_string(&rules).map_err(|e| { - serde::de::Error::custom(format!("Failed to serialize rules to JSON: {e}")) - }) + match serde_json::to_string(&rules) { + Ok(json) => Ok(json), + Err(e) => { + tracing::error!("Failed to convert YAML rules to JSON: {}", e); + Ok(String::new()) + } + } } } @@ -55,8 +61,11 @@ where { let json_string = deserializer.deserialize_any(StringOrReplaceRulesVisitor)?; - let rules = parse_rules_from_string(&json_string) - .map_err(|e| serde::de::Error::custom(format!("Parse error: {e}")))?; - - Ok(Some(rules)) + match parse_rules_from_string(&json_string) { + Ok(rules) => Ok(Some(rules)), + Err(e) => { + tracing::error!("Failed to parse APM replace rule, ignoring: {}", e); + Ok(None) + } + } } diff --git a/env.rs b/env.rs index 7d31424e..6b50178d 100644 --- a/env.rs +++ b/env.rs @@ -11,9 +11,10 @@ use crate::{ additional_endpoints::deserialize_additional_endpoints, apm_replace_rule::deserialize_apm_replace_rules, deserialize_apm_filter_tags, deserialize_array_from_comma_separated_string, - deserialize_key_value_pairs, deserialize_optional_bool_from_anything, - deserialize_optional_duration_from_microseconds, - deserialize_optional_duration_from_seconds, deserialize_string_or_int, + deserialize_key_value_pairs, deserialize_option_lossless, + deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, + deserialize_optional_duration_from_seconds, deserialize_optional_string, + deserialize_string_or_int, flush_strategy::FlushStrategy, log_level::LogLevel, logs_additional_endpoints::{ @@ -34,10 +35,12 @@ pub struct EnvConfig { /// @env `DD_SITE` /// /// The Datadog site to send telemetry to + #[serde(deserialize_with = "deserialize_optional_string")] pub site: Option, /// @env `DD_API_KEY` /// /// The Datadog API key used to submit telemetry to Datadog + #[serde(deserialize_with = "deserialize_optional_string")] pub api_key: Option, /// @env `DD_LOG_LEVEL` /// @@ -50,12 +53,14 @@ pub struct EnvConfig { /// Flush timeout in seconds /// todo(duncanista): find out where this comes from /// todo(?): go agent adds jitter too + #[serde(deserialize_with = "deserialize_option_lossless")] pub flush_timeout: Option, // Proxy /// @env `DD_PROXY_HTTPS` /// /// Proxy endpoint for HTTPS connections (most Datadog traffic) + #[serde(deserialize_with = "deserialize_optional_string")] pub proxy_https: Option, /// @env `DD_PROXY_NO_PROXY` /// @@ -66,6 +71,7 @@ pub struct EnvConfig { /// /// The HTTP protocol to use for the Datadog Agent. /// The transport type to use for sending logs. Possible values are "auto" or "http1". + #[serde(deserialize_with = "deserialize_optional_string")] pub http_protocol: Option, // Metrics @@ -79,10 +85,12 @@ pub struct EnvConfig { /// or Live Process intake which have their own "*_`dd_url`" settings. /// /// If `DD_DD_URL` and `DD_URL` are both set, `DD_DD_URL` is used in priority. + #[serde(deserialize_with = "deserialize_optional_string")] pub dd_url: Option, /// @env `DD_URL` /// /// @default `https://app.datadoghq.com` + #[serde(deserialize_with = "deserialize_optional_string")] pub url: Option, /// @env `DD_ADDITIONAL_ENDPOINTS` /// @@ -112,12 +120,14 @@ pub struct EnvConfig { /// Global level `compression_level` parameter accepts values from 0 (no compression) /// to 9 (maximum compression but higher resource usage). This value is effective only if /// the individual component doesn't specify its own. + #[serde(deserialize_with = "deserialize_option_lossless")] pub compression_level: Option, // Logs /// @env `DD_LOGS_CONFIG_LOGS_DD_URL` /// /// Define the endpoint and port to hit when using a proxy for logs. + #[serde(deserialize_with = "deserialize_optional_string")] pub logs_config_logs_dd_url: Option, /// @env `DD_LOGS_CONFIG_PROCESSING_RULES` /// @@ -136,6 +146,7 @@ pub struct EnvConfig { /// The `compression_level` parameter accepts values from 0 (no compression) /// to 9 (maximum compression but higher resource usage). Only takes effect if /// `use_compression` is set to `true`. + #[serde(deserialize_with = "deserialize_option_lossless")] pub logs_config_compression_level: Option, /// @env `DD_LOGS_CONFIG_ADDITIONAL_ENDPOINTS` /// @@ -151,6 +162,7 @@ pub struct EnvConfig { /// @env `DD_OBSERVABILITY_PIPELINES_WORKER_LOGS_URL` /// /// The URL endpoint for sending logs to Observability Pipelines Worker + #[serde(deserialize_with = "deserialize_optional_string")] pub observability_pipelines_worker_logs_url: Option, // APM @@ -162,6 +174,7 @@ pub struct EnvConfig { /// @env `DD_APM_DD_URL` /// /// Define the endpoint and port to hit when using a proxy for APM. + #[serde(deserialize_with = "deserialize_optional_string")] pub apm_dd_url: Option, /// @env `DD_APM_REPLACE_TAGS` /// @@ -186,6 +199,7 @@ pub struct EnvConfig { /// The Agent compresses traces before sending them. The `compression_level` parameter /// accepts values from 0 (no compression) to 9 (maximum compression but /// higher resource usage). + #[serde(deserialize_with = "deserialize_option_lossless")] pub apm_config_compression_level: Option, /// @env `DD_APM_FEATURES` #[serde(deserialize_with = "deserialize_array_from_comma_separated_string")] @@ -248,6 +262,7 @@ pub struct EnvConfig { /// The metrics compresses traces before sending them. The `compression_level` parameter /// accepts values from 0 (no compression) to 9 (maximum compression but /// higher resource usage). + #[serde(deserialize_with = "deserialize_option_lossless")] pub metrics_config_compression_level: Option, // OTLP @@ -268,15 +283,19 @@ pub struct EnvConfig { // // - Receiver / HTTP /// @env `DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT` + #[serde(deserialize_with = "deserialize_optional_string")] pub otlp_config_receiver_protocols_http_endpoint: Option, // - Unsupported Configuration // // - Receiver / GRPC /// @env `DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT` + #[serde(deserialize_with = "deserialize_optional_string")] pub otlp_config_receiver_protocols_grpc_endpoint: Option, /// @env `DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_TRANSPORT` + #[serde(deserialize_with = "deserialize_optional_string")] pub otlp_config_receiver_protocols_grpc_transport: Option, /// @env `DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_MAX_RECV_MSG_SIZE_MIB` + #[serde(deserialize_with = "deserialize_option_lossless")] pub otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib: Option, // - Metrics /// @env `DD_OTLP_CONFIG_METRICS_ENABLED` @@ -289,10 +308,13 @@ pub struct EnvConfig { #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub otlp_config_metrics_instrumentation_scope_metadata_as_tags: Option, /// @env `DD_OTLP_CONFIG_METRICS_TAG_CARDINALITY` + #[serde(deserialize_with = "deserialize_optional_string")] pub otlp_config_metrics_tag_cardinality: Option, /// @env `DD_OTLP_CONFIG_METRICS_DELTA_TTL` + #[serde(deserialize_with = "deserialize_option_lossless")] pub otlp_config_metrics_delta_ttl: Option, /// @env `DD_OTLP_CONFIG_METRICS_HISTOGRAMS_MODE` + #[serde(deserialize_with = "deserialize_optional_string")] pub otlp_config_metrics_histograms_mode: Option, /// @env `DD_OTLP_CONFIG_METRICS_HISTOGRAMS_SEND_COUNT_SUM_METRICS` #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] @@ -300,13 +322,17 @@ pub struct EnvConfig { /// @env `DD_OTLP_CONFIG_METRICS_HISTOGRAMS_SEND_AGGREGATION_METRICS` #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub otlp_config_metrics_histograms_send_aggregation_metrics: Option, + #[serde(deserialize_with = "deserialize_optional_string")] pub otlp_config_metrics_sums_cumulative_monotonic_mode: Option, /// @env `DD_OTLP_CONFIG_METRICS_SUMS_INITIAL_CUMULATIVE_MONOTONIC_VALUE` + #[serde(deserialize_with = "deserialize_optional_string")] pub otlp_config_metrics_sums_initial_cumulativ_monotonic_value: Option, /// @env `DD_OTLP_CONFIG_METRICS_SUMMARIES_MODE` + #[serde(deserialize_with = "deserialize_optional_string")] pub otlp_config_metrics_summaries_mode: Option, // - Traces /// @env `DD_OTLP_CONFIG_TRACES_PROBABILISTIC_SAMPLER_SAMPLING_PERCENTAGE` + #[serde(deserialize_with = "deserialize_option_lossless")] pub otlp_config_traces_probabilistic_sampler_sampling_percentage: Option, // - Logs /// @env `DD_OTLP_CONFIG_LOGS_ENABLED` @@ -317,10 +343,12 @@ pub struct EnvConfig { /// @env `DD_API_KEY_SECRET_ARN` /// /// The AWS ARN of the secret containing the Datadog API key. + #[serde(deserialize_with = "deserialize_optional_string")] pub api_key_secret_arn: Option, /// @env `DD_KMS_API_KEY` /// /// The AWS KMS API key to use for the Datadog Agent. + #[serde(deserialize_with = "deserialize_optional_string")] pub kms_api_key: Option, /// @env `DD_SERVERLESS_LOGS_ENABLED` /// @@ -346,6 +374,7 @@ pub struct EnvConfig { /// - File descriptor count /// - Thread count /// - Temp directory usage + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub lambda_proc_enhanced_metrics: Option, /// @env `DD_CAPTURE_LAMBDA_PAYLOAD` /// @@ -357,6 +386,7 @@ pub struct EnvConfig { /// /// The maximum depth of the Lambda payload to capture. /// Default is `10`. Requires `capture_lambda_payload` to be `true`. + #[serde(deserialize_with = "deserialize_option_lossless")] pub capture_lambda_payload_max_depth: Option, /// @env `DD_COMPUTE_TRACE_STATS_ON_EXTENSION` /// @@ -375,6 +405,7 @@ pub struct EnvConfig { /// @env `DD_APPSEC_RULES` /// /// The path to a user-configured App & API Protection ruleset (in JSON format). + #[serde(deserialize_with = "deserialize_optional_string")] pub appsec_rules: Option, /// @env `DD_APPSEC_WAF_TIMEOUT` /// @@ -391,11 +422,6 @@ pub struct EnvConfig { /// The delay between two samples of the API Security schema collection, in seconds. #[serde(deserialize_with = "deserialize_optional_duration_from_seconds")] pub api_security_sample_delay: Option, - /// @env `DD_EXTENSION_VERSION` - /// - /// Used to decide which version of the Datadog Lambda Extension to use. - /// When set to `compatibility`, the extension will boot up in legacy mode. - pub extension_version: Option, } #[allow(clippy::too_many_lines)] @@ -577,7 +603,6 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option_to_value!(config, env_config, appsec_waf_timeout); merge_option_to_value!(config, env_config, api_security_enabled); merge_option_to_value!(config, env_config, api_security_sample_delay); - merge_option!(config, env_config, extension_version); } #[derive(Debug, PartialEq, Clone, Copy)] @@ -771,7 +796,6 @@ mod tests { jail.set_env("DD_APPSEC_WAF_TIMEOUT", "1000000"); // Microseconds jail.set_env("DD_API_SECURITY_ENABLED", "0"); // Seconds jail.set_env("DD_API_SECURITY_SAMPLE_DELAY", "60"); // Seconds - jail.set_env("DD_EXTENSION_VERSION", "compatibility"); let mut config = Config::default(); let env_config_source = EnvConfigSource; @@ -921,7 +945,6 @@ mod tests { appsec_waf_timeout: Duration::from_secs(1), api_security_enabled: false, api_security_sample_delay: Duration::from_secs(60), - extension_version: Some("compatibility".to_string()), }; assert_eq!(config, expected_config); diff --git a/mod.rs b/mod.rs index a940b6e0..e8a1a6f9 100644 --- a/mod.rs +++ b/mod.rs @@ -32,7 +32,6 @@ use crate::config::{ trace_propagation_style::TracePropagationStyle, yaml::YamlConfigSource, }; -use crate::proc::has_dotnet_binary; /// Helper macro to merge Option fields to String fields /// @@ -350,8 +349,6 @@ pub struct Config { pub appsec_waf_timeout: Duration, pub api_security_enabled: bool, pub api_security_sample_delay: Duration, - - pub extension_version: Option, } impl Default for Config { @@ -453,90 +450,19 @@ impl Default for Config { appsec_waf_timeout: Duration::from_millis(5), api_security_enabled: true, api_security_sample_delay: Duration::from_secs(30), - - extension_version: None, } } } -fn log_fallback_reason(reason: &str) { - println!("{{\"DD_EXTENSION_FALLBACK_REASON\":\"{reason}\"}}"); -} - -fn fallback(config: &Config) -> Result<(), ConfigError> { - // Customer explicitly opted out of the Next Gen extension - let opted_out = match config.extension_version.as_deref() { - Some("compatibility") => true, - // We want customers using the `next` to not be affected - _ => false, - }; - - if opted_out { - log_fallback_reason("extension_version"); - return Err(ConfigError::UnsupportedField( - "extension_version".to_string(), - )); - } - - // ASM / .NET - // todo(duncanista): Remove once the .NET runtime is fixed - if config.serverless_appsec_enabled && has_dotnet_binary() { - log_fallback_reason("serverless_appsec_enabled_dotnet"); - return Err(ConfigError::UnsupportedField( - "serverless_appsec_enabled_dotnet".to_string(), - )); - } - - // OTLP - let has_otlp_config = config - .otlp_config_receiver_protocols_grpc_endpoint - .is_some() - || config - .otlp_config_receiver_protocols_grpc_transport - .is_some() - || config - .otlp_config_receiver_protocols_grpc_max_recv_msg_size_mib - .is_some() - || config.otlp_config_metrics_enabled - || config.otlp_config_metrics_resource_attributes_as_tags - || config.otlp_config_metrics_instrumentation_scope_metadata_as_tags - || config.otlp_config_metrics_tag_cardinality.is_some() - || config.otlp_config_metrics_delta_ttl.is_some() - || config.otlp_config_metrics_histograms_mode.is_some() - || config.otlp_config_metrics_histograms_send_count_sum_metrics - || config.otlp_config_metrics_histograms_send_aggregation_metrics - || config - .otlp_config_metrics_sums_cumulative_monotonic_mode - .is_some() - || config - .otlp_config_metrics_sums_initial_cumulativ_monotonic_value - .is_some() - || config.otlp_config_metrics_summaries_mode.is_some() - || config - .otlp_config_traces_probabilistic_sampler_sampling_percentage - .is_some() - || config.otlp_config_logs_enabled; - - if has_otlp_config { - log_fallback_reason("otel"); - return Err(ConfigError::UnsupportedField("otel".to_string())); - } - - Ok(()) -} - #[allow(clippy::module_name_repetitions)] -pub fn get_config(config_directory: &Path) -> Result { +#[inline] +#[must_use] +pub fn get_config(config_directory: &Path) -> Config { let path: std::path::PathBuf = config_directory.join("datadog.yaml"); - let mut config_builder = ConfigBuilder::default() + ConfigBuilder::default() .add_source(Box::new(YamlConfigSource { path })) - .add_source(Box::new(EnvConfigSource)); - - let config = config_builder.build(); - - fallback(&config)?; - - Ok(config) + .add_source(Box::new(EnvConfigSource)) + .build() } #[inline] @@ -545,6 +471,22 @@ fn build_fqdn_logs(site: String) -> String { format!("https://http-intake.logs.{site}") } +pub fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + match Value::deserialize(deserializer)? { + Value::String(s) => Ok(Some(s)), + other => { + error!( + "Failed to parse value, expected a string, got: {}, ignoring", + other + ); + Ok(None) + } + } +} + pub fn deserialize_string_or_int<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -559,7 +501,10 @@ where } } Value::Number(n) => Ok(Some(n.to_string())), - _ => Err(serde::de::Error::custom("expected a string or an integer")), + _ => { + error!("Failed to parse value, expected a string or an integer, ignoring"); + Ok(None) + } } } @@ -574,13 +519,13 @@ where match opt { None => Ok(None), - Some(value) => { - // Use your existing method by deserializing the value - let bool_result = deserialize_bool_from_anything(value).map_err(|e| { - serde::de::Error::custom(format!("Failed to deserialize bool: {e}")) - })?; - Ok(Some(bool_result)) - } + Some(value) => match deserialize_bool_from_anything(value) { + Ok(bool_result) => Ok(Some(bool_result)), + Err(e) => { + error!("Failed to parse bool value: {}, ignoring", e); + Ok(None) + } + }, } } @@ -609,14 +554,63 @@ where let parts = tag.split(':').collect::>(); if parts.len() == 2 { map.insert(parts[0].to_string(), parts[1].to_string()); + } else { + error!( + "Failed to parse tag '{}', expected format 'key:value', ignoring", + tag.trim() + ); } } Ok(map) } + + fn visit_u64(self, value: u64) -> Result + where + E: serde::de::Error, + { + error!( + "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", + value + ); + Ok(HashMap::new()) + } + + fn visit_i64(self, value: i64) -> Result + where + E: serde::de::Error, + { + error!( + "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", + value + ); + Ok(HashMap::new()) + } + + fn visit_f64(self, value: f64) -> Result + where + E: serde::de::Error, + { + error!( + "Failed to parse tags: expected string in format 'key:value', got number {}, ignoring", + value + ); + Ok(HashMap::new()) + } + + fn visit_bool(self, value: bool) -> Result + where + E: serde::de::Error, + { + error!( + "Failed to parse tags: expected string in format 'key:value', got boolean {}, ignoring", + value + ); + Ok(HashMap::new()) + } } - deserializer.deserialize_str(KeyValueVisitor) + deserializer.deserialize_any(KeyValueVisitor) } pub fn deserialize_array_from_comma_separated_string<'de, D>( @@ -644,6 +638,11 @@ where let parts = s.split(':').collect::>(); if parts.len() == 2 { map.insert(parts[0].to_string(), parts[1].to_string()); + } else { + error!( + "Failed to parse tag '{}', expected format 'key:value', ignoring", + s.trim() + ); } } Ok(map) @@ -696,6 +695,20 @@ where } } +pub fn deserialize_option_lossless<'de, D, T>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + match Option::::deserialize(deserializer) { + Ok(value) => Ok(value), + Err(e) => { + error!("Failed to deserialize optional value: {}, ignoring", e); + Ok(None) + } + } +} + pub fn deserialize_optional_duration_from_microseconds<'de, D: Deserializer<'de>>( deserializer: D, ) -> Result, D::Error> { @@ -716,13 +729,15 @@ pub fn deserialize_optional_duration_from_seconds<'de, D: Deserializer<'de>>( } fn visit_i64(self, v: i64) -> Result { if v < 0 { - return Err(E::custom("negative durations are not allowed")); + error!("Failed to parse duration: negative durations are not allowed, ignoring"); + return Ok(None); } self.visit_u64(u64::try_from(v).expect("positive i64 to u64 conversion never fails")) } fn visit_f64(self, v: f64) -> Result { if v < 0f64 { - return Err(E::custom("negative durations are not allowed")); + error!("Failed to parse duration: negative durations are not allowed, ignoring"); + return Ok(None); } Ok(Some(Duration::from_secs_f64(v))) } @@ -744,62 +759,12 @@ pub mod tests { trace_propagation_style::TracePropagationStyle, }; - #[test] - fn test_reject_on_opted_out() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env("DD_EXTENSION_VERSION", "compatibility"); - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!( - config, - ConfigError::UnsupportedField("extension_version".to_string()) - ); - Ok(()) - }); - } - - #[test] - fn test_fallback_on_otel() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env( - "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", - "localhost:4138", - ); - - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); - Ok(()) - }); - } - - #[test] - fn test_fallback_on_otel_yaml() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.create_file( - "datadog.yaml", - r" - otlp_config: - receiver: - protocols: - grpc: - endpoint: localhost:4138 - ", - )?; - - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); - Ok(()) - }); - } - #[test] fn test_default_logs_intake_url() { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_logs_dd_url, "https://http-intake.logs.datadoghq.com".to_string() @@ -817,7 +782,7 @@ pub mod tests { "agent-http-intake-pci.logs.datadoghq.com:443", ); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_logs_dd_url, "agent-http-intake-pci.logs.datadoghq.com:443".to_string() @@ -832,7 +797,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_APM_DD_URL", "https://trace-pci.agent.datadoghq.com"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.apm_dd_url, "https://trace-pci.agent.datadoghq.com/api/v0.2/traces".to_string() @@ -847,7 +812,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_DD_URL", "custom_proxy:3128"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.dd_url, "custom_proxy:3128".to_string()); Ok(()) }); @@ -859,7 +824,7 @@ pub mod tests { jail.clear_env(); jail.set_env("DD_URL", "custom_proxy:3128"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.url, "custom_proxy:3128".to_string()); Ok(()) }); @@ -870,27 +835,12 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.dd_url, String::new()); Ok(()) }); } - #[test] - fn test_allowed_but_disabled() { - figment::Jail::expect_with(|jail| { - jail.clear_env(); - jail.set_env( - "DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT", - "localhost:4138", - ); - - let config = get_config(Path::new("")).expect_err("should reject unknown fields"); - assert_eq!(config, ConfigError::UnsupportedField("otel".to_string())); - Ok(()) - }); - } - #[test] fn test_precedence() { figment::Jail::expect_with(|jail| { @@ -902,7 +852,7 @@ pub mod tests { ", )?; jail.set_env("DD_SITE", "datad0g.com"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "datad0g.com"); Ok(()) }); @@ -918,7 +868,7 @@ pub mod tests { r" ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "datadoghq.com"); Ok(()) }); @@ -929,7 +879,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SITE", "datadoghq.eu"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "datadoghq.eu"); Ok(()) }); @@ -940,7 +890,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_LOG_LEVEL", "TRACE"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.log_level, LogLevel::Trace); Ok(()) }); @@ -950,7 +900,7 @@ pub mod tests { fn test_parse_default() { figment::Jail::expect_with(|jail| { jail.clear_env(); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config, Config { @@ -974,7 +924,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_PROXY_HTTPS", "my-proxy:3128"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -990,7 +940,7 @@ pub mod tests { "NO_PROXY", "127.0.0.1,localhost,172.16.0.0/12,us-east-1.amazonaws.com,datadoghq.eu", ); - let config = get_config(Path::new("")).expect("should parse noproxy"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, None); Ok(()) }); @@ -1008,7 +958,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect("should parse weird proxy config"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, Some("my-proxy:3128".to_string())); Ok(()) }); @@ -1028,7 +978,7 @@ pub mod tests { ", )?; - let config = get_config(Path::new("")).expect("should parse weird proxy config"); + let config = get_config(Path::new("")); assert_eq!(config.proxy_https, None); // Assertion to ensure config.site runs before proxy // because we chenck that noproxy contains the site @@ -1042,7 +992,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "end"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.serverless_flush_strategy, FlushStrategy::End); Ok(()) }); @@ -1053,7 +1003,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "periodically,100000"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.serverless_flush_strategy, FlushStrategy::Periodically(PeriodicStrategy { interval: 100_000 }) @@ -1067,7 +1017,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_SERVERLESS_FLUSH_STRATEGY", "invalid_strategy"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -1081,7 +1031,7 @@ pub mod tests { "DD_SERVERLESS_FLUSH_STRATEGY", "periodically,invalid_interval", ); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.serverless_flush_strategy, FlushStrategy::Default); Ok(()) }); @@ -1094,7 +1044,7 @@ pub mod tests { jail.set_env("DD_VERSION", "123"); jail.set_env("DD_ENV", "123456890"); jail.set_env("DD_SERVICE", "123456"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.version.expect("failed to parse DD_VERSION"), "123"); assert_eq!(config.env.expect("failed to parse DD_ENV"), "123456890"); assert_eq!( @@ -1116,7 +1066,6 @@ pub mod tests { jail.create_file( "datadog.yaml", r" - extension_version: next logs_config: processing_rules: - type: exclude_at_match @@ -1124,7 +1073,7 @@ pub mod tests { pattern: exclude-me-yaml ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -1153,7 +1102,7 @@ pub mod tests { pattern: exclude ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.logs_config_processing_rules, Some(vec![ProcessingRule { @@ -1182,7 +1131,7 @@ pub mod tests { repl: 'REDACTED' ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); let rule = parse_rules_from_string( r#"[ {"name": "*", "pattern": "foo", "repl": "REDACTED"} @@ -1213,7 +1162,7 @@ pub mod tests { repl: 'REDACTED-YAML' ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); let rule = parse_rules_from_string( r#"[ {"name": "*", "pattern": "foo", "repl": "REDACTED-ENV"} @@ -1240,7 +1189,7 @@ pub mod tests { remove_paths_with_digits: true ", )?; - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert!(config.apm_config_obfuscation_http_remove_query_string,); assert!(config.apm_config_obfuscation_http_remove_paths_with_digits,); Ok(()) @@ -1254,8 +1203,7 @@ pub mod tests { "DD_TRACE_PROPAGATION_STYLE", "datadog,tracecontext,b3,b3multi", ); - jail.set_env("DD_EXTENSION_VERSION", "next"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); let expected_styles = vec![ TracePropagationStyle::Datadog, @@ -1274,7 +1222,7 @@ pub mod tests { figment::Jail::expect_with(|jail| { jail.clear_env(); jail.set_env("DD_TRACE_PROPAGATION_STYLE_EXTRACT", "datadog"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!( config.trace_propagation_style, @@ -1292,15 +1240,12 @@ pub mod tests { } #[test] - fn test_ignore_apm_replace_tags() { + fn test_bad_tags() { figment::Jail::expect_with(|jail| { jail.clear_env(); - jail.set_env( - "DD_APM_REPLACE_TAGS", - r#"[{"name":"resource.name","pattern":"(.*)/(foo[:%].+)","repl":"$1/{foo}"}]"#, - ); + jail.set_env("DD_TAGS", 123); let config = get_config(Path::new("")); - assert!(config.is_ok()); + assert_eq!(config.tags, HashMap::new()); Ok(()) }); } @@ -1313,7 +1258,7 @@ pub mod tests { jail.set_env("DD_ENHANCED_METRICS", "1"); jail.set_env("DD_LOGS_CONFIG_USE_COMPRESSION", "TRUE"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "0"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert!(config.serverless_logs_enabled); assert!(config.enhanced_metrics); assert!(config.logs_config_use_compression); @@ -1337,7 +1282,7 @@ pub mod tests { jail.set_env("DD_SITE", "us5.datadoghq.com"); jail.set_env("DD_API_KEY", "env-api-key"); jail.set_env("DD_FLUSH_TIMEOUT", "10"); - let config = get_config(Path::new("")).expect("should parse config"); + let config = get_config(Path::new("")); assert_eq!(config.site, "us5.datadoghq.com"); assert_eq!(config.api_key, "env-api-key"); @@ -1387,16 +1332,20 @@ pub mod tests { serde_json::from_str::("{}").expect("failed to parse JSON"), Value { duration: None } ); - serde_json::from_str::(r#"{"duration":-1}"#) - .expect_err("should have failed parsing"); + assert_eq!( + serde_json::from_str::(r#"{"duration":-1}"#).expect("failed to parse JSON"), + Value { duration: None } + ); assert_eq!( serde_json::from_str::(r#"{"duration":1}"#).expect("failed to parse JSON"), Value { duration: Some(Duration::from_secs(1)) } ); - serde_json::from_str::(r#"{"duration":-1.5}"#) - .expect_err("should have failed parsing"); + assert_eq!( + serde_json::from_str::(r#"{"duration":-1.5}"#).expect("failed to parse JSON"), + Value { duration: None } + ); assert_eq!( serde_json::from_str::(r#"{"duration":1.5}"#).expect("failed to parse JSON"), Value { diff --git a/processing_rule.rs b/processing_rule.rs index f494d02f..cae8a5ad 100644 --- a/processing_rule.rs +++ b/processing_rule.rs @@ -28,21 +28,28 @@ where let value: JsonValue = Deserialize::deserialize(deserializer)?; match value { - JsonValue::String(s) => { - let values: Vec = serde_json::from_str(&s).map_err(|e| { - serde::de::Error::custom(format!("Failed to deserialize processing rules: {e}")) - })?; - Ok(Some(values)) - } + JsonValue::String(s) => match serde_json::from_str(&s) { + Ok(values) => Ok(Some(values)), + Err(e) => { + tracing::error!("Failed to parse processing rules: {}, ignoring", e); + Ok(None) + } + }, JsonValue::Array(a) => { let mut values = Vec::new(); for v in a { - let rule: ProcessingRule = serde_json::from_value(v).map_err(|e| { - serde::de::Error::custom(format!("Failed to deserialize processing rule: {e}")) - })?; - values.push(rule); + match serde_json::from_value(v.clone()) { + Ok(rule) => values.push(rule), + Err(e) => { + tracing::error!("Failed to parse processing rule: {}, ignoring", e); + } + } + } + if values.is_empty() { + Ok(None) + } else { + Ok(Some(values)) } - Ok(Some(values)) } _ => Ok(None), } diff --git a/service_mapping.rs b/service_mapping.rs index 4deda11f..5b133989 100644 --- a/service_mapping.rs +++ b/service_mapping.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use serde::{Deserialize, Deserializer}; -use tracing::debug; #[allow(clippy::module_name_repetitions)] pub fn deserialize_service_mapping<'de, D>( @@ -14,22 +13,20 @@ where let map = s .split(',') - .map(|pair| { + .filter_map(|pair| { let mut split = pair.split(':'); let service = split.next(); let to_map = split.next(); if let (Some(service), Some(to_map)) = (service, to_map) { - Ok((service.trim().to_string(), to_map.trim().to_string())) + Some((service.trim().to_string(), to_map.trim().to_string())) } else { - debug!("Ignoring invalid service mapping pair: {pair}"); - Err(serde::de::Error::custom(format!( - "Failed to deserialize service mapping for pair: {pair}" - ))) + tracing::error!("Failed to parse service mapping '{}', expected format 'service:mapped_service', ignoring", pair.trim()); + None } }) .collect(); - map + Ok(map) } diff --git a/trace_propagation_style.rs b/trace_propagation_style.rs index 6ebc9dc7..e1a4f609 100644 --- a/trace_propagation_style.rs +++ b/trace_propagation_style.rs @@ -48,11 +48,15 @@ where { let s: String = String::deserialize(deserializer)?; - s.split(',') - .map(|style| { - TracePropagationStyle::from_str(style.trim()).map_err(|e| { - serde::de::Error::custom(format!("Failed to deserialize propagation style: {e}")) - }) - }) - .collect() + Ok(s.split(',') + .filter_map( + |style| match TracePropagationStyle::from_str(style.trim()) { + Ok(parsed_style) => Some(parsed_style), + Err(e) => { + tracing::error!("Failed to parse trace propagation style: {}, ignoring", e); + None + } + }, + ) + .collect()) } diff --git a/yaml.rs b/yaml.rs index 0c0e39b4..bf814d68 100644 --- a/yaml.rs +++ b/yaml.rs @@ -6,9 +6,10 @@ use crate::{ Config, ConfigError, ConfigSource, ProcessingRule, additional_endpoints::deserialize_additional_endpoints, deserialize_apm_replace_rules, deserialize_key_value_pair_array_to_hashmap, - deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, - deserialize_optional_duration_from_seconds, deserialize_processing_rules, - deserialize_string_or_int, + deserialize_option_lossless, deserialize_optional_bool_from_anything, + deserialize_optional_duration_from_microseconds, + deserialize_optional_duration_from_seconds, deserialize_optional_string, + deserialize_processing_rules, deserialize_string_or_int, flush_strategy::FlushStrategy, log_level::LogLevel, logs_additional_endpoints::LogsAdditionalEndpoint, @@ -32,18 +33,24 @@ use serde::Deserialize; #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct YamlConfig { + #[serde(deserialize_with = "deserialize_optional_string")] pub site: Option, + #[serde(deserialize_with = "deserialize_optional_string")] pub api_key: Option, pub log_level: Option, + #[serde(deserialize_with = "deserialize_option_lossless")] pub flush_timeout: Option, + #[serde(deserialize_with = "deserialize_option_lossless")] pub compression_level: Option, // Proxy pub proxy: ProxyConfig, // nit: this should probably be in the endpoints section + #[serde(deserialize_with = "deserialize_optional_string")] pub dd_url: Option, + #[serde(deserialize_with = "deserialize_optional_string")] pub http_protocol: Option, // Endpoints @@ -87,28 +94,33 @@ pub struct YamlConfig { pub otlp_config: Option, // AWS Lambda + #[serde(deserialize_with = "deserialize_optional_string")] pub api_key_secret_arn: Option, + #[serde(deserialize_with = "deserialize_optional_string")] pub kms_api_key: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub serverless_logs_enabled: Option, pub serverless_flush_strategy: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub enhanced_metrics: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub lambda_proc_enhanced_metrics: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub capture_lambda_payload: Option, + #[serde(deserialize_with = "deserialize_option_lossless")] pub capture_lambda_payload_max_depth: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub compute_trace_stats_on_extension: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub serverless_appsec_enabled: Option, + #[serde(deserialize_with = "deserialize_optional_string")] pub appsec_rules: Option, #[serde(deserialize_with = "deserialize_optional_duration_from_microseconds")] pub appsec_waf_timeout: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub api_security_enabled: Option, #[serde(deserialize_with = "deserialize_optional_duration_from_seconds")] pub api_security_sample_delay: Option, - pub extension_version: Option, } /// Proxy Config @@ -134,6 +146,7 @@ pub struct LogsConfig { pub processing_rules: Option>, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub use_compression: Option, + #[serde(deserialize_with = "deserialize_option_lossless")] pub compression_level: Option, pub additional_endpoints: Vec, } @@ -144,6 +157,7 @@ pub struct LogsConfig { #[serde(default)] #[allow(clippy::module_name_repetitions)] pub struct MetricsConfig { + #[serde(deserialize_with = "deserialize_option_lossless")] pub compression_level: Option, } @@ -158,6 +172,7 @@ pub struct ApmConfig { #[serde(deserialize_with = "deserialize_apm_replace_rules")] pub replace_tags: Option>, pub obfuscation: Option, + #[serde(deserialize_with = "deserialize_option_lossless")] pub compression_level: Option, pub features: Vec, #[serde(deserialize_with = "deserialize_additional_endpoints")] @@ -242,6 +257,7 @@ pub struct OtlpReceiverHttpConfig { pub struct OtlpReceiverGrpcConfig { pub endpoint: Option, pub transport: Option, + #[serde(deserialize_with = "deserialize_option_lossless")] pub max_recv_msg_size_mib: Option, } @@ -263,6 +279,7 @@ pub struct OtlpTracesConfig { #[derive(Debug, PartialEq, Clone, Deserialize, Default, Copy)] pub struct OtlpTracesProbabilisticSampler { + #[serde(deserialize_with = "deserialize_option_lossless")] pub sampling_percentage: Option, } @@ -276,6 +293,7 @@ pub struct OtlpMetricsConfig { #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub instrumentation_scope_metadata_as_tags: Option, pub tag_cardinality: Option, + #[serde(deserialize_with = "deserialize_option_lossless")] pub delta_ttl: Option, pub histograms: Option, pub sums: Option, @@ -662,7 +680,6 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option_to_value!(config, yaml_config, appsec_waf_timeout); merge_option_to_value!(config, yaml_config, api_security_enabled); merge_option_to_value!(config, yaml_config, api_security_sample_delay); - merge_option!(config, yaml_config, extension_version); } #[derive(Debug, PartialEq, Clone)] @@ -832,7 +849,6 @@ appsec_rules: "/path/to/rules.json" appsec_waf_timeout: 1000000 # Microseconds api_security_enabled: false api_security_sample_delay: 60 # Seconds -extension_version: "compatibility" "#, )?; @@ -967,7 +983,6 @@ extension_version: "compatibility" api_security_enabled: false, api_security_sample_delay: Duration::from_secs(60), - extension_version: Some("compatibility".to_string()), apm_filter_tags_require: None, apm_filter_tags_reject: None, apm_filter_tags_regex_require: None, From 03d73d9717806df07a047036b602615ee7fe4cf3 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Thu, 9 Oct 2025 14:44:54 -0400 Subject: [PATCH 091/112] fix: use datadog as default propagation style if supplied version is malformed (#891) Fixes an issue where config parsing fails if this is invalid --- trace_propagation_style.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/trace_propagation_style.rs b/trace_propagation_style.rs index e1a4f609..704d7244 100644 --- a/trace_propagation_style.rs +++ b/trace_propagation_style.rs @@ -1,6 +1,7 @@ use std::{fmt::Display, str::FromStr}; use serde::{Deserialize, Deserializer}; +use tracing::error; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TracePropagationStyle { @@ -21,7 +22,10 @@ impl FromStr for TracePropagationStyle { "b3" => Ok(TracePropagationStyle::B3), "tracecontext" => Ok(TracePropagationStyle::TraceContext), "none" => Ok(TracePropagationStyle::None), - _ => Err(format!("Unknown trace propagation style: {s}")), + _ => { + error!("Trace propagation style is invalid: {:?}, using Datadog", s); + Ok(TracePropagationStyle::Datadog) + } } } } From ce36e8275f03d890905df99fe6ac97f06d7fb2b6 Mon Sep 17 00:00:00 2001 From: AJ Stuyvenberg Date: Wed, 15 Oct 2025 09:23:38 -0400 Subject: [PATCH 092/112] fix: use None if propagation style is invalid (#895) After internal discussion we determined that the tracing libraries use None of the trace propagation style is invalid or malformed. This brings us into alignment. --- trace_propagation_style.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trace_propagation_style.rs b/trace_propagation_style.rs index 704d7244..65971bfa 100644 --- a/trace_propagation_style.rs +++ b/trace_propagation_style.rs @@ -23,8 +23,8 @@ impl FromStr for TracePropagationStyle { "tracecontext" => Ok(TracePropagationStyle::TraceContext), "none" => Ok(TracePropagationStyle::None), _ => { - error!("Trace propagation style is invalid: {:?}, using Datadog", s); - Ok(TracePropagationStyle::Datadog) + error!("Trace propagation style is invalid: {:?}, using None", s); + Ok(TracePropagationStyle::None) } } } From fef101241353599c340becf9dd1876dec587857c Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Wed, 15 Oct 2025 15:01:05 -0400 Subject: [PATCH 093/112] feat: Support periodic reload for api key secret (#893) # This PR Supports the env var `DD_API_KEY_SECRET_RELOAD_INTERVAL`, in seconds. It applies when Datadog API Key is set using `DD_API_KEY_SECRET_ARN`. For example: - if it's `120`, then api key will be reloaded about every 120 seconds. Note that reload can only be triggered when api key is used, usually when data is being flushed. If there is no invocation and no data needs to be flushed, then reload won't happen. - If it's not set or set to `0`, then api key will only be loaded once the first time it is used, and won't be reloaded. # Motivation Some customers regularly rotate their api key in a secret. We need to provide a way for them to update our cached key. https://github.com/DataDog/datadog-lambda-extension/issues/834 # Testing ## Steps 1. Set the env var `DD_API_KEY_SECRET_RELOAD_INTERVAL` to `120` 2. Invoke the Lambda every minute ## Result The reload interval is passed to the `ApiKeyFactory` image Reload happens roughly every 120 seconds. It's sometimes longer than 120 seconds due to the reason explained above. image # Notes to Users When you use this env var, please also keep a grace period for the old api key after you update the secret to the new key, and make the grace period longer than the reload interval to give the extension sufficient time to reload the secret. # Internal Notes Jira: https://datadoghq.atlassian.net/browse/SVLS-7572 --- env.rs | 13 ++++++++++++- mod.rs | 35 +++++++++++++++++++++++++++++++++++ yaml.rs | 8 +++++++- 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/env.rs b/env.rs index 6b50178d..1ba53dff 100644 --- a/env.rs +++ b/env.rs @@ -13,7 +13,8 @@ use crate::{ deserialize_apm_filter_tags, deserialize_array_from_comma_separated_string, deserialize_key_value_pairs, deserialize_option_lossless, deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, - deserialize_optional_duration_from_seconds, deserialize_optional_string, + deserialize_optional_duration_from_seconds, + deserialize_optional_duration_from_seconds_ignore_zero, deserialize_optional_string, deserialize_string_or_int, flush_strategy::FlushStrategy, log_level::LogLevel, @@ -395,6 +396,13 @@ pub struct EnvConfig { /// Default is `false`. #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub compute_trace_stats_on_extension: Option, + /// @env `DD_API_KEY_SECRET_RELOAD_INTERVAL` + /// + /// The interval at which the Datadog API key is reloaded, in seconds. + /// If None, the API key will not be reloaded. + /// Default is `None`. + #[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")] + pub api_key_secret_reload_interval: Option, /// @env `DD_SERVERLESS_APPSEC_ENABLED` /// /// Enable Application and API Protection (AAP), previously known as AppSec/ASM, for AWS Lambda. @@ -598,6 +606,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option_to_value!(config, env_config, capture_lambda_payload); merge_option_to_value!(config, env_config, capture_lambda_payload_max_depth); merge_option_to_value!(config, env_config, compute_trace_stats_on_extension); + merge_option!(config, env_config, api_key_secret_reload_interval); merge_option_to_value!(config, env_config, serverless_appsec_enabled); merge_option!(config, env_config, appsec_rules); merge_option_to_value!(config, env_config, appsec_waf_timeout); @@ -791,6 +800,7 @@ mod tests { jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "true"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "5"); jail.set_env("DD_COMPUTE_TRACE_STATS_ON_EXTENSION", "true"); + jail.set_env("DD_API_KEY_SECRET_RELOAD_INTERVAL", "10"); jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); jail.set_env("DD_APPSEC_RULES", "/path/to/rules.json"); jail.set_env("DD_APPSEC_WAF_TIMEOUT", "1000000"); // Microseconds @@ -940,6 +950,7 @@ mod tests { capture_lambda_payload: true, capture_lambda_payload_max_depth: 5, compute_trace_stats_on_extension: true, + api_key_secret_reload_interval: Some(Duration::from_secs(10)), serverless_appsec_enabled: true, appsec_rules: Some("/path/to/rules.json".to_string()), appsec_waf_timeout: Duration::from_secs(1), diff --git a/mod.rs b/mod.rs index e8a1a6f9..3d5e4fc6 100644 --- a/mod.rs +++ b/mod.rs @@ -343,6 +343,7 @@ pub struct Config { pub capture_lambda_payload: bool, pub capture_lambda_payload_max_depth: u32, pub compute_trace_stats_on_extension: bool, + pub api_key_secret_reload_interval: Option, pub serverless_appsec_enabled: bool, pub appsec_rules: Option, @@ -444,6 +445,7 @@ impl Default for Config { capture_lambda_payload: false, capture_lambda_payload_max_depth: 10, compute_trace_stats_on_extension: false, + api_key_secret_reload_interval: None, serverless_appsec_enabled: false, appsec_rules: None, @@ -745,6 +747,17 @@ pub fn deserialize_optional_duration_from_seconds<'de, D: Deserializer<'de>>( deserializer.deserialize_any(DurationVisitor) } +// Like deserialize_optional_duration_from_seconds(), but return None if the value is 0 +pub fn deserialize_optional_duration_from_seconds_ignore_zero<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + let duration: Option = deserialize_optional_duration_from_seconds(deserializer)?; + if duration.is_some_and(|d| d.as_secs() == 0) { + return Ok(None); + } + Ok(duration) +} + #[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] pub mod tests { @@ -1353,4 +1366,26 @@ pub mod tests { } ); } + + #[test] + fn test_parse_duration_from_seconds_ignore_zero() { + #[derive(Deserialize, Debug, PartialEq, Eq)] + struct Value { + #[serde(default)] + #[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")] + duration: Option, + } + + assert_eq!( + serde_json::from_str::(r#"{"duration":1}"#).expect("failed to parse JSON"), + Value { + duration: Some(Duration::from_secs(1)) + } + ); + + assert_eq!( + serde_json::from_str::(r#"{"duration":0}"#).expect("failed to parse JSON"), + Value { duration: None } + ); + } } diff --git a/yaml.rs b/yaml.rs index bf814d68..4df1f85b 100644 --- a/yaml.rs +++ b/yaml.rs @@ -8,7 +8,8 @@ use crate::{ deserialize_apm_replace_rules, deserialize_key_value_pair_array_to_hashmap, deserialize_option_lossless, deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, - deserialize_optional_duration_from_seconds, deserialize_optional_string, + deserialize_optional_duration_from_seconds, + deserialize_optional_duration_from_seconds_ignore_zero, deserialize_optional_string, deserialize_processing_rules, deserialize_string_or_int, flush_strategy::FlushStrategy, log_level::LogLevel, @@ -111,6 +112,8 @@ pub struct YamlConfig { pub capture_lambda_payload_max_depth: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub compute_trace_stats_on_extension: Option, + #[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")] + pub api_key_secret_reload_interval: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub serverless_appsec_enabled: Option, #[serde(deserialize_with = "deserialize_optional_string")] @@ -675,6 +678,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option_to_value!(config, yaml_config, capture_lambda_payload); merge_option_to_value!(config, yaml_config, capture_lambda_payload_max_depth); merge_option_to_value!(config, yaml_config, compute_trace_stats_on_extension); + merge_option!(config, yaml_config, api_key_secret_reload_interval); merge_option_to_value!(config, yaml_config, serverless_appsec_enabled); merge_option!(config, yaml_config, appsec_rules); merge_option_to_value!(config, yaml_config, appsec_waf_timeout); @@ -844,6 +848,7 @@ lambda_proc_enhanced_metrics: false capture_lambda_payload: true capture_lambda_payload_max_depth: 5 compute_trace_stats_on_extension: true +api_key_secret_reload_interval: 0 serverless_appsec_enabled: true appsec_rules: "/path/to/rules.json" appsec_waf_timeout: 1000000 # Microseconds @@ -976,6 +981,7 @@ api_security_sample_delay: 60 # Seconds capture_lambda_payload: true, capture_lambda_payload_max_depth: 5, compute_trace_stats_on_extension: true, + api_key_secret_reload_interval: None, serverless_appsec_enabled: true, appsec_rules: Some("/path/to/rules.json".to_string()), From 5fc8b859f075cd70efe8c88135b271c40bc7dc2c Mon Sep 17 00:00:00 2001 From: jchrostek-dd Date: Fri, 7 Nov 2025 09:11:49 -0500 Subject: [PATCH 094/112] [SVLS-7885] update tag splitting to allow for ',' and ' ' (#916) ## Overview We currently split the`DD_TAGS` only by `,`. Customer is asking if we can also split by spaces since that is common for container images and lambda lets you deploy images. (https://docs.datadoghq.com/getting_started/tagging/assigning_tags/?tab=noncontainerizedenvironments) --- mod.rs | 96 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 92 insertions(+), 4 deletions(-) diff --git a/mod.rs b/mod.rs index 3d5e4fc6..66fd0054 100644 --- a/mod.rs +++ b/mod.rs @@ -551,15 +551,17 @@ where E: serde::de::Error, { let mut map = HashMap::new(); - - for tag in value.split(',') { + for tag in value.split(&[',', ' ']) { + if tag.is_empty() { + continue; + } let parts = tag.split(':').collect::>(); - if parts.len() == 2 { + if parts.len() == 2 && !parts[0].is_empty() && !parts[1].is_empty() { map.insert(parts[0].to_string(), parts[1].to_string()); } else { error!( "Failed to parse tag '{}', expected format 'key:value', ignoring", - tag.trim() + tag ); } } @@ -1263,6 +1265,62 @@ pub mod tests { }); } + #[test] + fn test_tags_comma_separated() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_TAGS", "team:serverless,env:prod,version:1.0"); + let config = get_config(Path::new("")); + assert_eq!(config.tags.get("team"), Some(&"serverless".to_string())); + assert_eq!(config.tags.get("env"), Some(&"prod".to_string())); + assert_eq!(config.tags.get("version"), Some(&"1.0".to_string())); + assert_eq!(config.tags.len(), 3); + Ok(()) + }); + } + + #[test] + fn test_tags_space_separated() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_TAGS", "team:serverless env:prod version:1.0"); + let config = get_config(Path::new("")); + assert_eq!(config.tags.get("team"), Some(&"serverless".to_string())); + assert_eq!(config.tags.get("env"), Some(&"prod".to_string())); + assert_eq!(config.tags.get("version"), Some(&"1.0".to_string())); + assert_eq!(config.tags.len(), 3); + Ok(()) + }); + } + + #[test] + fn test_tags_space_separated_with_extra_spaces() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_TAGS", "team:serverless env:prod version:1.0"); + let config = get_config(Path::new("")); + assert_eq!(config.tags.get("team"), Some(&"serverless".to_string())); + assert_eq!(config.tags.get("env"), Some(&"prod".to_string())); + assert_eq!(config.tags.get("version"), Some(&"1.0".to_string())); + assert_eq!(config.tags.len(), 3); + Ok(()) + }); + } + + #[test] + fn test_tags_mixed_separators() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_TAGS", "team:serverless,env:prod version:1.0"); + let config = get_config(Path::new("")); + assert_eq!(config.tags.get("team"), Some(&"serverless".to_string())); + assert_eq!(config.tags.get("env"), Some(&"prod".to_string())); + assert_eq!(config.tags.get("version"), Some(&"1.0".to_string())); + assert_eq!(config.tags.len(), 3); + Ok(()) + }); + } + #[test] fn test_parse_bool_from_anything() { figment::Jail::expect_with(|jail| { @@ -1388,4 +1446,34 @@ pub mod tests { Value { duration: None } ); } + + #[test] + fn test_deserialize_key_value_pairs_ignores_empty_keys() { + #[derive(Deserialize, Debug, PartialEq)] + struct TestStruct { + #[serde(deserialize_with = "deserialize_key_value_pairs")] + tags: HashMap, + } + + let result = serde_json::from_str::(r#"{"tags": ":value,valid:tag"}"#) + .expect("failed to parse JSON"); + let mut expected = HashMap::new(); + expected.insert("valid".to_string(), "tag".to_string()); + assert_eq!(result.tags, expected); + } + + #[test] + fn test_deserialize_key_value_pairs_ignores_empty_values() { + #[derive(Deserialize, Debug, PartialEq)] + struct TestStruct { + #[serde(deserialize_with = "deserialize_key_value_pairs")] + tags: HashMap, + } + + let result = serde_json::from_str::(r#"{"tags": "key:,valid:tag"}"#) + .expect("failed to parse JSON"); + let mut expected = HashMap::new(); + expected.insert("valid".to_string(), "tag".to_string()); + assert_eq!(result.tags, expected); + } } From 2dfbf7ca7dd30031243f5e43f2b87b4046b72c98 Mon Sep 17 00:00:00 2001 From: jchrostek-dd Date: Tue, 11 Nov 2025 14:30:09 -0500 Subject: [PATCH 095/112] [SLES-2547] add metric namespace for DogStatsD (#920) Follow up from https://github.com/DataDog/serverless-components/pull/48 What does this PR do? Add support for DD_STATSD_METRIC_NAMESPACE. Motivation This was brought up by a customer, they noticed issues migrating to bottlecap. Our docs show we should support this, but we currently don't have it implemented - https://docs.datadoghq.com/serverless/guide/agent_configuration/#dogstatsd-custom-metrics. Additional Notes Requires changes in agent/extension. Will follow up with those PRs. Describe how to test/QA your changes Deployed changes to extension and tested with / without the custom namespace env variable. Confirmed that metrics are getting the prefix attached, [metrics](https://ddserverless.datadoghq.com/metric/explorer?fromUser=false&graph_layout=stacked&start=1762783238873&end=1762784138873&paused=false#N4Ig7glgJg5gpgFxALlAGwIYE8D2BXJVEADxQEYAaELcqyKBAC1pEbghkcLIF8qo4AMwgA7CAgg4RKUAiwAHOChASAtnADOcAE4RNIKtrgBHPJoQaUAbVBGN8qVoD6gnNtUZCKiOq279VKY6epbINiAiGOrKQdpYZAYgUJ4YThr42gDGSsgg6gi6mZaBZnHKGABuMMiZeBoIOKoAdPJYTFJNcMRwtRIdmfgiCMAAVDwgfKCR0bmxWABMickIqel4WTl5iIXFIHPlVcgAVjiMIk3TmvIY2U219Y0tbYwdXT0EkucDeEOj4zwAXSornceEwoXCINUYIwMVK8QmFFAUJhcJ0CwmQJA9SwaByoGueIQCE2UBwMCcmXBGggmUSaFEcCcckUynSDKg9MZTnoTGUIjcHjQiKSEHsmCwzIUmwZIiUgJ4fGx8gZCAAwlJhDAUCIwWgeEA) --- env.rs | 10 +++++++ mod.rs | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ yaml.rs | 1 + 3 files changed, 95 insertions(+) diff --git a/env.rs b/env.rs index 1ba53dff..40783737 100644 --- a/env.rs +++ b/env.rs @@ -266,6 +266,11 @@ pub struct EnvConfig { #[serde(deserialize_with = "deserialize_option_lossless")] pub metrics_config_compression_level: Option, + /// @env `DD_STATSD_METRIC_NAMESPACE` + /// Prefix all `StatsD` metrics with a namespace. + #[serde(deserialize_with = "deserialize_optional_string")] + pub statsd_metric_namespace: Option, + // OTLP // // - APM / Traces @@ -521,6 +526,10 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { ); merge_option_to_value!(config, env_config, metrics_config_compression_level); + if let Some(namespace) = &env_config.statsd_metric_namespace { + config.statsd_metric_namespace = super::validate_metric_namespace(namespace); + } + // OTLP merge_option_to_value!(config, env_config, otlp_config_traces_enabled); merge_option_to_value!( @@ -938,6 +947,7 @@ mod tests { otlp_config_metrics_summaries_mode: Some("quantiles".to_string()), otlp_config_traces_probabilistic_sampler_sampling_percentage: Some(50), otlp_config_logs_enabled: true, + statsd_metric_namespace: None, api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" .to_string(), kms_api_key: "test-kms-key".to_string(), diff --git a/mod.rs b/mod.rs index 66fd0054..9e55cc07 100644 --- a/mod.rs +++ b/mod.rs @@ -298,6 +298,7 @@ pub struct Config { // Metrics pub metrics_config_compression_level: i32, + pub statsd_metric_namespace: Option, // OTLP // @@ -411,6 +412,7 @@ impl Default for Config { // Metrics metrics_config_compression_level: 3, + statsd_metric_namespace: None, // OTLP otlp_config_traces_enabled: true, @@ -699,6 +701,39 @@ where } } +fn validate_metric_namespace(namespace: &str) -> Option { + let trimmed = namespace.trim(); + if trimmed.is_empty() { + return None; + } + + let mut chars = trimmed.chars(); + + if let Some(first_char) = chars.next() { + if !first_char.is_ascii_alphabetic() { + error!( + "DD_STATSD_METRIC_NAMESPACE must start with a letter, got: '{}'. Ignoring namespace.", + trimmed + ); + return None; + } + } else { + return None; + } + + if let Some(invalid_char) = + chars.find(|&ch| !ch.is_ascii_alphanumeric() && ch != '_' && ch != '.') + { + error!( + "DD_STATSD_METRIC_NAMESPACE contains invalid character '{}' in '{}'. Only ASCII alphanumerics, underscores, and periods are allowed. Ignoring namespace.", + invalid_char, trimmed + ); + return None; + } + + Some(trimmed.to_string()) +} + pub fn deserialize_option_lossless<'de, D, T>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -1476,4 +1511,53 @@ pub mod tests { expected.insert("valid".to_string(), "tag".to_string()); assert_eq!(result.tags, expected); } + + #[test] + fn test_validate_metric_namespace_valid() { + assert_eq!( + validate_metric_namespace("myapp"), + Some("myapp".to_string()) + ); + assert_eq!( + validate_metric_namespace("my_app"), + Some("my_app".to_string()) + ); + assert_eq!( + validate_metric_namespace("my.app"), + Some("my.app".to_string()) + ); + assert_eq!( + validate_metric_namespace("MyApp123"), + Some("MyApp123".to_string()) + ); + assert_eq!( + validate_metric_namespace(" myapp "), + Some("myapp".to_string()) + ); + } + + #[test] + fn test_validate_metric_namespace_empty() { + assert_eq!(validate_metric_namespace(""), None); + assert_eq!(validate_metric_namespace(" "), None); + assert_eq!(validate_metric_namespace("\t\n"), None); + } + + #[test] + fn test_validate_metric_namespace_invalid_first_char() { + assert_eq!(validate_metric_namespace("1myapp"), None); + assert_eq!(validate_metric_namespace("_myapp"), None); + assert_eq!(validate_metric_namespace(".myapp"), None); + assert_eq!(validate_metric_namespace("-myapp"), None); + } + + #[test] + fn test_validate_metric_namespace_invalid_chars() { + assert_eq!(validate_metric_namespace("my-app"), None); + assert_eq!(validate_metric_namespace("my app"), None); + assert_eq!(validate_metric_namespace("my@app"), None); + assert_eq!(validate_metric_namespace("my#app"), None); + assert_eq!(validate_metric_namespace("my$app"), None); + assert_eq!(validate_metric_namespace("my!app"), None); + } } diff --git a/yaml.rs b/yaml.rs index 4df1f85b..c800455f 100644 --- a/yaml.rs +++ b/yaml.rs @@ -993,6 +993,7 @@ api_security_sample_delay: 60 # Seconds apm_filter_tags_reject: None, apm_filter_tags_regex_require: None, apm_filter_tags_regex_reject: None, + statsd_metric_namespace: None, }; // Assert that From 2f2c69cb4578f8199776b605afcc7b5a35c1e13a Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Tue, 11 Nov 2025 17:23:51 -0500 Subject: [PATCH 096/112] refactor: Move metric namespace validation to dogstatsd util (#921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://datadoghq.atlassian.net/browse/SLES-2547 - Updates dependency to use centralized parse_metric_namespace function. - Removes duplicate code in favor of the shared implementation. Test: - Deploy the extension and config w/ [DD_STATSD_METRIC_NAMESPACE](https://us-east-1.console.aws.amazon.com/lambda/home?region=us-east-1#/functions/ltn-fullinstrument-bn-10bst-node22-lambda?subtab=envVars&tab=configure) image - Invoke the function and expect to see the metric using this custom prefix namespace Screenshot 2025-11-11 at 4 59 57 PM [Metric link](https://ddserverless.datadoghq.com/metric/explorer?fromUser=true&graph_layout=stacked&start=1762897808375&end=1762898083375&paused=true#N4Ig7glgJg5gpgFxALlAGwIYE8D2BXJVEADxQEYAaELcqyKBAC1pEbghkcLIF8qo4AMwgA7CAgg4RKUAiwAHOChASAtnADOcAE4RNIKtrgBHPJoQaUAbVBGN8qVoD6gnNtUZCKiOq279VKY6epbINiAiGOrKQdpYZAYgUJ4YThr42gDGSsgg6gi6mZaBZnHKGABuMMhsaGg4YG5oUAB0WmiCLapS4m6iMMAAVDwgPAC6VBpyaDmg8hgzCAg5STgwTpmYGhoQmYloonBOcorK6QdQ+4dO9EzKIm4eaKP8EPaYWMcKKwciSuM8Pggd7iADCUmEMBQIjwdR4QA) --- env.rs | 3 ++- mod.rs | 82 ---------------------------------------------------------- 2 files changed, 2 insertions(+), 83 deletions(-) diff --git a/env.rs b/env.rs index 40783737..f0b6c075 100644 --- a/env.rs +++ b/env.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use std::time::Duration; use datadog_trace_obfuscation::replacer::ReplaceRule; +use dogstatsd::util::parse_metric_namespace; use crate::{ config::{ @@ -527,7 +528,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option_to_value!(config, env_config, metrics_config_compression_level); if let Some(namespace) = &env_config.statsd_metric_namespace { - config.statsd_metric_namespace = super::validate_metric_namespace(namespace); + config.statsd_metric_namespace = parse_metric_namespace(namespace); } // OTLP diff --git a/mod.rs b/mod.rs index 9e55cc07..369e6ee7 100644 --- a/mod.rs +++ b/mod.rs @@ -701,39 +701,6 @@ where } } -fn validate_metric_namespace(namespace: &str) -> Option { - let trimmed = namespace.trim(); - if trimmed.is_empty() { - return None; - } - - let mut chars = trimmed.chars(); - - if let Some(first_char) = chars.next() { - if !first_char.is_ascii_alphabetic() { - error!( - "DD_STATSD_METRIC_NAMESPACE must start with a letter, got: '{}'. Ignoring namespace.", - trimmed - ); - return None; - } - } else { - return None; - } - - if let Some(invalid_char) = - chars.find(|&ch| !ch.is_ascii_alphanumeric() && ch != '_' && ch != '.') - { - error!( - "DD_STATSD_METRIC_NAMESPACE contains invalid character '{}' in '{}'. Only ASCII alphanumerics, underscores, and periods are allowed. Ignoring namespace.", - invalid_char, trimmed - ); - return None; - } - - Some(trimmed.to_string()) -} - pub fn deserialize_option_lossless<'de, D, T>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -1511,53 +1478,4 @@ pub mod tests { expected.insert("valid".to_string(), "tag".to_string()); assert_eq!(result.tags, expected); } - - #[test] - fn test_validate_metric_namespace_valid() { - assert_eq!( - validate_metric_namespace("myapp"), - Some("myapp".to_string()) - ); - assert_eq!( - validate_metric_namespace("my_app"), - Some("my_app".to_string()) - ); - assert_eq!( - validate_metric_namespace("my.app"), - Some("my.app".to_string()) - ); - assert_eq!( - validate_metric_namespace("MyApp123"), - Some("MyApp123".to_string()) - ); - assert_eq!( - validate_metric_namespace(" myapp "), - Some("myapp".to_string()) - ); - } - - #[test] - fn test_validate_metric_namespace_empty() { - assert_eq!(validate_metric_namespace(""), None); - assert_eq!(validate_metric_namespace(" "), None); - assert_eq!(validate_metric_namespace("\t\n"), None); - } - - #[test] - fn test_validate_metric_namespace_invalid_first_char() { - assert_eq!(validate_metric_namespace("1myapp"), None); - assert_eq!(validate_metric_namespace("_myapp"), None); - assert_eq!(validate_metric_namespace(".myapp"), None); - assert_eq!(validate_metric_namespace("-myapp"), None); - } - - #[test] - fn test_validate_metric_namespace_invalid_chars() { - assert_eq!(validate_metric_namespace("my-app"), None); - assert_eq!(validate_metric_namespace("my app"), None); - assert_eq!(validate_metric_namespace("my@app"), None); - assert_eq!(validate_metric_namespace("my#app"), None); - assert_eq!(validate_metric_namespace("my$app"), None); - assert_eq!(validate_metric_namespace("my!app"), None); - } } From 3c44ef624ac5cb285aa5255f52725aaf0a37000a Mon Sep 17 00:00:00 2001 From: jchrostek-dd Date: Wed, 19 Nov 2025 14:58:30 -0500 Subject: [PATCH 097/112] [SVLS-7704] add support for SSM Parameter API key (#924) ## Overview * Add support for customers storing Datadog API Key in SSM Parameter Store. ## Testing * Deployed changes and confirmed this work with Parameter Store String and SecureString. --- env.rs | 7 +++++++ mod.rs | 2 ++ yaml.rs | 1 + 3 files changed, 10 insertions(+) diff --git a/env.rs b/env.rs index f0b6c075..3b1a0bcc 100644 --- a/env.rs +++ b/env.rs @@ -357,6 +357,11 @@ pub struct EnvConfig { /// The AWS KMS API key to use for the Datadog Agent. #[serde(deserialize_with = "deserialize_optional_string")] pub kms_api_key: Option, + /// @env `DD_API_KEY_SSM_ARN` + /// + /// The AWS Systems Manager Parameter Store parameter ARN containing the Datadog API key. + #[serde(deserialize_with = "deserialize_optional_string")] + pub api_key_ssm_arn: Option, /// @env `DD_SERVERLESS_LOGS_ENABLED` /// /// Enable logs for AWS Lambda. Default is `true`. @@ -609,6 +614,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { // AWS Lambda merge_string!(config, env_config, api_key_secret_arn); merge_string!(config, env_config, kms_api_key); + merge_string!(config, env_config, api_key_ssm_arn); merge_option_to_value!(config, env_config, serverless_logs_enabled); merge_option_to_value!(config, env_config, serverless_flush_strategy); merge_option_to_value!(config, env_config, enhanced_metrics); @@ -952,6 +958,7 @@ mod tests { api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" .to_string(), kms_api_key: "test-kms-key".to_string(), + api_key_ssm_arn: String::default(), serverless_logs_enabled: false, serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { interval: 60000, diff --git a/mod.rs b/mod.rs index 369e6ee7..86366184 100644 --- a/mod.rs +++ b/mod.rs @@ -337,6 +337,7 @@ pub struct Config { // AWS Lambda pub api_key_secret_arn: String, pub kms_api_key: String, + pub api_key_ssm_arn: String, pub serverless_logs_enabled: bool, pub serverless_flush_strategy: FlushStrategy, pub enhanced_metrics: bool, @@ -440,6 +441,7 @@ impl Default for Config { // AWS Lambda api_key_secret_arn: String::default(), kms_api_key: String::default(), + api_key_ssm_arn: String::default(), serverless_logs_enabled: true, serverless_flush_strategy: FlushStrategy::Default, enhanced_metrics: true, diff --git a/yaml.rs b/yaml.rs index c800455f..022d0066 100644 --- a/yaml.rs +++ b/yaml.rs @@ -972,6 +972,7 @@ api_security_sample_delay: 60 # Seconds api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" .to_string(), kms_api_key: "test-kms-key".to_string(), + api_key_ssm_arn: String::default(), serverless_logs_enabled: false, serverless_flush_strategy: FlushStrategy::Periodically(PeriodicStrategy { interval: 60000, From 5dedfa12d2cc371b9202c01dd533444e13d460f5 Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Mon, 24 Nov 2025 10:15:37 -0500 Subject: [PATCH 098/112] feat: Add support for DD_LOGS_ENABLED as alias for DD_SERVERLESS_LOGS_ENABLED (#928) https://datadoghq.atlassian.net/browse/SVLS-7818 ## Overview Add DD_LOGS_ENABLED environment variable and YAML config option as an alias for DD_SERVERLESS_LOGS_ENABLED. Both variables now use OR logic, meaning logs are enabled if either variable is set to true. Changes: - Add logs_enabled field to EnvConfig and YamlConfig structs - Implement OR logic in merge_config functions: logs are enabled if either DD_LOGS_ENABLED or DD_SERVERLESS_LOGS_ENABLED is true - Add comprehensive test coverage with 9 test cases covering all combinations of the two variables - Maintain backward compatibility with existing configurations - Default value remains true when neither variable is set ## Testing Set DD_LOGS_ENABLED and DD_SERVERLESS_LOGS_ENABLED to false and expect: - [Log can be found in AWS console](https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups/log-group/$252Faws$252Flambda$252Fltn-fullinstrument-bn-cold-node22-lambda/log-events/2025$252F11$252F13$252F$255B$2524LATEST$255D455478dcbc944055b5be933e2e099f6a$3FfilterPattern$3DREPORT+RequestId) - [Log could NOT be found in DD console](https://ddserverless.datadoghq.com/logs?query=source%3Alambda%20%40lambda.arn%3A%22arn%3Aaws%3Alambda%3Aus-east-1%3A425362996713%3Afunction%3Altn-fullinstrument-bn-cold-node22-lambda%22%20AND%20%22REPORT%20RequestId%22&agg_m=count&agg_m_source=base&agg_t=count&clustering_pattern_field_path=message&cols=host%2Cservice%2C%40lambda.request_id&fromUser=true&messageDisplay=inline&refresh_mode=paused&storage=hot&stream_sort=desc&viz=stream&from_ts=1763063694206&to_ts=1763065424700&live=false) Otherwise the log should be available in DD console. --- env.rs | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ yaml.rs | 10 +++- 2 files changed, 180 insertions(+), 1 deletion(-) diff --git a/env.rs b/env.rs index 3b1a0bcc..c4ed5cd4 100644 --- a/env.rs +++ b/env.rs @@ -367,6 +367,11 @@ pub struct EnvConfig { /// Enable logs for AWS Lambda. Default is `true`. #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub serverless_logs_enabled: Option, + /// @env `DD_LOGS_ENABLED` + /// + /// Enable logs for AWS Lambda. Alias for `DD_SERVERLESS_LOGS_ENABLED`. Default is `true`. + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub logs_enabled: Option, /// @env `DD_SERVERLESS_FLUSH_STRATEGY` /// /// The flush strategy to use for AWS Lambda. @@ -616,6 +621,13 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_string!(config, env_config, kms_api_key); merge_string!(config, env_config, api_key_ssm_arn); merge_option_to_value!(config, env_config, serverless_logs_enabled); + + // Handle serverless_logs_enabled with OR logic: if either DD_LOGS_ENABLED or DD_SERVERLESS_LOGS_ENABLED is true, enable logs + if env_config.serverless_logs_enabled.is_some() || env_config.logs_enabled.is_some() { + config.serverless_logs_enabled = env_config.serverless_logs_enabled.unwrap_or(false) + || env_config.logs_enabled.unwrap_or(false); + } + merge_option_to_value!(config, env_config, serverless_flush_strategy); merge_option_to_value!(config, env_config, enhanced_metrics); merge_option_to_value!(config, env_config, lambda_proc_enhanced_metrics); @@ -981,4 +993,163 @@ mod tests { Ok(()) }); } + + #[test] + fn test_dd_logs_enabled_true() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_LOGS_ENABLED", "true"); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + assert!(config.serverless_logs_enabled); + Ok(()) + }); + } + + #[test] + fn test_dd_logs_enabled_false() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_LOGS_ENABLED", "false"); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + assert!(!config.serverless_logs_enabled); + Ok(()) + }); + } + + #[test] + fn test_dd_serverless_logs_enabled_true() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + assert!(config.serverless_logs_enabled); + Ok(()) + }); + } + + #[test] + fn test_dd_serverless_logs_enabled_false() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + assert!(!config.serverless_logs_enabled); + Ok(()) + }); + } + + #[test] + fn test_both_logs_enabled_true() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_LOGS_ENABLED", "true"); + jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + assert!(config.serverless_logs_enabled); + Ok(()) + }); + } + + #[test] + fn test_both_logs_enabled_false() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_LOGS_ENABLED", "false"); + jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + assert!(!config.serverless_logs_enabled); + Ok(()) + }); + } + + #[test] + fn test_logs_enabled_true_serverless_logs_enabled_false() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_LOGS_ENABLED", "true"); + jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "false"); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + // OR logic: if either is true, logs are enabled + assert!(config.serverless_logs_enabled); + Ok(()) + }); + } + + #[test] + fn test_logs_enabled_false_serverless_logs_enabled_true() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_LOGS_ENABLED", "false"); + jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + // OR logic: if either is true, logs are enabled + assert!(config.serverless_logs_enabled); + Ok(()) + }); + } + + #[test] + fn test_neither_logs_enabled_set_uses_default() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + // Default value is true + assert!(config.serverless_logs_enabled); + Ok(()) + }); + } } diff --git a/yaml.rs b/yaml.rs index 022d0066..cddaeb32 100644 --- a/yaml.rs +++ b/yaml.rs @@ -101,6 +101,8 @@ pub struct YamlConfig { pub kms_api_key: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub serverless_logs_enabled: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub logs_enabled: Option, pub serverless_flush_strategy: Option, #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub enhanced_metrics: Option, @@ -671,7 +673,13 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { // AWS Lambda merge_string!(config, yaml_config, api_key_secret_arn); merge_string!(config, yaml_config, kms_api_key); - merge_option_to_value!(config, yaml_config, serverless_logs_enabled); + + // Handle serverless_logs_enabled with OR logic: if either logs_enabled or serverless_logs_enabled is true, enable logs + if yaml_config.serverless_logs_enabled.is_some() || yaml_config.logs_enabled.is_some() { + config.serverless_logs_enabled = yaml_config.serverless_logs_enabled.unwrap_or(false) + || yaml_config.logs_enabled.unwrap_or(false); + } + merge_option_to_value!(config, yaml_config, serverless_flush_strategy); merge_option_to_value!(config, yaml_config, enhanced_metrics); merge_option_to_value!(config, yaml_config, lambda_proc_enhanced_metrics); From 3409263f4d6288974c86cd9b0409da01b9e48fe7 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 25 Nov 2025 13:36:16 -0500 Subject: [PATCH 099/112] chore: Upgrade libdatadog and construct http client for traces (#917) Upgrade libdatadog. Including: - Rename a few creates: - `ddcommon` -> `libdd-common` - `datadog-trace-protobuf` -> `libdd-trace-protobuf` - `datadog-trace-utils` -> `libdd-trace-utils` - `datadog-trace-normalization` -> `libdd-trace-normalization` - `datadog-trace-stats` -> `libdd-trace-stats` - Use the new API to send traces, which takes in an http_client object instead of proxy url string GitHub issue: https://github.com/DataDog/datadog-lambda-extension/issues/860 Jira: https://datadoghq.atlassian.net/browse/SLES-2499 Slack discussion: https://dd.slack.com/archives/C01TCF143GB/p1762526199549409 --- mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod.rs b/mod.rs index 86366184..df527a67 100644 --- a/mod.rs +++ b/mod.rs @@ -11,7 +11,7 @@ pub mod trace_propagation_style; pub mod yaml; use datadog_trace_obfuscation::replacer::ReplaceRule; -use datadog_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; +use libdd_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; use serde::{Deserialize, Deserializer}; use serde_aux::prelude::deserialize_bool_from_anything; From 13d5bb1b623daa823ce9db4bd9eb61dd29461b22 Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Mon, 1 Dec 2025 12:06:38 -0500 Subject: [PATCH 100/112] Merge Lambda Managed Instance feature branch (#947) https://datadoghq.atlassian.net/browse/SVLS-8080 ## Overview Merge Lambda Managed Instance feature branch ## Testing Covered by individual commits Co-authored-by: shreyamalpani Co-authored-by: duncanista <30836115+duncanista@users.noreply.github.com> Co-authored-by: astuyve Co-authored-by: jchrostek-dd Co-authored-by: tianning.li --- aws.rs | 11 +++++++++++ flush_strategy.rs | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/aws.rs b/aws.rs index 62100e47..d1af40a8 100644 --- a/aws.rs +++ b/aws.rs @@ -11,6 +11,9 @@ const AWS_LAMBDA_FUNCTION_NAME: &str = "AWS_LAMBDA_FUNCTION_NAME"; const AWS_LAMBDA_RUNTIME_API: &str = "AWS_LAMBDA_RUNTIME_API"; const AWS_LWA_LAMBDA_RUNTIME_API_PROXY: &str = "AWS_LWA_LAMBDA_RUNTIME_API_PROXY"; const AWS_LAMBDA_EXEC_WRAPPER: &str = "AWS_LAMBDA_EXEC_WRAPPER"; +const AWS_LAMBDA_INITIALIZATION_TYPE: &str = "AWS_LAMBDA_INITIALIZATION_TYPE"; + +pub const LAMBDA_MANAGED_INSTANCES_INIT_TYPE: &str = "lambda-managed-instances"; #[allow(clippy::module_name_repetitions)] #[derive(Debug, Clone)] @@ -21,6 +24,7 @@ pub struct AwsConfig { pub runtime_api: String, pub sandbox_init_time: Instant, pub exec_wrapper: Option, + pub initialization_type: String, } impl AwsConfig { @@ -33,8 +37,15 @@ impl AwsConfig { runtime_api: env::var(AWS_LAMBDA_RUNTIME_API).unwrap_or_default(), sandbox_init_time: start_time, exec_wrapper: env::var(AWS_LAMBDA_EXEC_WRAPPER).ok(), + initialization_type: env::var(AWS_LAMBDA_INITIALIZATION_TYPE).unwrap_or_default(), } } + + #[must_use] + pub fn is_managed_instance_mode(&self) -> bool { + self.initialization_type + .eq(LAMBDA_MANAGED_INSTANCES_INIT_TYPE) + } } #[allow(clippy::module_name_repetitions)] diff --git a/flush_strategy.rs b/flush_strategy.rs index 51e9710e..0a09e822 100644 --- a/flush_strategy.rs +++ b/flush_strategy.rs @@ -21,6 +21,20 @@ pub enum FlushStrategy { Continuously(PeriodicStrategy), } +impl FlushStrategy { + /// Returns the name of the flush strategy as a string slice. + #[must_use] + pub const fn name(&self) -> &'static str { + match self { + FlushStrategy::Default => "default", + FlushStrategy::End => "end", + FlushStrategy::Periodically(_) => "periodically", + FlushStrategy::EndPeriodically(_) => "end-periodically", + FlushStrategy::Continuously(_) => "continuously", + } + } +} + // A restricted subset of `FlushStrategy`. The Default strategy is now allowed, which is required to be // translated into a concrete strategy. #[allow(clippy::module_name_repetitions)] @@ -121,4 +135,34 @@ mod tests { let flush_strategy: FlushStrategy = serde_json::from_str("\"end,invalid\"").unwrap(); assert_eq!(flush_strategy, FlushStrategy::Default); } + + #[test] + fn test_flush_strategy_name_default() { + let strategy = FlushStrategy::Default; + assert_eq!(strategy.name(), "default"); + } + + #[test] + fn test_flush_strategy_name_end() { + let strategy = FlushStrategy::End; + assert_eq!(strategy.name(), "end"); + } + + #[test] + fn test_flush_strategy_name_periodically() { + let strategy = FlushStrategy::Periodically(PeriodicStrategy { interval: 1000 }); + assert_eq!(strategy.name(), "periodically"); + } + + #[test] + fn test_flush_strategy_name_end_periodically() { + let strategy = FlushStrategy::EndPeriodically(PeriodicStrategy { interval: 2000 }); + assert_eq!(strategy.name(), "end-periodically"); + } + + #[test] + fn test_flush_strategy_name_continuously() { + let strategy = FlushStrategy::Continuously(PeriodicStrategy { interval: 30000 }); + assert_eq!(strategy.name(), "continuously"); + } } From 90acf2ad8b58bda3184128d2fd7a76ecf3d0e98a Mon Sep 17 00:00:00 2001 From: Tianning Li Date: Thu, 4 Dec 2025 14:27:18 -0500 Subject: [PATCH 101/112] fix(config): support colons in tag values (URLs, etc.) (#953) https://datadoghq.atlassian.net/browse/SVLS-8095 ## Overview Tag parsing previously used split(':') which broke values containing colons like URLs (git.repository_url:https://...). Changed to usesplitn(2, ':') to split only on the first colon, preserving the rest as the value. Changes: - Add parse_key_value_tag() helper to centralize parsing logic - Refactor deserialize_key_value_pairs to use helper - Refactor deserialize_key_value_pair_array_to_hashmap to use helper - Add comprehensive test coverage for URL values and edge cases ## Testing unit test and expect e2e tests to pass Co-authored-by: tianning.li --- mod.rs | 109 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 93 insertions(+), 16 deletions(-) diff --git a/mod.rs b/mod.rs index df527a67..336fcd19 100644 --- a/mod.rs +++ b/mod.rs @@ -535,6 +535,21 @@ where } } +/// Parse a single "key:value" string into a (key, value) tuple +/// Returns None if the string is invalid (e.g., missing colon, empty key/value) +fn parse_key_value_tag(tag: &str) -> Option<(String, String)> { + let parts: Vec<&str> = tag.splitn(2, ':').collect(); + if parts.len() == 2 && !parts[0].is_empty() && !parts[1].is_empty() { + Some((parts[0].to_string(), parts[1].to_string())) + } else { + error!( + "Failed to parse tag '{}', expected format 'key:value', ignoring", + tag + ); + None + } +} + pub fn deserialize_key_value_pairs<'de, D>( deserializer: D, ) -> Result, D::Error> @@ -559,14 +574,8 @@ where if tag.is_empty() { continue; } - let parts = tag.split(':').collect::>(); - if parts.len() == 2 && !parts[0].is_empty() && !parts[1].is_empty() { - map.insert(parts[0].to_string(), parts[1].to_string()); - } else { - error!( - "Failed to parse tag '{}', expected format 'key:value', ignoring", - tag - ); + if let Some((key, val)) = parse_key_value_tag(tag) { + map.insert(key, val); } } @@ -643,14 +652,8 @@ where let array: Vec = Vec::deserialize(deserializer)?; let mut map = HashMap::new(); for s in array { - let parts = s.split(':').collect::>(); - if parts.len() == 2 { - map.insert(parts[0].to_string(), parts[1].to_string()); - } else { - error!( - "Failed to parse tag '{}', expected format 'key:value', ignoring", - s.trim() - ); + if let Some((key, val)) = parse_key_value_tag(&s) { + map.insert(key, val); } } Ok(map) @@ -1480,4 +1483,78 @@ pub mod tests { expected.insert("valid".to_string(), "tag".to_string()); assert_eq!(result.tags, expected); } + + #[test] + fn test_deserialize_key_value_pairs_with_url_values() { + #[derive(Deserialize, Debug, PartialEq)] + struct TestStruct { + #[serde(deserialize_with = "deserialize_key_value_pairs")] + tags: HashMap, + } + + let result = serde_json::from_str::( + r#"{"tags": "git.repository_url:https://gitlab.ddbuild.io/DataDog/serverless-e2e-tests.git,env:prod"}"# + ) + .expect("failed to parse JSON"); + let mut expected = HashMap::new(); + expected.insert( + "git.repository_url".to_string(), + "https://gitlab.ddbuild.io/DataDog/serverless-e2e-tests.git".to_string(), + ); + expected.insert("env".to_string(), "prod".to_string()); + assert_eq!(result.tags, expected); + } + + #[test] + fn test_deserialize_key_value_pair_array_with_urls() { + #[derive(Deserialize, Debug, PartialEq)] + struct TestStruct { + #[serde(deserialize_with = "deserialize_key_value_pair_array_to_hashmap")] + tags: HashMap, + } + + let result = serde_json::from_str::( + r#"{"tags": ["git.repository_url:https://gitlab.ddbuild.io/DataDog/serverless-e2e-tests.git", "env:prod", "version:1.2.3"]}"# + ) + .expect("failed to parse JSON"); + let mut expected = HashMap::new(); + expected.insert( + "git.repository_url".to_string(), + "https://gitlab.ddbuild.io/DataDog/serverless-e2e-tests.git".to_string(), + ); + expected.insert("env".to_string(), "prod".to_string()); + expected.insert("version".to_string(), "1.2.3".to_string()); + assert_eq!(result.tags, expected); + } + + #[test] + fn test_deserialize_key_value_pair_array_ignores_invalid() { + #[derive(Deserialize, Debug, PartialEq)] + struct TestStruct { + #[serde(deserialize_with = "deserialize_key_value_pair_array_to_hashmap")] + tags: HashMap, + } + + let result = serde_json::from_str::( + r#"{"tags": ["valid:tag", "invalid_no_colon", "another:good:value:with:colons"]}"#, + ) + .expect("failed to parse JSON"); + let mut expected = HashMap::new(); + expected.insert("valid".to_string(), "tag".to_string()); + expected.insert("another".to_string(), "good:value:with:colons".to_string()); + assert_eq!(result.tags, expected); + } + + #[test] + fn test_deserialize_key_value_pair_array_empty() { + #[derive(Deserialize, Debug, PartialEq)] + struct TestStruct { + #[serde(deserialize_with = "deserialize_key_value_pair_array_to_hashmap")] + tags: HashMap, + } + + let result = + serde_json::from_str::(r#"{"tags": []}"#).expect("failed to parse JSON"); + assert_eq!(result.tags, HashMap::new()); + } } From 155a5a345eecf0e6653da5f030282b5f9732f735 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Mon, 15 Dec 2025 11:27:15 -0500 Subject: [PATCH 102/112] [SVLS-7934] feat: Support TLS certificate for trace/stats flusher (#961) ## Problem A customer reported that their Lambda is behind a proxy, and the Rust-based extension can't send traces to Datadog via the proxy, while the previous go-based extension worked. ## This PR Supports the env var `DD_TLS_CERT_FILE`: The path to a file of concatenated CA certificates in PEM format. Example: `DD_TLS_CERT_FILE=/opt/ca-cert.pem`, so the when the extension flushes traces/stats to Datadog, the HTTP client created can load and use this cert, and connect the proxy properly. ## Testing ### Steps 1. Create a Lambda in a VPC with an NGINX proxy. 2. Add a layer to the Lambda, which includes the CA certificate `ca-cert.pem` 3. Set env vars: - `DD_TLS_CERT_FILE=/opt/ca-cert.pem` - `DD_PROXY_HTTPS=http://10.0.0.30:3128`, where `10.0.0.30` is the private IP of the proxy EC2 instance - `DD_LOG_LEVEL=debug` 4. Update routing rules of security groups so the Lambda can reach `http://10.0.0.30:3128` 5. Invoke the Lambda ### Result **Before** Trace flush failed with error logs: > DD_EXTENSION | ERROR | Max retries exceeded, returning request error error=Network error: client error (Connect) attempts=1 DD_EXTENSION | ERROR | TRACES | Request failed: No requests sent **After** Trace flush is successful: > DD_EXTENSION | DEBUG | TRACES | Flushing 1 traces DD_EXTENSION | DEBUG | TRACES | Added root certificate from /opt/ca-cert.pem DD_EXTENSION | DEBUG | TRACES | Proxy connector created with proxy: Some("http://10.0.0.30:3128") DD_EXTENSION | DEBUG | Sending with retry url=https://trace.agent.datadoghq.com/api/v0.2/traces payload_size=1120 max_retries=1 DD_EXTENSION | DEBUG | Received response status=202 Accepted attempt=1 DD_EXTENSION | DEBUG | Request succeeded status=202 Accepted attempts=1 DD_EXTENSION | DEBUG | TRACES | Flushing took 1609 ms ## Notes This fix only covers trace flusher and stats flusher, which use `ServerlessTraceFlusher::get_http_client()` to create the HTTP client. It doesn't cover logs flusher and proxy flusher, which use a different function (http.rs:get_client()) to create the HTTP client. However, logs flushing was successful in my tests, even if no certificate was added. We can come back to logs/proxy flusher if someone reports an error. --- env.rs | 8 ++++++++ mod.rs | 2 ++ yaml.rs | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/env.rs b/env.rs index c4ed5cd4..04d4e27e 100644 --- a/env.rs +++ b/env.rs @@ -75,6 +75,11 @@ pub struct EnvConfig { /// The transport type to use for sending logs. Possible values are "auto" or "http1". #[serde(deserialize_with = "deserialize_optional_string")] pub http_protocol: Option, + /// @env `DD_TLS_CERT_FILE` + /// The path to a file of concatenated CA certificates in PEM format. + /// Example: `/opt/ca-cert.pem` + #[serde(deserialize_with = "deserialize_optional_string")] + pub tls_cert_file: Option, // Metrics /// @env `DD_DD_URL` @@ -466,6 +471,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option!(config, env_config, proxy_https); merge_vec!(config, env_config, proxy_no_proxy); merge_option!(config, env_config, http_protocol); + merge_option!(config, env_config, tls_cert_file); // Endpoints merge_string!(config, env_config, dd_url); @@ -695,6 +701,7 @@ mod tests { jail.set_env("DD_PROXY_HTTPS", "https://proxy.example.com"); jail.set_env("DD_PROXY_NO_PROXY", "localhost,127.0.0.1"); jail.set_env("DD_HTTP_PROTOCOL", "http1"); + jail.set_env("DD_TLS_CERT_FILE", "/opt/ca-cert.pem"); // Metrics jail.set_env("DD_DD_URL", "https://metrics.datadoghq.com"); @@ -850,6 +857,7 @@ mod tests { proxy_https: Some("https://proxy.example.com".to_string()), proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), + tls_cert_file: Some("/opt/ca-cert.pem".to_string()), dd_url: "https://metrics.datadoghq.com".to_string(), url: "https://app.datadoghq.com".to_string(), additional_endpoints: HashMap::from([ diff --git a/mod.rs b/mod.rs index 336fcd19..9c1ecdd4 100644 --- a/mod.rs +++ b/mod.rs @@ -252,6 +252,7 @@ pub struct Config { pub proxy_https: Option, pub proxy_no_proxy: Vec, pub http_protocol: Option, + pub tls_cert_file: Option, // Endpoints pub dd_url: String, @@ -366,6 +367,7 @@ impl Default for Config { proxy_https: None, proxy_no_proxy: vec![], http_protocol: None, + tls_cert_file: None, // Endpoints dd_url: String::default(), diff --git a/yaml.rs b/yaml.rs index cddaeb32..5dbc9536 100644 --- a/yaml.rs +++ b/yaml.rs @@ -53,6 +53,8 @@ pub struct YamlConfig { pub dd_url: Option, #[serde(deserialize_with = "deserialize_optional_string")] pub http_protocol: Option, + #[serde(deserialize_with = "deserialize_optional_string")] + pub tls_cert_file: Option, // Endpoints #[serde(deserialize_with = "deserialize_additional_endpoints")] @@ -417,6 +419,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option!(config, proxy_https, yaml_config.proxy, https); merge_option_to_value!(config, proxy_no_proxy, yaml_config.proxy, no_proxy); merge_option!(config, yaml_config, http_protocol); + merge_option!(config, yaml_config, tls_cert_file); // Endpoints merge_hashmap!(config, yaml_config, additional_endpoints); @@ -747,6 +750,7 @@ proxy: no_proxy: ["localhost", "127.0.0.1"] dd_url: "https://metrics.datadoghq.com" http_protocol: "http1" +tls_cert_file: "/opt/ca-cert.pem" # Endpoints additional_endpoints: @@ -882,6 +886,7 @@ api_security_sample_delay: 60 # Seconds proxy_https: Some("https://proxy.example.com".to_string()), proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), + tls_cert_file: Some("/opt/ca-cert.pem".to_string()), dd_url: "https://metrics.datadoghq.com".to_string(), url: String::new(), // doesnt exist in yaml additional_endpoints: HashMap::from([ From 4ab1ae4251fac3a04bd14938508b7dae8278d311 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Mon, 15 Dec 2025 15:40:34 -0500 Subject: [PATCH 103/112] chore: Upgrade libdatadog (#964) ## Overview The crate `datadog-trace-obfuscation` has been renamed as `libdd-trace-obfuscation`. This PR updates this dependency. ## Testing --- apm_replace_rule.rs | 2 +- env.rs | 4 ++-- mod.rs | 4 ++-- yaml.rs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apm_replace_rule.rs b/apm_replace_rule.rs index 0e49161c..41b13594 100644 --- a/apm_replace_rule.rs +++ b/apm_replace_rule.rs @@ -1,4 +1,4 @@ -use datadog_trace_obfuscation::replacer::{ReplaceRule, parse_rules_from_string}; +use libdd_trace_obfuscation::replacer::{ReplaceRule, parse_rules_from_string}; use serde::de::{Deserializer, SeqAccess, Visitor}; use serde::{Deserialize, Serialize}; use serde_json; diff --git a/env.rs b/env.rs index 04d4e27e..af68a7a3 100644 --- a/env.rs +++ b/env.rs @@ -3,8 +3,8 @@ use serde::Deserialize; use std::collections::HashMap; use std::time::Duration; -use datadog_trace_obfuscation::replacer::ReplaceRule; use dogstatsd::util::parse_metric_namespace; +use libdd_trace_obfuscation::replacer::ReplaceRule; use crate::{ config::{ @@ -900,7 +900,7 @@ mod tests { )]), apm_dd_url: "https://apm.datadoghq.com".to_string(), apm_replace_tags: Some( - datadog_trace_obfuscation::replacer::parse_rules_from_string( + libdd_trace_obfuscation::replacer::parse_rules_from_string( r#"[{"name":"test-tag","pattern":"test-pattern","repl":"replacement"}]"#, ) .expect("Failed to parse replace rules"), diff --git a/mod.rs b/mod.rs index 9c1ecdd4..997e2580 100644 --- a/mod.rs +++ b/mod.rs @@ -10,7 +10,7 @@ pub mod service_mapping; pub mod trace_propagation_style; pub mod yaml; -use datadog_trace_obfuscation::replacer::ReplaceRule; +use libdd_trace_obfuscation::replacer::ReplaceRule; use libdd_trace_utils::config_utils::{trace_intake_url, trace_intake_url_prefixed}; use serde::{Deserialize, Deserializer}; @@ -772,7 +772,7 @@ pub fn deserialize_optional_duration_from_seconds_ignore_zero<'de, D: Deserializ #[cfg_attr(coverage_nightly, coverage(off))] // Test modules skew coverage metrics #[cfg(test)] pub mod tests { - use datadog_trace_obfuscation::replacer::parse_rules_from_string; + use libdd_trace_obfuscation::replacer::parse_rules_from_string; use super::*; diff --git a/yaml.rs b/yaml.rs index 5dbc9536..8075ef36 100644 --- a/yaml.rs +++ b/yaml.rs @@ -19,11 +19,11 @@ use crate::{ }, merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, }; -use datadog_trace_obfuscation::replacer::ReplaceRule; use figment::{ Figment, providers::{Format, Yaml}, }; +use libdd_trace_obfuscation::replacer::ReplaceRule; use serde::Deserialize; /// `YamlConfig` is a struct that represents some of the fields in the `datadog.yaml` file. From 2049503dcec71fc1b5c9911f89ebc6a134d32a29 Mon Sep 17 00:00:00 2001 From: Yiming Luo Date: Tue, 13 Jan 2026 13:11:50 -0500 Subject: [PATCH 104/112] [SVLS-8211] feat: Add timeout for requests to span_dedup_service (#986) ## Problem Span dedup service sometimes fails to return the result and thus logs the error: > DD_EXTENSION | ERROR | Failed to send check_and_add response: true I see this error in our Self Monitoring and a customer's account. Also I believe it causes extension to fail to receive traces from the tracer, causing missing traces. This is because the caller of span dedup is in `process_traces()`, which is the function that handles the tracer's HTTP request to send traces. If this function fails to get span dedup result and gets stuck, the HTTP request will time out. ## This PR While I don't yet know what causes the error, this PR adds a patch to mitigate the impact: 1. Change log level from `error` to `warn` 2. Add a timeout of 5 seconds to the span dedup check, so that if the caller doesn't get an answer soon, it defaults to treating the trace as not a duplicate, which is the most common case. ## Testing To merge this PR then check log in self monitoring, as it's hard to run high-volume tests in self monitoring from a non-main branch. --- env.rs | 10 ++++++++++ mod.rs | 2 ++ yaml.rs | 1 + 3 files changed, 13 insertions(+) diff --git a/env.rs b/env.rs index af68a7a3..0bbc199e 100644 --- a/env.rs +++ b/env.rs @@ -417,6 +417,13 @@ pub struct EnvConfig { /// Default is `false`. #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] pub compute_trace_stats_on_extension: Option, + /// @env `DD_SPAN_DEDUP_TIMEOUT` + /// + /// The timeout for the span deduplication service to check if a span key exists, in seconds. + /// For now, this is a temporary field added to debug the failure of `check_and_add()` in span dedup service. + /// Do not use this field extensively in production. + #[serde(deserialize_with = "deserialize_optional_duration_from_seconds_ignore_zero")] + pub span_dedup_timeout: Option, /// @env `DD_API_KEY_SECRET_RELOAD_INTERVAL` /// /// The interval at which the Datadog API key is reloaded, in seconds. @@ -640,6 +647,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_option_to_value!(config, env_config, capture_lambda_payload); merge_option_to_value!(config, env_config, capture_lambda_payload_max_depth); merge_option_to_value!(config, env_config, compute_trace_stats_on_extension); + merge_option!(config, env_config, span_dedup_timeout); merge_option!(config, env_config, api_key_secret_reload_interval); merge_option_to_value!(config, env_config, serverless_appsec_enabled); merge_option!(config, env_config, appsec_rules); @@ -835,6 +843,7 @@ mod tests { jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD", "true"); jail.set_env("DD_CAPTURE_LAMBDA_PAYLOAD_MAX_DEPTH", "5"); jail.set_env("DD_COMPUTE_TRACE_STATS_ON_EXTENSION", "true"); + jail.set_env("DD_SPAN_DEDUP_TIMEOUT", "5"); jail.set_env("DD_API_KEY_SECRET_RELOAD_INTERVAL", "10"); jail.set_env("DD_SERVERLESS_APPSEC_ENABLED", "true"); jail.set_env("DD_APPSEC_RULES", "/path/to/rules.json"); @@ -988,6 +997,7 @@ mod tests { capture_lambda_payload: true, capture_lambda_payload_max_depth: 5, compute_trace_stats_on_extension: true, + span_dedup_timeout: Some(Duration::from_secs(5)), api_key_secret_reload_interval: Some(Duration::from_secs(10)), serverless_appsec_enabled: true, appsec_rules: Some("/path/to/rules.json".to_string()), diff --git a/mod.rs b/mod.rs index 997e2580..48bc3c30 100644 --- a/mod.rs +++ b/mod.rs @@ -346,6 +346,7 @@ pub struct Config { pub capture_lambda_payload: bool, pub capture_lambda_payload_max_depth: u32, pub compute_trace_stats_on_extension: bool, + pub span_dedup_timeout: Option, pub api_key_secret_reload_interval: Option, pub serverless_appsec_enabled: bool, @@ -451,6 +452,7 @@ impl Default for Config { capture_lambda_payload: false, capture_lambda_payload_max_depth: 10, compute_trace_stats_on_extension: false, + span_dedup_timeout: None, api_key_secret_reload_interval: None, serverless_appsec_enabled: false, diff --git a/yaml.rs b/yaml.rs index 8075ef36..dbac692b 100644 --- a/yaml.rs +++ b/yaml.rs @@ -995,6 +995,7 @@ api_security_sample_delay: 60 # Seconds capture_lambda_payload: true, capture_lambda_payload_max_depth: 5, compute_trace_stats_on_extension: true, + span_dedup_timeout: None, api_key_secret_reload_interval: None, serverless_appsec_enabled: true, From 3ea5aca67f9151c0cda2be4befb3c4d768f855ee Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Mon, 9 Feb 2026 16:11:22 -0500 Subject: [PATCH 105/112] [SVLS-8150] fix(config): ensure logs intake URL is correctly prefixed (#1021) ## Overview Ensures `DD_LOGS_CONFIG_LOGS_DD_URL` is correctly prefixed with `https://` ## Testing Manually tested that logs get sent to alternate logs intake --- mod.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/mod.rs b/mod.rs index 48bc3c30..26481608 100644 --- a/mod.rs +++ b/mod.rs @@ -217,8 +217,11 @@ impl ConfigBuilder { } // If Logs URL is not set, set it to the default - if self.config.logs_config_logs_dd_url.is_empty() { + if self.config.logs_config_logs_dd_url.trim().is_empty() { self.config.logs_config_logs_dd_url = build_fqdn_logs(self.config.site.clone()); + } else { + self.config.logs_config_logs_dd_url = + logs_intake_url(self.config.logs_config_logs_dd_url.as_str()); } // If APM URL is not set, set it to the default @@ -481,6 +484,19 @@ fn build_fqdn_logs(site: String) -> String { format!("https://http-intake.logs.{site}") } +#[inline] +#[must_use] +fn logs_intake_url(url: &str) -> String { + let url = url.trim(); + if url.is_empty() { + return url.to_string(); + } + if url.starts_with("https://") || url.starts_with("http://") { + return url.to_string(); + } + format!("https://{url}") +} + pub fn deserialize_optional_string<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, @@ -811,7 +827,44 @@ pub mod tests { let config = get_config(Path::new("")); assert_eq!( config.logs_config_logs_dd_url, - "agent-http-intake-pci.logs.datadoghq.com:443".to_string() + "https://agent-http-intake-pci.logs.datadoghq.com:443".to_string() + ); + Ok(()) + }); + } + + #[test] + fn test_logs_intake_url_adds_prefix() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env( + "DD_LOGS_CONFIG_LOGS_DD_URL", + "dr-test-failover-http-intake.logs.datadoghq.com:443", + ); + + let config = get_config(Path::new("")); + // ensure host:port URL is prefixed with https:// + assert_eq!( + config.logs_config_logs_dd_url, + "https://dr-test-failover-http-intake.logs.datadoghq.com:443".to_string() + ); + Ok(()) + }); + } + + #[test] + fn test_prefixed_logs_intake_url() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env( + "DD_LOGS_CONFIG_LOGS_DD_URL", + "https://custom-intake.logs.datadoghq.com:443", + ); + + let config = get_config(Path::new("")); + assert_eq!( + config.logs_config_logs_dd_url, + "https://custom-intake.logs.datadoghq.com:443".to_string() ); Ok(()) }); From d2c3cd8bd48abf0788242c76ad528e70863c011b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Wed, 18 Feb 2026 14:11:22 -0800 Subject: [PATCH 106/112] chore(deps): upgrade dogstatsd (#1020) ## Overview Continuation of #1018 removing unnecessary mut lock on callers for dogstatsd --- env.rs | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ mod.rs | 19 ++++++++++++++++ yaml.rs | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+) diff --git a/env.rs b/env.rs index 0bbc199e..59f67357 100644 --- a/env.rs +++ b/env.rs @@ -277,6 +277,24 @@ pub struct EnvConfig { #[serde(deserialize_with = "deserialize_optional_string")] pub statsd_metric_namespace: Option, + /// @env `DD_DOGSTATSD_SO_RCVBUF` + /// Size of the receive buffer for `DogStatsD` UDP packets, in bytes (`SO_RCVBUF`). + /// Increase to reduce packet loss under high-throughput metric bursts. + #[serde(deserialize_with = "deserialize_option_lossless")] + pub dogstatsd_so_rcvbuf: Option, + + /// @env `DD_DOGSTATSD_BUFFER_SIZE` + /// Maximum size of a single read from any transport (UDP or named pipe), in bytes. + /// Defaults to 8192. + #[serde(deserialize_with = "deserialize_option_lossless")] + pub dogstatsd_buffer_size: Option, + + /// @env `DD_DOGSTATSD_QUEUE_SIZE` + /// Internal queue capacity between the socket reader and metric processor. + /// Defaults to 1024. Increase if the processor can't keep up with burst traffic. + #[serde(deserialize_with = "deserialize_option_lossless")] + pub dogstatsd_queue_size: Option, + // OTLP // // - APM / Traces @@ -554,6 +572,11 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { config.statsd_metric_namespace = parse_metric_namespace(namespace); } + // DogStatsD + merge_option!(config, env_config, dogstatsd_so_rcvbuf); + merge_option!(config, env_config, dogstatsd_buffer_size); + merge_option!(config, env_config, dogstatsd_queue_size); + // OTLP merge_option_to_value!(config, env_config, otlp_config_traces_enabled); merge_option_to_value!( @@ -830,6 +853,11 @@ mod tests { ); jail.set_env("DD_OTLP_CONFIG_LOGS_ENABLED", "true"); + // DogStatsD + jail.set_env("DD_DOGSTATSD_SO_RCVBUF", "1048576"); + jail.set_env("DD_DOGSTATSD_BUFFER_SIZE", "65507"); + jail.set_env("DD_DOGSTATSD_QUEUE_SIZE", "2048"); + // AWS Lambda jail.set_env( "DD_API_KEY_SECRET_ARN", @@ -984,6 +1012,9 @@ mod tests { otlp_config_traces_probabilistic_sampler_sampling_percentage: Some(50), otlp_config_logs_enabled: true, statsd_metric_namespace: None, + dogstatsd_so_rcvbuf: Some(1048576), + dogstatsd_buffer_size: Some(65507), + dogstatsd_queue_size: Some(2048), api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" .to_string(), kms_api_key: "test-kms-key".to_string(), @@ -1170,4 +1201,43 @@ mod tests { Ok(()) }); } + + #[test] + fn test_dogstatsd_config_from_env() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_DOGSTATSD_SO_RCVBUF", "1048576"); + jail.set_env("DD_DOGSTATSD_BUFFER_SIZE", "65507"); + jail.set_env("DD_DOGSTATSD_QUEUE_SIZE", "2048"); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + assert_eq!(config.dogstatsd_so_rcvbuf, Some(1048576)); + assert_eq!(config.dogstatsd_buffer_size, Some(65507)); + assert_eq!(config.dogstatsd_queue_size, Some(2048)); + Ok(()) + }); + } + + #[test] + fn test_dogstatsd_config_defaults_to_none() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + + let mut config = Config::default(); + let env_config_source = EnvConfigSource; + env_config_source + .load(&mut config) + .expect("Failed to load config"); + + assert_eq!(config.dogstatsd_so_rcvbuf, None); + assert_eq!(config.dogstatsd_buffer_size, None); + assert_eq!(config.dogstatsd_queue_size, None); + Ok(()) + }); + } } diff --git a/mod.rs b/mod.rs index 26481608..7ef0d40b 100644 --- a/mod.rs +++ b/mod.rs @@ -303,6 +303,17 @@ pub struct Config { // Metrics pub metrics_config_compression_level: i32, pub statsd_metric_namespace: Option, + /// Size of the receive buffer for `DogStatsD` UDP packets, in bytes (`SO_RCVBUF`). + /// Increase to reduce packet loss under high-throughput metric bursts. + /// If None, uses the OS default. + pub dogstatsd_so_rcvbuf: Option, + /// Maximum size of a single read from any transport (UDP or named pipe), in bytes. + /// Defaults to 8192. For UDP, the client must batch metrics into packets of + /// this size for the increase to take effect. + pub dogstatsd_buffer_size: Option, + /// Internal queue capacity between the socket reader and metric processor. + /// Defaults to 1024. Increase if the processor can't keep up with burst traffic. + pub dogstatsd_queue_size: Option, // OTLP // @@ -421,6 +432,14 @@ impl Default for Config { metrics_config_compression_level: 3, statsd_metric_namespace: None, + // DogStatsD + // Defaults to None, which uses the OS default. + dogstatsd_so_rcvbuf: None, + // Defaults to 8192 internally. + dogstatsd_buffer_size: None, + // Defaults to 1024 internally. + dogstatsd_queue_size: None, + // OTLP otlp_config_traces_enabled: true, otlp_config_traces_span_name_as_resource_name: false, diff --git a/yaml.rs b/yaml.rs index dbac692b..31278db8 100644 --- a/yaml.rs +++ b/yaml.rs @@ -93,6 +93,17 @@ pub struct YamlConfig { // Metrics pub metrics_config: MetricsConfig, + // DogStatsD + /// Size of the receive buffer for `DogStatsD` UDP packets, in bytes (`SO_RCVBUF`). + #[serde(deserialize_with = "deserialize_option_lossless")] + pub dogstatsd_so_rcvbuf: Option, + /// Maximum size of a single read from any transport (UDP or named pipe), in bytes. + #[serde(deserialize_with = "deserialize_option_lossless")] + pub dogstatsd_buffer_size: Option, + /// Internal queue capacity between the socket reader and metric processor. + #[serde(deserialize_with = "deserialize_option_lossless")] + pub dogstatsd_queue_size: Option, + // OTLP pub otlp_config: Option, @@ -477,6 +488,11 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { compression_level ); + // DogStatsD + merge_option!(config, yaml_config, dogstatsd_so_rcvbuf); + merge_option!(config, yaml_config, dogstatsd_buffer_size); + merge_option!(config, yaml_config, dogstatsd_queue_size); + // APM merge_hashmap!(config, yaml_config, service_mapping); merge_string!(config, apm_dd_url, yaml_config.apm_config, apm_dd_url); @@ -814,6 +830,10 @@ trace_aws_service_representation_enabled: true metrics_config: compression_level: 3 +dogstatsd_so_rcvbuf: 1048576 +dogstatsd_buffer_size: 65507 +dogstatsd_queue_size: 2048 + # OTLP otlp_config: receiver: @@ -1009,6 +1029,9 @@ api_security_sample_delay: 60 # Seconds apm_filter_tags_regex_require: None, apm_filter_tags_regex_reject: None, statsd_metric_namespace: None, + dogstatsd_so_rcvbuf: Some(1048576), + dogstatsd_buffer_size: Some(65507), + dogstatsd_queue_size: Some(2048), }; // Assert that @@ -1017,4 +1040,51 @@ api_security_sample_delay: 60 # Seconds Ok(()) }); } + + #[test] + fn test_yaml_dogstatsd_config() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file( + "datadog.yaml", + r#" +dogstatsd_so_rcvbuf: 524288 +dogstatsd_buffer_size: 16384 +dogstatsd_queue_size: 512 +"#, + )?; + let mut config = Config::default(); + let yaml_config_source = YamlConfigSource { + path: Path::new("datadog.yaml").to_path_buf(), + }; + yaml_config_source + .load(&mut config) + .expect("Failed to load config"); + + assert_eq!(config.dogstatsd_so_rcvbuf, Some(524288)); + assert_eq!(config.dogstatsd_buffer_size, Some(16384)); + assert_eq!(config.dogstatsd_queue_size, Some(512)); + Ok(()) + }); + } + + #[test] + fn test_yaml_dogstatsd_config_defaults_to_none() { + figment::Jail::expect_with(|jail| { + jail.clear_env(); + jail.create_file("datadog.yaml", "")?; + let mut config = Config::default(); + let yaml_config_source = YamlConfigSource { + path: Path::new("datadog.yaml").to_path_buf(), + }; + yaml_config_source + .load(&mut config) + .expect("Failed to load config"); + + assert_eq!(config.dogstatsd_so_rcvbuf, None); + assert_eq!(config.dogstatsd_buffer_size, None); + assert_eq!(config.dogstatsd_queue_size, None); + Ok(()) + }); + } } From ea26ede1d167233d2872edc38001db3d2ace7b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 19 Feb 2026 12:21:26 -0800 Subject: [PATCH 107/112] chore(deps): upgrade rust to `v1.93.1` (#1034) ## What? Upgrade rust to latest stable 1.93.1 ## Why? `time` vulnerability fix is only available on rust >= 1.88.0 --- env.rs | 4 ++-- mod.rs | 10 +++++----- yaml.rs | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/env.rs b/env.rs index 59f67357..9f66587a 100644 --- a/env.rs +++ b/env.rs @@ -1012,7 +1012,7 @@ mod tests { otlp_config_traces_probabilistic_sampler_sampling_percentage: Some(50), otlp_config_logs_enabled: true, statsd_metric_namespace: None, - dogstatsd_so_rcvbuf: Some(1048576), + dogstatsd_so_rcvbuf: Some(1_048_576), dogstatsd_buffer_size: Some(65507), dogstatsd_queue_size: Some(2048), api_key_secret_arn: "arn:aws:secretsmanager:region:account:secret:datadog-api-key" @@ -1216,7 +1216,7 @@ mod tests { .load(&mut config) .expect("Failed to load config"); - assert_eq!(config.dogstatsd_so_rcvbuf, Some(1048576)); + assert_eq!(config.dogstatsd_so_rcvbuf, Some(1_048_576)); assert_eq!(config.dogstatsd_buffer_size, Some(65507)); assert_eq!(config.dogstatsd_queue_size, Some(2048)); Ok(()) diff --git a/mod.rs b/mod.rs index 7ef0d40b..86e886a5 100644 --- a/mod.rs +++ b/mod.rs @@ -188,10 +188,10 @@ impl ConfigBuilder { // If `proxy_https` is not set, set it from `HTTPS_PROXY` environment variable // if it exists - if let Ok(https_proxy) = std::env::var("HTTPS_PROXY") { - if self.config.proxy_https.is_none() { - self.config.proxy_https = Some(https_proxy); - } + if let Ok(https_proxy) = std::env::var("HTTPS_PROXY") + && self.config.proxy_https.is_none() + { + self.config.proxy_https = Some(https_proxy); } // If `proxy_https` is set, check if the site is in `NO_PROXY` environment variable @@ -1008,7 +1008,7 @@ pub mod tests { TracePropagationStyle::TraceContext ], logs_config_logs_dd_url: "https://http-intake.logs.datadoghq.com".to_string(), - apm_dd_url: trace_intake_url("datadoghq.com").to_string(), + apm_dd_url: trace_intake_url("datadoghq.com").clone(), dd_url: String::new(), // We add the prefix in main.rs ..Config::default() } diff --git a/yaml.rs b/yaml.rs index 31278db8..406d73c3 100644 --- a/yaml.rs +++ b/yaml.rs @@ -1029,7 +1029,7 @@ api_security_sample_delay: 60 # Seconds apm_filter_tags_regex_require: None, apm_filter_tags_regex_reject: None, statsd_metric_namespace: None, - dogstatsd_so_rcvbuf: Some(1048576), + dogstatsd_so_rcvbuf: Some(1_048_576), dogstatsd_buffer_size: Some(65507), dogstatsd_queue_size: Some(2048), }; @@ -1047,11 +1047,11 @@ api_security_sample_delay: 60 # Seconds jail.clear_env(); jail.create_file( "datadog.yaml", - r#" + r" dogstatsd_so_rcvbuf: 524288 dogstatsd_buffer_size: 16384 dogstatsd_queue_size: 512 -"#, +", )?; let mut config = Config::default(); let yaml_config_source = YamlConfigSource { @@ -1061,7 +1061,7 @@ dogstatsd_queue_size: 512 .load(&mut config) .expect("Failed to load config"); - assert_eq!(config.dogstatsd_so_rcvbuf, Some(524288)); + assert_eq!(config.dogstatsd_so_rcvbuf, Some(524_288)); assert_eq!(config.dogstatsd_buffer_size, Some(16384)); assert_eq!(config.dogstatsd_queue_size, Some(512)); Ok(()) From b0e7f5ac6137605b860ab34489279a7ba4aa6820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jordan=20gonz=C3=A1lez?= <30836115+duncanista@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:26:44 -0500 Subject: [PATCH 108/112] feat(http): allow skip ssl validation (#1064) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Overview Add DD_SKIP_SSL_VALIDATION support, parsed from both env and YAML, matching the datadog-agent's behavior — applied to all outgoing HTTP clients (reqwest via danger_accept_invalid_certs, hyper via custom ServerCertVerifier). ## Motivation Customers in environments with corporate proxies or custom CA setups need the ability to disable TLS certificate validation, matching the existing datadog-agent config option. The Go agent applies tls.Config{InsecureSkipVerify: true} to all HTTP transports via a central CreateHTTPTransport() — we mirror this by wiring the config through to both client builders. And [SLES-2710](https://datadoghq.atlassian.net/browse/SLES-2710) ## Changes Config (config/mod.rs, config/env.rs, config/yaml.rs): - Add skip_ssl_validation: bool to Config, EnvConfig, and YamlConfig with default false reqwest client (http.rs): - .danger_accept_invalid_certs(config.skip_ssl_validation) on the shared client builder hyper client (traces/http_client.rs): - Custom NoVerifier implementing rustls::client::danger::ServerCertVerifier that accepts all certificates - Uses CryptoProvider::get_default() (not hardcoded aws_lc_rs) for FIPS-safe signature scheme reporting - New skip_ssl_validation parameter on create_client() ## Testing Unit tests and self monitoring [SLES-2710]: https://datadoghq.atlassian.net/browse/SLES-2710?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ --- env.rs | 8 ++++++++ mod.rs | 2 ++ yaml.rs | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/env.rs b/env.rs index 9f66587a..f48d2c8d 100644 --- a/env.rs +++ b/env.rs @@ -80,6 +80,11 @@ pub struct EnvConfig { /// Example: `/opt/ca-cert.pem` #[serde(deserialize_with = "deserialize_optional_string")] pub tls_cert_file: Option, + /// @env `DD_SKIP_SSL_VALIDATION` + /// + /// If set to true, the Agent will skip TLS certificate validation for outgoing connections. + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub skip_ssl_validation: Option, // Metrics /// @env `DD_DD_URL` @@ -497,6 +502,7 @@ fn merge_config(config: &mut Config, env_config: &EnvConfig) { merge_vec!(config, env_config, proxy_no_proxy); merge_option!(config, env_config, http_protocol); merge_option!(config, env_config, tls_cert_file); + merge_option_to_value!(config, env_config, skip_ssl_validation); // Endpoints merge_string!(config, env_config, dd_url); @@ -733,6 +739,7 @@ mod tests { jail.set_env("DD_PROXY_NO_PROXY", "localhost,127.0.0.1"); jail.set_env("DD_HTTP_PROTOCOL", "http1"); jail.set_env("DD_TLS_CERT_FILE", "/opt/ca-cert.pem"); + jail.set_env("DD_SKIP_SSL_VALIDATION", "true"); // Metrics jail.set_env("DD_DD_URL", "https://metrics.datadoghq.com"); @@ -895,6 +902,7 @@ mod tests { proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), tls_cert_file: Some("/opt/ca-cert.pem".to_string()), + skip_ssl_validation: true, dd_url: "https://metrics.datadoghq.com".to_string(), url: "https://app.datadoghq.com".to_string(), additional_endpoints: HashMap::from([ diff --git a/mod.rs b/mod.rs index 86e886a5..23fe110a 100644 --- a/mod.rs +++ b/mod.rs @@ -256,6 +256,7 @@ pub struct Config { pub proxy_no_proxy: Vec, pub http_protocol: Option, pub tls_cert_file: Option, + pub skip_ssl_validation: bool, // Endpoints pub dd_url: String, @@ -383,6 +384,7 @@ impl Default for Config { proxy_no_proxy: vec![], http_protocol: None, tls_cert_file: None, + skip_ssl_validation: false, // Endpoints dd_url: String::default(), diff --git a/yaml.rs b/yaml.rs index 406d73c3..0934d966 100644 --- a/yaml.rs +++ b/yaml.rs @@ -55,6 +55,8 @@ pub struct YamlConfig { pub http_protocol: Option, #[serde(deserialize_with = "deserialize_optional_string")] pub tls_cert_file: Option, + #[serde(deserialize_with = "deserialize_optional_bool_from_anything")] + pub skip_ssl_validation: Option, // Endpoints #[serde(deserialize_with = "deserialize_additional_endpoints")] @@ -431,6 +433,7 @@ fn merge_config(config: &mut Config, yaml_config: &YamlConfig) { merge_option_to_value!(config, proxy_no_proxy, yaml_config.proxy, no_proxy); merge_option!(config, yaml_config, http_protocol); merge_option!(config, yaml_config, tls_cert_file); + merge_option_to_value!(config, yaml_config, skip_ssl_validation); // Endpoints merge_hashmap!(config, yaml_config, additional_endpoints); @@ -767,6 +770,7 @@ proxy: dd_url: "https://metrics.datadoghq.com" http_protocol: "http1" tls_cert_file: "/opt/ca-cert.pem" +skip_ssl_validation: true # Endpoints additional_endpoints: @@ -907,6 +911,7 @@ api_security_sample_delay: 60 # Seconds proxy_no_proxy: vec!["localhost".to_string(), "127.0.0.1".to_string()], http_protocol: Some("http1".to_string()), tls_cert_file: Some("/opt/ca-cert.pem".to_string()), + skip_ssl_validation: true, dd_url: "https://metrics.datadoghq.com".to_string(), url: String::new(), // doesnt exist in yaml additional_endpoints: HashMap::from([ From 93bea111f1f575333fee4f9e032effc3ccd85147 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Tue, 10 Mar 2026 16:24:29 -0400 Subject: [PATCH 109/112] add Cargo.toml for datadog-agent-config --- Cargo.lock | 155 ++++++++++++++++++++++++- crates/datadog-agent-config/Cargo.toml | 26 +++++ crates/datadog-agent-config/env.rs | 36 +++--- crates/datadog-agent-config/mod.rs | 4 +- crates/datadog-agent-config/yaml.rs | 29 +++-- 5 files changed, 211 insertions(+), 39 deletions(-) create mode 100644 crates/datadog-agent-config/Cargo.toml diff --git a/Cargo.lock b/Cargo.lock index 19805168..5056e700 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -77,6 +77,15 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -188,6 +197,12 @@ version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + [[package]] name = "byteorder" version = "1.5.0" @@ -380,6 +395,22 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "datadog-agent-config" +version = "0.1.0" +dependencies = [ + "dogstatsd", + "figment", + "libdd-trace-obfuscation", + "libdd-trace-utils", + "log", + "serde", + "serde-aux", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "datadog-fips" version = "0.1.0" @@ -454,7 +485,7 @@ source = "git+https://github.com/DataDog/saluki/?rev=f863626dbfe3c59bb390985fa65 dependencies = [ "datadog-protos", "float-cmp", - "ordered-float", + "ordered-float 4.6.0", "smallvec", ] @@ -592,6 +623,22 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "parking_lot", + "pear", + "serde", + "serde_yaml", + "tempfile", + "uncased", + "version_check", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1145,6 +1192,12 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + [[package]] name = "ipnet" version = "2.11.0" @@ -1506,6 +1559,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "4.6.0" @@ -1553,6 +1615,29 @@ dependencies = [ "smallvec", ] +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.114", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1668,6 +1753,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "version_check", + "yansi", +] + [[package]] name = "proptest" version = "1.9.0" @@ -2240,6 +2338,27 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-aux" +version = "4.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "207f67b28fe90fb596503a9bf0bf1ea5e831e21307658e177c5dfcdfc3ab8a0a" +dependencies = [ + "serde", + "serde-value", + "serde_json", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float 2.10.1", + "serde", +] + [[package]] name = "serde_bytes" version = "0.11.19" @@ -2305,6 +2424,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serial_test" version = "2.0.0" @@ -2803,6 +2935,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-ident" version = "1.0.22" @@ -2821,6 +2962,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -3200,6 +3347,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.1" diff --git a/crates/datadog-agent-config/Cargo.toml b/crates/datadog-agent-config/Cargo.toml new file mode 100644 index 00000000..f5d15d88 --- /dev/null +++ b/crates/datadog-agent-config/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "datadog-agent-config" +version = "0.1.0" +edition = "2024" +license.workspace = true + +[lib] +path = "mod.rs" + +[dependencies] +figment = { version = "0.10", default-features = false, features = ["yaml", "env"] } +libdd-trace-obfuscation = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } +libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95" } +log = { version = "0.4", default-features = false } +serde = { version = "1.0", default-features = false, features = ["derive"] } +serde-aux = { version = "4.7", default-features = false } +serde_json = { version = "1.0", default-features = false, features = ["alloc"] } +tracing = { version = "0.1", default-features = false } +dogstatsd = { path = "../dogstatsd" } +tokio = { version = "1.47", default-features = false, features = ["time"] } + +[dev-dependencies] +figment = { version = "0.10", default-features = false, features = ["yaml", "env", "test"] } + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage,coverage_nightly)'] } diff --git a/crates/datadog-agent-config/env.rs b/crates/datadog-agent-config/env.rs index f48d2c8d..f9d66b69 100644 --- a/crates/datadog-agent-config/env.rs +++ b/crates/datadog-agent-config/env.rs @@ -7,26 +7,22 @@ use dogstatsd::util::parse_metric_namespace; use libdd_trace_obfuscation::replacer::ReplaceRule; use crate::{ - config::{ - Config, ConfigError, ConfigSource, - additional_endpoints::deserialize_additional_endpoints, - apm_replace_rule::deserialize_apm_replace_rules, - deserialize_apm_filter_tags, deserialize_array_from_comma_separated_string, - deserialize_key_value_pairs, deserialize_option_lossless, - deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, - deserialize_optional_duration_from_seconds, - deserialize_optional_duration_from_seconds_ignore_zero, deserialize_optional_string, - deserialize_string_or_int, - flush_strategy::FlushStrategy, - log_level::LogLevel, - logs_additional_endpoints::{ - LogsAdditionalEndpoint, deserialize_logs_additional_endpoints, - }, - processing_rule::{ProcessingRule, deserialize_processing_rules}, - service_mapping::deserialize_service_mapping, - trace_propagation_style::{TracePropagationStyle, deserialize_trace_propagation_style}, - }, + Config, ConfigError, ConfigSource, + additional_endpoints::deserialize_additional_endpoints, + apm_replace_rule::deserialize_apm_replace_rules, + deserialize_apm_filter_tags, deserialize_array_from_comma_separated_string, + deserialize_key_value_pairs, deserialize_option_lossless, + deserialize_optional_bool_from_anything, deserialize_optional_duration_from_microseconds, + deserialize_optional_duration_from_seconds, + deserialize_optional_duration_from_seconds_ignore_zero, deserialize_optional_string, + deserialize_string_or_int, + flush_strategy::FlushStrategy, + log_level::LogLevel, + logs_additional_endpoints::{LogsAdditionalEndpoint, deserialize_logs_additional_endpoints}, merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, + processing_rule::{ProcessingRule, deserialize_processing_rules}, + service_mapping::deserialize_service_mapping, + trace_propagation_style::{TracePropagationStyle, deserialize_trace_propagation_style}, }; #[derive(Debug, PartialEq, Deserialize, Clone, Default)] @@ -714,7 +710,7 @@ mod tests { use std::time::Duration; use super::*; - use crate::config::{ + use crate::{ Config, flush_strategy::{FlushStrategy, PeriodicStrategy}, log_level::LogLevel, diff --git a/crates/datadog-agent-config/mod.rs b/crates/datadog-agent-config/mod.rs index 23fe110a..97e5991f 100644 --- a/crates/datadog-agent-config/mod.rs +++ b/crates/datadog-agent-config/mod.rs @@ -22,7 +22,7 @@ use std::time::Duration; use std::{collections::HashMap, fmt}; use tracing::{debug, error}; -use crate::config::{ +use crate::{ apm_replace_rule::deserialize_apm_replace_rules, env::EnvConfigSource, flush_strategy::FlushStrategy, @@ -815,7 +815,7 @@ pub mod tests { use super::*; - use crate::config::{ + use crate::{ flush_strategy::{FlushStrategy, PeriodicStrategy}, log_level::LogLevel, processing_rule::ProcessingRule, diff --git a/crates/datadog-agent-config/yaml.rs b/crates/datadog-agent-config/yaml.rs index 0934d966..0a7a52cd 100644 --- a/crates/datadog-agent-config/yaml.rs +++ b/crates/datadog-agent-config/yaml.rs @@ -2,22 +2,19 @@ use std::time::Duration; use std::{collections::HashMap, path::PathBuf}; use crate::{ - config::{ - Config, ConfigError, ConfigSource, ProcessingRule, - additional_endpoints::deserialize_additional_endpoints, - deserialize_apm_replace_rules, deserialize_key_value_pair_array_to_hashmap, - deserialize_option_lossless, deserialize_optional_bool_from_anything, - deserialize_optional_duration_from_microseconds, - deserialize_optional_duration_from_seconds, - deserialize_optional_duration_from_seconds_ignore_zero, deserialize_optional_string, - deserialize_processing_rules, deserialize_string_or_int, - flush_strategy::FlushStrategy, - log_level::LogLevel, - logs_additional_endpoints::LogsAdditionalEndpoint, - service_mapping::deserialize_service_mapping, - trace_propagation_style::{TracePropagationStyle, deserialize_trace_propagation_style}, - }, + Config, ConfigError, ConfigSource, ProcessingRule, + additional_endpoints::deserialize_additional_endpoints, + deserialize_apm_replace_rules, deserialize_key_value_pair_array_to_hashmap, + deserialize_option_lossless, deserialize_optional_bool_from_anything, + deserialize_optional_duration_from_microseconds, deserialize_optional_duration_from_seconds, + deserialize_optional_duration_from_seconds_ignore_zero, deserialize_optional_string, + deserialize_processing_rules, deserialize_string_or_int, + flush_strategy::FlushStrategy, + log_level::LogLevel, + logs_additional_endpoints::LogsAdditionalEndpoint, merge_hashmap, merge_option, merge_option_to_value, merge_string, merge_vec, + service_mapping::deserialize_service_mapping, + trace_propagation_style::{TracePropagationStyle, deserialize_trace_propagation_style}, }; use figment::{ Figment, @@ -745,7 +742,7 @@ mod tests { use std::path::Path; use std::time::Duration; - use crate::config::{flush_strategy::PeriodicStrategy, processing_rule::Kind}; + use crate::{flush_strategy::PeriodicStrategy, processing_rule::Kind}; use super::*; From ff987e1a4d128488d584f6ee2e64d230aeed5281 Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Tue, 10 Mar 2026 16:42:54 -0400 Subject: [PATCH 110/112] update licenses --- LICENSE-3rdparty.csv | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 26f606f9..c94550e4 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -7,6 +7,7 @@ assert-json-diff,https://github.com/davidpdrsn/assert-json-diff,MIT,David Peders async-lock,https://github.com/smol-rs/async-lock,Apache-2.0 OR MIT,Stjepan Glavina async-object-pool,https://github.com/alexliesenfeld/async-object-pool,MIT,Alexander Liesenfeld async-trait,https://github.com/dtolnay/async-trait,MIT OR Apache-2.0,David Tolnay +atomic,https://github.com/Amanieu/atomic-rs,Apache-2.0 OR MIT,Amanieu d'Antras atomic-waker,https://github.com/smol-rs/atomic-waker,Apache-2.0 OR MIT,"Stjepan Glavina , Contributors to futures-rs" aws-lc-rs,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC),AWS-LibCrypto aws-lc-sys,https://github.com/aws/aws-lc-rs,ISC AND (Apache-2.0 OR ISC) AND OpenSSL,AWS-LC @@ -16,6 +17,7 @@ bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessn bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers bumpalo,https://github.com/fitzgen/bumpalo,MIT OR Apache-2.0,Nick Fitzgerald +bytemuck,https://github.com/Lokathor/bytemuck,Zlib OR Apache-2.0 OR MIT,Lokathor byteorder,https://github.com/BurntSushi/byteorder,Unlicense OR MIT,Andrew Gallant bytes,https://github.com/tokio-rs/bytes,MIT,"Carl Lerche , Sean McArthur " camino,https://github.com/camino-rs/camino,MIT OR Apache-2.0,"Without Boats , Ashley Williams , Steve Klabnik , Rain " @@ -45,6 +47,7 @@ errno,https://github.com/lambda-fairy/rust-errno,MIT OR Apache-2.0,"Chris Wong < event-listener,https://github.com/smol-rs/event-listener,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " event-listener-strategy,https://github.com/smol-rs/event-listener-strategy,Apache-2.0 OR MIT,John Nunley fastrand,https://github.com/smol-rs/fastrand,Apache-2.0 OR MIT,Stjepan Glavina +figment,https://github.com/SergioBenitez/Figment,MIT OR Apache-2.0,Sergio Benitez find-msvc-tools,https://github.com/rust-lang/cc-rs,MIT OR Apache-2.0,The find-msvc-tools Authors fixedbitset,https://github.com/petgraph/fixedbitset,MIT OR Apache-2.0,bluss flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " @@ -91,6 +94,7 @@ icu_provider,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project idna,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers idna_adapter,https://github.com/hsivonen/idna_adapter,Apache-2.0 OR MIT,The rust-url developers indexmap,https://github.com/indexmap-rs/indexmap,Apache-2.0 OR MIT,The indexmap Authors +inlinable_string,https://github.com/fitzgen/inlinable_string,Apache-2.0 OR MIT,Nick Fitzgerald ipnet,https://github.com/krisprice/ipnet,MIT OR Apache-2.0,Kris Price iri-string,https://github.com/lo48576/iri-string,MIT OR Apache-2.0,YOSHIOKA Takuma itertools,https://github.com/rust-itertools/itertools,MIT OR Apache-2.0,bluss @@ -126,6 +130,8 @@ parking,https://github.com/smol-rs/parking,Apache-2.0 OR MIT,"Stjepan Glavina parking_lot_core,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras path-tree,https://github.com/viz-rs/path-tree,MIT OR Apache-2.0,Fangdun Tsai +pear,https://github.com/SergioBenitez/Pear,MIT OR Apache-2.0,Sergio Benitez +pear_codegen,https://github.com/SergioBenitez/Pear,MIT OR Apache-2.0,Sergio Benitez percent-encoding,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers petgraph,https://github.com/petgraph/petgraph,MIT OR Apache-2.0,"bluss, mitchmindtree" pin-project,https://github.com/taiki-e/pin-project,Apache-2.0 OR MIT,The pin-project Authors @@ -138,6 +144,7 @@ prettyplease,https://github.com/dtolnay/prettyplease,MIT OR Apache-2.0,David Tol proc-macro-error,https://gitlab.com/CreepySkeleton/proc-macro-error,MIT OR Apache-2.0,CreepySkeleton proc-macro-error-attr,https://gitlab.com/CreepySkeleton/proc-macro-error,MIT OR Apache-2.0,CreepySkeleton proc-macro2,https://github.com/dtolnay/proc-macro2,MIT OR Apache-2.0,"David Tolnay , Alex Crichton " +proc-macro2-diagnostics,https://github.com/SergioBenitez/proc-macro2-diagnostics,MIT OR Apache-2.0,Sergio Benitez prost,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " prost-build,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " prost-derive,https://github.com/tokio-rs/prost,Apache-2.0,"Dan Burkert , Lucio Franco , Casper Meijn , Tokio Contributors " @@ -178,12 +185,15 @@ security-framework,https://github.com/kornelski/rust-security-framework,MIT OR A security-framework-sys,https://github.com/kornelski/rust-security-framework,MIT OR Apache-2.0,"Steven Fackler , Kornel " semver,https://github.com/dtolnay/semver,MIT OR Apache-2.0,David Tolnay serde,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " +serde-aux,https://github.com/iddm/serde-aux,MIT,Victor Polevoy +serde-value,https://github.com/arcnmx/serde-value,MIT,arcnmx serde_bytes,https://github.com/serde-rs/bytes,MIT OR Apache-2.0,David Tolnay serde_core,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_derive,https://github.com/serde-rs/serde,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_json,https://github.com/serde-rs/json,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_regex,https://github.com/tailhook/serde-regex,MIT OR Apache-2.0,paul@colomiets.name serde_urlencoded,https://github.com/nox/serde_urlencoded,MIT OR Apache-2.0,Anthony Ramine +serde_yaml,https://github.com/dtolnay/serde-yaml,MIT OR Apache-2.0,David Tolnay serial_test_derive,https://github.com/palfrey/serial_test,MIT,Tom Parker-Shemilt sha1,https://github.com/RustCrypto/hashes,MIT OR Apache-2.0,RustCrypto Developers sharded-slab,https://github.com/hawkw/sharded-slab,MIT,Eliza Weisman @@ -228,9 +238,11 @@ tracing-test-macro,https://github.com/dbrgn/tracing-test,MIT,Danilo Bargen typenum,https://github.com/paholg/typenum,MIT OR Apache-2.0,"Paho Lurie-Gregg , Andre Bogus " unarray,https://github.com/cameron1024/unarray,MIT OR Apache-2.0,The unarray Authors +uncased,https://github.com/SergioBenitez/uncased,MIT OR Apache-2.0,Sergio Benitez unicode-ident,https://github.com/dtolnay/unicode-ident,(MIT OR Apache-2.0) AND Unicode-3.0,David Tolnay unicode-width,https://github.com/unicode-rs/unicode-width,MIT OR Apache-2.0,"kwantam , Manish Goregaokar " unicode-xid,https://github.com/unicode-rs/unicode-xid,MIT OR Apache-2.0,"erick.tryzelaar , kwantam , Manish Goregaokar " +unsafe-libyaml,https://github.com/dtolnay/unsafe-libyaml,MIT,David Tolnay untrusted,https://github.com/briansmith/untrusted,ISC,Brian Smith url,https://github.com/servo/rust-url,MIT OR Apache-2.0,The rust-url developers urlencoding,https://github.com/kornelski/rust_urlencoding,MIT,"Kornel , Bertram Truong " @@ -273,6 +285,7 @@ windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,Mi windows_x86_64_msvc,https://github.com/microsoft/windows-rs,MIT OR Apache-2.0,The windows_x86_64_msvc Authors wit-bindgen,https://github.com/bytecodealliance/wit-bindgen,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,Alex Crichton writeable,https://github.com/unicode-org/icu4x,Unicode-3.0,The ICU4X Project Developers +yansi,https://github.com/SergioBenitez/yansi,MIT OR Apache-2.0,Sergio Benitez yoke,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar yoke-derive,https://github.com/unicode-org/icu4x,Unicode-3.0,Manish Goregaokar zerocopy,https://github.com/google/zerocopy,BSD-2-Clause OR Apache-2.0 OR MIT,"Joshua Liebow-Feeser , Jack Wrenn " From 1bff52845b829a67eab0b7f8ab842c42be56dd9d Mon Sep 17 00:00:00 2001 From: Duncan Harvey Date: Wed, 11 Mar 2026 11:27:54 -0400 Subject: [PATCH 111/112] remove aws.rs from datadog-agent-config --- crates/datadog-agent-config/aws.rs | 89 ------------------------------ crates/datadog-agent-config/mod.rs | 1 - 2 files changed, 90 deletions(-) delete mode 100644 crates/datadog-agent-config/aws.rs diff --git a/crates/datadog-agent-config/aws.rs b/crates/datadog-agent-config/aws.rs deleted file mode 100644 index d1af40a8..00000000 --- a/crates/datadog-agent-config/aws.rs +++ /dev/null @@ -1,89 +0,0 @@ -use std::env; -use tokio::time::Instant; - -const AWS_DEFAULT_REGION: &str = "AWS_DEFAULT_REGION"; -const AWS_ACCESS_KEY_ID: &str = "AWS_ACCESS_KEY_ID"; -const AWS_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY"; -const AWS_SESSION_TOKEN: &str = "AWS_SESSION_TOKEN"; -const AWS_CONTAINER_CREDENTIALS_FULL_URI: &str = "AWS_CONTAINER_CREDENTIALS_FULL_URI"; -const AWS_CONTAINER_AUTHORIZATION_TOKEN: &str = "AWS_CONTAINER_AUTHORIZATION_TOKEN"; -const AWS_LAMBDA_FUNCTION_NAME: &str = "AWS_LAMBDA_FUNCTION_NAME"; -const AWS_LAMBDA_RUNTIME_API: &str = "AWS_LAMBDA_RUNTIME_API"; -const AWS_LWA_LAMBDA_RUNTIME_API_PROXY: &str = "AWS_LWA_LAMBDA_RUNTIME_API_PROXY"; -const AWS_LAMBDA_EXEC_WRAPPER: &str = "AWS_LAMBDA_EXEC_WRAPPER"; -const AWS_LAMBDA_INITIALIZATION_TYPE: &str = "AWS_LAMBDA_INITIALIZATION_TYPE"; - -pub const LAMBDA_MANAGED_INSTANCES_INIT_TYPE: &str = "lambda-managed-instances"; - -#[allow(clippy::module_name_repetitions)] -#[derive(Debug, Clone)] -pub struct AwsConfig { - pub region: String, - pub aws_lwa_proxy_lambda_runtime_api: Option, - pub function_name: String, - pub runtime_api: String, - pub sandbox_init_time: Instant, - pub exec_wrapper: Option, - pub initialization_type: String, -} - -impl AwsConfig { - #[must_use] - pub fn from_env(start_time: Instant) -> Self { - Self { - region: env::var(AWS_DEFAULT_REGION).unwrap_or("us-east-1".to_string()), - aws_lwa_proxy_lambda_runtime_api: env::var(AWS_LWA_LAMBDA_RUNTIME_API_PROXY).ok(), - function_name: env::var(AWS_LAMBDA_FUNCTION_NAME).unwrap_or_default(), - runtime_api: env::var(AWS_LAMBDA_RUNTIME_API).unwrap_or_default(), - sandbox_init_time: start_time, - exec_wrapper: env::var(AWS_LAMBDA_EXEC_WRAPPER).ok(), - initialization_type: env::var(AWS_LAMBDA_INITIALIZATION_TYPE).unwrap_or_default(), - } - } - - #[must_use] - pub fn is_managed_instance_mode(&self) -> bool { - self.initialization_type - .eq(LAMBDA_MANAGED_INSTANCES_INIT_TYPE) - } -} - -#[allow(clippy::module_name_repetitions)] -#[derive(Debug, Clone)] -pub struct AwsCredentials { - pub aws_access_key_id: String, - pub aws_secret_access_key: String, - pub aws_session_token: String, - pub aws_container_credentials_full_uri: String, - pub aws_container_authorization_token: String, -} - -impl AwsCredentials { - #[must_use] - pub fn from_env() -> Self { - Self { - aws_access_key_id: env::var(AWS_ACCESS_KEY_ID).unwrap_or_default(), - aws_secret_access_key: env::var(AWS_SECRET_ACCESS_KEY).unwrap_or_default(), - aws_session_token: env::var(AWS_SESSION_TOKEN).unwrap_or_default(), - aws_container_credentials_full_uri: env::var(AWS_CONTAINER_CREDENTIALS_FULL_URI) - .unwrap_or_default(), - aws_container_authorization_token: env::var(AWS_CONTAINER_AUTHORIZATION_TOKEN) - .unwrap_or_default(), - } - } -} - -#[must_use] -pub fn get_aws_partition_by_region(region: &str) -> String { - match region { - r if r.starts_with("us-gov-") => "aws-us-gov".to_string(), - r if r.starts_with("cn-") => "aws-cn".to_string(), - _ => "aws".to_string(), - } -} - -#[must_use] -pub fn build_lambda_function_arn(account_id: &str, region: &str, function_name: &str) -> String { - let aws_partition = get_aws_partition_by_region(region); - format!("arn:{aws_partition}:lambda:{region}:{account_id}:function:{function_name}") -} diff --git a/crates/datadog-agent-config/mod.rs b/crates/datadog-agent-config/mod.rs index 97e5991f..1d27be6d 100644 --- a/crates/datadog-agent-config/mod.rs +++ b/crates/datadog-agent-config/mod.rs @@ -1,6 +1,5 @@ pub mod additional_endpoints; pub mod apm_replace_rule; -pub mod aws; pub mod env; pub mod flush_strategy; pub mod log_level; From b013122ed417ce4f953e24d4d1f6579cdd178759 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:55:28 -0400 Subject: [PATCH 112/112] chore: upgrade workspace rust edition to 2024 (#96) * upgrade rust edition to 2024 for workspace * apply formatting --- Cargo.lock | 10 + Cargo.toml | 2 +- crates/datadog-agent-config/Cargo.toml | 2 +- crates/datadog-serverless-compat/src/main.rs | 4 +- crates/datadog-trace-agent/Cargo.toml | 1 + crates/datadog-trace-agent/src/config.rs | 490 +++++++++++------- .../datadog-trace-agent/src/env_verifier.rs | 60 ++- crates/datadog-trace-agent/src/http_utils.rs | 5 +- crates/datadog-trace-agent/src/mini_agent.rs | 2 +- .../datadog-trace-agent/src/stats_flusher.rs | 2 +- .../src/stats_processor.rs | 2 +- .../datadog-trace-agent/src/trace_flusher.rs | 4 +- .../src/trace_processor.rs | 24 +- .../tests/common/mock_server.rs | 2 +- .../tests/integration_test.rs | 2 +- crates/dogstatsd/src/aggregator/core.rs | 66 ++- crates/dogstatsd/src/aggregator/service.rs | 2 +- crates/dogstatsd/src/api_key.rs | 2 +- crates/dogstatsd/src/dogstatsd.rs | 18 +- crates/dogstatsd/src/flusher.rs | 11 +- crates/dogstatsd/src/metric.rs | 22 +- crates/dogstatsd/src/util.rs | 3 +- crates/dogstatsd/tests/integration_test.rs | 8 +- 23 files changed, 464 insertions(+), 280 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5056e700..c625dfba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -473,6 +473,7 @@ dependencies = [ "serde", "serde_json", "serial_test", + "temp-env", "tempfile", "tokio", "tracing", @@ -2606,6 +2607,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "temp-env" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96374855068f47402c3121c6eed88d29cb1de8f3ab27090e273e420bdabcf050" +dependencies = [ + "parking_lot", +] + [[package]] name = "tempfile" version = "3.24.0" diff --git a/Cargo.toml b/Cargo.toml index cb438b99..0ce470ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = [ ] [workspace.package] -edition = "2021" +edition = "2024" license = "Apache-2.0" homepage = "https://github.com/DataDog/serverless-components" repository = "https://github.com/DataDog/serverless-components" diff --git a/crates/datadog-agent-config/Cargo.toml b/crates/datadog-agent-config/Cargo.toml index f5d15d88..50678993 100644 --- a/crates/datadog-agent-config/Cargo.toml +++ b/crates/datadog-agent-config/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "datadog-agent-config" version = "0.1.0" -edition = "2024" +edition.workspace = true license.workspace = true [lib] diff --git a/crates/datadog-serverless-compat/src/main.rs b/crates/datadog-serverless-compat/src/main.rs index 6d764815..d50798f0 100644 --- a/crates/datadog-serverless-compat/src/main.rs +++ b/crates/datadog-serverless-compat/src/main.rs @@ -10,7 +10,7 @@ use std::{env, sync::Arc}; use tokio::{ sync::Mutex as TokioMutex, - time::{interval, Duration}, + time::{Duration, interval}, }; use tracing::{debug, error, info}; use tracing_subscriber::EnvFilter; @@ -36,7 +36,7 @@ use dogstatsd::{ util::parse_metric_namespace, }; -use dogstatsd::metric::{SortedTags, EMPTY_TAGS}; +use dogstatsd::metric::{EMPTY_TAGS, SortedTags}; use tokio_util::sync::CancellationToken; const DOGSTATSD_FLUSH_INTERVAL: u64 = 10; diff --git a/crates/datadog-trace-agent/Cargo.toml b/crates/datadog-trace-agent/Cargo.toml index aec60c93..4cc5ed73 100644 --- a/crates/datadog-trace-agent/Cargo.toml +++ b/crates/datadog-trace-agent/Cargo.toml @@ -38,6 +38,7 @@ bytes = "1.10.1" rmp-serde = "1.1.1" serial_test = "2.0.0" duplicate = "0.4.1" +temp-env = "0.3.6" tempfile = "3.3.0" libdd-trace-utils = { git = "https://github.com/DataDog/libdatadog", rev = "d52ee90209cb12a28bdda0114535c1a985a29d95", features = [ "test-utils", diff --git a/crates/datadog-trace-agent/src/config.rs b/crates/datadog-trace-agent/src/config.rs index 2d94b4f3..5a7b8a8c 100644 --- a/crates/datadog-trace-agent/src/config.rs +++ b/crates/datadog-trace-agent/src/config.rs @@ -260,54 +260,57 @@ mod tests { use duplicate::duplicate_item; use serial_test::serial; use std::collections::HashMap; - use std::env; use crate::config; #[test] #[serial] fn test_error_if_unable_to_identify_env() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - - let config = config::Config::new(); - assert!(config.is_err()); - assert_eq!( - config.unwrap_err().to_string(), - "Unable to identify environment. Shutting down Mini Agent." - ); - env::remove_var("DD_API_KEY"); + temp_env::with_vars([("DD_API_KEY", Some("_not_a_real_key_"))], || { + let config = config::Config::new(); + assert!(config.is_err()); + assert_eq!( + config.unwrap_err().to_string(), + "Unable to identify environment. Shutting down Mini Agent." + ); + }); } #[test] #[serial] fn test_error_if_no_api_key_env_var() { - env::remove_var("DD_API_KEY"); - let config = config::Config::new(); - assert!(config.is_err()); - assert_eq!( - config.unwrap_err().to_string(), - "DD_API_KEY environment variable is not set" - ); + temp_env::with_vars([("DD_API_KEY", None::<&str>)], || { + let config = config::Config::new(); + assert!(config.is_err()); + assert_eq!( + config.unwrap_err().to_string(), + "DD_API_KEY environment variable is not set" + ); + }); } #[test] #[serial] fn test_default_trace_and_trace_stats_urls() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("K_SERVICE", "function_name"); - let config_res = config::Config::new(); - assert!(config_res.is_ok()); - let config = config_res.unwrap(); - assert_eq!( - config.trace_intake.url, - "https://trace.agent.datadoghq.com/api/v0.2/traces" - ); - assert_eq!( - config.trace_stats_intake.url, - "https://trace.agent.datadoghq.com/api/v0.2/stats" + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ("K_SERVICE", Some("function_name")), + ], + || { + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!( + config.trace_intake.url, + "https://trace.agent.datadoghq.com/api/v0.2/traces" + ); + assert_eq!( + config.trace_stats_intake.url, + "https://trace.agent.datadoghq.com/api/v0.2/stats" + ); + }, ); - env::remove_var("DD_API_KEY"); - env::remove_var("K_SERVICE"); } #[duplicate_item( @@ -322,16 +325,19 @@ mod tests { #[test] #[serial] fn test_name() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("K_SERVICE", "function_name"); - env::set_var("DD_SITE", dd_site); - let config_res = config::Config::new(); - assert!(config_res.is_ok()); - let config = config_res.unwrap(); - assert_eq!(config.trace_intake.url, expected_url); - env::remove_var("DD_API_KEY"); - env::remove_var("DD_SITE"); - env::remove_var("K_SERVICE"); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ("K_SERVICE", Some("function_name")), + ("DD_SITE", Some(dd_site)), + ], + || { + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!(config.trace_intake.url, expected_url); + }, + ); } #[duplicate_item( @@ -346,193 +352,267 @@ mod tests { #[test] #[serial] fn test_name() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("K_SERVICE", "function_name"); - env::set_var("DD_SITE", dd_site); - let config_res = config::Config::new(); - assert!(config_res.is_ok()); - let config = config_res.unwrap(); - assert_eq!(config.trace_stats_intake.url, expected_url); - env::remove_var("DD_API_KEY"); - env::remove_var("DD_SITE"); - env::remove_var("K_SERVICE"); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ("K_SERVICE", Some("function_name")), + ("DD_SITE", Some(dd_site)), + ], + || { + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!(config.trace_stats_intake.url, expected_url); + }, + ); } #[test] #[serial] fn test_set_custom_trace_and_trace_stats_intake_url() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("K_SERVICE", "function_name"); - env::set_var("DD_APM_DD_URL", "http://127.0.0.1:3333"); - let config_res = config::Config::new(); - assert!(config_res.is_ok()); - let config = config_res.unwrap(); - assert_eq!( - config.trace_intake.url, - "http://127.0.0.1:3333/api/v0.2/traces" - ); - assert_eq!( - config.trace_stats_intake.url, - "http://127.0.0.1:3333/api/v0.2/stats" + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ("K_SERVICE", Some("function_name")), + ("DD_APM_DD_URL", Some("http://127.0.0.1:3333")), + ], + || { + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!( + config.trace_intake.url, + "http://127.0.0.1:3333/api/v0.2/traces" + ); + assert_eq!( + config.trace_stats_intake.url, + "http://127.0.0.1:3333/api/v0.2/stats" + ); + }, ); - env::remove_var("DD_API_KEY"); - env::remove_var("DD_APM_DD_URL"); - env::remove_var("K_SERVICE"); } #[test] #[serial] #[cfg(any(all(windows, feature = "windows-pipes"), test))] fn test_apm_windows_pipe_name() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("ASCSVCRT_SPRING__APPLICATION__NAME", "test-spring-app"); - env::set_var("DD_APM_WINDOWS_PIPE_NAME", r"test_pipe"); - let config_res = config::Config::new(); - assert!(config_res.is_ok()); - let config = config_res.unwrap(); - assert_eq!( - config.dd_apm_windows_pipe_name, - Some(r"\\.\pipe\test_pipe".to_string()) + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_APM_WINDOWS_PIPE_NAME", Some(r"test_pipe")), + ], + || { + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!( + config.dd_apm_windows_pipe_name, + Some(r"\\.\pipe\test_pipe".to_string()) + ); + + // Port should be overridden to 0 when pipe is set + assert_eq!(config.dd_apm_receiver_port, 0); + }, ); - - // Port should be overridden to 0 when pipe is set - assert_eq!(config.dd_apm_receiver_port, 0); - env::remove_var("DD_API_KEY"); - env::remove_var("ASCSVCRT_SPRING__APPLICATION__NAME"); - env::remove_var("DD_APM_WINDOWS_PIPE_NAME"); } #[test] #[serial] #[cfg(any(all(windows, feature = "windows-pipes"), test))] fn test_dogstatsd_windows_pipe_name() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("ASCSVCRT_SPRING__APPLICATION__NAME", "test-spring-app"); - env::set_var("DD_DOGSTATSD_WINDOWS_PIPE_NAME", r"test_pipe"); - let config_res = config::Config::new(); - assert!(config_res.is_ok()); - let config = config_res.unwrap(); - assert_eq!( - config.dd_dogstatsd_windows_pipe_name, - Some(r"\\.\pipe\test_pipe".to_string()) + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_DOGSTATSD_WINDOWS_PIPE_NAME", Some(r"test_pipe")), + ], + || { + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!( + config.dd_dogstatsd_windows_pipe_name, + Some(r"\\.\pipe\test_pipe".to_string()) + ); + + // Port should be overridden to 0 when pipe is set + assert_eq!(config.dd_dogstatsd_port, 0); + }, ); - - // Port should be overridden to 0 when pipe is set - assert_eq!(config.dd_dogstatsd_port, 0); - env::remove_var("DD_API_KEY"); - env::remove_var("ASCSVCRT_SPRING__APPLICATION__NAME"); - env::remove_var("DD_DOGSTATSD_WINDOWS_PIPE_NAME"); } #[test] #[serial] fn test_default_dogstatsd_port() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("ASCSVCRT_SPRING__APPLICATION__NAME", "test-spring-app"); - let config_res = config::Config::new(); - assert!(config_res.is_ok()); - let config = config_res.unwrap(); - assert_eq!(config.dd_dogstatsd_port, 8125); - env::remove_var("DD_API_KEY"); - env::remove_var("ASCSVCRT_SPRING__APPLICATION__NAME"); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ], + || { + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!(config.dd_dogstatsd_port, 8125); + }, + ); } #[test] #[serial] fn test_custom_dogstatsd_port() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("ASCSVCRT_SPRING__APPLICATION__NAME", "test-spring-app"); - env::set_var("DD_DOGSTATSD_PORT", "18125"); - let config_res = config::Config::new(); - println!("{:?}", config_res); - assert!(config_res.is_ok()); - let config = config_res.unwrap(); - assert_eq!(config.dd_dogstatsd_port, 18125); - env::remove_var("DD_API_KEY"); - env::remove_var("ASCSVCRT_SPRING__APPLICATION__NAME"); - env::remove_var("DD_DOGSTATSD_PORT"); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_DOGSTATSD_PORT", Some("18125")), + ], + || { + let config_res = config::Config::new(); + println!("{:?}", config_res); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!(config.dd_dogstatsd_port, 18125); + }, + ); } #[test] #[serial] fn test_default_apm_receiver_port() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("ASCSVCRT_SPRING__APPLICATION__NAME", "test-spring-app"); - let config_res = config::Config::new(); - assert!(config_res.is_ok()); - let config = config_res.unwrap(); - assert_eq!(config.dd_apm_receiver_port, 8126); - #[cfg(any(all(windows, feature = "windows-pipes"), test))] - assert_eq!(config.dd_apm_windows_pipe_name, None); - env::remove_var("DD_API_KEY"); - env::remove_var("ASCSVCRT_SPRING__APPLICATION__NAME"); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ], + || { + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!(config.dd_apm_receiver_port, 8126); + #[cfg(any(all(windows, feature = "windows-pipes"), test))] + assert_eq!(config.dd_apm_windows_pipe_name, None); + }, + ); } #[test] #[serial] fn test_custom_apm_receiver_port() { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("ASCSVCRT_SPRING__APPLICATION__NAME", "test-spring-app"); - env::set_var("DD_APM_RECEIVER_PORT", "18126"); - let config_res = config::Config::new(); - assert!(config_res.is_ok()); - let config = config_res.unwrap(); - assert_eq!(config.dd_apm_receiver_port, 18126); - env::remove_var("DD_API_KEY"); - env::remove_var("ASCSVCRT_SPRING__APPLICATION__NAME"); - env::remove_var("DD_APM_RECEIVER_PORT"); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_APM_RECEIVER_PORT", Some("18126")), + ], + || { + let config_res = config::Config::new(); + assert!(config_res.is_ok()); + let config = config_res.unwrap(); + assert_eq!(config.dd_apm_receiver_port, 18126); + }, + ); } - fn test_config_with_dd_tags(dd_tags: &str) -> config::Config { - env::set_var("DD_API_KEY", "_not_a_real_key_"); - env::set_var("ASCSVCRT_SPRING__APPLICATION__NAME", "test-spring-app"); - env::set_var("DD_TAGS", dd_tags); + /// Call from within temp_env::with_vars that set DD_API_KEY, ASCSVCRT_SPRING__APPLICATION__NAME, and DD_TAGS. + fn test_config_with_dd_tags() -> config::Config { let config_res = config::Config::new(); assert!(config_res.is_ok()); - let config = config_res.unwrap(); - env::remove_var("DD_API_KEY"); - env::remove_var("ASCSVCRT_SPRING__APPLICATION__NAME"); - env::remove_var("DD_TAGS"); - config + config_res.unwrap() } #[test] #[serial] fn test_dd_tags_comma_separated() { - let config = test_config_with_dd_tags("some:tag,another:thing,invalid:thing:here"); - let expected_tags = HashMap::from([ - ("some".to_string(), "tag".to_string()), - ("another".to_string(), "thing".to_string()), - ]); - assert_eq!(config.tags.tags(), &expected_tags); - assert_eq!(config.tags.function_tags(), Some("another:thing,some:tag")); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_TAGS", Some("some:tag,another:thing,invalid:thing:here")), + ], + || { + let config = test_config_with_dd_tags(); + let expected_tags = HashMap::from([ + ("some".to_string(), "tag".to_string()), + ("another".to_string(), "thing".to_string()), + ]); + assert_eq!(config.tags.tags(), &expected_tags); + assert_eq!(config.tags.function_tags(), Some("another:thing,some:tag")); + }, + ); } #[test] #[serial] fn test_dd_tags_space_separated() { - let config = test_config_with_dd_tags("some:tag another:thing invalid:thing:here"); - let expected_tags = HashMap::from([ - ("some".to_string(), "tag".to_string()), - ("another".to_string(), "thing".to_string()), - ]); - assert_eq!(config.tags.tags(), &expected_tags); - assert_eq!(config.tags.function_tags(), Some("another:thing,some:tag")); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_TAGS", Some("some:tag another:thing invalid:thing:here")), + ], + || { + let config = test_config_with_dd_tags(); + let expected_tags = HashMap::from([ + ("some".to_string(), "tag".to_string()), + ("another".to_string(), "thing".to_string()), + ]); + assert_eq!(config.tags.tags(), &expected_tags); + assert_eq!(config.tags.function_tags(), Some("another:thing,some:tag")); + }, + ); } #[test] #[serial] fn test_dd_tags_mixed_separators() { - let config = test_config_with_dd_tags("some:tag,another:thing extra:value"); - let expected_tags = HashMap::from([ - ("some".to_string(), "tag".to_string()), - ("another".to_string(), "thing".to_string()), - ("extra".to_string(), "value".to_string()), - ]); - assert_eq!(config.tags.tags(), &expected_tags); - assert_eq!( - config.tags.function_tags(), - Some("another:thing,extra:value,some:tag") + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_TAGS", Some("some:tag,another:thing extra:value")), + ], + || { + let config = test_config_with_dd_tags(); + let expected_tags = HashMap::from([ + ("some".to_string(), "tag".to_string()), + ("another".to_string(), "thing".to_string()), + ("extra".to_string(), "value".to_string()), + ]); + assert_eq!(config.tags.tags(), &expected_tags); + assert_eq!( + config.tags.function_tags(), + Some("another:thing,extra:value,some:tag") + ); + }, ); } @@ -540,24 +620,72 @@ mod tests { #[serial] fn test_dd_tags_no_valid_tags() { // Test with only invalid tags - let config = test_config_with_dd_tags("invalid:thing:here,also-bad"); - assert_eq!(config.tags.tags(), &HashMap::new()); - assert_eq!(config.tags.function_tags(), None); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_TAGS", Some("invalid:thing:here,also-bad")), + ], + || { + let config = test_config_with_dd_tags(); + assert_eq!(config.tags.tags(), &HashMap::new()); + assert_eq!(config.tags.function_tags(), None); + }, + ); // Test with empty string - let config = test_config_with_dd_tags(""); - assert_eq!(config.tags.tags(), &HashMap::new()); - assert_eq!(config.tags.function_tags(), None); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_TAGS", Some("")), + ], + || { + let config = test_config_with_dd_tags(); + assert_eq!(config.tags.tags(), &HashMap::new()); + assert_eq!(config.tags.function_tags(), None); + }, + ); // Test with just whitespace - let config = test_config_with_dd_tags(" "); - assert_eq!(config.tags.tags(), &HashMap::new()); - assert_eq!(config.tags.function_tags(), None); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_TAGS", Some(" ")), + ], + || { + let config = test_config_with_dd_tags(); + assert_eq!(config.tags.tags(), &HashMap::new()); + assert_eq!(config.tags.function_tags(), None); + }, + ); // Test with just commas and spaces - let config = test_config_with_dd_tags(" , , "); - assert_eq!(config.tags.tags(), &HashMap::new()); - assert_eq!(config.tags.function_tags(), None); + temp_env::with_vars( + [ + ("DD_API_KEY", Some("_not_a_real_key_")), + ( + "ASCSVCRT_SPRING__APPLICATION__NAME", + Some("test-spring-app"), + ), + ("DD_TAGS", Some(" , , ")), + ], + || { + let config = test_config_with_dd_tags(); + assert_eq!(config.tags.tags(), &HashMap::new()); + assert_eq!(config.tags.function_tags(), None); + }, + ); } } diff --git a/crates/datadog-trace-agent/src/env_verifier.rs b/crates/datadog-trace-agent/src/env_verifier.rs index 74bed2cb..29fcc5c7 100644 --- a/crates/datadog-trace-agent/src/env_verifier.rs +++ b/crates/datadog-trace-agent/src/env_verifier.rs @@ -110,12 +110,16 @@ impl ServerlessEnvVerifier { metadata } Err(err) => { - error!("The Mini Agent can only be run in Google Cloud Functions & Azure Functions. Verification has failed, shutting down now. Error: {err}"); + error!( + "The Mini Agent can only be run in Google Cloud Functions & Azure Functions. Verification has failed, shutting down now. Error: {err}" + ); process::exit(1); } }, Err(_) => { - error!("Google Metadata request timeout of {verify_env_timeout} ms exceeded. Using default values."); + error!( + "Google Metadata request timeout of {verify_env_timeout} ms exceeded. Using default values." + ); GCPMetadata::default() } }; @@ -262,7 +266,9 @@ async fn verify_azure_environment_or_exit(os: &str) { debug!("Successfully verified Azure Function Environment."); } Err(e) => { - error!("The Mini Agent can only be run in Google Cloud Functions & Azure Functions. Verification has failed, shutting down now. Error: {e}"); + error!( + "The Mini Agent can only be run in Google Cloud Functions & Azure Functions. Verification has failed, shutting down now. Error: {e}" + ); process::exit(1); } } @@ -354,19 +360,19 @@ async fn ensure_azure_function_environment( #[cfg(test)] mod tests { use async_trait::async_trait; - use hyper::{body::Bytes, Response, StatusCode}; + use hyper::{Response, StatusCode, body::Bytes}; use libdd_common::hyper_migration; use libdd_trace_utils::trace_utils; use serde_json::json; use serial_test::serial; - use std::{env, fs, path::Path, time::Duration}; + use std::{fs, path::Path, time::Duration}; use crate::env_verifier::{ - ensure_azure_function_environment, ensure_gcp_function_environment, - get_region_from_gcp_region_string, is_azure_flex_without_resource_group, - AzureVerificationClient, AzureVerificationClientWrapper, GCPInstance, GCPMetadata, - GCPProject, GoogleMetadataClient, AZURE_FUNCTION_JSON_NAME, AZURE_HOST_JSON_NAME, - DD_AZURE_RESOURCE_GROUP, WEBSITE_SKU, + AZURE_FUNCTION_JSON_NAME, AZURE_HOST_JSON_NAME, AzureVerificationClient, + AzureVerificationClientWrapper, DD_AZURE_RESOURCE_GROUP, GCPInstance, GCPMetadata, + GCPProject, GoogleMetadataClient, WEBSITE_SKU, ensure_azure_function_environment, + ensure_gcp_function_environment, get_region_from_gcp_region_string, + is_azure_flex_without_resource_group, }; use super::{EnvVerifier, ServerlessEnvVerifier}; @@ -642,28 +648,36 @@ mod tests { #[test] #[serial] fn test_is_azure_flex_without_resource_group_true() { - env::remove_var(DD_AZURE_RESOURCE_GROUP); - env::set_var(WEBSITE_SKU, "FlexConsumption"); - assert!(is_azure_flex_without_resource_group()); - env::remove_var(WEBSITE_SKU); + temp_env::with_vars( + [ + (DD_AZURE_RESOURCE_GROUP, None::<&str>), + (WEBSITE_SKU, Some("FlexConsumption")), + ], + || assert!(is_azure_flex_without_resource_group()), + ); } #[test] #[serial] fn test_is_azure_flex_without_resource_group_false_resource_group_set() { - env::set_var(DD_AZURE_RESOURCE_GROUP, "test-resource-group"); - env::set_var(WEBSITE_SKU, "FlexConsumption"); - assert!(!is_azure_flex_without_resource_group()); - env::remove_var(DD_AZURE_RESOURCE_GROUP); - env::remove_var(WEBSITE_SKU); + temp_env::with_vars( + [ + (DD_AZURE_RESOURCE_GROUP, Some("test-resource-group")), + (WEBSITE_SKU, Some("FlexConsumption")), + ], + || assert!(!is_azure_flex_without_resource_group()), + ); } #[test] #[serial] fn test_is_azure_flex_without_resource_group_false_not_flex() { - env::remove_var(DD_AZURE_RESOURCE_GROUP); - env::set_var(WEBSITE_SKU, "ElasticPremium"); - assert!(!is_azure_flex_without_resource_group()); - env::remove_var(WEBSITE_SKU); + temp_env::with_vars( + [ + (DD_AZURE_RESOURCE_GROUP, None::<&str>), + (WEBSITE_SKU, Some("ElasticPremium")), + ], + || assert!(!is_azure_flex_without_resource_group()), + ); } } diff --git a/crates/datadog-trace-agent/src/http_utils.rs b/crates/datadog-trace-agent/src/http_utils.rs index c330cf20..74bc5103 100644 --- a/crates/datadog-trace-agent/src/http_utils.rs +++ b/crates/datadog-trace-agent/src/http_utils.rs @@ -4,9 +4,8 @@ use core::time::Duration; use datadog_fips::reqwest_adapter::create_reqwest_client_builder; use hyper::{ - header, + Response, StatusCode, header, http::{self, HeaderMap}, - Response, StatusCode, }; use libdd_common::hyper_migration; use serde_json::json; @@ -130,9 +129,9 @@ pub fn build_client( #[cfg(test)] mod tests { use http_body_util::BodyExt; - use hyper::header; use hyper::HeaderMap; use hyper::StatusCode; + use hyper::header; use libdd_common::hyper_migration; use super::verify_request_content_length; diff --git a/crates/datadog-trace-agent/src/mini_agent.rs b/crates/datadog-trace-agent/src/mini_agent.rs index 6af32b12..855290c7 100644 --- a/crates/datadog-trace-agent/src/mini_agent.rs +++ b/crates/datadog-trace-agent/src/mini_agent.rs @@ -3,7 +3,7 @@ use http_body_util::BodyExt; use hyper::service::service_fn; -use hyper::{http, Method, Response, StatusCode}; +use hyper::{Method, Response, StatusCode, http}; use libdd_common::hyper_migration; use serde_json::json; use std::io; diff --git a/crates/datadog-trace-agent/src/stats_flusher.rs b/crates/datadog-trace-agent/src/stats_flusher.rs index 573bbeb0..6c6e5805 100644 --- a/crates/datadog-trace-agent/src/stats_flusher.rs +++ b/crates/datadog-trace-agent/src/stats_flusher.rs @@ -3,7 +3,7 @@ use async_trait::async_trait; use std::{sync::Arc, time}; -use tokio::sync::{mpsc::Receiver, Mutex}; +use tokio::sync::{Mutex, mpsc::Receiver}; use tracing::{debug, error}; use libdd_trace_protobuf::pb; diff --git a/crates/datadog-trace-agent/src/stats_processor.rs b/crates/datadog-trace-agent/src/stats_processor.rs index d15be854..889e5f2c 100644 --- a/crates/datadog-trace-agent/src/stats_processor.rs +++ b/crates/datadog-trace-agent/src/stats_processor.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use std::time::UNIX_EPOCH; use async_trait::async_trait; -use hyper::{http, StatusCode}; +use hyper::{StatusCode, http}; use libdd_common::hyper_migration; use tokio::sync::mpsc::Sender; use tracing::debug; diff --git a/crates/datadog-trace-agent/src/trace_flusher.rs b/crates/datadog-trace-agent/src/trace_flusher.rs index 03b78224..cf2619e0 100644 --- a/crates/datadog-trace-agent/src/trace_flusher.rs +++ b/crates/datadog-trace-agent/src/trace_flusher.rs @@ -3,10 +3,10 @@ use async_trait::async_trait; use std::{error::Error, sync::Arc, time}; -use tokio::sync::{mpsc::Receiver, Mutex}; +use tokio::sync::{Mutex, mpsc::Receiver}; use tracing::{debug, error}; -use libdd_common::{hyper_migration, GenericHttpClient}; +use libdd_common::{GenericHttpClient, hyper_migration}; use libdd_trace_utils::trace_utils; use libdd_trace_utils::trace_utils::SendData; diff --git a/crates/datadog-trace-agent/src/trace_processor.rs b/crates/datadog-trace-agent/src/trace_processor.rs index 41f6e9e8..16851371 100644 --- a/crates/datadog-trace-agent/src/trace_processor.rs +++ b/crates/datadog-trace-agent/src/trace_processor.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use async_trait::async_trait; -use hyper::{http, StatusCode}; +use hyper::{StatusCode, http}; use libdd_common::hyper_migration; use tokio::sync::mpsc::Sender; use tracing::debug; @@ -123,19 +123,19 @@ impl TraceProcessor for ServerlessTraceProcessor { return log_and_create_traces_success_http_response( &format!("Error processing trace chunks: {err}"), StatusCode::INTERNAL_SERVER_ERROR, - ) + ); } }; // Add function_tags to payload if we can - if let Some(function_tags) = config.tags.function_tags() { - if let TracerPayloadCollection::V07(ref mut tracer_payloads) = payload { - for tracer_payload in tracer_payloads { - tracer_payload.tags.insert( - TRACER_PAYLOAD_FUNCTION_TAGS_TAG_KEY.to_string(), - function_tags.to_string(), - ); - } + if let Some(function_tags) = config.tags.function_tags() + && let TracerPayloadCollection::V07(ref mut tracer_payloads) = payload + { + for tracer_payload in tracer_payloads { + tracer_payload.tags.insert( + TRACER_PAYLOAD_FUNCTION_TAGS_TAG_KEY.to_string(), + function_tags.to_string(), + ); } } @@ -168,9 +168,9 @@ mod tests { use crate::{ config::{Config, Tags}, - trace_processor::{self, TraceProcessor, TRACER_PAYLOAD_FUNCTION_TAGS_TAG_KEY}, + trace_processor::{self, TRACER_PAYLOAD_FUNCTION_TAGS_TAG_KEY, TraceProcessor}, }; - use libdd_common::{hyper_migration, Endpoint}; + use libdd_common::{Endpoint, hyper_migration}; use libdd_trace_protobuf::pb; use libdd_trace_utils::test_utils::{create_test_gcp_json_span, create_test_gcp_span}; use libdd_trace_utils::trace_utils::MiniAgentMetadata; diff --git a/crates/datadog-trace-agent/tests/common/mock_server.rs b/crates/datadog-trace-agent/tests/common/mock_server.rs index 5e0b485d..b78b96c0 100644 --- a/crates/datadog-trace-agent/tests/common/mock_server.rs +++ b/crates/datadog-trace-agent/tests/common/mock_server.rs @@ -4,7 +4,7 @@ //! Simple mock HTTP server for testing flushers use http_body_util::BodyExt; -use hyper::{body::Incoming, Request, Response}; +use hyper::{Request, Response, body::Incoming}; use hyper_util::rt::TokioIo; use libdd_common::hyper_migration; use std::net::SocketAddr; diff --git a/crates/datadog-trace-agent/tests/integration_test.rs b/crates/datadog-trace-agent/tests/integration_test.rs index a240b812..bf28d4f8 100644 --- a/crates/datadog-trace-agent/tests/integration_test.rs +++ b/crates/datadog-trace-agent/tests/integration_test.rs @@ -7,7 +7,7 @@ use common::helpers::{create_test_trace_payload, send_tcp_request}; use common::mock_server::MockServer; use common::mocks::{MockEnvVerifier, MockStatsFlusher, MockStatsProcessor, MockTraceFlusher}; use datadog_trace_agent::{ - config::{test_helpers::create_tcp_test_config, Config}, + config::{Config, test_helpers::create_tcp_test_config}, mini_agent::MiniAgent, proxy_flusher::ProxyFlusher, trace_flusher::TraceFlusher, diff --git a/crates/dogstatsd/src/aggregator/core.rs b/crates/dogstatsd/src/aggregator/core.rs index 15799e59..62b53a0e 100644 --- a/crates/dogstatsd/src/aggregator/core.rs +++ b/crates/dogstatsd/src/aggregator/core.rs @@ -153,7 +153,10 @@ impl Aggregator { || (this_batch_size + next_chunk_size >= self.max_batch_bytes_sketch_metric) { if this_batch_size == 0 { - warn!("Only one distribution exceeds max batch size, adding it anyway: {:?} with {}", sketch.metric, next_chunk_size); + warn!( + "Only one distribution exceeds max batch size, adding it anyway: {:?} with {}", + sketch.metric, next_chunk_size + ); } else { batched_payloads.push(sketch_payload); sketch_payload = SketchPayload::new(); @@ -218,7 +221,10 @@ impl Aggregator { >= self.max_batch_bytes_single_metric) { if this_batch_size == 0 { - warn!("Only one metric exceeds max batch size, adding it anyway: {:?} with {}", metric.metric, serialized_metric_size); + warn!( + "Only one metric exceeds max batch size, adding it anyway: {:?} with {}", + metric.metric, serialized_metric_size + ); } else { batched_payloads.push(series_payload); series_payload = Series { @@ -320,7 +326,7 @@ fn build_metric(entry: &Metric, mut base_tag_vec: SortedTags) -> Option { if series_failed.is_empty() && sketches_failed.is_empty() { - debug!("Successfully flushed {n_series} series and {n_distributions} distributions"); + debug!( + "Successfully flushed {n_series} series and {n_distributions} distributions" + ); None // Return None to indicate success } else if series_had_error || sketches_had_error { // Only return the metrics if there was an actual shipping error - error!("Failed to flush some metrics due to shipping errors: {} series and {} sketches", - series_failed.len(), sketches_failed.len()); + error!( + "Failed to flush some metrics due to shipping errors: {} series and {} sketches", + series_failed.len(), + sketches_failed.len() + ); // Return the failed metrics for potential retry Some((series_failed, sketches_failed)) } else { diff --git a/crates/dogstatsd/src/metric.rs b/crates/dogstatsd/src/metric.rs index 72fe630e..7698e163 100644 --- a/crates/dogstatsd/src/metric.rs +++ b/crates/dogstatsd/src/metric.rs @@ -406,7 +406,7 @@ mod tests { use proptest::{collection, option, strategy::Strategy, string::string_regex}; use ustr::Ustr; - use crate::metric::{id, parse, timestamp_to_bucket, MetricValue, SortedTags}; + use crate::metric::{MetricValue, SortedTags, id, parse, timestamp_to_bucket}; use super::ParseError; @@ -629,15 +629,19 @@ mod tests { #[test] fn parse_tag_no_value() { - let result = parse("datadog.tracer.flush_triggered:1|c|#lang:go,lang_version:go1.22.10,_dd.origin:lambda,runtime-id:d66f501c-d09b-4d0d-970f-515235c4eb56,v1.65.1,service:aws.lambda,reason:scheduled"); + let result = parse( + "datadog.tracer.flush_triggered:1|c|#lang:go,lang_version:go1.22.10,_dd.origin:lambda,runtime-id:d66f501c-d09b-4d0d-970f-515235c4eb56,v1.65.1,service:aws.lambda,reason:scheduled", + ); assert!(result.is_ok()); - assert!(result - .unwrap() - .tags - .unwrap() - .values - .iter() - .any(|(k, v)| k == "v1.65.1" && v.is_empty())); + assert!( + result + .unwrap() + .tags + .unwrap() + .values + .iter() + .any(|(k, v)| k == "v1.65.1" && v.is_empty()) + ); } #[test] diff --git a/crates/dogstatsd/src/util.rs b/crates/dogstatsd/src/util.rs index c0137383..8a16ef0b 100644 --- a/crates/dogstatsd/src/util.rs +++ b/crates/dogstatsd/src/util.rs @@ -58,7 +58,8 @@ pub fn parse_metric_namespace(namespace: &str) -> Option { { tracing::error!( "DD_STATSD_METRIC_NAMESPACE contains invalid character '{}' in '{}'. Only ASCII alphanumerics, underscores, and periods are allowed. Ignoring namespace.", - invalid_char, trimmed + invalid_char, + trimmed ); return None; } diff --git a/crates/dogstatsd/tests/integration_test.rs b/crates/dogstatsd/tests/integration_test.rs index 49a6f1c2..3155ef87 100644 --- a/crates/dogstatsd/tests/integration_test.rs +++ b/crates/dogstatsd/tests/integration_test.rs @@ -15,7 +15,7 @@ use mockito::Server; use std::sync::Arc; use tokio::{ net::UdpSocket, - time::{sleep, timeout, Duration}, + time::{Duration, sleep, timeout}, }; use tokio_util::sync::CancellationToken; use zstd::zstd_safe::CompressionLevel; @@ -133,7 +133,7 @@ async fn start_dogstatsd_on_port( #[tokio::test] async fn test_send_with_retry_immediate_failure() { use dogstatsd::datadog::{DdApi, DdDdUrl, RetryStrategy}; - use dogstatsd::metric::{parse, SortedTags}; + use dogstatsd::metric::{SortedTags, parse}; let mut server = Server::new_async().await; let mock = server @@ -182,7 +182,7 @@ async fn test_send_with_retry_immediate_failure() { #[tokio::test] async fn test_send_with_retry_linear_backoff_success() { use dogstatsd::datadog::{DdApi, DdDdUrl, RetryStrategy}; - use dogstatsd::metric::{parse, SortedTags}; + use dogstatsd::metric::{SortedTags, parse}; let mut server = Server::new_async().await; let mock = server @@ -246,7 +246,7 @@ async fn test_send_with_retry_linear_backoff_success() { async fn test_send_with_retry_immediate_failure_after_one_attempt() { use dogstatsd::datadog::{DdApi, DdDdUrl, RetryStrategy}; use dogstatsd::flusher::ShippingError; - use dogstatsd::metric::{parse, SortedTags}; + use dogstatsd::metric::{SortedTags, parse}; let mut server = Server::new_async().await; let mock = server