From b872d766e129d4a93c69bf2e1e3f59e701221949 Mon Sep 17 00:00:00 2001 From: Brendan Hy Date: Thu, 31 Jul 2025 09:35:35 -0700 Subject: [PATCH 1/8] feat(eap): add downsampled_event_retention to project configs --- relay-dynamic-config/src/project.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/relay-dynamic-config/src/project.rs b/relay-dynamic-config/src/project.rs index b40e28c0256..9b4331df5ab 100644 --- a/relay-dynamic-config/src/project.rs +++ b/relay-dynamic-config/src/project.rs @@ -44,6 +44,9 @@ pub struct ProjectConfig { /// Maximum event retention for the organization. #[serde(skip_serializing_if = "Option::is_none")] pub event_retention: Option, + /// Maximum sampled event retention for the organization. + #[serde(skip_serializing_if = "Option::is_none")] + pub downsampled_event_retention: Option, /// Usage quotas for this project. #[serde(skip_serializing_if = "Vec::is_empty")] pub quotas: Vec, @@ -138,6 +141,7 @@ impl Default for ProjectConfig { filter_settings: ProjectFiltersConfig::default(), datascrubbing_settings: DataScrubbingConfig::default(), event_retention: None, + downsampled_event_retention: None, quotas: Vec::new(), sampling: None, measurements: None, From 1d9db0f39ed0cc4b3b30019e89e9c123d29f3867 Mon Sep 17 00:00:00 2001 From: Brendan Hy Date: Wed, 6 Aug 2025 08:09:48 -0700 Subject: [PATCH 2/8] add changelog --- CHANGELOG.md | 1 + py/CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fad6ccff794..4f180f5257e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Implements basic inbound filters for logs. ([#5011](https://github.com/getsentry/relay/pull/5011)) - Always emit a span usage metric, independent of span feature flags. ([#4976](https://github.com/getsentry/relay/pull/4976)) - Improve PII scrubbing for `logentry.formatted` by ensuring only sensitive data is redacted, rather than replacing the entire field value. ([#4985](https://github.com/getsentry/relay/pull/4985)) +- Add `downsampled_event_retention` to the project configuration. ([#5013](https://github.com/getsentry/relay/pull/5013)) **Bug Fixes**: diff --git a/py/CHANGELOG.md b/py/CHANGELOG.md index 52255dd1c8f..80908315b49 100644 --- a/py/CHANGELOG.md +++ b/py/CHANGELOG.md @@ -3,6 +3,7 @@ # Unreleased - Add `trusted_relay_settings` to the project configuration. ([#4772](https://github.com/getsentry/relay/pull/4772)) +- Add `downsampled_event_retention` to the project configuration. ([#5013](https://github.com/getsentry/relay/pull/5013)) ## 0.9.10 From 667fb224ecbaa3d3caeba8e33ec0789320eb0985 Mon Sep 17 00:00:00 2001 From: Pierre Massat Date: Thu, 7 Aug 2025 16:39:26 -0700 Subject: [PATCH 3/8] feat(eap): Pass downsampled retention to TraceItem --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- relay-server/src/envelope/mod.rs | 22 +++++++++++++++++++++ relay-server/src/processing/logs/mod.rs | 9 +++++++-- relay-server/src/processing/logs/process.rs | 2 ++ relay-server/src/processing/logs/store.rs | 7 +++++++ relay-server/src/services/processor.rs | 8 ++++++++ relay-server/src/services/store.rs | 22 ++++++++++++++++----- 8 files changed, 67 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ccde2392ad0..bfc1ef3bb7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2279,7 +2279,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -4829,9 +4829,9 @@ dependencies = [ [[package]] name = "sentry_protos" -version = "0.3.0" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d688c4b51b06ef8bfd4c77c61e30c678cc3d68464877603ce94434f797cf00e" +checksum = "2736920441e5f518ac0e79c27b91e2203b7c240afbd6e76771dbf78e208dbc12" dependencies = [ "prost", "prost-types", diff --git a/Cargo.toml b/Cargo.toml index 2089721fed4..0a27e5eb6d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,7 +176,7 @@ sentry-core = "0.41.0" sentry-kafka-schemas = { version = "1.3.2", default-features = false } sentry-release-parser = { version = "1.3.2", default-features = false } sentry-types = "0.41.0" -sentry_protos = "0.3.0" +sentry_protos = "0.3.3" serde = { version = "1.0.215", features = ["derive", "rc"] } serde-transcode = "1.1.1" serde-vars = "0.2.0" diff --git a/relay-server/src/envelope/mod.rs b/relay-server/src/envelope/mod.rs index e70e6c82d9f..1b692322326 100644 --- a/relay-server/src/envelope/mod.rs +++ b/relay-server/src/envelope/mod.rs @@ -100,6 +100,13 @@ pub struct EnvelopeHeaders { #[serde(default, skip_serializing_if = "Option::is_none")] retention: Option, + /// Data retention in days for the items of this envelope. + /// + /// This value is always overwritten in processing mode by the value specified in the project + /// configuration. + #[serde(default, skip_serializing_if = "Option::is_none")] + downsampled_retention: Option, + /// Timestamp when the event has been sent, according to the SDK. /// /// This can be used to perform drift correction. @@ -155,6 +162,7 @@ impl EnvelopeHeaders { event_id: self.event_id, meta: meta.copy_to(request_meta), retention: self.retention, + downsampled_retention: self.downsampled_retention, sent_at: self.sent_at, trace: self.trace, required_features: self.required_features, @@ -205,6 +213,7 @@ impl Envelope { event_id, meta, retention: None, + downsampled_retention: None, sent_at: None, other: BTreeMap::new(), trace: None, @@ -303,6 +312,14 @@ impl Envelope { self.headers.retention.unwrap_or(DEFAULT_EVENT_RETENTION) } + /// Returns the data retention in days for items in this envelope. + #[cfg_attr(not(feature = "processing"), allow(dead_code))] + pub fn downsampled_retention(&self) -> u16 { + self.headers + .downsampled_retention + .unwrap_or(DEFAULT_EVENT_RETENTION) + } + /// When the event has been sent, according to the SDK. pub fn sent_at(&self) -> Option> { self.headers.sent_at @@ -360,6 +377,11 @@ impl Envelope { self.headers.retention = Some(retention); } + /// Sets the data retention in days for items in this envelope. + pub fn set_downsampled_retention(&mut self, retention: u16) { + self.headers.downsampled_retention = Some(retention); + } + /// Runs transaction parametrization on the DSC trace transaction. /// /// The purpose is for trace rules to match on the parametrized version of the transaction. diff --git a/relay-server/src/processing/logs/mod.rs b/relay-server/src/processing/logs/mod.rs index 7aa13a793c1..bd0718b44e1 100644 --- a/relay-server/src/processing/logs/mod.rs +++ b/relay-server/src/processing/logs/mod.rs @@ -190,11 +190,13 @@ impl Forward for LogOutput { let scoping = logs.scoping(); let received_at = logs.received_at(); - let (logs, retention) = logs.split_with_context(|logs| (logs.logs, logs.retention)); + let (logs, retentions) = logs + .split_with_context(|logs| (logs.logs, (logs.retention, logs.downsampled_retention))); let ctx = store::Context { scoping, received_at, - retention, + retention: retentions.0, + downsampled_retention: retentions.1, }; for log in logs { @@ -301,6 +303,9 @@ pub struct ExpandedLogs { /// Retention in days. #[cfg(feature = "processing")] retention: Option, + /// Downsampled retention in days. + #[cfg(feature = "processing")] + downsampled_retention: Option, } impl Counted for ExpandedLogs { diff --git a/relay-server/src/processing/logs/process.rs b/relay-server/src/processing/logs/process.rs index 7653c56b0c8..9434a7527eb 100644 --- a/relay-server/src/processing/logs/process.rs +++ b/relay-server/src/processing/logs/process.rs @@ -45,6 +45,8 @@ pub fn expand(logs: Managed, _ctx: Context<'_>) -> Managed, + /// Storage retention for downsampled data in days + pub downsampled_retention: Option, } pub fn convert(log: WithHeader, ctx: &Context) -> Result { @@ -63,6 +65,10 @@ pub fn convert(log: WithHeader, ctx: &Context) -> Result project_id: ctx.scoping.project_id.value(), received: Some(ts(ctx.received_at)), retention_days: ctx.retention.unwrap_or(DEFAULT_EVENT_RETENTION).into(), + downsampled_retention_days: ctx + .downsampled_retention + .unwrap_or(DEFAULT_EVENT_RETENTION) + .into(), timestamp: Some(ts(timestamp.0)), trace_id: required!(log.trace_id).to_string(), item_id: Uuid::new_v7(timestamp.into()).as_bytes().to_vec(), @@ -352,6 +358,7 @@ mod tests { key_id: Some(3), }, retention: Some(42), + downsampled_retention: Some(42), } } diff --git a/relay-server/src/services/processor.rs b/relay-server/src/services/processor.rs index 18cf207acd0..8a114b156d6 100644 --- a/relay-server/src/services/processor.rs +++ b/relay-server/src/services/processor.rs @@ -2265,6 +2265,14 @@ impl EnvelopeProcessorService { managed_envelope.envelope_mut().set_retention(retention); } + // Set the event retention. Effectively, this value will only be available in processing + // mode when the full project config is queried from the upstream. + if let Some(retention) = project_info.config.downsampled_event_retention { + managed_envelope + .envelope_mut() + .set_downsampled_retention(retention); + } + // Ensure the project ID is updated to the stored instance for this project cache. This can // differ in two cases: // 1. The envelope was sent to the legacy `/store/` endpoint without a project ID. diff --git a/relay-server/src/services/store.rs b/relay-server/src/services/store.rs index 15fae32ab11..b6c6eb434da 100644 --- a/relay-server/src/services/store.rs +++ b/relay-server/src/services/store.rs @@ -250,6 +250,7 @@ impl StoreService { scoping: Scoping, ) -> Result<(), StoreError> { let retention = envelope.retention(); + let downsampled_retention = envelope.downsampled_retention(); let event_id = envelope.event_id(); let event_item = envelope.as_mut().take_item_by(|item| { @@ -340,9 +341,14 @@ impl StoreService { let client = envelope.meta().client(); self.produce_check_in(scoping.project_id, received_at, client, retention, item)? } - ItemType::Span => { - self.produce_span(scoping, received_at, event_id, retention, item)? - } + ItemType::Span => self.produce_span( + scoping, + received_at, + event_id, + retention, + downsampled_retention, + item, + )?, ty @ ItemType::Log => { debug_assert!( false, @@ -983,6 +989,7 @@ impl StoreService { received_at: DateTime, event_id: Option, retention_days: u16, + downsampled_retention_days: u16, item: &Item, ) -> Result<(), StoreError> { relay_log::trace!("Producing span"); @@ -1021,6 +1028,7 @@ impl StoreService { span.organization_id = scoping.organization_id.value(); span.project_id = scoping.project_id.value(); span.retention_days = retention_days; + span.downsampled_retention_days = downsampled_retention_days; span.start_timestamp_ms = (span.start_timestamp_precise * 1e3) as u64; span.key_id = scoping.key_id; @@ -1075,7 +1083,7 @@ impl StoreService { scoping: Scoping, received_at: DateTime, event_id: Option, - retention_days: u16, + _retention_days: u16, span: SpanKafkaMessage, ) -> Result<(), StoreError> { let mut trace_item = TraceItem { @@ -1086,7 +1094,8 @@ impl StoreService { seconds: safe_timestamp(received_at) as i64, nanos: 0, }), - retention_days: retention_days.into(), + retention_days: span.retention_days.into(), + downsampled_retention_days: span.downsampled_retention_days.into(), timestamp: Some(Timestamp { seconds: span.start_timestamp_precise as i64, nanos: 0, @@ -1647,6 +1656,9 @@ struct SpanKafkaMessage<'a> { /// Number of days until these data should be deleted. #[serde(default)] retention_days: u16, + /// Number of days until the downsampled version of this data should be deleted. + #[serde(default)] + downsampled_retention_days: u16, #[serde(default, skip_serializing_if = "Option::is_none")] segment_id: Option>, #[serde( From 8592f7d0c49794f273486a1108d0aef1c67435e6 Mon Sep 17 00:00:00 2001 From: Pierre Massat Date: Thu, 7 Aug 2025 16:42:44 -0700 Subject: [PATCH 4/8] Add a CHANGELOG entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f180f5257e..66d92062524 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ - Implements basic inbound filters for logs. ([#5011](https://github.com/getsentry/relay/pull/5011)) - Always emit a span usage metric, independent of span feature flags. ([#4976](https://github.com/getsentry/relay/pull/4976)) - Improve PII scrubbing for `logentry.formatted` by ensuring only sensitive data is redacted, rather than replacing the entire field value. ([#4985](https://github.com/getsentry/relay/pull/4985)) -- Add `downsampled_event_retention` to the project configuration. ([#5013](https://github.com/getsentry/relay/pull/5013)) +- Pass `downsampled_event_retention` to `Traceitem` where appropriate. ([#5013](https://github.com/getsentry/relay/pull/5013), [#5041](https://github.com/getsentry/relay/pull/5041)) **Bug Fixes**: From b2647f986426a5eff6c2a6a7ced0d484988ce0f3 Mon Sep 17 00:00:00 2001 From: Pierre Massat Date: Thu, 7 Aug 2025 18:05:22 -0700 Subject: [PATCH 5/8] Fix integration tests --- tests/integration/test_spans.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/integration/test_spans.py b/tests/integration/test_spans.py index acf5b48d6d3..9bffe3aa29f 100644 --- a/tests/integration/test_spans.py +++ b/tests/integration/test_spans.py @@ -134,6 +134,7 @@ def test_span_extraction( "sentry.transaction.op": "hi", }, "description": "GET /api/0/organizations/?member=1", + "downsampled_retention_days": 90, "duration_ms": int(duration.total_seconds() * 1e3), "event_id": "cbf6960622e14a45abc1f03b2055b186", "exclusive_time_ms": 500.0, @@ -201,6 +202,7 @@ def test_span_extraction( "sentry.transaction.op": "hi", }, "description": "hi", + "downsampled_retention_days": 90, "duration_ms": duration_ms, "event_id": "cbf6960622e14a45abc1f03b2055b186", "exclusive_time_ms": 1500.0, @@ -775,6 +777,7 @@ def test_span_ingestion( "sentry.status": "unknown", }, "description": "my 1st OTel span", + "downsampled_retention_days": 90, "duration_ms": 500, "exclusive_time_ms": 500.0, "is_segment": True, @@ -819,6 +822,7 @@ def test_span_ingestion( "sentry.status": "unknown", }, "description": "my 1st V2 span", + "downsampled_retention_days": 90, "duration_ms": 500, "exclusive_time_ms": 500.0, "is_segment": True, @@ -867,6 +871,7 @@ def test_span_ingestion( "score.total": 0.12121616, }, "description": "https://example.com/p/blah.js", + "downsampled_retention_days": 90, "duration_ms": 1500, "exclusive_time_ms": 345.0, "is_segment": True, @@ -921,6 +926,7 @@ def test_span_ingestion( "sentry.status": "ok", }, "description": "https://example.com/p/blah.js", + "downsampled_retention_days": 90, "duration_ms": 1500, "exclusive_time_ms": 161.0, "is_segment": True, @@ -968,6 +974,7 @@ def test_span_ingestion( "sentry.op": "default", }, "description": r"test \" with \" escaped \" chars", + "downsampled_retention_days": 90, "duration_ms": 1500, "exclusive_time_ms": 345.0, "is_segment": False, @@ -1038,6 +1045,7 @@ def test_span_ingestion( "sentry.browser.name": "Chrome", "sentry.op": "default", }, + "downsampled_retention_days": 90, "duration_ms": 1500, "exclusive_time_ms": 345.0, "is_segment": False, @@ -1071,6 +1079,7 @@ def test_span_ingestion( "sentry.status": "unknown", }, "description": "my 3rd protobuf OTel span", + "downsampled_retention_days": 90, "duration_ms": 500, "exclusive_time_ms": 500.0, "is_segment": False, From 702402124c4325df5550e6733a0ee659d09afa59 Mon Sep 17 00:00:00 2001 From: Pierre Massat Date: Fri, 8 Aug 2025 09:20:58 -0700 Subject: [PATCH 6/8] Fix more tests --- tests/integration/test_spans.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_spans.py b/tests/integration/test_spans.py index 9bffe3aa29f..95c162ec58d 100644 --- a/tests/integration/test_spans.py +++ b/tests/integration/test_spans.py @@ -1003,6 +1003,7 @@ def test_span_ingestion( "sentry.status": "unknown", }, "description": "my 2nd OTel span", + "downsampled_retention_days": 90, "duration_ms": 500, "exclusive_time_ms": 500.0, "is_segment": True, @@ -1619,6 +1620,7 @@ def test_span_ingestion_with_performance_scores( "ttfb": 500.0, "score.cls": 0.0, }, + "downsampled_retention_days": 90, "duration_ms": 1500, "exclusive_time_ms": 345.0, "is_segment": False, @@ -1699,6 +1701,7 @@ def test_span_ingestion_with_performance_scores( "score.total": 0.9948129113413748, "score.weight.inp": 1.0, }, + "downsampled_retention_days": 90, "duration_ms": 1500, "exclusive_time_ms": 345.0, "is_segment": False, @@ -2355,6 +2358,7 @@ def test_scrubs_ip_addresses( "extra_info": "added by user", }, "description": "GET /api/0/organizations/?member=1", + "downsampled_retention_days": 90, "duration_ms": int(duration.total_seconds() * 1e3), "event_id": "cbf6960622e14a45abc1f03b2055b186", "exclusive_time_ms": 500.0, @@ -2432,6 +2436,7 @@ def test_scrubs_ip_addresses( "sentry.user.username": "my_user", }, "description": "hi", + "downsampled_retention_days": 90, "duration_ms": duration_ms, "event_id": "cbf6960622e14a45abc1f03b2055b186", "exclusive_time_ms": 1500.0, From aeddee6b189f4e638c8e742984264e67453dade1 Mon Sep 17 00:00:00 2001 From: Pierre Massat Date: Fri, 8 Aug 2025 09:42:18 -0700 Subject: [PATCH 7/8] Change to default to the retention instead of an arbitrary one --- relay-server/src/envelope/mod.rs | 2 +- relay-server/src/processing/logs/store.rs | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/relay-server/src/envelope/mod.rs b/relay-server/src/envelope/mod.rs index 1b692322326..471b0929e54 100644 --- a/relay-server/src/envelope/mod.rs +++ b/relay-server/src/envelope/mod.rs @@ -317,7 +317,7 @@ impl Envelope { pub fn downsampled_retention(&self) -> u16 { self.headers .downsampled_retention - .unwrap_or(DEFAULT_EVENT_RETENTION) + .unwrap_or(self.retention()) } /// When the event has been sent, according to the SDK. diff --git a/relay-server/src/processing/logs/store.rs b/relay-server/src/processing/logs/store.rs index 879bf06e53b..8ea220a6a0a 100644 --- a/relay-server/src/processing/logs/store.rs +++ b/relay-server/src/processing/logs/store.rs @@ -58,17 +58,15 @@ pub fn convert(log: WithHeader, ctx: &Context) -> Result body: required!(log.body), span_id: log.span_id.into_value(), }; + let retention_days: u16 = ctx.retention.unwrap_or(DEFAULT_EVENT_RETENTION).into(); let trace_item = TraceItem { item_type: TraceItemType::Log.into(), organization_id: ctx.scoping.organization_id.value(), project_id: ctx.scoping.project_id.value(), received: Some(ts(ctx.received_at)), - retention_days: ctx.retention.unwrap_or(DEFAULT_EVENT_RETENTION).into(), - downsampled_retention_days: ctx - .downsampled_retention - .unwrap_or(DEFAULT_EVENT_RETENTION) - .into(), + retention_days, + downsampled_retention_days: ctx.downsampled_retention.unwrap_or(retention_days).into(), timestamp: Some(ts(timestamp.0)), trace_id: required!(log.trace_id).to_string(), item_id: Uuid::new_v7(timestamp.into()).as_bytes().to_vec(), From 9d9e242677ba3b8c77aed7c414ec1fee22d627a1 Mon Sep 17 00:00:00 2001 From: Pierre Massat Date: Fri, 8 Aug 2025 10:08:37 -0700 Subject: [PATCH 8/8] Lint and format --- relay-server/src/processing/logs/store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relay-server/src/processing/logs/store.rs b/relay-server/src/processing/logs/store.rs index 8ea220a6a0a..cd9105c2945 100644 --- a/relay-server/src/processing/logs/store.rs +++ b/relay-server/src/processing/logs/store.rs @@ -58,14 +58,14 @@ pub fn convert(log: WithHeader, ctx: &Context) -> Result body: required!(log.body), span_id: log.span_id.into_value(), }; - let retention_days: u16 = ctx.retention.unwrap_or(DEFAULT_EVENT_RETENTION).into(); + let retention_days = ctx.retention.unwrap_or(DEFAULT_EVENT_RETENTION); let trace_item = TraceItem { item_type: TraceItemType::Log.into(), organization_id: ctx.scoping.organization_id.value(), project_id: ctx.scoping.project_id.value(), received: Some(ts(ctx.received_at)), - retention_days, + retention_days: retention_days.into(), downsampled_retention_days: ctx.downsampled_retention.unwrap_or(retention_days).into(), timestamp: Some(ts(timestamp.0)), trace_id: required!(log.trace_id).to_string(),