Enable back pressure monitor in Potel (#3602)

Author: antonpirker

sentry_sdk/integrations/opentelemetry/potel_span_processor.py

@@ -62,11 +62,12 @@ def on_end(self, span):
         if is_sentry_span(span):
             return
 
-        if span.parent and not span.parent.is_remote:
-            self._children_spans[span.parent.span_id].append(span)
-        else:
+        is_root_span = not span.parent or span.parent.is_remote
+        if is_root_span:
             # if have a root span ending, we build a transaction and send it
             self._flush_root_span(span)
+        else:
+            self._children_spans[span.parent.span_id].append(span)
 
     # TODO-neel-potel not sure we need a clear like JS
     def shutdown(self):

sentry_sdk/integrations/opentelemetry/sampler.py

@@ -1,5 +1,5 @@
+import random
 from typing import cast
-from random import random
 
 from opentelemetry import trace
 
@@ -46,17 +46,33 @@ def get_parent_sampled(parent_context, trace_id):
     return None
 
 
-def dropped_result(span_context, attributes, sample_rate=None):
+def dropped_result(parent_span_context, attributes, sample_rate=None):
     # type: (SpanContext, Attributes, Optional[float]) -> SamplingResult
     # these will only be added the first time in a root span sampling decision
-    trace_state = span_context.trace_state
+    trace_state = parent_span_context.trace_state
 
     if TRACESTATE_SAMPLED_KEY not in trace_state:
         trace_state = trace_state.add(TRACESTATE_SAMPLED_KEY, "false")
 
     if sample_rate and TRACESTATE_SAMPLE_RATE_KEY not in trace_state:
         trace_state = trace_state.add(TRACESTATE_SAMPLE_RATE_KEY, str(sample_rate))
 
+    is_root_span = not (
+        parent_span_context.is_valid and not parent_span_context.is_remote
+    )
+    if is_root_span:
+        # Tell Sentry why we dropped the transaction/root-span
+        client = sentry_sdk.get_client()
+        if client.monitor and client.monitor.downsample_factor > 0:
+            reason = "backpressure"
+        else:
+            reason = "sample_rate"
+
+        client.transport.record_lost_event(reason, data_category="transaction")
+
+        # Only one span (the transaction itself) is discarded, since we did not record any spans here.
+        client.transport.record_lost_event(reason, data_category="span")
+
     return SamplingResult(
         Decision.DROP,
         attributes=attributes,
@@ -136,9 +152,14 @@ def should_sample(
             )
             return dropped_result(parent_span_context, attributes)
 
+        # Down-sample in case of back pressure monitor says so
+        # TODO: this should only be done for transactions (aka root spans)
+        if client.monitor:
+            sample_rate /= 2**client.monitor.downsample_factor
+
         # Roll the dice on sample rate
         sample_rate = float(cast("Union[bool, float, int]", sample_rate))
-        sampled = random() < sample_rate
+        sampled = random.random() < sample_rate
 
         if sampled:
             return sampled_result(parent_span_context, attributes, sample_rate)

tests/test_monitor.py

@@ -55,14 +55,16 @@ def test_monitor_unhealthy(sentry_init):
         assert monitor.downsample_factor == (i + 1 if i < 10 else 10)
 
 
-def test_transaction_uses_downsampled_rate(
-    sentry_init, capture_record_lost_event_calls, monkeypatch
+def test_transaction_uses_downsample_rate(
+    sentry_init, capture_envelopes, capture_record_lost_event_calls, monkeypatch
 ):
     sentry_init(
         traces_sample_rate=1.0,
         transport=UnhealthyTestTransport(),
     )
 
+    envelopes = capture_envelopes()
+
     record_lost_event_calls = capture_record_lost_event_calls()
 
     monitor = sentry_sdk.get_client().monitor
@@ -77,13 +79,32 @@ def test_transaction_uses_downsampled_rate(
     assert monitor.downsample_factor == 1
 
     with sentry_sdk.start_transaction(name="foobar") as transaction:
+        with sentry_sdk.start_span(name="foospan"):
+            with sentry_sdk.start_span(name="foospan2"):
+                with sentry_sdk.start_span(name="foospan3"):
+                    ...
+
         assert transaction.sampled is False
-        assert transaction.sample_rate == 0.5
+        assert (
+            transaction.sample_rate == 0.5
+        )  # TODO: this fails until we put the sample_rate in the POTelSpan
+
+    assert len(envelopes) == 0
 
     assert Counter(record_lost_event_calls) == Counter(
         [
-            ("backpressure", "transaction", None, 1),
-            ("backpressure", "span", None, 1),
+            (
+                "backpressure",
+                "transaction",
+                None,
+                1,
+            ),
+            (
+                "backpressure",
+                "span",
+                None,
+                1,
+            ),  # Only one span (the transaction itself) is counted, since we did not record any spans in the first place.
         ]
     )