feat(tracing): opt-in SGP_OBS_SUPPRESS_EXPORT to suppress self-tracing of span export

Once outbound HTTP is instrumented, the library's own span export (to egp /
Agentex) becomes an instrumented call whose spans nest into the trace being
exported and drag in egp authz/identity/DB work — dominating the trace ("camera
pointed at its own monitor"). Add suppress_export_instrumentation(): when
SGP_OBS_SUPPRESS_EXPORT is enabled it attaches OTel's suppress-instrumentation
context key around the export so it makes no spans and injects no traceparent.
Applied at the processor-invocation boundaries (sync _run_on_span_*, async
span_queue._handle) so it covers both the SGP and Agentex processors. Opt-in,
default off, fail-open (no-op if disabled or if OTel instrumentation isn't
importable).

TEST/PREVIEW branch — not for merge as-is (belongs in its own PR).

Author: NiteshDhanpal

src/agentex/lib/core/tracing/span_queue.py

@@ -9,6 +9,7 @@
 from agentex.types.span import Span
 from agentex.lib.utils.logging import make_logger
 from agentex.lib.core.observability import tracing_metrics_recording as _metrics
+from agentex.lib.core.tracing.suppression import suppress_export_instrumentation
 from agentex.lib.core.tracing.processors.tracing_processor_interface import (
     AsyncTracingProcessor,
 )
@@ -341,11 +342,14 @@ async def _handle(
         spans = [item.span for item in items]
         try:
             # Hold a concurrency slot only for the duration of the HTTP call.
+            # Suppress instrumentation for the export so the export's own HTTP
+            # call to egp/Agentex isn't traced back into the trace being exported.
             async with self._send_sema:
-                if event_type == SpanEventType.START:
-                    await p.on_spans_start(spans)
-                else:
-                    await p.on_spans_end(spans)
+                with suppress_export_instrumentation():
+                    if event_type == SpanEventType.START:
+                        await p.on_spans_start(spans)
+                    else:
+                        await p.on_spans_end(spans)
         except Exception as exc:
             self._handle_failure(p, items, event_type, exc)
 

src/agentex/lib/core/tracing/suppression.py

@@ -0,0 +1,63 @@
+"""Suppress instrumentation while the tracing library exports its own telemetry.
+
+Once a service instruments outbound HTTP, the library's own span export (shipping
+business spans to egp / the Agentex control plane) becomes an instrumented HTTP
+call. Those export spans nest into the very trace being exported and can dominate
+it -- exporting a span gets traced, which triggers egp authz/identity/DB work
+that is itself traced, and so on ("the camera pointed at its own monitor").
+
+When ``SGP_OBS_SUPPRESS_EXPORT`` is enabled, wrapping the export in
+``suppress_export_instrumentation()`` disables OTel instrumentation for the
+duration: the export makes no spans and injects no ``traceparent``, so it can't
+pollute or hijack the trace. The persistence still happens -- it just stops
+being traced.
+
+Opt-in, default OFF. Fail-open: a no-op if disabled or if OpenTelemetry's
+instrumentation utils aren't importable, so observability can never break the
+app path.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Iterator
+from contextlib import contextmanager
+
+_ENV = "SGP_OBS_SUPPRESS_EXPORT"
+
+
+def suppress_export_enabled() -> bool:
+    """True when ``SGP_OBS_SUPPRESS_EXPORT`` is set to a truthy value."""
+    return os.environ.get(_ENV, "").strip().lower() in ("1", "true", "yes", "on")
+
+
+@contextmanager
+def suppress_export_instrumentation() -> Iterator[None]:
+    """Disable OTel instrumentation for the wrapped telemetry-export block.
+
+    Attaches OpenTelemetry's suppress-instrumentation context key so instrumentors
+    (httpx, requests, ...) skip span creation AND context injection for any call
+    made inside the block. The key rides the current contextvar, so it applies to
+    an awaited export running in the same task. No-op when disabled or when OTel
+    instrumentation isn't importable; never raises.
+    """
+    if not suppress_export_enabled():
+        yield
+        return
+    try:
+        from opentelemetry import context as _otel_context
+
+        # Only present when opentelemetry-instrumentation is installed (agents),
+        # not in the bare SDK env -- hence the guarded import + fail-open above.
+        from opentelemetry.instrumentation.utils import _SUPPRESS_INSTRUMENTATION_KEY  # type: ignore[import-not-found]
+    except Exception:  # pragma: no cover - obs must never break the app path
+        yield
+        return
+    token = _otel_context.attach(_otel_context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True))
+    try:
+        yield
+    finally:
+        try:
+            _otel_context.detach(token)
+        except Exception:  # pragma: no cover - best-effort
+            pass

src/agentex/lib/core/tracing/trace.py

@@ -25,6 +25,7 @@
     AsyncSpanQueue,
     get_default_span_queue,
 )
+from agentex.lib.core.tracing.suppression import suppress_export_instrumentation
 from agentex.lib.core.tracing.processors.tracing_processor_interface import (
     SyncTracingProcessor,
     AsyncTracingProcessor,
@@ -81,7 +82,10 @@ def _run_on_span_start(processor: SyncTracingProcessor, span: Span) -> None:
     swallowing here, start_span returns normally and the standard end_span path
     pops and closes the handle -- no leak, no app-path failure."""
     try:
-        processor.on_span_start(span)
+        # Suppress instrumentation while the processor exports so the export's
+        # own HTTP call to egp/Agentex isn't traced back into this trace.
+        with suppress_export_instrumentation():
+            processor.on_span_start(span)
     except Exception:
         logger.warning(
             "on_span_start raised for processor %r; skipping (observability must not fail the app path)",
@@ -97,7 +101,8 @@ def _run_on_span_end(processor: SyncTracingProcessor, span: Span) -> None:
     before this runs (see end_span), so this only guards the app path against a
     buggy processor -- there is no handle left to leak here."""
     try:
-        processor.on_span_end(span)
+        with suppress_export_instrumentation():
+            processor.on_span_end(span)
     except Exception:
         logger.warning(
             "on_span_end raised for processor %r; skipping (observability must not fail the app path)",