diff --git a/docs/Configuring.md b/docs/Configuring.md index 75e68a6965..ebdf671b80 100644 --- a/docs/Configuring.md +++ b/docs/Configuring.md @@ -343,6 +343,12 @@ For OTLP provider: - `server.trace.provider.otlp.insecure`: Whether to use an insecure connection - `server.trace.provider.otlp.headers`: Headers to include in OTLP requests +Tracing covers incoming Connect RPCs, outbound IPC/ERS/external-core RPCs, +and PostgreSQL queries and connection-pool acquisition under recorded requests. +Database spans include SQL statements but do not include bind parameter values. +Query duration includes reading the result rows, not just database execution. +Setting `server.trace.enabled` to `false` disables span export for these paths. + Example: ```yaml diff --git a/service/go.mod b/service/go.mod index bad65d2842..da6939d721 100644 --- a/service/go.mod +++ b/service/go.mod @@ -17,6 +17,7 @@ require ( github.com/dgraph-io/ristretto/v2 v2.4.0 github.com/eko/gocache/lib/v4 v4.2.0 github.com/eko/gocache/store/ristretto/v4 v4.3.2 + github.com/exaring/otelpgx v0.10.0 github.com/fsnotify/fsnotify v1.9.0 github.com/go-chi/cors v1.2.1 github.com/go-playground/validator/v10 v10.26.0 diff --git a/service/go.sum b/service/go.sum index 98cd2b0b35..e49da1a1c7 100644 --- a/service/go.sum +++ b/service/go.sum @@ -103,6 +103,8 @@ github.com/eko/gocache/lib/v4 v4.2.0 h1:MNykyi5Xw+5Wu3+PUrvtOCaKSZM1nUSVftbzmeC7 github.com/eko/gocache/lib/v4 v4.2.0/go.mod h1:7ViVmbU+CzDHzRpmB4SXKyyzyuJ8A3UW3/cszpcqB4M= github.com/eko/gocache/store/ristretto/v4 v4.3.2 h1:DfvjqmB6hPHJ9oduReMohe8rZCVtxmY8OqTkmIu+dk0= github.com/eko/gocache/store/ristretto/v4 v4.3.2/go.mod h1:1F6nJFAY6fTx/UVd66iYr26V2GzZbVJqQJSl+CkRGh4= +github.com/exaring/otelpgx v0.10.0 h1:NGGegdoBQM3jNZDKG8ENhigUcgBN7d7943L0YlcIpZc= +github.com/exaring/otelpgx v0.10.0/go.mod h1:R5/M5LWsPPBZc1SrRE5e0DiU48bI78C1/GPTWs6I66U= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= diff --git a/service/pkg/db/db.go b/service/pkg/db/db.go index c24e586981..420fd60701 100644 --- a/service/pkg/db/db.go +++ b/service/pkg/db/db.go @@ -12,6 +12,7 @@ import ( "time" sq "github.com/Masterminds/squirrel" + "github.com/exaring/otelpgx" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" @@ -184,6 +185,9 @@ func New(ctx context.Context, config Config, logCfg logger.Config, tracer *trace if err != nil { return nil, fmt.Errorf("failed to parse pgx config: %w", err) } + if c.Tracer != nil { + dbConfig.ConnConfig.Tracer = otelpgx.NewTracer() + } dbConfig.ConnConfig.OnNotice = func(_ *pgconn.PgConn, n *pgconn.Notice) { switch n.Severity { diff --git a/service/pkg/server/start.go b/service/pkg/server/start.go index 55b356989f..355391b12e 100644 --- a/service/pkg/server/start.go +++ b/service/pkg/server/start.go @@ -548,6 +548,12 @@ func setupIPCSDK(cfg *config.Config, oidcconfig *auth.OIDCConfiguration, otdf *s // setupExternalSDK configures and creates SDK client for external mode func setupExternalSDK(cfg *config.Config, logger *logger.Logger, sdkOptions []sdk.Option) (*sdk.SDK, error) { + clientTraceInt, err := tracing.ConnectClientTraceInterceptor() + if err != nil { + return nil, fmt.Errorf("failed to create external SDK trace interceptor: %w", err) + } + sdkOptions = append(sdkOptions, sdk.WithExtraClientOptions(connect.WithInterceptors(clientTraceInt))) + // Use the provided SDK config if cfg.SDKConfig.CorePlatformConnection.Insecure { sdkOptions = append(sdkOptions, sdk.WithInsecureSkipVerifyConn()) diff --git a/service/tracing/otel.go b/service/tracing/otel.go index 28a486fa7b..2fef8f3de4 100644 --- a/service/tracing/otel.go +++ b/service/tracing/otel.go @@ -182,8 +182,8 @@ func InitTracer(ctx context.Context, cfg Config) (func(), error) { // 6. Return Shutdown Function return func() { logger.InfoContext(ctx, "shutting down tracing...") - // Use a separate context for shutdown, typically context.Background() or a context with a timeout - shutdownCtx, cancel := context.WithTimeout(ctx, ShutdownTimeout) // Example timeout + // Flush queued spans even if the initialization context was canceled. + shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), ShutdownTimeout) defer cancel() if err := tp.Shutdown(shutdownCtx); err != nil { diff --git a/service/tracing/otel_test.go b/service/tracing/otel_test.go index 3a99a2c030..fd1a1cde73 100644 --- a/service/tracing/otel_test.go +++ b/service/tracing/otel_test.go @@ -2,10 +2,12 @@ package tracing import ( "context" + "os" "path/filepath" "testing" "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" ) func Test_InitTracer_CompatibleResourceSchemas_Succeeds(t *testing.T) { @@ -26,3 +28,34 @@ func Test_InitTracer_CompatibleResourceSchemas_Succeeds(t *testing.T) { shutdown() } + +func Test_InitTracer_ShutdownFlushesAfterContextCancellation(t *testing.T) { + previousProvider := otel.GetTracerProvider() + previousPropagator := otel.GetTextMapPropagator() + t.Cleanup(func() { + otel.SetTracerProvider(previousProvider) + otel.SetTextMapPropagator(previousPropagator) + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + path := filepath.Join(t.TempDir(), "traces.json") + shutdown, err := InitTracer(ctx, Config{ + Enabled: true, + Provider: ProviderConfig{ + Name: ProviderFile, + File: &FileConfig{Path: path}, + }, + }) + require.NoError(t, err) + t.Cleanup(shutdown) + + _, span := otel.Tracer(ServiceName).Start(ctx, "pending-at-shutdown") + span.End() + cancel() + shutdown() + + data, err := os.ReadFile(path) + require.NoError(t, err) + require.Contains(t, string(data), `"Name":"pending-at-shutdown"`) +}