fix(core): DSPX-4999 restore database and external-core tracing

docs/Configuring.md

@@ -343,6 +343,12 @@ For OTLP provider:
 - `server.trace.provider.otlp.insecure`: Whether to use an insecure connection
 - `server.trace.provider.otlp.headers`: Headers to include in OTLP requests
 
+Tracing covers incoming Connect RPCs, outbound IPC/ERS/external-core RPCs,
+and PostgreSQL queries and connection-pool acquisition under recorded requests.
+Database spans include SQL statements but do not include bind parameter values.
+Query duration includes reading the result rows, not just database execution.
+Setting `server.trace.enabled` to `false` disables span export for these paths.
+
 Example:
 
 ```yaml

service/go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/dgraph-io/ristretto/v2 v2.4.0
 	github.com/eko/gocache/lib/v4 v4.2.0
 	github.com/eko/gocache/store/ristretto/v4 v4.3.2
+	github.com/exaring/otelpgx v0.10.0
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/go-chi/cors v1.2.1
 	github.com/go-playground/validator/v10 v10.26.0

service/go.sum

@@ -103,6 +103,8 @@ github.com/eko/gocache/lib/v4 v4.2.0 h1:MNykyi5Xw+5Wu3+PUrvtOCaKSZM1nUSVftbzmeC7
 github.com/eko/gocache/lib/v4 v4.2.0/go.mod h1:7ViVmbU+CzDHzRpmB4SXKyyzyuJ8A3UW3/cszpcqB4M=
 github.com/eko/gocache/store/ristretto/v4 v4.3.2 h1:DfvjqmB6hPHJ9oduReMohe8rZCVtxmY8OqTkmIu+dk0=
 github.com/eko/gocache/store/ristretto/v4 v4.3.2/go.mod h1:1F6nJFAY6fTx/UVd66iYr26V2GzZbVJqQJSl+CkRGh4=
+github.com/exaring/otelpgx v0.10.0 h1:NGGegdoBQM3jNZDKG8ENhigUcgBN7d7943L0YlcIpZc=
+github.com/exaring/otelpgx v0.10.0/go.mod h1:R5/M5LWsPPBZc1SrRE5e0DiU48bI78C1/GPTWs6I66U=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=

service/pkg/db/db.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	sq "github.com/Masterminds/squirrel"
+	"github.com/exaring/otelpgx"
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgconn"
 	"github.com/jackc/pgx/v5/pgxpool"
@@ -184,6 +185,9 @@ func New(ctx context.Context, config Config, logCfg logger.Config, tracer *trace
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse pgx config: %w", err)
 	}
+	if c.Tracer != nil {
+		dbConfig.ConnConfig.Tracer = otelpgx.NewTracer()
+	}
 
 	dbConfig.ConnConfig.OnNotice = func(_ *pgconn.PgConn, n *pgconn.Notice) {
 		switch n.Severity {

service/pkg/server/start.go

@@ -548,6 +548,12 @@ func setupIPCSDK(cfg *config.Config, oidcconfig *auth.OIDCConfiguration, otdf *s
 
 // setupExternalSDK configures and creates SDK client for external mode
 func setupExternalSDK(cfg *config.Config, logger *logger.Logger, sdkOptions []sdk.Option) (*sdk.SDK, error) {
+	clientTraceInt, err := tracing.ConnectClientTraceInterceptor()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create external SDK trace interceptor: %w", err)
+	}
+	sdkOptions = append(sdkOptions, sdk.WithExtraClientOptions(connect.WithInterceptors(clientTraceInt)))
+
 	// Use the provided SDK config
 	if cfg.SDKConfig.CorePlatformConnection.Insecure {
 		sdkOptions = append(sdkOptions, sdk.WithInsecureSkipVerifyConn())

service/tracing/otel.go

@@ -182,8 +182,8 @@ func InitTracer(ctx context.Context, cfg Config) (func(), error) {
 	// 6. Return Shutdown Function
 	return func() {
 		logger.InfoContext(ctx, "shutting down tracing...")
-		// Use a separate context for shutdown, typically context.Background() or a context with a timeout
-		shutdownCtx, cancel := context.WithTimeout(ctx, ShutdownTimeout) // Example timeout
+		// Flush queued spans even if the initialization context was canceled.
+		shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), ShutdownTimeout)
 		defer cancel()
 
 		if err := tp.Shutdown(shutdownCtx); err != nil {

service/tracing/otel_test.go

@@ -2,10 +2,12 @@ package tracing
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel"
 )
 
 func Test_InitTracer_CompatibleResourceSchemas_Succeeds(t *testing.T) {
@@ -26,3 +28,34 @@ func Test_InitTracer_CompatibleResourceSchemas_Succeeds(t *testing.T) {
 
 	shutdown()
 }
+
+func Test_InitTracer_ShutdownFlushesAfterContextCancellation(t *testing.T) {
+	previousProvider := otel.GetTracerProvider()
+	previousPropagator := otel.GetTextMapPropagator()
+	t.Cleanup(func() {
+		otel.SetTracerProvider(previousProvider)
+		otel.SetTextMapPropagator(previousPropagator)
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	path := filepath.Join(t.TempDir(), "traces.json")
+	shutdown, err := InitTracer(ctx, Config{
+		Enabled: true,
+		Provider: ProviderConfig{
+			Name: ProviderFile,
+			File: &FileConfig{Path: path},
+		},
+	})
+	require.NoError(t, err)
+	t.Cleanup(shutdown)
+
+	_, span := otel.Tracer(ServiceName).Start(ctx, "pending-at-shutdown")
+	span.End()
+	cancel()
+	shutdown()
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Contains(t, string(data), `"Name":"pending-at-shutdown"`)
+}