Sample high-volume analytics events

cli/src/utils/analytics.ts

@@ -9,6 +9,7 @@ import {
   IS_PROD as defaultIsProd,
   DEBUG_ANALYTICS,
 } from '@codebuff/common/env'
+import { shouldTrackAnalyticsEvent } from '@codebuff/common/util/analytics-sampling'
 
 import type { AnalyticsEvent } from '@codebuff/common/constants/analytics-events'
 
@@ -211,6 +212,10 @@ export function trackEvent(
     return
   }
 
+  if (!shouldTrackAnalyticsEvent({ event, distinctId, properties })) {
+    return
+  }
+
   try {
     client.capture({
       distinctId,

cli/src/utils/logger.ts

@@ -7,6 +7,10 @@ import { AnalyticsEvent } from '@codebuff/common/constants/analytics-events'
 import { env, IS_DEV, IS_TEST, IS_CI } from '@codebuff/common/env'
 import { createAnalyticsDispatcher } from '@codebuff/common/util/analytics-dispatcher'
 import { getAnalyticsEventId } from '@codebuff/common/util/analytics-log'
+import {
+  isFullTelemetryEnabled,
+  summarizeAnalyticsValue,
+} from '@codebuff/common/util/analytics-sampling'
 import { pino } from 'pino'
 
 import {
@@ -169,10 +173,23 @@ function sendAnalyticsAndLog(
   // Skip if the log already has an eventId (to avoid duplicate tracking)
   const hasEventId = includeData && getAnalyticsEventId(normalizedData) !== null
   if (!IS_DEV && !IS_TEST && !IS_CI && !hasEventId) {
+    const fullTelemetry = isFullTelemetryEnabled({
+      distinctId: loggerContext.userId,
+      properties: loggerContext,
+    })
+    const includeRawData =
+      fullTelemetry || level === 'error' || level === 'fatal'
+    const dataProperties =
+      includeData && includeRawData
+        ? { data: normalizedData }
+        : includeData
+          ? { dataSummary: summarizeAnalyticsValue(normalizedData) }
+          : {}
+
     trackEvent(AnalyticsEvent.CLI_LOG, {
       level,
       msg: stringFormat(normalizedMsg ?? '', ...args),
-      ...(includeData ? { data: normalizedData } : {}),
+      ...dataProperties,
       ...loggerContext,
     })
   }

common/src/util/__tests__/analytics-sampling.test.ts

@@ -0,0 +1,119 @@
+import { afterEach, describe, expect, it } from 'bun:test'
+
+import { AnalyticsEvent } from '@codebuff/common/constants/analytics-events'
+
+import {
+  isFullTelemetryEnabled,
+  shouldTrackAnalyticsEvent,
+  summarizeAnalyticsValue,
+} from '../analytics-sampling'
+
+const ORIGINAL_ENV = {
+  CODEBUFF_FULL_TELEMETRY: process.env.CODEBUFF_FULL_TELEMETRY,
+  CODEBUFF_FULL_TELEMETRY_IDS: process.env.CODEBUFF_FULL_TELEMETRY_IDS,
+  CODEBUFF_FULL_TELEMETRY_USER_IDS:
+    process.env.CODEBUFF_FULL_TELEMETRY_USER_IDS,
+}
+
+function restoreEnv() {
+  for (const [key, value] of Object.entries(ORIGINAL_ENV)) {
+    if (value === undefined) {
+      delete process.env[key]
+    } else {
+      process.env[key] = value
+    }
+  }
+}
+
+describe('analytics sampling', () => {
+  afterEach(() => {
+    restoreEnv()
+  })
+
+  it('always tracks core CLI lifecycle events', () => {
+    expect(
+      shouldTrackAnalyticsEvent({
+        event: AnalyticsEvent.APP_LAUNCHED,
+        distinctId: 'user-1',
+      }),
+    ).toBe(true)
+    expect(
+      shouldTrackAnalyticsEvent({
+        event: AnalyticsEvent.USER_INPUT_COMPLETE,
+        distinctId: 'user-1',
+      }),
+    ).toBe(true)
+  })
+
+  it('always tracks CLI error logs', () => {
+    expect(
+      shouldTrackAnalyticsEvent({
+        event: AnalyticsEvent.CLI_LOG,
+        distinctId: 'user-1',
+        properties: { level: 'error' },
+      }),
+    ).toBe(true)
+  })
+
+  it('samples high-volume events deterministically', () => {
+    const first = shouldTrackAnalyticsEvent({
+      event: AnalyticsEvent.TOOL_USE,
+      distinctId: 'user-1',
+    })
+    const second = shouldTrackAnalyticsEvent({
+      event: AnalyticsEvent.TOOL_USE,
+      distinctId: 'user-1',
+    })
+    const otherEvent = shouldTrackAnalyticsEvent({
+      event: AnalyticsEvent.AGENT_STEP,
+      distinctId: 'user-1',
+    })
+
+    expect(second).toBe(first)
+    expect(typeof otherEvent).toBe('boolean')
+  })
+
+  it('honors full telemetry env flags and allowlists', () => {
+    process.env.CODEBUFF_FULL_TELEMETRY = 'true'
+    expect(
+      isFullTelemetryEnabled({
+        distinctId: 'anyone',
+      }),
+    ).toBe(true)
+
+    delete process.env.CODEBUFF_FULL_TELEMETRY
+    process.env.CODEBUFF_FULL_TELEMETRY_IDS = 'user-2,person@example.com'
+
+    expect(
+      isFullTelemetryEnabled({
+        distinctId: 'user-2',
+      }),
+    ).toBe(true)
+    expect(
+      isFullTelemetryEnabled({
+        properties: { userEmail: 'person@example.com' },
+      }),
+    ).toBe(true)
+    expect(
+      isFullTelemetryEnabled({
+        distinctId: 'user-3',
+      }),
+    ).toBe(false)
+  })
+
+  it('summarizes values without retaining raw contents', () => {
+    expect(summarizeAnalyticsValue('secret text')).toEqual({
+      kind: 'string',
+      length: 11,
+    })
+    expect(summarizeAnalyticsValue(['a', 'b'])).toEqual({
+      kind: 'array',
+      length: 2,
+    })
+    expect(summarizeAnalyticsValue({ prompt: 'secret', count: 1 })).toEqual({
+      kind: 'object',
+      keyCount: 2,
+      keys: ['prompt', 'count'],
+    })
+  })
+})

common/src/util/analytics-sampling.ts

@@ -0,0 +1,200 @@
+import { AnalyticsEvent } from '../constants/analytics-events'
+
+const DEFAULT_SAMPLED_RATE = 0.01
+
+const SAMPLED_EVENT_RATES: Partial<Record<AnalyticsEvent, number>> = {
+  [AnalyticsEvent.AGENT_STEP]: DEFAULT_SAMPLED_RATE,
+  [AnalyticsEvent.CHATGPT_OAUTH_REQUEST]: DEFAULT_SAMPLED_RATE,
+  [AnalyticsEvent.CLI_LOG]: DEFAULT_SAMPLED_RATE,
+  [AnalyticsEvent.FEEDBACK_BUTTON_HOVERED]: DEFAULT_SAMPLED_RATE,
+  [AnalyticsEvent.FOLLOWUP_CLICKED]: DEFAULT_SAMPLED_RATE,
+  [AnalyticsEvent.SLASH_COMMAND_USED]: DEFAULT_SAMPLED_RATE,
+  [AnalyticsEvent.SLASH_MENU_ACTIVATED]: DEFAULT_SAMPLED_RATE,
+  [AnalyticsEvent.TOOL_USE]: DEFAULT_SAMPLED_RATE,
+}
+
+const ALWAYS_TRACK_EVENTS = new Set<AnalyticsEvent>([
+  AnalyticsEvent.APP_LAUNCHED,
+  AnalyticsEvent.CHANGE_DIRECTORY,
+  AnalyticsEvent.CHATGPT_OAUTH_AUTH_ERROR,
+  AnalyticsEvent.CHATGPT_OAUTH_RATE_LIMITED,
+  AnalyticsEvent.FINGERPRINT_GENERATED,
+  AnalyticsEvent.INVALID_COMMAND,
+  AnalyticsEvent.KNOWLEDGE_FILE_UPDATED,
+  AnalyticsEvent.LOGIN,
+  AnalyticsEvent.TERMINAL_COMMAND_COMPLETED,
+  AnalyticsEvent.UPDATE_CODEBUFF_FAILED,
+  AnalyticsEvent.USER_INPUT,
+  AnalyticsEvent.USER_INPUT_COMPLETE,
+])
+
+type AnalyticsProperties = Record<string, unknown> | undefined
+
+function getStringProperty(
+  properties: AnalyticsProperties,
+  key: string,
+): string | undefined {
+  const value = properties?.[key]
+  return typeof value === 'string' && value.trim() ? value : undefined
+}
+
+function getPropertyUserId(properties: AnalyticsProperties): string | undefined {
+  const direct =
+    getStringProperty(properties, 'userId') ??
+    getStringProperty(properties, 'user_id') ??
+    getStringProperty(properties, 'distinct_id')
+  if (direct) {
+    return direct
+  }
+
+  const user = properties?.user
+  if (user && typeof user === 'object') {
+    const id = (user as { id?: unknown }).id
+    return typeof id === 'string' && id.trim() ? id : undefined
+  }
+
+  return undefined
+}
+
+function splitEnvList(value: string | undefined): Set<string> {
+  return new Set(
+    (value ?? '')
+      .split(',')
+      .map((item) => item.trim())
+      .filter(Boolean),
+  )
+}
+
+function isTruthyEnv(value: string | undefined): boolean {
+  return value === '1' || value === 'true' || value === 'yes'
+}
+
+export function isFullTelemetryEnabled(params: {
+  distinctId?: string
+  properties?: AnalyticsProperties
+}): boolean {
+  if (isTruthyEnv(process.env.CODEBUFF_FULL_TELEMETRY)) {
+    return true
+  }
+
+  const ids = splitEnvList(
+    process.env.CODEBUFF_FULL_TELEMETRY_IDS ??
+      process.env.CODEBUFF_FULL_TELEMETRY_USER_IDS,
+  )
+  if (ids.size === 0) {
+    return false
+  }
+
+  const candidates = [
+    params.distinctId,
+    getPropertyUserId(params.properties),
+    getStringProperty(params.properties, 'userEmail'),
+    getStringProperty(params.properties, 'email'),
+  ].filter(
+    (value): value is string =>
+      typeof value === 'string' && value.length > 0,
+  )
+
+  return candidates.some((candidate) => ids.has(candidate))
+}
+
+function getEventSampleRate(
+  event: AnalyticsEvent,
+  properties: AnalyticsProperties,
+): number {
+  const level = getStringProperty(properties, 'level')?.toLowerCase()
+  if (
+    event === AnalyticsEvent.CLI_LOG &&
+    (level === 'error' || level === 'fatal')
+  ) {
+    return 1
+  }
+
+  if (ALWAYS_TRACK_EVENTS.has(event)) {
+    return 1
+  }
+
+  return SAMPLED_EVENT_RATES[event] ?? 1
+}
+
+function hashString(input: string): number {
+  let hash = 2166136261
+  for (let i = 0; i < input.length; i++) {
+    hash ^= input.charCodeAt(i)
+    hash = Math.imul(hash, 16777619)
+  }
+  return hash >>> 0
+}
+
+function getSamplingKey(params: {
+  event: AnalyticsEvent
+  distinctId?: string
+  properties?: AnalyticsProperties
+}): string {
+  return (
+    params.distinctId ??
+    getPropertyUserId(params.properties) ??
+    getStringProperty(params.properties, 'clientSessionId') ??
+    getStringProperty(params.properties, 'userInputId') ??
+    params.event
+  )
+}
+
+export function shouldTrackAnalyticsEvent(params: {
+  event: AnalyticsEvent
+  distinctId?: string
+  properties?: AnalyticsProperties
+}): boolean {
+  if (isFullTelemetryEnabled(params)) {
+    return true
+  }
+
+  const rate = getEventSampleRate(params.event, params.properties)
+  if (rate >= 1) {
+    return true
+  }
+  if (rate <= 0) {
+    return false
+  }
+
+  const bucket =
+    hashString(`${params.event}:${getSamplingKey(params)}`) / 0xffffffff
+  return bucket < rate
+}
+
+function valueKind(value: unknown): string {
+  if (Array.isArray(value)) {
+    return 'array'
+  }
+  if (value === null) {
+    return 'null'
+  }
+  return typeof value
+}
+
+export function summarizeAnalyticsValue(
+  value: unknown,
+): Record<string, unknown> {
+  if (value === null || value === undefined) {
+    return { kind: valueKind(value) }
+  }
+
+  if (typeof value === 'string') {
+    return { kind: 'string', length: value.length }
+  }
+
+  if (Array.isArray(value)) {
+    return { kind: 'array', length: value.length }
+  }
+
+  if (typeof value === 'object') {
+    const keys = Object.keys(value as Record<string, unknown>)
+    return {
+      kind: 'object',
+      keyCount: keys.length,
+      keys: keys.slice(0, 25),
+    }
+  }
+
+  return { kind: valueKind(value) }
+}

docs/environment-variables.md

@@ -6,6 +6,8 @@
 - Server secrets: validated in `packages/internal/src/env-schema.ts` (used via `@codebuff/internal/env`).
 - Runtime/OS env: pass typed snapshots instead of reading `process.env` throughout the codebase.
 - `IPINFO_TOKEN` is required; free-mode country gating uses it to check IPinfo privacy signals for VPN/proxy/Tor/relay/hosting traffic.
+- `CODEBUFF_FULL_TELEMETRY=true` or `CODEBUFF_FULL_TELEMETRY_IDS=user-id,email@example.com`
+  disables client analytics sampling for targeted debugging. Use sparingly because it can send full CLI log payloads.
 
 ## Env DI Helpers