From 881b07c8782facd7ec03a7890afd4c6395fff430 Mon Sep 17 00:00:00 2001 From: Ankur Goyal Date: Fri, 8 May 2026 15:46:54 -0700 Subject: [PATCH 1/4] opt out by default --- js/src/gitutil.ts | 8 +++++--- js/src/logger.ts | 20 ++++++++++---------- js/util/git_fields.ts | 20 +++++++++++++++++++- js/util/index.ts | 5 ++++- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/js/src/gitutil.ts b/js/src/gitutil.ts index ef7a26bcb..b82d177c0 100644 --- a/js/src/gitutil.ts +++ b/js/src/gitutil.ts @@ -4,6 +4,7 @@ import { } from "./generated_types"; import { debugLogger } from "./debug-logger"; import { simpleGit } from "simple-git"; +import { defaultGitMetadataSettings } from "../util/git_fields"; const COMMON_BASE_BRANCHES = ["main", "master", "develop"]; @@ -149,17 +150,18 @@ function truncateToByteLimit(s: string, byteLimit: number = 65536): string { } export async function getRepoInfo(settings?: GitMetadataSettings) { - if (settings && settings.collect === "none") { + const resolvedSettings = settings ?? defaultGitMetadataSettings(); + if (resolvedSettings.collect === "none") { return undefined; } const repo = await repoInfo(); - if (!repo || !settings || settings.collect === "all") { + if (!repo || resolvedSettings.collect === "all") { return repo; } let sanitized: RepoInfo = {}; - settings.fields?.forEach((field) => { + resolvedSettings.fields?.forEach((field) => { sanitized = { ...sanitized, [field]: repo[field] }; }); diff --git a/js/src/logger.ts b/js/src/logger.ts index 81da807fe..dee2c3342 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -31,6 +31,7 @@ import { IdField, IS_MERGE_FIELD, LogFeedbackFullArgs, + defaultGitMetadataSettings, mergeDicts, mergeGitMetadataSettings, mergeRowBatch, @@ -3535,7 +3536,7 @@ type InitializedExperiment = * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API key is specified, will prompt the user to login. * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple. * @param options.metadata (Optional) A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings. - * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect all git metadata fields allowed in org-level settings. + * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in. * @param setCurrent If true (the default), set the global current-experiment to the newly-created one. * @param options.open If the experiment already exists, open it in read-only mode. Throws an error if the experiment does not already exist. * @param options.projectId The id of the project to create the experiment in. This takes precedence over `project` if specified. @@ -3689,9 +3690,7 @@ export function init( return repoInfo; } let mergedGitMetadataSettings = { - ...(state.gitMetadataSettings || { - collect: "all", - }), + ...(state.gitMetadataSettings || defaultGitMetadataSettings()), }; if (gitMetadataSettings) { mergedGitMetadataSettings = mergeGitMetadataSettings( @@ -4263,8 +4262,8 @@ export function initDataset< legacy, _internal_btql, resolvedVersion instanceof LazyValue || - normalizedEnvironment !== undefined || - normalizedSnapshotName !== undefined + normalizedEnvironment !== undefined || + normalizedSnapshotName !== undefined ? { ...(resolvedVersion instanceof LazyValue ? { @@ -5699,7 +5698,8 @@ function _saveOrgInfo( state.orgName = org.name; state.apiUrl = iso.getEnv("BRAINTRUST_API_URL") ?? org.api_url; state.proxyUrl = iso.getEnv("BRAINTRUST_PROXY_URL") ?? org.proxy_url; - state.gitMetadataSettings = org.git_metadata || undefined; + state.gitMetadataSettings = + org.git_metadata || defaultGitMetadataSettings(); break; } } @@ -6041,9 +6041,9 @@ export type WithTransactionId = R & { export const DEFAULT_FETCH_BATCH_SIZE = 1000; export const MAX_BTQL_ITERATIONS = 10000; -export class ObjectFetcher implements AsyncIterable< - WithTransactionId -> { +export class ObjectFetcher + implements AsyncIterable> +{ private _fetchedData: WithTransactionId[] | undefined = undefined; constructor( diff --git a/js/util/git_fields.ts b/js/util/git_fields.ts index 8eb06d625..0ea592ab8 100644 --- a/js/util/git_fields.ts +++ b/js/util/git_fields.ts @@ -1,4 +1,22 @@ -import { GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; +import { type GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; + +export const DEFAULT_GIT_METADATA_FIELDS = [ + "commit", + "branch", + "tag", + "dirty", + "author_name", + "author_email", + "commit_message", + "commit_time", +]; + +export function defaultGitMetadataSettings(): GitMetadataSettings { + return { + collect: "some", + fields: [...DEFAULT_GIT_METADATA_FIELDS], + }; +} export function mergeGitMetadataSettings( s1: GitMetadataSettings, diff --git a/js/util/index.ts b/js/util/index.ts index 0d1d0b1a4..83d2b76c5 100644 --- a/js/util/index.ts +++ b/js/util/index.ts @@ -136,7 +136,10 @@ export { spanTypeAttributeValues, } from "./span_types"; -export { mergeGitMetadataSettings } from "./git_fields"; +export { + defaultGitMetadataSettings, + mergeGitMetadataSettings, +} from "./git_fields"; export { loadPrettyXact, prettifyXact } from "./xact-ids"; From d361db1c567ae56f831d9d657fcc1cc590563623 Mon Sep 17 00:00:00 2001 From: Ankur Goyal Date: Fri, 8 May 2026 16:12:52 -0700 Subject: [PATCH 2/4] bump --- js/util/git_fields.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/js/util/git_fields.ts b/js/util/git_fields.ts index 0ea592ab8..6001516f4 100644 --- a/js/util/git_fields.ts +++ b/js/util/git_fields.ts @@ -1,6 +1,8 @@ import { type GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; -export const DEFAULT_GIT_METADATA_FIELDS = [ +export const DEFAULT_GIT_METADATA_FIELDS: NonNullable< + GitMetadataSettings["fields"] +> = [ "commit", "branch", "tag", From c863e2e0e477efba92203376865f394c3157b727 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Thu, 21 May 2026 01:22:44 -0700 Subject: [PATCH 3/4] cleanup --- .changeset/soft-eagles-protect.md | 5 +++++ js/src/framework.ts | 2 +- js/src/logger.ts | 10 +++++----- 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 .changeset/soft-eagles-protect.md diff --git a/.changeset/soft-eagles-protect.md b/.changeset/soft-eagles-protect.md new file mode 100644 index 000000000..16c60f222 --- /dev/null +++ b/.changeset/soft-eagles-protect.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +Make the default git metadata settings exclude diff content unless the org opts in. diff --git a/js/src/framework.ts b/js/src/framework.ts index eb22fbb8b..7f00ea055 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -344,7 +344,7 @@ export interface Evaluator< baseExperimentId?: string; /** - * Optional settings for collecting git metadata. By default, will collect all git metadata fields allowed in org-level settings. + * Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in. */ gitMetadataSettings?: GitMetadataSettings; diff --git a/js/src/logger.ts b/js/src/logger.ts index 5d83cc509..91c4e4e61 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -4272,8 +4272,8 @@ export function initDataset< legacy, _internal_btql, resolvedVersion instanceof LazyValue || - normalizedEnvironment !== undefined || - normalizedSnapshotName !== undefined + normalizedEnvironment !== undefined || + normalizedSnapshotName !== undefined ? { ...(resolvedVersion instanceof LazyValue ? { @@ -6051,9 +6051,9 @@ export type WithTransactionId = R & { export const DEFAULT_FETCH_BATCH_SIZE = 1000; export const MAX_BTQL_ITERATIONS = 10000; -export class ObjectFetcher - implements AsyncIterable> -{ +export class ObjectFetcher implements AsyncIterable< + WithTransactionId +> { private _fetchedData: WithTransactionId[] | undefined = undefined; constructor( From 966f324fe5b9c1d8c827165a8bee27dd6413aaf2 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Thu, 21 May 2026 10:27:46 -0700 Subject: [PATCH 4/4] remove git default metadata settings and keep source of truth in control plane --- .changeset/soft-eagles-protect.md | 2 +- js/src/framework.ts | 2 +- js/src/gitutil.ts | 8 +++----- js/src/logger.ts | 8 +++----- js/util/git_fields.ts | 22 +--------------------- js/util/index.ts | 5 +---- 6 files changed, 10 insertions(+), 37 deletions(-) diff --git a/.changeset/soft-eagles-protect.md b/.changeset/soft-eagles-protect.md index 16c60f222..1f72dad5a 100644 --- a/.changeset/soft-eagles-protect.md +++ b/.changeset/soft-eagles-protect.md @@ -2,4 +2,4 @@ "braintrust": patch --- -Make the default git metadata settings exclude diff content unless the org opts in. +Do not collect git metadata when org-level metadata settings are absent. diff --git a/js/src/framework.ts b/js/src/framework.ts index 7f00ea055..c9a834ebd 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -344,7 +344,7 @@ export interface Evaluator< baseExperimentId?: string; /** - * Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in. + * Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings. If org settings are absent, git metadata is not collected. */ gitMetadataSettings?: GitMetadataSettings; diff --git a/js/src/gitutil.ts b/js/src/gitutil.ts index b82d177c0..43a6d8005 100644 --- a/js/src/gitutil.ts +++ b/js/src/gitutil.ts @@ -4,7 +4,6 @@ import { } from "./generated_types"; import { debugLogger } from "./debug-logger"; import { simpleGit } from "simple-git"; -import { defaultGitMetadataSettings } from "../util/git_fields"; const COMMON_BASE_BRANCHES = ["main", "master", "develop"]; @@ -150,18 +149,17 @@ function truncateToByteLimit(s: string, byteLimit: number = 65536): string { } export async function getRepoInfo(settings?: GitMetadataSettings) { - const resolvedSettings = settings ?? defaultGitMetadataSettings(); - if (resolvedSettings.collect === "none") { + if (!settings || settings.collect === "none") { return undefined; } const repo = await repoInfo(); - if (!repo || resolvedSettings.collect === "all") { + if (!repo || settings.collect === "all") { return repo; } let sanitized: RepoInfo = {}; - resolvedSettings.fields?.forEach((field) => { + settings.fields?.forEach((field) => { sanitized = { ...sanitized, [field]: repo[field] }; }); diff --git a/js/src/logger.ts b/js/src/logger.ts index 91c4e4e61..2fe305dec 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -31,7 +31,6 @@ import { IdField, IS_MERGE_FIELD, LogFeedbackFullArgs, - defaultGitMetadataSettings, mergeDicts, mergeGitMetadataSettings, mergeRowBatch, @@ -3546,7 +3545,7 @@ type InitializedExperiment = * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API key is specified, will prompt the user to login. * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple. * @param options.metadata (Optional) A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings. - * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in. + * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings. If org settings are absent, git metadata is not collected. * @param setCurrent If true (the default), set the global current-experiment to the newly-created one. * @param options.open If the experiment already exists, open it in read-only mode. Throws an error if the experiment does not already exist. * @param options.projectId The id of the project to create the experiment in. This takes precedence over `project` if specified. @@ -3700,7 +3699,7 @@ export function init( return repoInfo; } let mergedGitMetadataSettings = { - ...(state.gitMetadataSettings || defaultGitMetadataSettings()), + ...(state.gitMetadataSettings || { collect: "none" as const }), }; if (gitMetadataSettings) { mergedGitMetadataSettings = mergeGitMetadataSettings( @@ -5708,8 +5707,7 @@ function _saveOrgInfo( state.orgName = org.name; state.apiUrl = iso.getEnv("BRAINTRUST_API_URL") ?? org.api_url; state.proxyUrl = iso.getEnv("BRAINTRUST_PROXY_URL") ?? org.proxy_url; - state.gitMetadataSettings = - org.git_metadata || defaultGitMetadataSettings(); + state.gitMetadataSettings = org.git_metadata || { collect: "none" }; break; } } diff --git a/js/util/git_fields.ts b/js/util/git_fields.ts index 6001516f4..8eb06d625 100644 --- a/js/util/git_fields.ts +++ b/js/util/git_fields.ts @@ -1,24 +1,4 @@ -import { type GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; - -export const DEFAULT_GIT_METADATA_FIELDS: NonNullable< - GitMetadataSettings["fields"] -> = [ - "commit", - "branch", - "tag", - "dirty", - "author_name", - "author_email", - "commit_message", - "commit_time", -]; - -export function defaultGitMetadataSettings(): GitMetadataSettings { - return { - collect: "some", - fields: [...DEFAULT_GIT_METADATA_FIELDS], - }; -} +import { GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; export function mergeGitMetadataSettings( s1: GitMetadataSettings, diff --git a/js/util/index.ts b/js/util/index.ts index 83d2b76c5..0d1d0b1a4 100644 --- a/js/util/index.ts +++ b/js/util/index.ts @@ -136,10 +136,7 @@ export { spanTypeAttributeValues, } from "./span_types"; -export { - defaultGitMetadataSettings, - mergeGitMetadataSettings, -} from "./git_fields"; +export { mergeGitMetadataSettings } from "./git_fields"; export { loadPrettyXact, prettifyXact } from "./xact-ids";