From 2085913c7e51acead18a4627b5df2f413a4658c8 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Thu, 21 May 2026 10:28:20 -0700 Subject: [PATCH] make git default metadata stricter --- py/src/braintrust/framework.py | 9 +++---- py/src/braintrust/git_fields.py | 19 --------------- py/src/braintrust/gitutil.py | 4 ++-- py/src/braintrust/logger.py | 4 ++-- py/src/braintrust/test_logger.py | 40 ++++++++++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 27 deletions(-) diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py index 11f652b3..49eee975 100644 --- a/py/src/braintrust/framework.py +++ b/py/src/braintrust/framework.py @@ -397,8 +397,9 @@ class Evaluator(Generic[Input, Output, Expected]): git_metadata_settings: GitMetadataSettings | None = None """ - Optional settings for collecting git metadata. By default, will collect all - git metadata fields allowed in org-level settings. + Optional settings for collecting git metadata. By default, defers entirely + to org-level settings returned by the control plane; no git metadata is + collected if the org has not configured any. """ repo_info: RepoInfo | None = None @@ -863,7 +864,7 @@ async def EvalAsync( summarized and compared to this experiment. :param base_experiment_id: An optional experiment id to use as a base. If specified, the new experiment will be summarized and compared to this experiment. This takes precedence over `base_experiment_name` if specified. - :param git_metadata_settings: Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in. + :param git_metadata_settings: Optional settings for collecting git metadata. By default, defers to org-level settings returned by the control plane. :param repo_info: Optionally explicitly specify the git metadata for this experiment. This takes precedence over `git_metadata_settings` if specified. :param error_score_handler: Optionally supply a custom function to specifically handle score values when tasks or scoring functions have errored. :param description: An optional description for the experiment. @@ -991,7 +992,7 @@ def Eval( summarized and compared to this experiment. :param base_experiment_id: An optional experiment id to use as a base. If specified, the new experiment will be summarized and compared to this experiment. This takes precedence over `base_experiment_name` if specified. - :param git_metadata_settings: Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in. + :param git_metadata_settings: Optional settings for collecting git metadata. By default, defers to org-level settings returned by the control plane. :param repo_info: Optionally explicitly specify the git metadata for this experiment. This takes precedence over `git_metadata_settings` if specified. :param error_score_handler: Optionally supply a custom function to specifically handle score values when tasks or scoring functions have errored. :param description: An optional description for the experiment. diff --git a/py/src/braintrust/git_fields.py b/py/src/braintrust/git_fields.py index 06e54e06..264102fa 100644 --- a/py/src/braintrust/git_fields.py +++ b/py/src/braintrust/git_fields.py @@ -43,22 +43,3 @@ def merge(cls, s1: "GitMetadataSettings", s2: "GitMetadataSettings") -> "GitMeta if not ret.fields: ret.collect = "none" return ret - - -DEFAULT_GIT_METADATA_FIELDS = [ - "commit", - "branch", - "tag", - "dirty", - "author_name", - "author_email", - "commit_message", - "commit_time", -] - - -def default_git_metadata_settings() -> GitMetadataSettings: - return GitMetadataSettings( - collect="some", - fields=list(DEFAULT_GIT_METADATA_FIELDS), - ) diff --git a/py/src/braintrust/gitutil.py b/py/src/braintrust/gitutil.py index 76b66405..bc96206c 100644 --- a/py/src/braintrust/gitutil.py +++ b/py/src/braintrust/gitutil.py @@ -5,7 +5,7 @@ import threading from functools import lru_cache as _cache -from .git_fields import GitMetadataSettings, RepoInfo, default_git_metadata_settings +from .git_fields import GitMetadataSettings, RepoInfo # https://stackoverflow.com/questions/48399498/git-executable-not-found-in-python @@ -123,7 +123,7 @@ def truncate_to_byte_limit(input_string, byte_limit=65536): def get_repo_info(settings: GitMetadataSettings | None = None): if settings is None: - settings = default_git_metadata_settings() + settings = GitMetadataSettings(collect="none") if settings.collect == "none": return None diff --git a/py/src/braintrust/logger.py b/py/src/braintrust/logger.py index 02457b59..8c4431a6 100644 --- a/py/src/braintrust/logger.py +++ b/py/src/braintrust/logger.py @@ -60,7 +60,7 @@ PromptOptions, SpanAttributes, ) -from .git_fields import GitMetadataSettings, RepoInfo, default_git_metadata_settings +from .git_fields import GitMetadataSettings, RepoInfo from .gitutil import get_past_n_ancestors, get_repo_info from .merge_row_batch import batch_items, merge_row_batch from .object import DEFAULT_IS_LEGACY_DATASET, ensure_dataset_record @@ -2669,7 +2669,7 @@ def _check_org_info(state, org_info, org_name): state.api_url = os.environ.get("BRAINTRUST_API_URL", orgs["api_url"]) state.proxy_url = os.environ.get("BRAINTRUST_PROXY_URL", orgs["proxy_url"]) state.git_metadata_settings = GitMetadataSettings( - **(orgs.get("git_metadata") or default_git_metadata_settings().as_dict()) + **(orgs.get("git_metadata") or GitMetadataSettings(collect="none").as_dict()) ) break diff --git a/py/src/braintrust/test_logger.py b/py/src/braintrust/test_logger.py index c341bead..fcfe78cc 100644 --- a/py/src/braintrust/test_logger.py +++ b/py/src/braintrust/test_logger.py @@ -22,9 +22,12 @@ init_logger, logger, ) +from braintrust.gitutil import get_repo_info from braintrust.id_gen import OTELIDGenerator, get_id_generator from braintrust.logger import ( + BraintrustState, RemoteEvalParameters, + _check_org_info, _extract_attachments, parent_context, render_message, @@ -3699,3 +3702,40 @@ def failing_function(): logs = with_memory_logger.pop() assert len(logs) == 1 _assert_test_exception_group_contents(logs[0].get("error", "")) + + +def test_check_org_info_no_git_metadata_defaults_to_none(): + """When org has no git_metadata, state.git_metadata_settings should be collect='none'.""" + state = BraintrustState() + org_info = [ + { + "id": "org-1", + "name": "org1", + "api_url": "https://api.example.com", + "proxy_url": "https://proxy.example.com", + } + ] + _check_org_info(state, org_info, None) + assert state.git_metadata_settings.collect == "none" + + +def test_check_org_info_with_git_metadata_uses_server_settings(): + """When org provides git_metadata, it is used as-is.""" + state = BraintrustState() + org_info = [ + { + "id": "org-1", + "name": "org1", + "api_url": "https://api.example.com", + "proxy_url": "https://proxy.example.com", + "git_metadata": {"collect": "some", "fields": ["commit", "branch"]}, + } + ] + _check_org_info(state, org_info, None) + assert state.git_metadata_settings.collect == "some" + assert set(state.git_metadata_settings.fields) == {"commit", "branch"} + + +def test_get_repo_info_without_settings_returns_none(): + """Direct call to get_repo_info with settings=None should return None.""" + assert get_repo_info(None) is None