Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions py/src/braintrust/framework.py
Original file line number Diff line number Diff line change
Expand Up @@ -397,8 +397,9 @@ class Evaluator(Generic[Input, Output, Expected]):

git_metadata_settings: GitMetadataSettings | None = None
"""
Optional settings for collecting git metadata. By default, will collect all
git metadata fields allowed in org-level settings.
Optional settings for collecting git metadata. By default, defers entirely
to org-level settings returned by the control plane; no git metadata is
collected if the org has not configured any.
"""

repo_info: RepoInfo | None = None
Expand Down Expand Up @@ -863,7 +864,7 @@ async def EvalAsync(
summarized and compared to this experiment.
:param base_experiment_id: An optional experiment id to use as a base. If specified, the new experiment will be
summarized and compared to this experiment. This takes precedence over `base_experiment_name` if specified.
:param git_metadata_settings: Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in.
:param git_metadata_settings: Optional settings for collecting git metadata. By default, defers to org-level settings returned by the control plane.
:param repo_info: Optionally explicitly specify the git metadata for this experiment. This takes precedence over `git_metadata_settings` if specified.
:param error_score_handler: Optionally supply a custom function to specifically handle score values when tasks or scoring functions have errored.
:param description: An optional description for the experiment.
Expand Down Expand Up @@ -991,7 +992,7 @@ def Eval(
summarized and compared to this experiment.
:param base_experiment_id: An optional experiment id to use as a base. If specified, the new experiment will be
summarized and compared to this experiment. This takes precedence over `base_experiment_name` if specified.
:param git_metadata_settings: Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in.
:param git_metadata_settings: Optional settings for collecting git metadata. By default, defers to org-level settings returned by the control plane.
:param repo_info: Optionally explicitly specify the git metadata for this experiment. This takes precedence over `git_metadata_settings` if specified.
:param error_score_handler: Optionally supply a custom function to specifically handle score values when tasks or scoring functions have errored.
:param description: An optional description for the experiment.
Expand Down
19 changes: 0 additions & 19 deletions py/src/braintrust/git_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,22 +43,3 @@ def merge(cls, s1: "GitMetadataSettings", s2: "GitMetadataSettings") -> "GitMeta
if not ret.fields:
ret.collect = "none"
return ret


DEFAULT_GIT_METADATA_FIELDS = [
"commit",
"branch",
"tag",
"dirty",
"author_name",
"author_email",
"commit_message",
"commit_time",
]


def default_git_metadata_settings() -> GitMetadataSettings:
return GitMetadataSettings(
collect="some",
fields=list(DEFAULT_GIT_METADATA_FIELDS),
)
4 changes: 2 additions & 2 deletions py/src/braintrust/gitutil.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import threading
from functools import lru_cache as _cache

from .git_fields import GitMetadataSettings, RepoInfo, default_git_metadata_settings
from .git_fields import GitMetadataSettings, RepoInfo


# https://stackoverflow.com/questions/48399498/git-executable-not-found-in-python
Expand Down Expand Up @@ -123,7 +123,7 @@ def truncate_to_byte_limit(input_string, byte_limit=65536):

def get_repo_info(settings: GitMetadataSettings | None = None):
if settings is None:
settings = default_git_metadata_settings()
settings = GitMetadataSettings(collect="none")

if settings.collect == "none":
return None
Expand Down
4 changes: 2 additions & 2 deletions py/src/braintrust/logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
PromptOptions,
SpanAttributes,
)
from .git_fields import GitMetadataSettings, RepoInfo, default_git_metadata_settings
from .git_fields import GitMetadataSettings, RepoInfo
from .gitutil import get_past_n_ancestors, get_repo_info
from .merge_row_batch import batch_items, merge_row_batch
from .object import DEFAULT_IS_LEGACY_DATASET, ensure_dataset_record
Expand Down Expand Up @@ -2670,7 +2670,7 @@ def _check_org_info(state, org_info, org_name):
state.api_url = os.environ.get("BRAINTRUST_API_URL", orgs["api_url"])
state.proxy_url = os.environ.get("BRAINTRUST_PROXY_URL", orgs["proxy_url"])
state.git_metadata_settings = GitMetadataSettings(
**(orgs.get("git_metadata") or default_git_metadata_settings().as_dict())
**(orgs.get("git_metadata") or GitMetadataSettings(collect="none").as_dict())
)
break

Expand Down
40 changes: 40 additions & 0 deletions py/src/braintrust/test_logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,12 @@
logger,
)
from braintrust.db_fields import AUDIT_METADATA_FIELD
from braintrust.gitutil import get_repo_info
from braintrust.id_gen import OTELIDGenerator, get_id_generator
from braintrust.logger import (
BraintrustState,
RemoteEvalParameters,
_check_org_info,
_extract_attachments,
parent_context,
render_message,
Expand Down Expand Up @@ -3831,3 +3834,40 @@ def failing_function():
logs = with_memory_logger.pop()
assert len(logs) == 1
_assert_test_exception_group_contents(logs[0].get("error", ""))


def test_check_org_info_no_git_metadata_defaults_to_none():
"""When org has no git_metadata, state.git_metadata_settings should be collect='none'."""
state = BraintrustState()
org_info = [
{
"id": "org-1",
"name": "org1",
"api_url": "https://api.example.com",
"proxy_url": "https://proxy.example.com",
}
]
_check_org_info(state, org_info, None)
assert state.git_metadata_settings.collect == "none"


def test_check_org_info_with_git_metadata_uses_server_settings():
"""When org provides git_metadata, it is used as-is."""
state = BraintrustState()
org_info = [
{
"id": "org-1",
"name": "org1",
"api_url": "https://api.example.com",
"proxy_url": "https://proxy.example.com",
"git_metadata": {"collect": "some", "fields": ["commit", "branch"]},
}
]
_check_org_info(state, org_info, None)
assert state.git_metadata_settings.collect == "some"
assert set(state.git_metadata_settings.fields) == {"commit", "branch"}


def test_get_repo_info_without_settings_returns_none():
"""Direct call to get_repo_info with settings=None should return None."""
assert get_repo_info(None) is None
Loading