From 3018471de7205e7324d82fc14b85a7a538b97044 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 26 Aug 2026 10:10:45 -0700 Subject: [PATCH] fix: row.cosine_similarity was always None on self-host The engine sends the field as vectorSimilarity (run-result rows and analysis statistics alike); the models read only cosineSimilarity, the hosted platform's name. Every self-host caller of run.results()[i].cosine_similarity got None with vector similarity enabled and computed. Found while verifying the openai v7 bump - the 0.866 was sitting in row.raw the whole time. Both models now accept both spellings via AliasChoices. Verified live against a v7 engine: cosine_similarity 0.87 on a real run. Co-Authored-By: Claude Fable 5 --- agentx/evaluations/models.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/agentx/evaluations/models.py b/agentx/evaluations/models.py index db5aaff..a98e8cf 100644 --- a/agentx/evaluations/models.py +++ b/agentx/evaluations/models.py @@ -1,7 +1,7 @@ from __future__ import annotations from typing import Any, Dict, List, Literal, Optional, Union -from pydantic import BaseModel, Field, model_validator +from pydantic import AliasChoices, BaseModel, Field, model_validator # --------------------------------------------------------------------------- # Observable trace @@ -418,7 +418,11 @@ class RunResultRow(BaseModel): latency_ms: Optional[float] = Field(default=None, alias="latencyMs") input_tokens: Optional[int] = Field(default=None, alias="inputTokens") output_tokens: Optional[int] = Field(default=None, alias="outputTokens") - cosine_similarity: Optional[float] = Field(default=None, alias="cosineSimilarity") + # Self-host sends vectorSimilarity, the hosted platform cosineSimilarity - accept both + # (row.cosine_similarity was silently None forever on self-host before this). + cosine_similarity: Optional[float] = Field( + default=None, validation_alias=AliasChoices("cosineSimilarity", "vectorSimilarity") + ) jaccard_similarity: Optional[float] = Field(default=None, alias="jaccardSimilarity") bleu_score: Optional[float] = Field(default=None, alias="bleuScore") rouge_score: Optional[float] = Field(default=None, alias="rougeScore") @@ -468,7 +472,11 @@ class ReportStatistics(BaseModel): average_rating: float = Field(default=0.0, alias="averageRating") min_rating: float = Field(default=0.0, alias="minRating") max_rating: float = Field(default=0.0, alias="maxRating") - cosine_similarity: Optional[float] = Field(default=None, alias="cosineSimilarity") + # Self-host sends vectorSimilarity, the hosted platform cosineSimilarity - accept both + # (row.cosine_similarity was silently None forever on self-host before this). + cosine_similarity: Optional[float] = Field( + default=None, validation_alias=AliasChoices("cosineSimilarity", "vectorSimilarity") + ) jaccard_similarity: Optional[float] = Field(default=None, alias="jaccardSimilarity") bleu_score: Optional[float] = Field(default=None, alias="bleuScore") rouge_score: Optional[float] = Field(default=None, alias="rougeScore")