Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 1 addition & 22 deletions viewer/ai_comparer.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from evalbench.generators.models import get_generator
from evalbench.util.config import load_yaml_config
from summarizer import get_summarizer
from paths import get_results_dir

# Setup logging
logging.basicConfig(level=logging.INFO)
Expand All @@ -19,28 +20,6 @@
# Global models dict for get_generator
global_models = {"lock": threading.Lock(), "registered_models": {}}

def get_results_dir():
# Try to read from environment variable
res_dir = os.environ.get("RESULTS_DIR")
if res_dir:
return res_dir

# Check multiple locations for results directory
results_dir_candidates = [
"/tmp_session_files/results",
os.path.join(os.path.dirname(os.path.dirname(__file__)), "results"),
os.path.join(os.getcwd(), "results"),
"/evalbench/results"
]

for candidate in results_dir_candidates:
if os.path.exists(candidate):
logger.info(f"Found results directory at: {candidate}")
return candidate

logger.warning("Results directory not found in candidates, defaulting to current directory results")
return os.path.join(os.getcwd(), "results")

def compare_evals(id1, id2):
"""Compares two evaluation runs using Gemini."""
results_dir = get_results_dir()
Expand Down
2 changes: 1 addition & 1 deletion viewer/dataset_quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@

import mesop as me

from paths import get_results_dir
from precompute_dataset_quality import (
CACHE_FILENAME,
DATASET_FORMAT_CONFIG_KEY,
DATASET_QUALITY_FORMAT,
SUMMARY_COMPARATOR,
artifact_paths,
get_results_dir,
)

csv.field_size_limit(10**9)
Expand Down
21 changes: 1 addition & 20 deletions viewer/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import dataset_quality
from summarizer import summarize_eval_scoring
from ai_comparer import compare_evals
from paths import get_results_dir

@me.stateclass
class State:
Expand Down Expand Up @@ -116,26 +117,6 @@ def df_to_config(df: pd.DataFrame) -> dict:



def get_results_dir():
# Try to read from environment variable
res_dir = os.environ.get("RESULTS_DIR")
if res_dir:
return res_dir

# Check multiple locations for results directory
results_dir_candidates = [
"/tmp_session_files/results",
os.path.join(os.path.dirname(os.path.dirname(__file__)), "results"),
os.path.join(os.getcwd(), "results"),
]

for candidate in results_dir_candidates:
if os.path.exists(candidate) and os.path.isdir(candidate):
return candidate

return results_dir_candidates[1] # Fallback to default


# (mtime, value). Each is replaced as a whole so a concurrent reader never sees a
# value that disagrees with the mtime it was built from.
_TRENDS_DF_CACHE = (None, None)
Expand Down
21 changes: 21 additions & 0 deletions viewer/paths.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import os


def get_results_dir():
res_dir = os.environ.get("RESULTS_DIR")
if res_dir:
return res_dir

# Candidate 1 is the repo root, which is /evalbench in the container, so the
# deployed image resolves it to the same /evalbench/results the runs write to.
candidates = [
"/tmp_session_files/results",
os.path.join(os.path.dirname(os.path.dirname(__file__)), "results"),
os.path.join(os.getcwd(), "results"),
]

for candidate in candidates:
if os.path.isdir(candidate):
return candidate

return candidates[1]
22 changes: 2 additions & 20 deletions viewer/precompute_dataset_quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
import logging
import os

from paths import get_results_dir

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

CACHE_FILENAME = "dataset_quality_cache.json"
Expand All @@ -31,26 +33,6 @@
csv.field_size_limit(10**9)


def get_results_dir():
# Try to read from environment variable
res_dir = os.environ.get("RESULTS_DIR")
if res_dir:
return res_dir

# Check multiple locations for results directory
results_dir_candidates = [
"/tmp_session_files/results",
os.path.join(os.path.dirname(os.path.dirname(__file__)), "results"),
os.path.join(os.getcwd(), "results"),
]

for candidate in results_dir_candidates:
if os.path.exists(candidate) and os.path.isdir(candidate):
return candidate

return results_dir_candidates[1] # Fallback to default


def _read_configs(configs_file):
"""Return {config_key: value} plus the run_time carried on every row."""
values = {}
Expand Down
22 changes: 2 additions & 20 deletions viewer/precompute_trends.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import argparse
import pandas as pd

from paths import get_results_dir

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

BATCH_SIZE = int(os.environ.get("PRECOMPUTE_BATCH_SIZE", 50))
Expand All @@ -15,26 +17,6 @@
MAX_WORKERS = int(os.environ.get("PRECOMPUTE_WORKERS", 50))


def get_results_dir():
# Try to read from environment variable
res_dir = os.environ.get("RESULTS_DIR")
if res_dir:
return res_dir

# Check multiple locations for results directory
results_dir_candidates = [
"/tmp_session_files/results",
os.path.join(os.path.dirname(os.path.dirname(__file__)), "results"),
os.path.join(os.getcwd(), "results"),
]

for candidate in results_dir_candidates:
if os.path.exists(candidate) and os.path.isdir(candidate):
return candidate

return results_dir_candidates[1] # Fallback to default


def process_directory(d, results_dir):
run_dir = os.path.join(results_dir, d)
configs_file = os.path.join(run_dir, "configs.csv")
Expand Down
19 changes: 1 addition & 18 deletions viewer/trends.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,25 +3,8 @@
import mesop as me
import pandas as pd
from main import State, list_run_dirs, load_trends_df
from paths import get_results_dir

def get_results_dir():
# Try to read from environment variable
res_dir = os.environ.get("RESULTS_DIR")
if res_dir:
return res_dir

# Check multiple locations for results directory
results_dir_candidates = [
"/tmp_session_files/results",
os.path.join(os.path.dirname(os.path.dirname(__file__)), "results"),
os.path.join(os.getcwd(), "results"),
]

for candidate in results_dir_candidates:
if os.path.exists(candidate) and os.path.isdir(candidate):
return candidate

return results_dir_candidates[1] # Fallback to default

def generate_d3_chart(df, x_col, y_col, hue_col, title, ylabel):
# job_id is unplotted but read by the tooltip in chart.js.
Expand Down
Loading