From 270c6ca5c3d8dbf589f75eb69095d820e2ec167c Mon Sep 17 00:00:00 2001 From: yassinnouh21 Date: Fri, 9 Oct 2026 17:54:36 +0200 Subject: [PATCH] fix: shared batch length check in BaseDetector Add BaseDetector._validate_batch_lengths(prompts, answers). It raises ValueError when the two lengths differ. The message shows both lengths. TransformerDetector, LLMDetector and RAGFactCheckerDetector now call the helper instead of an inline check. The old message text stays as a prefix, so the #113 tests pass without edits. RAGFactCheckerDetector.predict_prompt_batch now zips the answers and the backend results with strict=True. Before this change, a short backend result dropped answers with no error. Add tests for the helper and a new test file for the RAG detector. Follow-up to #113, refs #114. Signed-off-by: yassinnouh21 --- lettucedetect/detectors/base.py | 14 +++++ lettucedetect/detectors/llm.py | 3 +- lettucedetect/detectors/rag_fact_checker.py | 5 +- lettucedetect/detectors/transformer.py | 3 +- tests/test_inference_pytest.py | 17 ++++++ tests/test_rag_fact_checker_pytest.py | 65 +++++++++++++++++++++ 6 files changed, 100 insertions(+), 7 deletions(-) create mode 100644 tests/test_rag_fact_checker_pytest.py diff --git a/lettucedetect/detectors/base.py b/lettucedetect/detectors/base.py index 2fee358..acb505c 100644 --- a/lettucedetect/detectors/base.py +++ b/lettucedetect/detectors/base.py @@ -71,6 +71,20 @@ def _validate_min_confidence(min_confidence: float) -> None: if not 0.0 <= min_confidence <= 1.0: raise ValueError(f"min_confidence must be in [0, 1], got {min_confidence}") + @staticmethod + def _validate_batch_lengths(prompts: list[str], answers: list[str]) -> None: + """Validate that ``prompts`` and ``answers`` have the same length. + + :param prompts: The prompt strings of the batch. + :param answers: The answer strings of the batch. + :raises ValueError: If ``len(prompts) != len(answers)``. + """ + if len(prompts) != len(answers): + raise ValueError( + "Number of prompts must match number of answers, " + f"got {len(prompts)} prompts and {len(answers)} answers" + ) + @staticmethod def _filter_spans_by_confidence( result: list, output_format: str, min_confidence: float diff --git a/lettucedetect/detectors/llm.py b/lettucedetect/detectors/llm.py index ef560b4..c83cc4f 100644 --- a/lettucedetect/detectors/llm.py +++ b/lettucedetect/detectors/llm.py @@ -603,8 +603,7 @@ def predict_prompt_batch( ``output_format="tokens"``. :raises ValueError: If ``len(prompts) != len(answers)``. """ - if len(prompts) != len(answers): - raise ValueError("Number of prompts must match number of answers") + self._validate_batch_lengths(prompts, answers) if output_format not in ["tokens", "spans"]: raise ValueError( f"LLMDetector doesn't support '{output_format}' format. Use 'tokens' or 'spans'" diff --git a/lettucedetect/detectors/rag_fact_checker.py b/lettucedetect/detectors/rag_fact_checker.py index c5b87d0..f22ac8b 100644 --- a/lettucedetect/detectors/rag_fact_checker.py +++ b/lettucedetect/detectors/rag_fact_checker.py @@ -107,8 +107,7 @@ def predict_prompt_batch( min_confidence: float = 0.0, ) -> list[list[dict[str, Any]]]: """Batch prediction using RAGFactChecker's batch processing.""" - if len(prompts) != len(answers): - raise ValueError("Number of prompts must match number of answers") + self._validate_batch_lengths(prompts, answers) self._validate_min_confidence(min_confidence) contexts = [[prompt] for prompt in prompts] # Convert prompts to context lists @@ -116,7 +115,7 @@ def predict_prompt_batch( # Convert each result to lettuceDetect format converted_results = [] - for i, (answer, rag_result) in enumerate(zip(answers, rag_results)): + for i, (answer, rag_result) in enumerate(zip(answers, rag_results, strict=True)): if output_format == "tokens": converted = self._convert_to_tokens(answer, rag_result) elif output_format == "spans": diff --git a/lettucedetect/detectors/transformer.py b/lettucedetect/detectors/transformer.py index c488ddf..9a0e1d6 100644 --- a/lettucedetect/detectors/transformer.py +++ b/lettucedetect/detectors/transformer.py @@ -546,8 +546,7 @@ def predict_prompt_batch( :returns: List of prediction lists, one per input pair. :raises ValueError: If ``len(prompts) != len(answers)``. """ - if len(prompts) != len(answers): - raise ValueError("Number of prompts must match number of answers") + self._validate_batch_lengths(prompts, answers) if not prompts: return [] if output_format not in ("tokens", "spans"): diff --git a/tests/test_inference_pytest.py b/tests/test_inference_pytest.py index f16eb6e..c711f52 100644 --- a/tests/test_inference_pytest.py +++ b/tests/test_inference_pytest.py @@ -454,6 +454,23 @@ def test_filter_keeps_spans_without_confidence(self): assert BaseDetector._filter_spans_by_confidence(spans, "spans", 0.9) == spans +class TestBatchLengthValidation: + """Unit tests for the shared BaseDetector._validate_batch_lengths helper.""" + + def test_mismatch_message_shows_both_lengths(self): + """A length mismatch raises a ValueError that shows the two lengths.""" + with pytest.raises(ValueError, match="got 2 prompts and 1 answers"): + BaseDetector._validate_batch_lengths(["p1", "p2"], ["a1"]) + + @pytest.mark.parametrize( + ("prompts", "answers"), + [([], []), (["p1"], ["a1"]), (["p1", "p2"], ["a1", "a2"])], + ) + def test_equal_lengths_raise_no_error(self, prompts, answers): + """The helper accepts equal lengths, including two empty lists.""" + BaseDetector._validate_batch_lengths(prompts, answers) + + class TestTransformerMinConfidence: """min_confidence behaviour wired through TransformerDetector.""" diff --git a/tests/test_rag_fact_checker_pytest.py b/tests/test_rag_fact_checker_pytest.py new file mode 100644 index 0000000..1513b3a --- /dev/null +++ b/tests/test_rag_fact_checker_pytest.py @@ -0,0 +1,65 @@ +"""Pytest tests for RAGFactCheckerDetector batch prediction.""" + +from unittest.mock import patch + +import pytest + +from lettucedetect.detectors.rag_fact_checker import RAGFactCheckerDetector + +HALLUCINATED_RESULT = {"hallucinated_triplets": [["Paris", "is capital of", "Germany"]]} +CLEAN_RESULT = {"hallucinated_triplets": []} + + +@pytest.fixture +def detector(): + """RAGFactCheckerDetector with a mock backend. The tests need no API key and no network.""" + with patch("lettucedetect.ragfactchecker.RAGFactChecker"): + return RAGFactCheckerDetector() + + +class TestPredictPromptBatchRAG: + """predict_prompt_batch() keeps one result for each (prompt, answer) pair.""" + + @pytest.mark.parametrize( + ("prompts", "answers"), + [ + (["p1", "p2"], ["a1"]), + (["p1"], ["a1", "a2"]), + (["p1"], []), + ([], ["a1"]), + ], + ) + def test_mismatched_lengths_raise_value_error(self, detector, prompts, answers): + """A length mismatch raises a ValueError before the backend gets a request.""" + with pytest.raises(ValueError, match="Number of prompts must match number of answers"): + detector.predict_prompt_batch(prompts, answers) + detector.rag.detect_hallucinations_batch.assert_not_called() + + def test_matched_lengths_return_spans_in_input_order(self, detector): + """Equal lengths give one spans result for each pair, in input order.""" + detector.rag.detect_hallucinations_batch.return_value = [HALLUCINATED_RESULT, CLEAN_RESULT] + answers = ["Paris is the capital of Germany.", "Berlin is the capital of Germany."] + + results = detector.predict_prompt_batch(["c1", "c2"], answers, output_format="spans") + + assert len(results) == 2 + assert [span["text"] for span in results[0]] == ["Germany"] + assert results[1] == [] + + def test_matched_lengths_return_tokens_in_input_order(self, detector): + """Equal lengths give one tokens result for each pair, in input order.""" + detector.rag.detect_hallucinations_batch.return_value = [HALLUCINATED_RESULT, CLEAN_RESULT] + answers = ["Paris is the capital of Germany.", "Berlin is the capital of Germany."] + + results = detector.predict_prompt_batch(["c1", "c2"], answers, output_format="tokens") + + assert [len(tokens) for tokens in results] == [6, 6] + assert any(token["pred"] == 1 for token in results[0]) + assert all(token["pred"] == 0 for token in results[1]) + + def test_short_backend_result_raises_instead_of_truncating(self, detector): + """The call raises a ValueError when the backend returns fewer results than answers.""" + detector.rag.detect_hallucinations_batch.return_value = [CLEAN_RESULT] + + with pytest.raises(ValueError, match="shorter"): + detector.predict_prompt_batch(["c1", "c2"], ["a1", "a2"], output_format="spans")