diff --git a/lettucedetect/detectors/base.py b/lettucedetect/detectors/base.py index 2fee358..acb505c 100644 --- a/lettucedetect/detectors/base.py +++ b/lettucedetect/detectors/base.py @@ -71,6 +71,20 @@ def _validate_min_confidence(min_confidence: float) -> None: if not 0.0 <= min_confidence <= 1.0: raise ValueError(f"min_confidence must be in [0, 1], got {min_confidence}") + @staticmethod + def _validate_batch_lengths(prompts: list[str], answers: list[str]) -> None: + """Validate that ``prompts`` and ``answers`` have the same length. + + :param prompts: The prompt strings of the batch. + :param answers: The answer strings of the batch. + :raises ValueError: If ``len(prompts) != len(answers)``. + """ + if len(prompts) != len(answers): + raise ValueError( + "Number of prompts must match number of answers, " + f"got {len(prompts)} prompts and {len(answers)} answers" + ) + @staticmethod def _filter_spans_by_confidence( result: list, output_format: str, min_confidence: float diff --git a/lettucedetect/detectors/llm.py b/lettucedetect/detectors/llm.py index ef560b4..c83cc4f 100644 --- a/lettucedetect/detectors/llm.py +++ b/lettucedetect/detectors/llm.py @@ -603,8 +603,7 @@ def predict_prompt_batch( ``output_format="tokens"``. :raises ValueError: If ``len(prompts) != len(answers)``. """ - if len(prompts) != len(answers): - raise ValueError("Number of prompts must match number of answers") + self._validate_batch_lengths(prompts, answers) if output_format not in ["tokens", "spans"]: raise ValueError( f"LLMDetector doesn't support '{output_format}' format. Use 'tokens' or 'spans'" diff --git a/lettucedetect/detectors/rag_fact_checker.py b/lettucedetect/detectors/rag_fact_checker.py index c5b87d0..f22ac8b 100644 --- a/lettucedetect/detectors/rag_fact_checker.py +++ b/lettucedetect/detectors/rag_fact_checker.py @@ -107,8 +107,7 @@ def predict_prompt_batch( min_confidence: float = 0.0, ) -> list[list[dict[str, Any]]]: """Batch prediction using RAGFactChecker's batch processing.""" - if len(prompts) != len(answers): - raise ValueError("Number of prompts must match number of answers") + self._validate_batch_lengths(prompts, answers) self._validate_min_confidence(min_confidence) contexts = [[prompt] for prompt in prompts] # Convert prompts to context lists @@ -116,7 +115,7 @@ def predict_prompt_batch( # Convert each result to lettuceDetect format converted_results = [] - for i, (answer, rag_result) in enumerate(zip(answers, rag_results)): + for i, (answer, rag_result) in enumerate(zip(answers, rag_results, strict=True)): if output_format == "tokens": converted = self._convert_to_tokens(answer, rag_result) elif output_format == "spans": diff --git a/lettucedetect/detectors/transformer.py b/lettucedetect/detectors/transformer.py index c488ddf..9a0e1d6 100644 --- a/lettucedetect/detectors/transformer.py +++ b/lettucedetect/detectors/transformer.py @@ -546,8 +546,7 @@ def predict_prompt_batch( :returns: List of prediction lists, one per input pair. :raises ValueError: If ``len(prompts) != len(answers)``. """ - if len(prompts) != len(answers): - raise ValueError("Number of prompts must match number of answers") + self._validate_batch_lengths(prompts, answers) if not prompts: return [] if output_format not in ("tokens", "spans"): diff --git a/tests/test_inference_pytest.py b/tests/test_inference_pytest.py index f16eb6e..c711f52 100644 --- a/tests/test_inference_pytest.py +++ b/tests/test_inference_pytest.py @@ -454,6 +454,23 @@ def test_filter_keeps_spans_without_confidence(self): assert BaseDetector._filter_spans_by_confidence(spans, "spans", 0.9) == spans +class TestBatchLengthValidation: + """Unit tests for the shared BaseDetector._validate_batch_lengths helper.""" + + def test_mismatch_message_shows_both_lengths(self): + """A length mismatch raises a ValueError that shows the two lengths.""" + with pytest.raises(ValueError, match="got 2 prompts and 1 answers"): + BaseDetector._validate_batch_lengths(["p1", "p2"], ["a1"]) + + @pytest.mark.parametrize( + ("prompts", "answers"), + [([], []), (["p1"], ["a1"]), (["p1", "p2"], ["a1", "a2"])], + ) + def test_equal_lengths_raise_no_error(self, prompts, answers): + """The helper accepts equal lengths, including two empty lists.""" + BaseDetector._validate_batch_lengths(prompts, answers) + + class TestTransformerMinConfidence: """min_confidence behaviour wired through TransformerDetector.""" diff --git a/tests/test_rag_fact_checker_pytest.py b/tests/test_rag_fact_checker_pytest.py new file mode 100644 index 0000000..1513b3a --- /dev/null +++ b/tests/test_rag_fact_checker_pytest.py @@ -0,0 +1,65 @@ +"""Pytest tests for RAGFactCheckerDetector batch prediction.""" + +from unittest.mock import patch + +import pytest + +from lettucedetect.detectors.rag_fact_checker import RAGFactCheckerDetector + +HALLUCINATED_RESULT = {"hallucinated_triplets": [["Paris", "is capital of", "Germany"]]} +CLEAN_RESULT = {"hallucinated_triplets": []} + + +@pytest.fixture +def detector(): + """RAGFactCheckerDetector with a mock backend. The tests need no API key and no network.""" + with patch("lettucedetect.ragfactchecker.RAGFactChecker"): + return RAGFactCheckerDetector() + + +class TestPredictPromptBatchRAG: + """predict_prompt_batch() keeps one result for each (prompt, answer) pair.""" + + @pytest.mark.parametrize( + ("prompts", "answers"), + [ + (["p1", "p2"], ["a1"]), + (["p1"], ["a1", "a2"]), + (["p1"], []), + ([], ["a1"]), + ], + ) + def test_mismatched_lengths_raise_value_error(self, detector, prompts, answers): + """A length mismatch raises a ValueError before the backend gets a request.""" + with pytest.raises(ValueError, match="Number of prompts must match number of answers"): + detector.predict_prompt_batch(prompts, answers) + detector.rag.detect_hallucinations_batch.assert_not_called() + + def test_matched_lengths_return_spans_in_input_order(self, detector): + """Equal lengths give one spans result for each pair, in input order.""" + detector.rag.detect_hallucinations_batch.return_value = [HALLUCINATED_RESULT, CLEAN_RESULT] + answers = ["Paris is the capital of Germany.", "Berlin is the capital of Germany."] + + results = detector.predict_prompt_batch(["c1", "c2"], answers, output_format="spans") + + assert len(results) == 2 + assert [span["text"] for span in results[0]] == ["Germany"] + assert results[1] == [] + + def test_matched_lengths_return_tokens_in_input_order(self, detector): + """Equal lengths give one tokens result for each pair, in input order.""" + detector.rag.detect_hallucinations_batch.return_value = [HALLUCINATED_RESULT, CLEAN_RESULT] + answers = ["Paris is the capital of Germany.", "Berlin is the capital of Germany."] + + results = detector.predict_prompt_batch(["c1", "c2"], answers, output_format="tokens") + + assert [len(tokens) for tokens in results] == [6, 6] + assert any(token["pred"] == 1 for token in results[0]) + assert all(token["pred"] == 0 for token in results[1]) + + def test_short_backend_result_raises_instead_of_truncating(self, detector): + """The call raises a ValueError when the backend returns fewer results than answers.""" + detector.rag.detect_hallucinations_batch.return_value = [CLEAN_RESULT] + + with pytest.raises(ValueError, match="shorter"): + detector.predict_prompt_batch(["c1", "c2"], ["a1", "a2"], output_format="spans")