From 1932d4921d1dd729a72f6e59877e5521f24f18f3 Mon Sep 17 00:00:00 2001
From: fei <204683769+feiiiiii5@users.noreply.github.com>
Date: Mon, 28 Sep 2026 16:47:39 +0800
Subject: [PATCH] fix(ragas): score 0 when Faithfulness gets no statements

An LLM judge that returns no statements (or no verdicts for them) left
`faithfulness` empty, and the scorer divided by `len(faithfulness)`, so the
whole eval run died with a ZeroDivisionError instead of reporting an
unfaithful answer. The TypeScript scorer already returns 0 in this case.

Guard both the async and sync paths the same way js/ragas.ts does, and cover
them with a regression test.
---
 py/autoevals/ragas.py      |  4 ++--
 py/autoevals/test_ragas.py | 39 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/py/autoevals/ragas.py b/py/autoevals/ragas.py
index 4726159..b5c27da 100644
--- a/py/autoevals/ragas.py
+++ b/py/autoevals/ragas.py
@@ -995,7 +995,7 @@ async def _run_eval_async(self, output, expected=None, input=None, context=None,
 
         return Score(
             name=self._name(),
-            score=sum([s["verdict"] for s in faithfulness]) / len(faithfulness),
+            score=(sum([s["verdict"] for s in faithfulness]) / len(faithfulness)) if faithfulness else 0,
             metadata={
                 "statements": statements,
                 "faithfulness": faithfulness,
@@ -1017,7 +1017,7 @@ def _run_eval_sync(self, output, expected=None, input=None, context=None, **kwar
 
         return Score(
             name=self._name(),
-            score=sum([s["verdict"] for s in faithfulness]) / len(faithfulness),
+            score=(sum([s["verdict"] for s in faithfulness]) / len(faithfulness)) if faithfulness else 0,
             metadata={
                 "statements": statements,
                 "faithfulness": faithfulness,
diff --git a/py/autoevals/test_ragas.py b/py/autoevals/test_ragas.py
index 74db4e5..cd636a5 100644
--- a/py/autoevals/test_ragas.py
+++ b/py/autoevals/test_ragas.py
@@ -171,6 +171,45 @@ def fake_extract_faithfulness(context, statements, client=None, **extra_args):
     assert captured_answer == "Paris is the capital of France."
 
 
+@pytest.mark.parametrize("is_async", [False, True])
+def test_faithfulness_scores_zero_when_judge_finds_no_statements(monkeypatch, is_async):
+    """A judge that finds no statements scores 0 instead of dividing by zero.
+
+    The TypeScript scorer already handles this: ``faithfulness.length ? ... : 0``
+    in ``js/ragas.ts``. The Python scorer divides by ``len(faithfulness)``
+    unguarded, so an empty verdict list aborts the whole eval run with a
+    ZeroDivisionError instead of reporting an unfaithful answer.
+    """
+
+    def fake_extract_statements(*args, **kwargs):
+        return {"statements": []}
+
+    def fake_extract_faithfulness(*args, **kwargs):
+        return {"faithfulness": []}
+
+    async def fake_aextract_statements(*args, **kwargs):
+        return {"statements": []}
+
+    async def fake_aextract_faithfulness(*args, **kwargs):
+        return {"faithfulness": []}
+
+    monkeypatch.setattr(ragas_module, "extract_statements", fake_extract_statements)
+    monkeypatch.setattr(ragas_module, "extract_faithfulness", fake_extract_faithfulness)
+    monkeypatch.setattr(ragas_module, "aextract_statements", fake_aextract_statements)
+    monkeypatch.setattr(ragas_module, "aextract_faithfulness", fake_aextract_faithfulness)
+
+    scorer = Faithfulness()
+    args = {
+        "input": "What is the capital of France?",
+        "output": "",
+        "context": "Paris is the capital of France.",
+    }
+    score = asyncio.run(scorer.eval_async(**args)) if is_async else scorer.eval(**args)
+
+    assert score.score == 0
+    assert score.metadata["faithfulness"] == []
+
+
 @respx.mock
 def test_answer_correctness_uses_custom_embedding_model():
     """Test that AnswerCorrectness passes embedding_model parameter through to embeddings API."""