fix(ragas): score 0 when AnswerCorrectness classification is empty

js/ragas.test.ts

@@ -327,3 +327,80 @@ describe("AnswerCorrectness custom embedding model", () => {
     expect(capturedEmbeddingModel).toBe("text-embedding-3-large");
   });
 });
+
+describe("AnswerCorrectness empty classification", () => {
+  const server = setupServer();
+
+  beforeAll(() => {
+    server.listen({
+      onUnhandledRequest: (req) => {
+        throw new Error(`Unhandled request ${req.method}, ${req.url}`);
+      },
+    });
+  });
+
+  afterEach(() => {
+    server.resetHandlers();
+    init();
+  });
+
+  afterAll(() => {
+    server.close();
+  });
+
+  test("scores 0 when the judge classifies nothing", async () => {
+    server.use(
+      http.post("https://api.openai.com/v1/chat/completions", async () => {
+        return HttpResponse.json({
+          id: "test-id",
+          object: "chat.completion",
+          created: Date.now(),
+          model: "gpt-5-mini",
+          choices: [
+            {
+              index: 0,
+              message: {
+                role: "assistant",
+                tool_calls: [
+                  {
+                    id: "call_test",
+                    type: "function",
+                    function: {
+                      name: "classify_statements",
+                      arguments: JSON.stringify({
+                        TP: [],
+                        FP: [],
+                        FN: [],
+                      }),
+                    },
+                  },
+                ],
+              },
+              finish_reason: "tool_calls",
+            },
+          ],
+        });
+      }),
+    );
+
+    init({
+      client: new OpenAI({
+        apiKey: "test-api-key",
+        baseURL: "https://api.openai.com/v1",
+      }),
+    });
+
+    // Regression test: computeF1Score divided by tp + 0.5 * (fp + fn)
+    // unguarded, so an empty classification returned NaN, which serializes
+    // to null and reads as a skipped score. answerSimilarityWeight 0 keeps
+    // the test off the embeddings API.
+    const result = await AnswerCorrectness({
+      input: "What is the capital of France?",
+      output: "I don't know.",
+      expected: "Paris is the capital of France",
+      answerSimilarityWeight: 0,
+    });
+
+    expect(result.score).toBe(0);
+  });
+});

js/ragas.ts

@@ -850,7 +850,11 @@ function computeF1Score(classification: AnswerCorrectnessClassification) {
   const tp = classification.TP.length;
   const fp = classification.FP.length;
   const fn = classification.FN.length;
-  return tp / (tp + 0.5 * (fp + fn));
+  const denominator = tp + 0.5 * (fp + fn);
+  // A judge can classify nothing at all (an answer with no checkable claims,
+  // or an empty classification). Score that 0 rather than dividing by zero
+  // and returning NaN, matching the guard in ContextRecall.
+  return denominator > 0 ? tp / denominator : 0;
 }
 
 /**

py/autoevals/ragas.py

@@ -1341,7 +1341,11 @@ def extract_correctness_request(question, answer, ground_truth, **extra_args):
 
 def compute_f1_score(factuality):
     tp, fp, fn = len(factuality["TP"]), len(factuality["FP"]), len(factuality["FN"])
-    return tp / (tp + 0.5 * (fp + fn))
+    denominator = tp + 0.5 * (fp + fn)
+    # A judge can classify nothing at all (an answer with no checkable claims,
+    # or an empty classification). Score that 0 rather than dividing by zero,
+    # matching the guard in ContextRecall._postprocess.
+    return tp / denominator if denominator > 0 else 0
 
 
 class AnswerCorrectness(OpenAILLMScorer):

py/autoevals/test_ragas.py

@@ -262,3 +262,47 @@ def mock_responses_api(request):
     )
 
     assert captured_embedding_model == "text-embedding-3-large"
+
+
+def test_compute_f1_score_empty_classification():
+    """An empty classification has no correct statements, so it scores 0.
+
+    Regression test: compute_f1_score divided by tp + 0.5 * (fp + fn)
+    unguarded, so a judge that classified nothing raised ZeroDivisionError.
+    """
+    assert ragas_module.compute_f1_score({"TP": [], "FP": [], "FN": []}) == 0
+    # Normal cases are unchanged.
+    assert ragas_module.compute_f1_score({"TP": ["a", "b"], "FP": ["c"], "FN": []}) == 0.8
+    assert ragas_module.compute_f1_score({"TP": [], "FP": ["a"], "FN": ["b"]}) == 0
+
+
+@pytest.mark.parametrize("is_async", [False, True])
+def test_answer_correctness_empty_classification_scores_zero(monkeypatch, is_async):
+    """A judge that classifies nothing scores 0 instead of crashing.
+
+    The TypeScript scorer returned NaN for the same input, which serializes
+    to null and reads as a skipped score.
+    """
+
+    def fake_load_function_call_request(client=None, **kwargs):
+        return {"TP": [], "FP": [], "FN": []}
+
+    async def fake_aload_function_call_request(client=None, **kwargs):
+        return {"TP": [], "FP": [], "FN": []}
+
+    monkeypatch.setattr(ragas_module, "load_function_call_request", fake_load_function_call_request)
+    monkeypatch.setattr(ragas_module, "aload_function_call_request", fake_aload_function_call_request)
+
+    # answer_similarity_weight=0 keeps the test off the embeddings API.
+    metric = AnswerCorrectness(answer_similarity_weight=0)
+    kwargs = dict(
+        input="What is the capital of France?",
+        output="I don't know.",
+        expected="Paris is the capital of France",
+    )
+    if is_async:
+        score = asyncio.run(metric.eval_async(**kwargs))
+    else:
+        score = metric.eval(**kwargs)
+
+    assert score.score == 0