fix(ragas): score empty retrieval contexts as zero

js/ragas.test.ts

@@ -1,7 +1,15 @@
 import { http, HttpResponse } from "msw";
 import { setupServer } from "msw/node";
 import { OpenAI } from "openai";
-import { afterAll, afterEach, beforeAll, describe, expect, test } from "vitest";
+import {
+  afterAll,
+  afterEach,
+  beforeAll,
+  describe,
+  expect,
+  test,
+  vi,
+} from "vitest";
 import {
   AnswerCorrectness,
   AnswerRelevancy,
@@ -109,6 +117,33 @@ describe("ContextRelevancy score clamping", () => {
     server.close();
   });
 
+  test.each([{ context: "" }, { context: [] }, { context: [""] }])(
+    "scores empty context $context as zero without calling the judge",
+    async ({ context }) => {
+      const client = new OpenAI({ apiKey: "test-api-key" });
+      const create = vi
+        .spyOn(client.chat.completions, "create")
+        .mockRejectedValue(
+          new Error("Empty context should not call the judge"),
+        );
+
+      const result = await ContextRelevancy({
+        input: "What is hello?",
+        output: "Hello world",
+        context,
+        client,
+      });
+
+      expect(result).toEqual({
+        name: "ContextRelevancy",
+        score: 0,
+        metadata: { relevantSentences: [] },
+      });
+      expect(create).not.toHaveBeenCalled();
+      expect(JSON.parse(JSON.stringify(result)).score).toBe(0);
+    },
+  );
+
   test("clamps score to 1.0 when LLM returns sentences longer than context", async () => {
     // Mock response where extracted sentences are LONGER than the context
     // This would produce a raw score > 1.0 without clamping

js/ragas.ts

@@ -253,6 +253,15 @@ export const ContextRelevancy: ScorerWithPartial<string, RagasArgs> =
       "ContextRelevancy",
     );
 
+    // An empty retrieval cannot contain relevant sentences.
+    if (context.length === 0) {
+      return {
+        name: "ContextRelevancy",
+        score: 0,
+        metadata: { relevantSentences: [] },
+      };
+    }
+
     const response = await client.chat.completions.create({
       ...chatArgs,
       messages: [

py/autoevals/ragas.py

@@ -364,6 +364,8 @@ def extract_sentences_request(question, context, **extra_args):
 class ContextRelevancy(OpenAILLMScorer):
     """Evaluates how relevant the context is to the input question.
 
+    An empty context scores zero without calling the LLM judge.
+
     Example:
         ```python
         from openai import OpenAI
@@ -414,6 +416,9 @@ async def _run_eval_async(self, output, expected=None, input=None, context=None,
         if isinstance(context, list):
             context = "\n".join(context)
 
+        if not context:
+            return Score(name=self._name(), score=0, metadata={"relevant_sentences": []})
+
         return self._postprocess(
             context,
             await arun_cached_request(
@@ -428,6 +433,9 @@ def _run_eval_sync(self, output, expected=None, input=None, context=None, **kwar
         if isinstance(context, list):
             context = "\n".join(context)
 
+        if not context:
+            return Score(name=self._name(), score=0, metadata={"relevant_sentences": []})
+
         return self._postprocess(
             context,
             run_cached_request(

py/autoevals/test_ragas.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+from unittest.mock import AsyncMock, Mock
 
 import pytest
 import respx
@@ -52,6 +53,25 @@ def test_ragas_retrieval(metric: OpenAILLMScorer, expected_score: float, is_asyn
             raise e
 
 
+@pytest.mark.parametrize("context", ["", [], [""]])
+@pytest.mark.parametrize("is_async", [False, True])
+def test_context_relevancy_empty_context(context, is_async, monkeypatch):
+    request = Mock(side_effect=AssertionError("Empty context should not call the judge"))
+    async_request = AsyncMock(side_effect=AssertionError("Empty context should not call the judge"))
+    monkeypatch.setattr(ragas_module, "run_cached_request", request)
+    monkeypatch.setattr(ragas_module, "arun_cached_request", async_request)
+    scorer = ContextRelevancy()
+    args = dict(input="What is hello?", output="Hello world", context=context)
+
+    result = asyncio.run(scorer.eval_async(**args)) if is_async else scorer.eval(**args)
+
+    assert result.name == "ContextRelevancy"
+    assert result.score == 0
+    assert result.metadata == {"relevant_sentences": []}
+    request.assert_not_called()
+    async_request.assert_not_called()
+
+
 def test_context_relevancy_score_clamping():
     """Test that ContextRelevancy clamps scores to [0, 1] range (#80).