fix(ragas): join list contexts in Faithfulness before judging

py/autoevals/ragas.py

@@ -981,6 +981,9 @@ def __init__(self, model: str | None = None, client: Client | None = None, **kwa
     async def _run_eval_async(self, output, expected=None, input=None, context=None, **kwargs):
         check_required("Faithfulness", input=input, output=output, context=context)
 
+        if isinstance(context, list):
+            context = "\n".join(context)
+
         statements = (
             await aextract_statements(
                 client=self.client, question=input, answer=output, model=self.model, **self.extra_args
@@ -1005,6 +1008,9 @@ async def _run_eval_async(self, output, expected=None, input=None, context=None,
     def _run_eval_sync(self, output, expected=None, input=None, context=None, **kwargs):
         check_required("Faithfulness", input=input, output=output, context=context)
 
+        if isinstance(context, list):
+            context = "\n".join(context)
+
         statements = (
             extract_statements(client=self.client, question=input, answer=output, model=self.model, **self.extra_args)
         )["statements"]

py/autoevals/test_ragas.py

@@ -171,6 +171,58 @@ def fake_extract_faithfulness(context, statements, client=None, **extra_args):
     assert captured_answer == "Paris is the capital of France."
 
 
+@pytest.mark.parametrize("is_async", [False, True])
+@pytest.mark.parametrize(
+    ["context", "expected_context"],
+    [
+        (["Paris is in France.", "It is the capital."], "Paris is in France.\nIt is the capital."),
+        ("Paris is in France.\nIt is the capital.", "Paris is in France.\nIt is the capital."),
+    ],
+)
+def test_faithfulness_joins_list_context(monkeypatch, is_async, context, expected_context):
+    """A list context reaches the judge joined with newlines, not as a list.
+
+    Regression test: Faithfulness passed the context through unjoined, so the
+    judge prompt contained the Python list repr (``['Paris is in France.',
+    'It is the capital.']``). Every other ragas scorer joins list contexts,
+    and the TypeScript scorer flattens them.
+    """
+    captured_context = None
+
+    def fake_extract_statements(question, answer, client=None, **extra_args):
+        return {"statements": ["Paris is the capital of France"]}
+
+    async def fake_aextract_statements(question, answer, client=None, **extra_args):
+        return fake_extract_statements(question, answer, client=client, **extra_args)
+
+    def fake_extract_faithfulness(context, statements, client=None, **extra_args):
+        nonlocal captured_context
+        captured_context = context
+        return {"faithfulness": [{"statement": statements[0], "verdict": 1, "reason": "Supported by context"}]}
+
+    async def fake_aextract_faithfulness(context, statements, client=None, **extra_args):
+        return fake_extract_faithfulness(context, statements, client=client, **extra_args)
+
+    monkeypatch.setattr(ragas_module, "extract_statements", fake_extract_statements)
+    monkeypatch.setattr(ragas_module, "aextract_statements", fake_aextract_statements)
+    monkeypatch.setattr(ragas_module, "extract_faithfulness", fake_extract_faithfulness)
+    monkeypatch.setattr(ragas_module, "aextract_faithfulness", fake_aextract_faithfulness)
+
+    scorer = Faithfulness()
+    kwargs = dict(
+        input="What is the capital of France?",
+        output="Paris is the capital of France.",
+        context=context,
+    )
+    if is_async:
+        score = asyncio.run(scorer.eval_async(**kwargs))
+    else:
+        score = scorer.eval(**kwargs)
+
+    assert captured_context == expected_context
+    assert score.score == 1
+
+
 @respx.mock
 def test_answer_correctness_uses_custom_embedding_model():
     """Test that AnswerCorrectness passes embedding_model parameter through to embeddings API."""