From 0a3907be158cdde263a88de65295f8c255f895d9 Mon Sep 17 00:00:00 2001
From: Yuriy H <jurayh@users.noreply.github.com>
Date: Fri, 9 Oct 2026 12:36:17 -1000
Subject: [PATCH] fix(ragas): join list contexts in Faithfulness before judging

Faithfulness passed the context to the judge unjoined, so a list context
reached the prompt as a Python list repr (['Paris is in France.', 'It is
the capital.']) instead of the documents themselves. Every other ragas
scorer joins list contexts with newlines, and the TypeScript scorer
flattens them. Join in both the sync and async paths and add regression
tests that capture the context the judge receives.
---
 py/autoevals/ragas.py      |  6 +++++
 py/autoevals/test_ragas.py | 52 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/py/autoevals/ragas.py b/py/autoevals/ragas.py
index 4726159..9739162 100644
--- a/py/autoevals/ragas.py
+++ b/py/autoevals/ragas.py
@@ -981,6 +981,9 @@ def __init__(self, model: str | None = None, client: Client | None = None, **kwa
     async def _run_eval_async(self, output, expected=None, input=None, context=None, **kwargs):
         check_required("Faithfulness", input=input, output=output, context=context)
 
+        if isinstance(context, list):
+            context = "\n".join(context)
+
         statements = (
             await aextract_statements(
                 client=self.client, question=input, answer=output, model=self.model, **self.extra_args
@@ -1005,6 +1008,9 @@ async def _run_eval_async(self, output, expected=None, input=None, context=None,
     def _run_eval_sync(self, output, expected=None, input=None, context=None, **kwargs):
         check_required("Faithfulness", input=input, output=output, context=context)
 
+        if isinstance(context, list):
+            context = "\n".join(context)
+
         statements = (
             extract_statements(client=self.client, question=input, answer=output, model=self.model, **self.extra_args)
         )["statements"]
diff --git a/py/autoevals/test_ragas.py b/py/autoevals/test_ragas.py
index 74db4e5..2854a4e 100644
--- a/py/autoevals/test_ragas.py
+++ b/py/autoevals/test_ragas.py
@@ -171,6 +171,58 @@ def fake_extract_faithfulness(context, statements, client=None, **extra_args):
     assert captured_answer == "Paris is the capital of France."
 
 
+@pytest.mark.parametrize("is_async", [False, True])
+@pytest.mark.parametrize(
+    ["context", "expected_context"],
+    [
+        (["Paris is in France.", "It is the capital."], "Paris is in France.\nIt is the capital."),
+        ("Paris is in France.\nIt is the capital.", "Paris is in France.\nIt is the capital."),
+    ],
+)
+def test_faithfulness_joins_list_context(monkeypatch, is_async, context, expected_context):
+    """A list context reaches the judge joined with newlines, not as a list.
+
+    Regression test: Faithfulness passed the context through unjoined, so the
+    judge prompt contained the Python list repr (``['Paris is in France.',
+    'It is the capital.']``). Every other ragas scorer joins list contexts,
+    and the TypeScript scorer flattens them.
+    """
+    captured_context = None
+
+    def fake_extract_statements(question, answer, client=None, **extra_args):
+        return {"statements": ["Paris is the capital of France"]}
+
+    async def fake_aextract_statements(question, answer, client=None, **extra_args):
+        return fake_extract_statements(question, answer, client=client, **extra_args)
+
+    def fake_extract_faithfulness(context, statements, client=None, **extra_args):
+        nonlocal captured_context
+        captured_context = context
+        return {"faithfulness": [{"statement": statements[0], "verdict": 1, "reason": "Supported by context"}]}
+
+    async def fake_aextract_faithfulness(context, statements, client=None, **extra_args):
+        return fake_extract_faithfulness(context, statements, client=client, **extra_args)
+
+    monkeypatch.setattr(ragas_module, "extract_statements", fake_extract_statements)
+    monkeypatch.setattr(ragas_module, "aextract_statements", fake_aextract_statements)
+    monkeypatch.setattr(ragas_module, "extract_faithfulness", fake_extract_faithfulness)
+    monkeypatch.setattr(ragas_module, "aextract_faithfulness", fake_aextract_faithfulness)
+
+    scorer = Faithfulness()
+    kwargs = dict(
+        input="What is the capital of France?",
+        output="Paris is the capital of France.",
+        context=context,
+    )
+    if is_async:
+        score = asyncio.run(scorer.eval_async(**kwargs))
+    else:
+        score = scorer.eval(**kwargs)
+
+    assert captured_context == expected_context
+    assert score.score == 1
+
+
 @respx.mock
 def test_answer_correctness_uses_custom_embedding_model():
     """Test that AnswerCorrectness passes embedding_model parameter through to embeddings API."""