fix(json): do not treat booleans as numbers in JSONDiff

py/autoevals/json.py

@@ -84,6 +84,7 @@ def _run_eval_sync(self, output, **kwargs):
 """
 
 import json
+from typing import Any
 
 from jsonschema import ValidationError, validate
 
@@ -94,6 +95,16 @@ def _run_eval_sync(self, output, **kwargs):
 from .string import Levenshtein
 
 
+def _is_number(value: Any) -> bool:
+    # `bool` is a subclass of `int`, so a bare isinstance check treats `True` as
+    # the number 1 and `True`/`1` as an exact match. The TypeScript
+    # implementation tests `typeof value === "number"`, which is false for
+    # booleans, so it compares them as the strings "true"/"false" instead. A
+    # boolean and an integer are different JSON values, so exclude them here to
+    # keep both implementations in agreement.
+    return isinstance(value, (int, float)) and not isinstance(value, bool)
+
+
 class JSONDiff(ScorerWithPartial):
     """Compare JSON objects for structural and content similarity.
 
@@ -180,7 +191,7 @@ def json_diff(self, o1, o2):
             return sum(base_scores) / max(len(o1), len(o2))
         elif isinstance(o1, str) and isinstance(o2, str):
             return self.string_scorer.eval(o1, o2).score
-        elif (isinstance(o1, int) or isinstance(o1, float)) and (isinstance(o2, int) or isinstance(o2, float)):
+        elif _is_number(o1) and _is_number(o2):
             return self.number_scorer.eval(o1, o2).score
         elif o1 is None and o2 is None:
             return 1

py/autoevals/test_json.py

@@ -166,3 +166,40 @@ def test_semantic_json():
                 assert abs(score - expected) < 0.0001
             else:
                 assert round(score * 100) <= round(expected * 100)
+
+
+def test_json_booleans_are_not_numbers():
+    # `bool` is a subclass of `int` in Python, so a bare isinstance check sends
+    # booleans to the numeric scorer. The TypeScript implementation tests
+    # `typeof v === "number"`, which is false for booleans, so it compares them
+    # as the strings "true"/"false" instead. A boolean and an integer are
+    # different JSON values, so neither implementation should call them equal.
+    cases = [
+        ({"flag": True}, {"flag": 1}),
+        ({"flag": 1}, {"flag": True}),
+        ({"flag": False}, {"flag": 0}),
+        ({"flag": 0}, {"flag": False}),
+        ([True], [1]),
+        (True, 1),
+    ]
+
+    evaluator = JSONDiff()
+    for output, expected in cases:
+        print(f"[{output}]", f"[{expected}]")
+        assert evaluator(output, expected).score < 1, f"{output!r} scored equal to {expected!r}"
+
+
+def test_json_equal_booleans_still_match():
+    # The fix must not break the case that is genuinely equal.
+    cases = [
+        ({"flag": True}, {"flag": True}),
+        ({"flag": False}, {"flag": False}),
+        (True, True),
+        (False, False),
+        ([True, False], [True, False]),
+    ]
+
+    evaluator = JSONDiff()
+    for output, expected in cases:
+        print(f"[{output}]", f"[{expected}]")
+        assert evaluator(output, expected).score == approx(1, abs=1e-4)