fix(python): honor ValidJSON schemas across scorer call paths

py/autoevals/json.py

@@ -250,10 +250,15 @@ async def validate_json():
     def __init__(self, schema=None):
         self.schema = schema
 
-    def _run_eval_sync(self, output, schema=None, **kwargs):
+    def _run_eval_sync(self, output, expected=None, schema=None, **kwargs):
+        # Preserve the existing second-positional-argument schema convention.
+        if schema is None:
+            schema = expected
         return Score(name=self._name(), score=self.valid_json(output, schema))
 
     def valid_json(self, output, schema=None):
+        if schema is None:
+            schema = self.schema
         try:
             parsed = json.loads(output) if isinstance(output, str) else output
 

py/autoevals/test_json.py

@@ -1,3 +1,4 @@
+import pytest
 from pytest import approx
 
 from autoevals.json import JSONDiff, ValidJSON
@@ -125,6 +126,38 @@ def test_valid_json():
         assert evaluator(output, schema).score == expected
 
 
+@pytest.mark.parametrize(
+    "output, expected_score", [({}, 0), ({"answer": "yes"}, 1), ("{}", 0), ('{"answer":"yes"}', 1)]
+)
+@pytest.mark.parametrize("schema_source", ["constructor", "keyword", "positional", "partial"])
+@pytest.mark.asyncio
+async def test_valid_json_schema_calling_conventions(output, expected_score, schema_source):
+    schema = {"type": "object", "required": ["answer"]}
+    scorer = ValidJSON()
+    args = ()
+    kwargs = {}
+    if schema_source == "constructor":
+        scorer = ValidJSON(schema=schema)
+    elif schema_source == "keyword":
+        kwargs["schema"] = schema
+    elif schema_source == "positional":
+        args = (schema,)
+    else:
+        scorer = ValidJSON.partial(schema=schema)()
+
+    assert scorer.eval(output, *args, **kwargs).score == expected_score
+    assert scorer(output, *args, **kwargs).score == expected_score
+    assert (await scorer.eval_async(output, *args, **kwargs)).score == expected_score
+
+
+@pytest.mark.parametrize("schema", [{}, True, False])
+def test_valid_json_call_schema_overrides_constructor(schema):
+    scorer = ValidJSON(schema={"type": "object", "required": ["answer"]})
+    assert scorer.eval({}, schema=schema).score == (0 if schema is False else 1)
+    assert scorer.eval({}).score == 0
+    assert scorer.valid_json({}) == 0
+
+
 def test_semantic_json():
     cases = [
         ('{"x": 1, "y": 2}', '{"y": 2, "x": 1}', 1),