Add force_exec_validation option to catch runtime errors across Python versions (#2719)

koxudaxi · web-flow · commit e1fe42ef0c5b · 2025-12-21T11:54:23.000+09:00
diff --git a/tests/main/conftest.py b/tests/main/conftest.py
@@ -53,6 +53,9 @@
     reason=f"Installed black ({black.__version__}) doesn't support Python 3.14",
 )
 
+CURRENT_PYTHON_VERSION = f"{sys.version_info[0]}.{sys.version_info[1]}"
+"""Current Python version as string (e.g., '3.13')."""
+
 DATA_PATH: Path = Path(__file__).parent.parent / "data"
 EXPECTED_MAIN_PATH: Path = DATA_PATH / "expected" / "main"
 
@@ -102,6 +105,21 @@ def output_dir(tmp_path: Path) -> Path:
     return tmp_path / "model"
 
 
+def get_current_version_args(*extra_args: str) -> list[str]:
+    """Create CLI args list with --target-python-version set to current version.
+
+    This is a convenience function for tests that want to use the current
+    Python version to enable exec() validation.
+
+    Example:
+        run_main_and_assert(
+            ...,
+            extra_args=get_current_version_args("--use-field-description"),
+        )
+    """
+    return ["--target-python-version", CURRENT_PYTHON_VERSION, *extra_args]
+
+
 def _copy_files(copy_files: CopyFilesMapping | None) -> None:
     """Copy files from source to destination paths."""
     if copy_files is not None:
@@ -223,6 +241,7 @@ def run_main_and_assert(  # noqa: PLR0912
     monkeypatch: pytest.MonkeyPatch | None = None,
     # Code validation options
     skip_code_validation: bool = False,
+    force_exec_validation: bool = False,
 ) -> None:
     """Execute main() and assert output.
 
@@ -259,6 +278,13 @@ def run_main_and_assert(  # noqa: PLR0912
         expected_stderr: Assert exact stderr match
         expected_stderr_contains: Assert stderr contains string
         assert_no_stderr: Assert stderr is empty
+
+    Code validation options:
+        skip_code_validation: Skip all code validation (compile and exec)
+        force_exec_validation: Run exec() even when target Python version differs from
+            the test environment (only effective when target <= runtime). This catches
+            runtime errors that would otherwise be missed. Has no effect when target >
+            runtime since compile is skipped in that case.
     """
     __tracebackhide__ = True
 
@@ -349,7 +375,7 @@ def run_main_and_assert(  # noqa: PLR0912
         assert_func(output_path, expected_file, transform=transform)
 
     if output_path is not None and not skip_code_validation:
-        _validate_output_files(output_path, extra_args)
+        _validate_output_files(output_path, extra_args, force_exec_validation=force_exec_validation)
 
 
 def _get_argument_value(arguments: Sequence[str] | None, argument_name: str) -> str | None:
@@ -380,8 +406,15 @@ def _should_skip_compile(extra_arguments: Sequence[str] | None) -> bool:
     return target_version > sys.version_info[:2]
 
 
-def _should_skip_exec(extra_arguments: Sequence[str] | None) -> bool:
-    """Check if exec should be skipped based on model type, pydantic version, and Python version."""
+def _should_skip_exec(extra_arguments: Sequence[str] | None, *, force_exec: bool = False) -> bool:
+    """Check if exec should be skipped based on model type, pydantic version, and Python version.
+
+    Args:
+        extra_arguments: CLI arguments passed to the test.
+        force_exec: If True, skip version mismatch check and allow exec on current Python version.
+            This only works when target version <= runtime version (older target on newer runtime).
+            When target > runtime, compile will be skipped entirely regardless of this flag.
+    """
     output_model_type = _get_argument_value(extra_arguments, "--output-model-type")
     is_pydantic_v1 = output_model_type is None or output_model_type == DataModelType.PydanticBaseModel.value
     if (is_pydantic_v1 and PYDANTIC_V2) or (
@@ -390,16 +423,30 @@ def _should_skip_exec(extra_arguments: Sequence[str] | None) -> bool:
         return True
     if (target_version := _parse_target_version(extra_arguments)) is None:
         return True
-    if target_version != sys.version_info[:2]:
+    if not force_exec and target_version != sys.version_info[:2]:
         return True
     return _get_argument_value(extra_arguments, "--base-class") is not None
 
 
-def _validate_output_files(output_path: Path, extra_arguments: Sequence[str] | None = None) -> None:
-    """Validate generated Python files by compiling/executing them."""
+def _validate_output_files(
+    output_path: Path,
+    extra_arguments: Sequence[str] | None = None,
+    *,
+    force_exec_validation: bool = False,
+) -> None:
+    """Validate generated Python files by compiling/executing them.
+
+    Args:
+        output_path: Path to output file or directory to validate.
+        extra_arguments: CLI arguments passed to the test.
+        force_exec_validation: If True, run exec even when target Python version differs from
+            the test environment (only when target <= runtime). This helps catch runtime errors
+            that would otherwise be missed. Has no effect when target > runtime since compile
+            is skipped in that case.
+    """
     if _should_skip_compile(extra_arguments):
         return
-    should_exec = not _should_skip_exec(extra_arguments)
+    should_exec = not _should_skip_exec(extra_arguments, force_exec=force_exec_validation)
     if output_path.is_file() and output_path.suffix == ".py":
         validate_generated_code(output_path.read_text(encoding="utf-8"), str(output_path), do_exec=should_exec)
     elif output_path.is_dir():  # pragma: no cover
@@ -471,6 +518,7 @@ def run_main_url_and_assert(
     expected_file: str | Path,
     extra_args: Sequence[str] | None = None,
     transform: Callable[[str], str] | None = None,
+    force_exec_validation: bool = False,
 ) -> None:
     """Execute main() with URL input and assert output.
 
@@ -482,10 +530,12 @@ def run_main_url_and_assert(
         expected_file: Expected output filename
         extra_args: Additional CLI arguments
         transform: Optional function to transform output before comparison
+        force_exec_validation: Run exec() even when target Python version differs from
+            the test environment (only effective when target <= runtime).
     """
     __tracebackhide__ = True
     return_code = _run_main_url(url, output_path, input_file_type, extra_args=extra_args)
     _assert_exit_code(return_code, Exit.OK, f"URL: {url}")
     assert_func(output_path, expected_file, transform=transform)
 
-    _validate_output_files(output_path, extra_args)
+    _validate_output_files(output_path, extra_args, force_exec_validation=force_exec_validation)
diff --git a/tests/main/test_exec_validation.py b/tests/main/test_exec_validation.py
@@ -0,0 +1,161 @@
+"""Tests that validate generated code execution on the current Python version.
+
+These tests specifically target the current Python runtime to catch runtime errors
+that may not be caught when --target-python-version differs from the test environment.
+
+See: _should_skip_exec() in conftest.py for the skip logic that these tests bypass.
+"""
+
+from __future__ import annotations
+
+import sys
+from typing import TYPE_CHECKING
+
+import pytest
+
+from datamodel_code_generator.format import PythonVersion, is_supported_in_black
+from datamodel_code_generator.util import PYDANTIC_V2
+
+from .conftest import (
+    CURRENT_PYTHON_VERSION,
+    JSON_SCHEMA_DATA_PATH,
+    OPEN_API_DATA_PATH,
+    get_current_version_args,
+    run_main_and_assert,
+)
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+_CURRENT_PY_VERSION = PythonVersion(CURRENT_PYTHON_VERSION)
+_SKIP_PYDANTIC = pytest.mark.skipif(not PYDANTIC_V2, reason="Pydantic v2 required")
+_SKIP_BLACK = pytest.mark.skipif(
+    not is_supported_in_black(_CURRENT_PY_VERSION),
+    reason=f"Installed black doesn't support Python {CURRENT_PYTHON_VERSION}",
+)
+
+
+@_SKIP_PYDANTIC
+@_SKIP_BLACK
+def test_openapi_api_exec_current_version(output_file: Path) -> None:
+    """Test that api.yaml schema generates executable code on current Python."""
+    run_main_and_assert(
+        input_path=OPEN_API_DATA_PATH / "api.yaml",
+        output_path=output_file,
+        extra_args=get_current_version_args("--output-model-type", "pydantic_v2.BaseModel"),
+        skip_code_validation=False,
+        force_exec_validation=True,
+    )
+
+
+@_SKIP_PYDANTIC
+@_SKIP_BLACK
+def test_openapi_with_refs_exec_current_version(output_file: Path) -> None:
+    """Test that OpenAPI schema with $ref generates executable code."""
+    run_main_and_assert(
+        input_path=OPEN_API_DATA_PATH / "body_and_parameters.yaml",
+        output_path=output_file,
+        extra_args=get_current_version_args("--output-model-type", "pydantic_v2.BaseModel"),
+        skip_code_validation=False,
+        force_exec_validation=True,
+    )
+
+
+@_SKIP_PYDANTIC
+@_SKIP_BLACK
+def test_openapi_allof_exec_current_version(output_file: Path) -> None:
+    """Test that OpenAPI schema with allOf generates executable code."""
+    run_main_and_assert(
+        input_path=OPEN_API_DATA_PATH / "allof.yaml",
+        output_path=output_file,
+        extra_args=get_current_version_args("--output-model-type", "pydantic_v2.BaseModel"),
+        skip_code_validation=False,
+        force_exec_validation=True,
+    )
+
+
+@_SKIP_PYDANTIC
+@_SKIP_BLACK
+def test_jsonschema_person_exec_current_version(output_file: Path) -> None:
+    """Test that person.json schema generates executable code on current Python."""
+    run_main_and_assert(
+        input_path=JSON_SCHEMA_DATA_PATH / "person.json",
+        output_path=output_file,
+        extra_args=get_current_version_args("--output-model-type", "pydantic_v2.BaseModel"),
+        skip_code_validation=False,
+        force_exec_validation=True,
+    )
+
+
+@_SKIP_PYDANTIC
+@_SKIP_BLACK
+def test_jsonschema_nested_array_exec_current_version(output_file: Path) -> None:
+    """Test that nested array JSON Schema generates executable code."""
+    run_main_and_assert(
+        input_path=JSON_SCHEMA_DATA_PATH / "nested_array.json",
+        output_path=output_file,
+        extra_args=get_current_version_args("--output-model-type", "pydantic_v2.BaseModel"),
+        skip_code_validation=False,
+        force_exec_validation=True,
+    )
+
+
+@_SKIP_PYDANTIC
+@_SKIP_BLACK
+def test_jsonschema_circular_reference_exec_current_version(output_file: Path) -> None:
+    """Test that circular reference JSON Schema generates executable code."""
+    run_main_and_assert(
+        input_path=JSON_SCHEMA_DATA_PATH / "circular_reference.json",
+        output_path=output_file,
+        extra_args=get_current_version_args("--output-model-type", "pydantic_v2.BaseModel"),
+        skip_code_validation=False,
+        force_exec_validation=True,
+    )
+
+
+@_SKIP_PYDANTIC
+@_SKIP_BLACK
+@pytest.mark.skipif(
+    sys.version_info[:2] <= (3, 10),
+    reason="Need runtime > target (3.10) to test force_exec_validation behavior",
+)
+def test_force_exec_with_different_target_version(output_file: Path) -> None:
+    """Test that force_exec_validation runs exec even with different target version.
+
+    This test uses --target-python-version 3.10 but force_exec_validation=True
+    to verify that exec still runs on the current Python version.
+
+    Requirements for this test to be meaningful:
+    - Runtime must be > 3.10 (so target < runtime, allowing compile to proceed)
+    - This ensures force_exec_validation can bypass the version mismatch skip
+    """
+    run_main_and_assert(
+        input_path=JSON_SCHEMA_DATA_PATH / "person.json",
+        output_path=output_file,
+        extra_args=["--target-python-version", "3.10", "--output-model-type", "pydantic_v2.BaseModel"],
+        skip_code_validation=False,
+        force_exec_validation=True,
+    )
+
+
+def test_get_current_version_args_basic() -> None:
+    """Test that get_current_version_args returns correct args."""
+    args = get_current_version_args()
+    assert args == ["--target-python-version", CURRENT_PYTHON_VERSION]
+
+
+def test_get_current_version_args_with_extra() -> None:
+    """Test that get_current_version_args includes extra args."""
+    args = get_current_version_args("--use-field-description", "--strict")
+    assert args == [
+        "--target-python-version",
+        CURRENT_PYTHON_VERSION,
+        "--use-field-description",
+        "--strict",
+    ]
+
+
+def test_current_python_version_format() -> None:
+    """Test that CURRENT_PYTHON_VERSION matches expected format."""
+    expected = f"{sys.version_info[0]}.{sys.version_info[1]}"
+    assert expected == CURRENT_PYTHON_VERSION
