From 69c484d26a0827022622c44e5ee717e44db992bf Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 02:44:37 +0530 Subject: [PATCH 01/11] Add immutable execution event model --- src/prooflight/events/__init__.py | 0 src/prooflight/events/event.py | 0 tests/events/test_event.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/prooflight/events/__init__.py create mode 100644 src/prooflight/events/event.py create mode 100644 tests/events/test_event.py diff --git a/src/prooflight/events/__init__.py b/src/prooflight/events/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/prooflight/events/event.py b/src/prooflight/events/event.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/events/test_event.py b/tests/events/test_event.py new file mode 100644 index 0000000..e69de29 From 297cc3b1504252b92d34569fce5d7b1a5eca1283 Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 03:07:15 +0530 Subject: [PATCH 02/11] Add immutable execution event model --- src/prooflight/events/__init__.py | 7 ++++ src/prooflight/events/event.py | 56 +++++++++++++++++++++++++++++++ tests/events/test_event.py | 23 +++++++++++++ 3 files changed, 86 insertions(+) diff --git a/src/prooflight/events/__init__.py b/src/prooflight/events/__init__.py index e69de29..a9bb3f6 100644 --- a/src/prooflight/events/__init__.py +++ b/src/prooflight/events/__init__.py @@ -0,0 +1,7 @@ +""" +Execution event models. +""" + +from .event import Event + +__all__ = ["Event"] diff --git a/src/prooflight/events/event.py b/src/prooflight/events/event.py index e69de29..4e9e260 100644 --- a/src/prooflight/events/event.py +++ b/src/prooflight/events/event.py @@ -0,0 +1,56 @@ +""" +Immutable execution event. + +Events are the fundamental record of everything that happens during an +experiment. They are intentionally generic so that new event types can be +introduced without changing the core event model. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any +from uuid import UUID, uuid4 + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class Event(BaseModel): + """Immutable execution event.""" + + model_config = ConfigDict( + frozen=True, + extra="forbid", + ) + + id: UUID = Field( + default_factory=uuid4, + description="Unique identifier for the event.", + ) + + timestamp: datetime = Field( + default_factory=lambda: datetime.now(UTC), + description="Time at which the event occurred.", + ) + + name: str = Field( + min_length=1, + description="Machine-readable event name.", + ) + + payload: dict[str, Any] = Field( + default_factory=dict, + description="Structured event data.", + ) + + @field_validator("name") + @classmethod + def validate_name(cls, value: str) -> str: + """Reject empty event names.""" + + value = value.strip() + + if not value: + raise ValueError("Event name cannot be empty.") + + return value diff --git a/tests/events/test_event.py b/tests/events/test_event.py index e69de29..90046d8 100644 --- a/tests/events/test_event.py +++ b/tests/events/test_event.py @@ -0,0 +1,23 @@ +from pydantic import ValidationError + +from prooflight.events import Event + + +def test_create_event() -> None: + """A valid event can be created.""" + + event = Event(name="runtime.started") + + assert event.name == "runtime.started" + assert event.payload == {} + + +def test_empty_name_is_rejected() -> None: + """Empty event names are invalid.""" + + try: + Event(name=" ") + except ValidationError: + return + + raise AssertionError("Expected ValidationError.") From b50bae9993f1c05dd699c9e79a9541547ad0ca3b Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 03:11:18 +0530 Subject: [PATCH 03/11] Add execution event recorder --- src/prooflight/recorder/__init__.py | 11 +++++ src/prooflight/recorder/recorder.py | 66 +++++++++++++++++++++++++++++ tests/recorder/test_recorder.py | 50 ++++++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 src/prooflight/recorder/__init__.py create mode 100644 src/prooflight/recorder/recorder.py create mode 100644 tests/recorder/test_recorder.py diff --git a/src/prooflight/recorder/__init__.py b/src/prooflight/recorder/__init__.py new file mode 100644 index 0000000..74acc3c --- /dev/null +++ b/src/prooflight/recorder/__init__.py @@ -0,0 +1,11 @@ +""" +Event recording utilities. + +The recorder captures the complete execution history of an experiment. +""" + +from .recorder import Recorder + +__all__ = [ + "Recorder", +] diff --git a/src/prooflight/recorder/recorder.py b/src/prooflight/recorder/recorder.py new file mode 100644 index 0000000..1ae87b5 --- /dev/null +++ b/src/prooflight/recorder/recorder.py @@ -0,0 +1,66 @@ +""" +Execution event recorder. + +The Recorder is responsible for collecting immutable events produced during +an experiment execution. + +It intentionally has a small interface. Persistence, exporting, and artifact +generation will be handled later by the Artifact Store layer. +""" + +from __future__ import annotations + +from prooflight.events import Event + + +class Recorder: + """ + Stores events generated during one execution. + + A Recorder belongs to a single execution context. It should not be shared + across independent experiments. + """ + + def __init__(self) -> None: + """ + Initialize an empty event history. + + Internally events are stored as a list because execution order matters. + The first event happened before the second event, and that ordering is + important for replay and debugging. + """ + + self._events: list[Event] = [] + + def record(self, event: Event) -> None: + """ + Append a new event to the execution history. + + Parameters + ---------- + event: + Immutable event produced by any execution component. + """ + + self._events.append(event) + + @property + def events(self) -> tuple[Event, ...]: + """ + Return all recorded events. + + A tuple is returned instead of the internal list so callers cannot + accidentally mutate the recorder's history. + """ + + return tuple(self._events) + + def clear(self) -> None: + """ + Remove all recorded events. + + This is mainly useful for testing. In normal execution, a recorder + represents the complete lifetime of one experiment. + """ + + self._events.clear() diff --git a/tests/recorder/test_recorder.py b/tests/recorder/test_recorder.py new file mode 100644 index 0000000..c13a0a3 --- /dev/null +++ b/tests/recorder/test_recorder.py @@ -0,0 +1,50 @@ +from prooflight.events import Event +from prooflight.recorder import Recorder + + +def test_recorder_stores_events() -> None: + """ + Recorder should preserve events in insertion order. + """ + + recorder = Recorder() + + first = Event(name="experiment.started") + second = Event(name="experiment.finished") + + recorder.record(first) + recorder.record(second) + + assert recorder.events == ( + first, + second, + ) + + +def test_recorder_does_not_expose_mutable_storage() -> None: + """ + Returned events should be immutable from the caller's perspective. + """ + + recorder = Recorder() + + recorder.record(Event(name="runtime.started")) + + events = recorder.events + + assert isinstance(events, tuple) + assert len(events) == 1 + + +def test_recorder_clear() -> None: + """ + Recorder can reset its internal state. + """ + + recorder = Recorder() + + recorder.record(Event(name="test.event")) + + recorder.clear() + + assert recorder.events == () From 23453cdca8cc892c55bba8367f2a6ccf20ed62b0 Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 03:47:34 +0530 Subject: [PATCH 04/11] Add execution context --- src/prooflight/execution/__init__.py | 12 ++++++ src/prooflight/execution/context.py | 56 ++++++++++++++++++++++++++++ tests/execution/test_context.py | 44 ++++++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 src/prooflight/execution/__init__.py create mode 100644 src/prooflight/execution/context.py create mode 100644 tests/execution/test_context.py diff --git a/src/prooflight/execution/__init__.py b/src/prooflight/execution/__init__.py new file mode 100644 index 0000000..556a156 --- /dev/null +++ b/src/prooflight/execution/__init__.py @@ -0,0 +1,12 @@ +""" +Execution lifecycle components. + +This package contains objects responsible for managing the lifecycle +of an experiment execution. +""" + +from .context import ExecutionContext + +__all__ = [ + "ExecutionContext", +] diff --git a/src/prooflight/execution/context.py b/src/prooflight/execution/context.py new file mode 100644 index 0000000..327fccd --- /dev/null +++ b/src/prooflight/execution/context.py @@ -0,0 +1,56 @@ +""" +Execution context. + +An ExecutionContext represents the state shared across a single experiment +execution. + +It acts as a dependency boundary. Instead of passing many independent +objects between execution components, the context provides a single stable +interface. + +The context currently contains: +- Experiment definition +- Event Recorder + +Future components such as: +- Runtime +- Artifact Store +- Telemetry +- Evaluation state + +will be added here when they become necessary. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from prooflight.domain import Experiment +from prooflight.recorder import Recorder + + +class ExecutionContext(BaseModel): + """ + Immutable container for one experiment execution. + + A context belongs to exactly one execution lifecycle. + + Once created, the execution dependencies should not be replaced. + This guarantees that every component participating in an execution + observes the same execution state. + """ + + model_config = ConfigDict( + frozen=True, + arbitrary_types_allowed=True, + ) + + experiment: Experiment + """ + Experiment specification being executed. + """ + + recorder: Recorder + """ + Recorder responsible for collecting execution events. + """ diff --git a/tests/execution/test_context.py b/tests/execution/test_context.py new file mode 100644 index 0000000..79893c6 --- /dev/null +++ b/tests/execution/test_context.py @@ -0,0 +1,44 @@ +from pathlib import Path + +from prooflight.domain import Experiment +from prooflight.execution import ExecutionContext +from prooflight.recorder import Recorder + + +def create_experiment() -> Experiment: + """ + Create a minimal valid experiment for testing. + """ + + return Experiment( + name="context-test", + runtime="mock", + agent="test-agent", + output_dir=Path("./artifacts"), + ) + + +def test_execution_context_stores_dependencies() -> None: + """ + ExecutionContext should store the dependencies required + for a single execution. + """ + + experiment = create_experiment() + recorder = Recorder() + + context = ExecutionContext( + experiment=experiment, + recorder=recorder, + ) + + assert context.experiment == experiment + assert context.recorder == recorder + + +def test_execution_context_is_frozen() -> None: + """ + ExecutionContext should enforce immutability. + """ + + assert ExecutionContext.model_config.get("frozen") is True From a5003071b8aec8bd8fef0d413e228c9c4d953ade Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 03:57:34 +0530 Subject: [PATCH 05/11] Add runtime abstraction --- src/prooflight/runtime/__init__.py | 11 ++++++ src/prooflight/runtime/base.py | 59 ++++++++++++++++++++++++++++ tests/runtime/test_runtime.py | 62 ++++++++++++++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 src/prooflight/runtime/__init__.py create mode 100644 src/prooflight/runtime/base.py create mode 100644 tests/runtime/test_runtime.py diff --git a/src/prooflight/runtime/__init__.py b/src/prooflight/runtime/__init__.py new file mode 100644 index 0000000..8ccafad --- /dev/null +++ b/src/prooflight/runtime/__init__.py @@ -0,0 +1,11 @@ +""" +Runtime abstractions. + +A runtime defines how an agent or model is executed. +""" + +from .base import Runtime + +__all__ = [ + "Runtime", +] diff --git a/src/prooflight/runtime/base.py b/src/prooflight/runtime/base.py new file mode 100644 index 0000000..f0b4167 --- /dev/null +++ b/src/prooflight/runtime/base.py @@ -0,0 +1,59 @@ +""" +Runtime abstraction. + +A Runtime represents an executable environment capable of running +an evaluation. + +Concrete implementations may wrap: + +- language models +- agent frameworks +- external APIs +- simulation environments + +The base Runtime does not implement execution logic. +It only defines the contract. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from prooflight.execution import ExecutionContext + + +class Runtime(ABC): + """ + Abstract execution backend. + + Every runtime must provide an execute method. + + Examples of future implementations: + + - TransformersRuntime + - OpenAIRuntime + - LangGraphRuntime + - LocalAgentRuntime + """ + + @abstractmethod + def execute( + self, + context: ExecutionContext, + ) -> None: + """ + Execute one evaluation. + + Parameters + ---------- + context: + Complete state of the current execution. + + Returns + ------- + None + Execution results are currently recorded through events. + Future versions will introduce structured results. + """ + + raise NotImplementedError diff --git a/tests/runtime/test_runtime.py b/tests/runtime/test_runtime.py new file mode 100644 index 0000000..acbb3b0 --- /dev/null +++ b/tests/runtime/test_runtime.py @@ -0,0 +1,62 @@ +from pathlib import Path + +from prooflight.domain import Experiment +from prooflight.execution import ExecutionContext +from prooflight.recorder import Recorder +from prooflight.runtime import Runtime + + +class DummyRuntime(Runtime): + """ + Minimal runtime implementation for testing. + + Real runtimes will provide concrete execution behaviour. + """ + + def execute( + self, + context: ExecutionContext, + ) -> None: + """ + Execute a dummy evaluation. + """ + + return None + + +def create_context() -> ExecutionContext: + """ + Create a minimal execution context. + """ + + experiment = Experiment( + name="runtime-test", + runtime="dummy", + agent="test-agent", + output_dir=Path("./artifacts"), + ) + + return ExecutionContext( + experiment=experiment, + recorder=Recorder(), + ) + + +def test_runtime_can_be_implemented() -> None: + """ + Concrete runtimes should satisfy the Runtime contract. + """ + + runtime = DummyRuntime() + + assert isinstance(runtime, Runtime) + + +def test_runtime_execute_accepts_context() -> None: + """ + Runtime execution should receive an ExecutionContext. + """ + + runtime = DummyRuntime() + + runtime.execute(create_context()) From c0d6865fb7caa06978f1f321d2e20d4e30daf5a1 Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 04:30:01 +0530 Subject: [PATCH 06/11] Add execution lifecycle events --- src/prooflight/execution/__init__.py | 2 + src/prooflight/execution/executor.py | 83 +++++++++++++++++++++++ tests/execution/test_executor.py | 61 +++++++++++++++++ tests/execution/test_executor_events.py | 88 +++++++++++++++++++++++++ 4 files changed, 234 insertions(+) create mode 100644 src/prooflight/execution/executor.py create mode 100644 tests/execution/test_executor.py create mode 100644 tests/execution/test_executor_events.py diff --git a/src/prooflight/execution/__init__.py b/src/prooflight/execution/__init__.py index 556a156..a7b0579 100644 --- a/src/prooflight/execution/__init__.py +++ b/src/prooflight/execution/__init__.py @@ -6,7 +6,9 @@ """ from .context import ExecutionContext +from .executor import Executor __all__ = [ "ExecutionContext", + "Executor", ] diff --git a/src/prooflight/execution/executor.py b/src/prooflight/execution/executor.py new file mode 100644 index 0000000..fd61d8e --- /dev/null +++ b/src/prooflight/execution/executor.py @@ -0,0 +1,83 @@ +""" +Execution orchestration. + +The Executor coordinates runtime execution and records +execution lifecycle events. +""" + +from __future__ import annotations + +from prooflight.events import Event +from prooflight.execution.context import ExecutionContext +from prooflight.runtime import Runtime + + +class Executor: + """ + Coordinates execution of one experiment. + """ + + def __init__( + self, + runtime: Runtime, + ) -> None: + """ + Initialize executor. + + Parameters + ---------- + runtime: + Concrete runtime implementation. + """ + + self.runtime = runtime + + def execute( + self, + context: ExecutionContext, + ) -> None: + """ + Execute one experiment. + + Lifecycle: + + execution.started + | + v + runtime.execute() + | + v + execution.completed + + If execution fails: + + execution.failed + """ + + context.recorder.record( + Event( + name="execution.started", + ) + ) + + try: + self.runtime.execute(context) + + except Exception as exc: + context.recorder.record( + Event( + name="execution.failed", + payload={ + "error": str(exc), + "error_type": type(exc).__name__, + }, + ) + ) + + raise + + context.recorder.record( + Event( + name="execution.completed", + ) + ) diff --git a/tests/execution/test_executor.py b/tests/execution/test_executor.py new file mode 100644 index 0000000..1a6463b --- /dev/null +++ b/tests/execution/test_executor.py @@ -0,0 +1,61 @@ +from pathlib import Path + +from prooflight.domain import Experiment +from prooflight.execution import ExecutionContext, Executor +from prooflight.recorder import Recorder +from prooflight.runtime import Runtime + + +class RecordingRuntime(Runtime): + """ + Runtime implementation used to verify Executor behaviour. + """ + + def __init__(self) -> None: + self.executed = False + + def execute( + self, + context: ExecutionContext, + ) -> None: + """ + Mark execution as completed. + """ + + self.executed = True + + +def create_context() -> ExecutionContext: + """ + Create a minimal execution context. + """ + + experiment = Experiment( + name="executor-test", + runtime="dummy", + agent="test-agent", + output_dir=Path("./artifacts"), + ) + + return ExecutionContext( + experiment=experiment, + recorder=Recorder(), + ) + + +def test_executor_delegates_to_runtime() -> None: + """ + Executor should delegate execution to Runtime. + """ + + runtime = RecordingRuntime() + + executor = Executor( + runtime=runtime, + ) + + executor.execute( + create_context(), + ) + + assert runtime.executed is True diff --git a/tests/execution/test_executor_events.py b/tests/execution/test_executor_events.py new file mode 100644 index 0000000..6163e89 --- /dev/null +++ b/tests/execution/test_executor_events.py @@ -0,0 +1,88 @@ +from pathlib import Path + +from prooflight.domain import Experiment +from prooflight.execution import ExecutionContext, Executor +from prooflight.recorder import Recorder +from prooflight.runtime import Runtime + + +class SuccessfulRuntime(Runtime): + """ + Runtime that completes successfully. + """ + + def execute( + self, + context: ExecutionContext, + ) -> None: + pass + + +class FailingRuntime(Runtime): + """ + Runtime that raises an error. + """ + + def execute( + self, + context: ExecutionContext, + ) -> None: + raise RuntimeError("failed execution") + + +def create_context() -> ExecutionContext: + """ + Create test execution context. + """ + + return ExecutionContext( + experiment=Experiment( + name="event-test", + runtime="dummy", + agent="test-agent", + output_dir=Path("./artifacts"), + ), + recorder=Recorder(), + ) + + +def test_executor_records_success_events() -> None: + """ + Successful execution should record started and completed events. + """ + + context = create_context() + + executor = Executor( + runtime=SuccessfulRuntime(), + ) + + executor.execute(context) + + assert [event.name for event in context.recorder.events] == [ + "execution.started", + "execution.completed", + ] + + +def test_executor_records_failure_event() -> None: + """ + Failed execution should record failed event. + """ + + context = create_context() + + executor = Executor( + runtime=FailingRuntime(), + ) + + try: + executor.execute(context) + + except RuntimeError: + pass + + assert [event.name for event in context.recorder.events] == [ + "execution.started", + "execution.failed", + ] From 2d85efec0a3d2c8fad3096b9b9353c283fe28497 Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 04:45:26 +0530 Subject: [PATCH 07/11] Add execution result model and formalize outcomes --- src/prooflight/execution/__init__.py | 2 + src/prooflight/execution/executor.py | 43 +++++++++---------- src/prooflight/execution/result.py | 62 ++++++++++++++++++++++++++++ tests/execution/test_result.py | 0 4 files changed, 83 insertions(+), 24 deletions(-) create mode 100644 src/prooflight/execution/result.py create mode 100644 tests/execution/test_result.py diff --git a/src/prooflight/execution/__init__.py b/src/prooflight/execution/__init__.py index a7b0579..2af921f 100644 --- a/src/prooflight/execution/__init__.py +++ b/src/prooflight/execution/__init__.py @@ -7,8 +7,10 @@ from .context import ExecutionContext from .executor import Executor +from .result import ExecutionResult __all__ = [ "ExecutionContext", "Executor", + "ExecutionResult", ] diff --git a/src/prooflight/execution/executor.py b/src/prooflight/execution/executor.py index fd61d8e..ff9c91c 100644 --- a/src/prooflight/execution/executor.py +++ b/src/prooflight/execution/executor.py @@ -1,14 +1,15 @@ """ Execution orchestration. -The Executor coordinates runtime execution and records -execution lifecycle events. +The Executor coordinates runtime execution, +records lifecycle events, and produces execution results. """ from __future__ import annotations from prooflight.events import Event from prooflight.execution.context import ExecutionContext +from prooflight.execution.result import ExecutionResult from prooflight.runtime import Runtime @@ -23,11 +24,6 @@ def __init__( ) -> None: """ Initialize executor. - - Parameters - ---------- - runtime: - Concrete runtime implementation. """ self.runtime = runtime @@ -35,23 +31,14 @@ def __init__( def execute( self, context: ExecutionContext, - ) -> None: + ) -> ExecutionResult: """ - Execute one experiment. - - Lifecycle: - - execution.started - | - v - runtime.execute() - | - v - execution.completed + Execute one experiment lifecycle. - If execution fails: - - execution.failed + Returns + ------- + ExecutionResult + Final execution outcome. """ context.recorder.record( @@ -61,7 +48,7 @@ def execute( ) try: - self.runtime.execute(context) + output = self.runtime.execute(context) except Exception as exc: context.recorder.record( @@ -74,10 +61,18 @@ def execute( ) ) - raise + return ExecutionResult( + status="failed", + error=str(exc), + ) context.recorder.record( Event( name="execution.completed", ) ) + + return ExecutionResult( + status="completed", + output=output, + ) diff --git a/src/prooflight/execution/result.py b/src/prooflight/execution/result.py new file mode 100644 index 0000000..22d7f48 --- /dev/null +++ b/src/prooflight/execution/result.py @@ -0,0 +1,62 @@ +""" +Execution result model. + +Represents the final outcome of one experiment execution. + +Events answer: + "What happened during execution?" + +ExecutionResult answers: + "What was the final outcome?" +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any +from uuid import UUID, uuid4 + +from pydantic import BaseModel, ConfigDict, Field + + +class ExecutionResult(BaseModel): + """ + Immutable result produced after execution completes. + + The result is intentionally separate from events. + + Events: + - chronological execution history + + Result: + - final execution summary + """ + + model_config = ConfigDict( + frozen=True, + extra="forbid", + ) + + id: UUID = Field( + default_factory=uuid4, + description="Unique identifier for this execution result.", + ) + + status: str = Field( + description="Final execution status.", + ) + + output: Any | None = Field( + default=None, + description="Execution output produced by runtime.", + ) + + error: str | None = Field( + default=None, + description="Error message if execution failed.", + ) + + created_at: datetime = Field( + default_factory=lambda: datetime.now(UTC), + description="Time when result was created.", + ) diff --git a/tests/execution/test_result.py b/tests/execution/test_result.py new file mode 100644 index 0000000..e69de29 From d7bcab241e976bca1075aed591f37b4ace226c7d Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 05:08:56 +0530 Subject: [PATCH 08/11] Update README for execution core milestone --- README.md | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index d498b20..3bad2f5 100644 --- a/README.md +++ b/README.md @@ -317,42 +317,39 @@ Completed: --- -## Milestone 2: Execution Runtime +## Milestone 2: Execution Core ✅ -Planned: +Completed: - runtime abstraction -- model adapters -- asynchronous execution -- agent interfaces -- environment interfaces -- event model +- execution context +- executor orchestration +- immutable event model +- execution recorder +- lifecycle event tracking +- execution result model Architecture: -``` Experiment - | ▼ - -Runtime Executor - +ExecutionContext | ▼ - -Agent - +Executor | - ▼ - -Environment + +----------------+ + | | + ▼ ▼ +Runtime Recorder + | + ▼ + Events - | - ▼ +Executor returns: -Events -``` +ExecutionResult --- From 20c749cedb270d1f855247f8baa8260c574ddc69 Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 06:16:54 +0530 Subject: [PATCH 09/11] Added tests/execution/test_result.py and Updated README.md --- README.md | 93 +++++++++++++++++++++++++++++----- tests/execution/test_result.py | 34 +++++++++++++ 2 files changed, 113 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3bad2f5..0a18d21 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Experiment # Current Status -Prooflight is currently in **Milestone 1: Foundation Layer**. +Prooflight is currently in **Milestone 2: Execution Core**. Implemented: @@ -87,6 +87,13 @@ Implemented: - static type checking - linting and formatting - CI-ready development workflow +- runtime abstraction +- execution context +- execution orchestration +- immutable execution events +- execution recorder +- lifecycle event tracking +- structured execution results The current implementation establishes the core abstraction that future evaluation components will build upon: @@ -185,14 +192,22 @@ Reproducibility is treated as a first-class engineering requirement. ## 4. Observability -Future versions will record: +Prooflight treats execution history as a first-class artifact. -- execution traces -- agent decisions -- tool usage -- failures +Current capabilities include: + +- execution lifecycle events +- ordered event recording +- execution failure tracking +- structured execution outcomes + +Future versions will extend this into: + +- agent trajectories +- tool interactions - resource consumption -- evaluation outcomes +- evaluation metrics +- replay systems The goal is that every evaluation result can be reconstructed. @@ -208,13 +223,43 @@ prooflight/ ├── src/ │ └── prooflight/ │ ├── __init__.py -│ └── domain/ +│ │ +│ ├── domain/ +│ │ ├── __init__.py +│ │ └── experiment.py +│ │ +│ ├── events/ +│ │ ├── __init__.py +│ │ └── event.py +│ │ +│ ├── recorder/ +│ │ ├── __init__.py +│ │ └── recorder.py +│ │ +│ ├── execution/ +│ │ ├── __init__.py +│ │ ├── context.py +│ │ ├── executor.py +│ │ └── result.py +│ │ +│ └── runtime/ │ ├── __init__.py -│ └── experiment.py +│ └── runtime.py │ ├── tests/ -│ └── domain/ -│ └── test_experiment.py +│ ├── domain/ +│ │ └── test_experiment.py +│ │ +│ ├── events/ +│ │ └── test_event.py +│ │ +│ ├── recorder/ +│ │ └── test_recorder.py +│ │ +│ └── execution/ +│ ├── test_context.py +│ ├── test_executor.py +│ └── test_result.py │ ├── .github/ │ └── workflows/ @@ -291,11 +336,13 @@ Current tests verify: - seed validation - path normalization - experiment immutability +- event validation +- recorder behaviour +- execution lifecycle +- execution result handling Future tests will cover: -- runtime execution -- event generation - benchmark execution - telemetry - replay @@ -329,7 +376,9 @@ Completed: - lifecycle event tracking - execution result model -Architecture: +### Execution Architecture + +Milestone 2 establishes the execution foundation: Experiment | @@ -351,6 +400,22 @@ Executor returns: ExecutionResult +The separation allows Prooflight to distinguish between: + +**Execution history** + +"What happened?" + +Captured through events. + +and: + +**Execution outcome** + +"What was the final result?" + +Captured through `ExecutionResult`. + --- ## Milestone 3: Evaluation Infrastructure diff --git a/tests/execution/test_result.py b/tests/execution/test_result.py index e69de29..4c50800 100644 --- a/tests/execution/test_result.py +++ b/tests/execution/test_result.py @@ -0,0 +1,34 @@ +from prooflight.execution import ExecutionResult + + +def test_execution_result_is_immutable() -> None: + """ + Execution results should not change after creation. + """ + + result = ExecutionResult( + status="completed", + output="success", + ) + + try: + result.status = "failed" # type: ignore[misc] + + except Exception: + return + + raise AssertionError("ExecutionResult should be immutable.") + + +def test_execution_result_stores_output() -> None: + """ + Result should preserve execution output. + """ + + result = ExecutionResult( + status="completed", + output={"answer": 42}, + ) + + assert result.status == "completed" + assert result.output == {"answer": 42} From fa996e41698c49a14cf9f59fadfdf66992f90c03 Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 06:39:20 +0530 Subject: [PATCH 10/11] Formatted README --- README.md | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0a18d21..c7940e7 100644 --- a/README.md +++ b/README.md @@ -380,25 +380,19 @@ Completed: Milestone 2 establishes the execution foundation: -Experiment - | - ▼ -ExecutionContext - | - ▼ -Executor - | - +----------------+ - | | - ▼ ▼ -Runtime Recorder - | - ▼ - Events - -Executor returns: +Experiment + ↓ +ExecutionContext + ↓ +Executor + ├───────────────┐ + ↓ ↓ +Runtime Recorder + ↓ +Events + +Executor returns: **ExecutionResult** -ExecutionResult The separation allows Prooflight to distinguish between: From ed57eaafe5161ecb5b6b12fc95ec95f762e797a5 Mon Sep 17 00:00:00 2001 From: Rishab Date: Thu, 6 Aug 2026 06:43:42 +0530 Subject: [PATCH 11/11] Formatted README --- README.md | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index c7940e7..0086c8b 100644 --- a/README.md +++ b/README.md @@ -380,19 +380,27 @@ Completed: Milestone 2 establishes the execution foundation: -Experiment - ↓ -ExecutionContext - ↓ -Executor - ├───────────────┐ - ↓ ↓ -Runtime Recorder - ↓ -Events - -Executor returns: **ExecutionResult** +```text +Experiment + | + ▼ +ExecutionContext + | + ▼ +Executor + | + +----------------+ + | | + ▼ ▼ + Runtime Recorder + | + ▼ + Events + +Executor returns: +ExecutionResult +``` The separation allows Prooflight to distinguish between: