diff --git a/README.md b/README.md index d498b20..0086c8b 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ Experiment # Current Status -Prooflight is currently in **Milestone 1: Foundation Layer**. +Prooflight is currently in **Milestone 2: Execution Core**. Implemented: @@ -87,6 +87,13 @@ Implemented: - static type checking - linting and formatting - CI-ready development workflow +- runtime abstraction +- execution context +- execution orchestration +- immutable execution events +- execution recorder +- lifecycle event tracking +- structured execution results The current implementation establishes the core abstraction that future evaluation components will build upon: @@ -185,14 +192,22 @@ Reproducibility is treated as a first-class engineering requirement. ## 4. Observability -Future versions will record: +Prooflight treats execution history as a first-class artifact. + +Current capabilities include: + +- execution lifecycle events +- ordered event recording +- execution failure tracking +- structured execution outcomes -- execution traces -- agent decisions -- tool usage -- failures +Future versions will extend this into: + +- agent trajectories +- tool interactions - resource consumption -- evaluation outcomes +- evaluation metrics +- replay systems The goal is that every evaluation result can be reconstructed. @@ -208,13 +223,43 @@ prooflight/ ├── src/ │ └── prooflight/ │ ├── __init__.py -│ └── domain/ +│ │ +│ ├── domain/ +│ │ ├── __init__.py +│ │ └── experiment.py +│ │ +│ ├── events/ +│ │ ├── __init__.py +│ │ └── event.py +│ │ +│ ├── recorder/ +│ │ ├── __init__.py +│ │ └── recorder.py +│ │ +│ ├── execution/ +│ │ ├── __init__.py +│ │ ├── context.py +│ │ ├── executor.py +│ │ └── result.py +│ │ +│ └── runtime/ │ ├── __init__.py -│ └── experiment.py +│ └── runtime.py │ ├── tests/ -│ └── domain/ -│ └── test_experiment.py +│ ├── domain/ +│ │ └── test_experiment.py +│ │ +│ ├── events/ +│ │ └── test_event.py +│ │ +│ ├── recorder/ +│ │ └── test_recorder.py +│ │ +│ └── execution/ +│ ├── test_context.py +│ ├── test_executor.py +│ └── test_result.py │ ├── .github/ │ └── workflows/ @@ -291,11 +336,13 @@ Current tests verify: - seed validation - path normalization - experiment immutability +- event validation +- recorder behaviour +- execution lifecycle +- execution result handling Future tests will cover: -- runtime execution -- event generation - benchmark execution - telemetry - replay @@ -317,42 +364,59 @@ Completed: --- -## Milestone 2: Execution Runtime +## Milestone 2: Execution Core ✅ -Planned: +Completed: - runtime abstraction -- model adapters -- asynchronous execution -- agent interfaces -- environment interfaces -- event model +- execution context +- executor orchestration +- immutable event model +- execution recorder +- lifecycle event tracking +- execution result model -Architecture: +### Execution Architecture -``` +Milestone 2 establishes the execution foundation: + +```text Experiment + | + ▼ +ExecutionContext + | + ▼ +Executor + | + +----------------+ + | | + ▼ ▼ + Runtime Recorder + | + ▼ + Events - | - ▼ +Executor returns: -Runtime Executor +ExecutionResult +``` - | - ▼ +The separation allows Prooflight to distinguish between: -Agent +**Execution history** - | - ▼ +"What happened?" -Environment +Captured through events. - | - ▼ +and: -Events -``` +**Execution outcome** + +"What was the final result?" + +Captured through `ExecutionResult`. --- diff --git a/src/prooflight/events/__init__.py b/src/prooflight/events/__init__.py new file mode 100644 index 0000000..a9bb3f6 --- /dev/null +++ b/src/prooflight/events/__init__.py @@ -0,0 +1,7 @@ +""" +Execution event models. +""" + +from .event import Event + +__all__ = ["Event"] diff --git a/src/prooflight/events/event.py b/src/prooflight/events/event.py new file mode 100644 index 0000000..4e9e260 --- /dev/null +++ b/src/prooflight/events/event.py @@ -0,0 +1,56 @@ +""" +Immutable execution event. + +Events are the fundamental record of everything that happens during an +experiment. They are intentionally generic so that new event types can be +introduced without changing the core event model. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any +from uuid import UUID, uuid4 + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class Event(BaseModel): + """Immutable execution event.""" + + model_config = ConfigDict( + frozen=True, + extra="forbid", + ) + + id: UUID = Field( + default_factory=uuid4, + description="Unique identifier for the event.", + ) + + timestamp: datetime = Field( + default_factory=lambda: datetime.now(UTC), + description="Time at which the event occurred.", + ) + + name: str = Field( + min_length=1, + description="Machine-readable event name.", + ) + + payload: dict[str, Any] = Field( + default_factory=dict, + description="Structured event data.", + ) + + @field_validator("name") + @classmethod + def validate_name(cls, value: str) -> str: + """Reject empty event names.""" + + value = value.strip() + + if not value: + raise ValueError("Event name cannot be empty.") + + return value diff --git a/src/prooflight/execution/__init__.py b/src/prooflight/execution/__init__.py new file mode 100644 index 0000000..2af921f --- /dev/null +++ b/src/prooflight/execution/__init__.py @@ -0,0 +1,16 @@ +""" +Execution lifecycle components. + +This package contains objects responsible for managing the lifecycle +of an experiment execution. +""" + +from .context import ExecutionContext +from .executor import Executor +from .result import ExecutionResult + +__all__ = [ + "ExecutionContext", + "Executor", + "ExecutionResult", +] diff --git a/src/prooflight/execution/context.py b/src/prooflight/execution/context.py new file mode 100644 index 0000000..327fccd --- /dev/null +++ b/src/prooflight/execution/context.py @@ -0,0 +1,56 @@ +""" +Execution context. + +An ExecutionContext represents the state shared across a single experiment +execution. + +It acts as a dependency boundary. Instead of passing many independent +objects between execution components, the context provides a single stable +interface. + +The context currently contains: +- Experiment definition +- Event Recorder + +Future components such as: +- Runtime +- Artifact Store +- Telemetry +- Evaluation state + +will be added here when they become necessary. +""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from prooflight.domain import Experiment +from prooflight.recorder import Recorder + + +class ExecutionContext(BaseModel): + """ + Immutable container for one experiment execution. + + A context belongs to exactly one execution lifecycle. + + Once created, the execution dependencies should not be replaced. + This guarantees that every component participating in an execution + observes the same execution state. + """ + + model_config = ConfigDict( + frozen=True, + arbitrary_types_allowed=True, + ) + + experiment: Experiment + """ + Experiment specification being executed. + """ + + recorder: Recorder + """ + Recorder responsible for collecting execution events. + """ diff --git a/src/prooflight/execution/executor.py b/src/prooflight/execution/executor.py new file mode 100644 index 0000000..ff9c91c --- /dev/null +++ b/src/prooflight/execution/executor.py @@ -0,0 +1,78 @@ +""" +Execution orchestration. + +The Executor coordinates runtime execution, +records lifecycle events, and produces execution results. +""" + +from __future__ import annotations + +from prooflight.events import Event +from prooflight.execution.context import ExecutionContext +from prooflight.execution.result import ExecutionResult +from prooflight.runtime import Runtime + + +class Executor: + """ + Coordinates execution of one experiment. + """ + + def __init__( + self, + runtime: Runtime, + ) -> None: + """ + Initialize executor. + """ + + self.runtime = runtime + + def execute( + self, + context: ExecutionContext, + ) -> ExecutionResult: + """ + Execute one experiment lifecycle. + + Returns + ------- + ExecutionResult + Final execution outcome. + """ + + context.recorder.record( + Event( + name="execution.started", + ) + ) + + try: + output = self.runtime.execute(context) + + except Exception as exc: + context.recorder.record( + Event( + name="execution.failed", + payload={ + "error": str(exc), + "error_type": type(exc).__name__, + }, + ) + ) + + return ExecutionResult( + status="failed", + error=str(exc), + ) + + context.recorder.record( + Event( + name="execution.completed", + ) + ) + + return ExecutionResult( + status="completed", + output=output, + ) diff --git a/src/prooflight/execution/result.py b/src/prooflight/execution/result.py new file mode 100644 index 0000000..22d7f48 --- /dev/null +++ b/src/prooflight/execution/result.py @@ -0,0 +1,62 @@ +""" +Execution result model. + +Represents the final outcome of one experiment execution. + +Events answer: + "What happened during execution?" + +ExecutionResult answers: + "What was the final outcome?" +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any +from uuid import UUID, uuid4 + +from pydantic import BaseModel, ConfigDict, Field + + +class ExecutionResult(BaseModel): + """ + Immutable result produced after execution completes. + + The result is intentionally separate from events. + + Events: + - chronological execution history + + Result: + - final execution summary + """ + + model_config = ConfigDict( + frozen=True, + extra="forbid", + ) + + id: UUID = Field( + default_factory=uuid4, + description="Unique identifier for this execution result.", + ) + + status: str = Field( + description="Final execution status.", + ) + + output: Any | None = Field( + default=None, + description="Execution output produced by runtime.", + ) + + error: str | None = Field( + default=None, + description="Error message if execution failed.", + ) + + created_at: datetime = Field( + default_factory=lambda: datetime.now(UTC), + description="Time when result was created.", + ) diff --git a/src/prooflight/recorder/__init__.py b/src/prooflight/recorder/__init__.py new file mode 100644 index 0000000..74acc3c --- /dev/null +++ b/src/prooflight/recorder/__init__.py @@ -0,0 +1,11 @@ +""" +Event recording utilities. + +The recorder captures the complete execution history of an experiment. +""" + +from .recorder import Recorder + +__all__ = [ + "Recorder", +] diff --git a/src/prooflight/recorder/recorder.py b/src/prooflight/recorder/recorder.py new file mode 100644 index 0000000..1ae87b5 --- /dev/null +++ b/src/prooflight/recorder/recorder.py @@ -0,0 +1,66 @@ +""" +Execution event recorder. + +The Recorder is responsible for collecting immutable events produced during +an experiment execution. + +It intentionally has a small interface. Persistence, exporting, and artifact +generation will be handled later by the Artifact Store layer. +""" + +from __future__ import annotations + +from prooflight.events import Event + + +class Recorder: + """ + Stores events generated during one execution. + + A Recorder belongs to a single execution context. It should not be shared + across independent experiments. + """ + + def __init__(self) -> None: + """ + Initialize an empty event history. + + Internally events are stored as a list because execution order matters. + The first event happened before the second event, and that ordering is + important for replay and debugging. + """ + + self._events: list[Event] = [] + + def record(self, event: Event) -> None: + """ + Append a new event to the execution history. + + Parameters + ---------- + event: + Immutable event produced by any execution component. + """ + + self._events.append(event) + + @property + def events(self) -> tuple[Event, ...]: + """ + Return all recorded events. + + A tuple is returned instead of the internal list so callers cannot + accidentally mutate the recorder's history. + """ + + return tuple(self._events) + + def clear(self) -> None: + """ + Remove all recorded events. + + This is mainly useful for testing. In normal execution, a recorder + represents the complete lifetime of one experiment. + """ + + self._events.clear() diff --git a/src/prooflight/runtime/__init__.py b/src/prooflight/runtime/__init__.py new file mode 100644 index 0000000..8ccafad --- /dev/null +++ b/src/prooflight/runtime/__init__.py @@ -0,0 +1,11 @@ +""" +Runtime abstractions. + +A runtime defines how an agent or model is executed. +""" + +from .base import Runtime + +__all__ = [ + "Runtime", +] diff --git a/src/prooflight/runtime/base.py b/src/prooflight/runtime/base.py new file mode 100644 index 0000000..f0b4167 --- /dev/null +++ b/src/prooflight/runtime/base.py @@ -0,0 +1,59 @@ +""" +Runtime abstraction. + +A Runtime represents an executable environment capable of running +an evaluation. + +Concrete implementations may wrap: + +- language models +- agent frameworks +- external APIs +- simulation environments + +The base Runtime does not implement execution logic. +It only defines the contract. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +from prooflight.execution import ExecutionContext + + +class Runtime(ABC): + """ + Abstract execution backend. + + Every runtime must provide an execute method. + + Examples of future implementations: + + - TransformersRuntime + - OpenAIRuntime + - LangGraphRuntime + - LocalAgentRuntime + """ + + @abstractmethod + def execute( + self, + context: ExecutionContext, + ) -> None: + """ + Execute one evaluation. + + Parameters + ---------- + context: + Complete state of the current execution. + + Returns + ------- + None + Execution results are currently recorded through events. + Future versions will introduce structured results. + """ + + raise NotImplementedError diff --git a/tests/events/test_event.py b/tests/events/test_event.py new file mode 100644 index 0000000..90046d8 --- /dev/null +++ b/tests/events/test_event.py @@ -0,0 +1,23 @@ +from pydantic import ValidationError + +from prooflight.events import Event + + +def test_create_event() -> None: + """A valid event can be created.""" + + event = Event(name="runtime.started") + + assert event.name == "runtime.started" + assert event.payload == {} + + +def test_empty_name_is_rejected() -> None: + """Empty event names are invalid.""" + + try: + Event(name=" ") + except ValidationError: + return + + raise AssertionError("Expected ValidationError.") diff --git a/tests/execution/test_context.py b/tests/execution/test_context.py new file mode 100644 index 0000000..79893c6 --- /dev/null +++ b/tests/execution/test_context.py @@ -0,0 +1,44 @@ +from pathlib import Path + +from prooflight.domain import Experiment +from prooflight.execution import ExecutionContext +from prooflight.recorder import Recorder + + +def create_experiment() -> Experiment: + """ + Create a minimal valid experiment for testing. + """ + + return Experiment( + name="context-test", + runtime="mock", + agent="test-agent", + output_dir=Path("./artifacts"), + ) + + +def test_execution_context_stores_dependencies() -> None: + """ + ExecutionContext should store the dependencies required + for a single execution. + """ + + experiment = create_experiment() + recorder = Recorder() + + context = ExecutionContext( + experiment=experiment, + recorder=recorder, + ) + + assert context.experiment == experiment + assert context.recorder == recorder + + +def test_execution_context_is_frozen() -> None: + """ + ExecutionContext should enforce immutability. + """ + + assert ExecutionContext.model_config.get("frozen") is True diff --git a/tests/execution/test_executor.py b/tests/execution/test_executor.py new file mode 100644 index 0000000..1a6463b --- /dev/null +++ b/tests/execution/test_executor.py @@ -0,0 +1,61 @@ +from pathlib import Path + +from prooflight.domain import Experiment +from prooflight.execution import ExecutionContext, Executor +from prooflight.recorder import Recorder +from prooflight.runtime import Runtime + + +class RecordingRuntime(Runtime): + """ + Runtime implementation used to verify Executor behaviour. + """ + + def __init__(self) -> None: + self.executed = False + + def execute( + self, + context: ExecutionContext, + ) -> None: + """ + Mark execution as completed. + """ + + self.executed = True + + +def create_context() -> ExecutionContext: + """ + Create a minimal execution context. + """ + + experiment = Experiment( + name="executor-test", + runtime="dummy", + agent="test-agent", + output_dir=Path("./artifacts"), + ) + + return ExecutionContext( + experiment=experiment, + recorder=Recorder(), + ) + + +def test_executor_delegates_to_runtime() -> None: + """ + Executor should delegate execution to Runtime. + """ + + runtime = RecordingRuntime() + + executor = Executor( + runtime=runtime, + ) + + executor.execute( + create_context(), + ) + + assert runtime.executed is True diff --git a/tests/execution/test_executor_events.py b/tests/execution/test_executor_events.py new file mode 100644 index 0000000..6163e89 --- /dev/null +++ b/tests/execution/test_executor_events.py @@ -0,0 +1,88 @@ +from pathlib import Path + +from prooflight.domain import Experiment +from prooflight.execution import ExecutionContext, Executor +from prooflight.recorder import Recorder +from prooflight.runtime import Runtime + + +class SuccessfulRuntime(Runtime): + """ + Runtime that completes successfully. + """ + + def execute( + self, + context: ExecutionContext, + ) -> None: + pass + + +class FailingRuntime(Runtime): + """ + Runtime that raises an error. + """ + + def execute( + self, + context: ExecutionContext, + ) -> None: + raise RuntimeError("failed execution") + + +def create_context() -> ExecutionContext: + """ + Create test execution context. + """ + + return ExecutionContext( + experiment=Experiment( + name="event-test", + runtime="dummy", + agent="test-agent", + output_dir=Path("./artifacts"), + ), + recorder=Recorder(), + ) + + +def test_executor_records_success_events() -> None: + """ + Successful execution should record started and completed events. + """ + + context = create_context() + + executor = Executor( + runtime=SuccessfulRuntime(), + ) + + executor.execute(context) + + assert [event.name for event in context.recorder.events] == [ + "execution.started", + "execution.completed", + ] + + +def test_executor_records_failure_event() -> None: + """ + Failed execution should record failed event. + """ + + context = create_context() + + executor = Executor( + runtime=FailingRuntime(), + ) + + try: + executor.execute(context) + + except RuntimeError: + pass + + assert [event.name for event in context.recorder.events] == [ + "execution.started", + "execution.failed", + ] diff --git a/tests/execution/test_result.py b/tests/execution/test_result.py new file mode 100644 index 0000000..4c50800 --- /dev/null +++ b/tests/execution/test_result.py @@ -0,0 +1,34 @@ +from prooflight.execution import ExecutionResult + + +def test_execution_result_is_immutable() -> None: + """ + Execution results should not change after creation. + """ + + result = ExecutionResult( + status="completed", + output="success", + ) + + try: + result.status = "failed" # type: ignore[misc] + + except Exception: + return + + raise AssertionError("ExecutionResult should be immutable.") + + +def test_execution_result_stores_output() -> None: + """ + Result should preserve execution output. + """ + + result = ExecutionResult( + status="completed", + output={"answer": 42}, + ) + + assert result.status == "completed" + assert result.output == {"answer": 42} diff --git a/tests/recorder/test_recorder.py b/tests/recorder/test_recorder.py new file mode 100644 index 0000000..c13a0a3 --- /dev/null +++ b/tests/recorder/test_recorder.py @@ -0,0 +1,50 @@ +from prooflight.events import Event +from prooflight.recorder import Recorder + + +def test_recorder_stores_events() -> None: + """ + Recorder should preserve events in insertion order. + """ + + recorder = Recorder() + + first = Event(name="experiment.started") + second = Event(name="experiment.finished") + + recorder.record(first) + recorder.record(second) + + assert recorder.events == ( + first, + second, + ) + + +def test_recorder_does_not_expose_mutable_storage() -> None: + """ + Returned events should be immutable from the caller's perspective. + """ + + recorder = Recorder() + + recorder.record(Event(name="runtime.started")) + + events = recorder.events + + assert isinstance(events, tuple) + assert len(events) == 1 + + +def test_recorder_clear() -> None: + """ + Recorder can reset its internal state. + """ + + recorder = Recorder() + + recorder.record(Event(name="test.event")) + + recorder.clear() + + assert recorder.events == () diff --git a/tests/runtime/test_runtime.py b/tests/runtime/test_runtime.py new file mode 100644 index 0000000..acbb3b0 --- /dev/null +++ b/tests/runtime/test_runtime.py @@ -0,0 +1,62 @@ +from pathlib import Path + +from prooflight.domain import Experiment +from prooflight.execution import ExecutionContext +from prooflight.recorder import Recorder +from prooflight.runtime import Runtime + + +class DummyRuntime(Runtime): + """ + Minimal runtime implementation for testing. + + Real runtimes will provide concrete execution behaviour. + """ + + def execute( + self, + context: ExecutionContext, + ) -> None: + """ + Execute a dummy evaluation. + """ + + return None + + +def create_context() -> ExecutionContext: + """ + Create a minimal execution context. + """ + + experiment = Experiment( + name="runtime-test", + runtime="dummy", + agent="test-agent", + output_dir=Path("./artifacts"), + ) + + return ExecutionContext( + experiment=experiment, + recorder=Recorder(), + ) + + +def test_runtime_can_be_implemented() -> None: + """ + Concrete runtimes should satisfy the Runtime contract. + """ + + runtime = DummyRuntime() + + assert isinstance(runtime, Runtime) + + +def test_runtime_execute_accepts_context() -> None: + """ + Runtime execution should receive an ExecutionContext. + """ + + runtime = DummyRuntime() + + runtime.execute(create_context())