Add reference spec framework for noise-free benchmark outputs (#924)

memory/MEMORY.md

@@ -0,0 +1,3 @@
+# Memory Index
+
+- [Issue #924 — reference spec contribution](project_issue924_reference_spec.md) — Maria's in-progress PR adding noise-free output specs to MQT Bench benchmarks

memory/project_issue924_reference_spec.md

@@ -0,0 +1,35 @@
+---
+name: project-issue924-reference-spec
+description: Maria's contribution to MQT Bench issue #924 — add noise-free reference outputs for benchmark circuits
+metadata:
+  type: project
+---
+
+Maria is contributing to munich-quantum-toolkit/bench issue #924, which adds expected noise-free output distributions to MQT Bench circuits so downstream users can validate experiments.
+
+**Why:** Without knowing how the circuit generator constructs the oracle (e.g., Grover), users have no ground-truth to test against.
+
+**What was built:**
+
+- `_reference.py` — type hierarchy: `SparseReference`, `UniformReference`, `SimulateReference`, `NoneReference`, `ObjectiveSpec`, `MetricApplicability`, `ReferenceSpec` with `to_dict()` for JSON serialisation
+- `_registry.py` — parallel reference registry with `register_reference` decorator and `get_reference_factory_by_name`
+- `__init__.py` — public `get_reference_spec(name, size, **kwargs)` and lazy-load-aware `has_reference(name)`
+- `create_reference` functions added to: `ghz` (sparse 50/50), `wstate` (uniform hamming_weight==1), `bv` (sparse deterministic), `dj` (sparse deterministic from fixed seed), `grover` (sparse + marked-states objective), `qpeexact` (sparse phase readout), `randomcircuit` (simulate)
+- `tests/test_reference.py` — 52 tests covering all kinds, JSON round-trip, and metric fields
+
+**Architecture decisions:**
+
+- `create_reference` mirrors `create_circuit` in each benchmark file, registered via `@register_reference`
+- Spec is always poly-size: sparse table, uniform predicate, or "simulate"
+- `bit_order: "qiskit-little-endian"` — only Qiskit is used in this repo (no tket/cirq)
+- Grover marked state = all-ones on search register (oracle is `mcp(π, q, flag)`)
+- DJ/BV/QPE output strings reproduced from the same fixed seeds without running circuits
+- Qiskit string convention: reversed hidden_string for BV, reversed b_str for DJ
+
+**What's left for maintainers:**
+
+- Add `create_reference` to the remaining ~25 benchmarks (QAOA, VQE, QFT, adders, AE, etc.)
+- QAOA/VQE probably get `NoneReference` (outputs depend on optimised angles)
+- Arithmetic circuits could get `SparseReference` if the maintainers know the inputs
+
+**How to apply:** When touching benchmark files, check if a `create_reference` companion is needed. Follow the pattern in `ghz.py` or `bv.py`.

src/mqt/bench/benchmarks/__init__.py

@@ -15,12 +15,26 @@
 from functools import cache
 from typing import TYPE_CHECKING, Any
 
+from ._reference import (
+    MetricApplicability,
+    NoneReference,
+    ObjectiveSpec,
+    ReferenceSpec,
+    SimulateReference,
+    SparseReference,
+    UniformReference,
+)
 from ._registry import (
     benchmark_catalog,
     benchmark_description,
     benchmark_names,
     get_benchmark_by_name,
+    get_reference_factory_by_name,
     register_benchmark,
+    register_reference,
+)
+from ._registry import (
+    has_reference as _registry_has_reference,
 )
 
 if TYPE_CHECKING:
@@ -38,11 +52,21 @@
 _IMPORTED_BENCHMARKS: set[str] = set()
 
 __all__ = [
+    "MetricApplicability",
+    "NoneReference",
+    "ObjectiveSpec",
+    "ReferenceSpec",
+    "SimulateReference",
+    "SparseReference",
+    "UniformReference",
     "create_circuit",
     "get_available_benchmark_names",
     "get_benchmark_catalog",
     "get_benchmark_description",
+    "get_reference_spec",
+    "has_reference",
     "register_benchmark",
+    "register_reference",
 ]
 
 
@@ -78,6 +102,22 @@ def get_available_benchmark_names() -> list[str]:
     return sorted(_DISCOVERED_BENCHMARKS | set(benchmark_names())).copy()
 
 
+def has_reference(benchmark_name: str) -> bool:
+    """Return ``True`` if a reference spec is available for *benchmark_name*.
+
+    Triggers lazy import of the benchmark module so that the reference registry
+    is populated before the check.
+
+    Args:
+        benchmark_name: The benchmark to query.
+    """
+    try:
+        _ensure_loaded(benchmark_name)
+    except ValueError:
+        return False
+    return _registry_has_reference(benchmark_name)
+
+
 @cache
 def get_benchmark_description(benchmark_name: str) -> str:
     """Return the benchmark description given a benchmark name."""
@@ -99,6 +139,49 @@ def _get_factory(benchmark_name: str) -> Callable[..., QuantumCircuit]:
     return get_benchmark_by_name(benchmark_name)
 
 
+@cache
+def _get_reference_factory(benchmark_name: str) -> Callable[..., ReferenceSpec] | None:
+    """Internal reference factory cache."""
+    _ensure_loaded(benchmark_name)
+    return get_reference_factory_by_name(benchmark_name)
+
+
+# ruff: noqa: ANN401
+def get_reference_spec(benchmark_name: str, circuit_size: int, /, *args: Any, **kwargs: Any) -> ReferenceSpec:
+    """Return the reference specification for a benchmark instance.
+
+    The spec describes the ideal, noise-free output distribution in a compact
+    form that stays poly-size even for exponentially large circuits.  Call
+    :meth:`ReferenceSpec.to_dict` on the result to get a JSON-serialisable dict.
+
+    Args:
+        benchmark_name: The name of the benchmark (must match the registry key).
+        circuit_size: The number of qubits — same value you would pass to
+            :func:`create_circuit`.
+        *args: Forwarded to the benchmark's ``create_reference`` function.
+        **kwargs: Forwarded to the benchmark's ``create_reference`` function.
+
+    Returns:
+        ReferenceSpec describing the noise-free output.
+
+    Raises:
+        ValueError: If the benchmark name is unknown or has no reference registered.
+    """
+    if circuit_size <= 0:
+        msg = "`circuit_size` must be a positive integer."
+        raise ValueError(msg)
+
+    factory = _get_reference_factory(benchmark_name)
+    if factory is None:
+        msg = (
+            f"No reference spec registered for '{benchmark_name}'. "
+            "Either the benchmark does not yet have a create_reference function, "
+            "or the relevant module has not been imported."
+        )
+        raise ValueError(msg)
+    return factory(circuit_size, *args, **kwargs)
+
+
 # ruff: noqa: ANN401
 def create_circuit(benchmark_name: str, circuit_size: int, /, *args: Any, **kwargs: Any) -> QuantumCircuit:
     """Creates and returns a quantum circuit based on the specified benchmark name and additional arguments.

src/mqt/bench/benchmarks/_reference.py

@@ -0,0 +1,183 @@
+# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM
+# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH
+# All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+#
+# Licensed under the MIT License
+
+"""Reference specification types for noise-free benchmark outputs.
+
+Each benchmark can optionally expose a ``create_reference`` function (registered
+via :func:`register_reference`) that returns a :class:`ReferenceSpec` describing
+the ideal, noise-free output distribution in a compact, evaluable form.
+
+The spec is deliberately *poly-size* even for exponentially large Hilbert spaces:
+it stores a *description* of the distribution (sparse table, uniform predicate,
+analytic form, or semantic objective) rather than a dense state vector.
+
+Calling :func:`~mqt.bench.benchmarks.get_reference_spec` returns the spec as a
+:class:`ReferenceSpec` object; call :meth:`ReferenceSpec.to_dict` to get a
+plain ``dict`` that serialises cleanly to JSON.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class SparseReference:
+    """Explicit probability table for circuits with a small support.
+
+    Suitable for: GHZ, BV, DJ, QPE, Grover (few high-probability states).
+    """
+
+    entries: dict[str, float]
+    normalized: bool = True
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"kind": "sparse", "normalized": self.normalized, "entries": self.entries}
+
+
+@dataclass
+class UniformReference:
+    """Uniform distribution over a predicate-defined support.
+
+    Stores only the predicate string and support size, so P(x) = 1/size
+    whenever the predicate holds and 0 otherwise.  Evaluable in O(1) per
+    bitstring without enumerating the full support.
+
+    Suitable for: W state (``hamming_weight == 1``), random Clifford states, etc.
+    """
+
+    predicate: str
+    size: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"kind": "uniform", "predicate": self.predicate, "size": self.size}
+
+
+@dataclass
+class SimulateReference:
+    """Reference to be obtained by classical statevector simulation.
+
+    Used when no compact closed-form is available but simulation is feasible.
+    ``max_qubits`` is advisory: indicate the largest instance the team has
+    pre-simulated (or considers tractable).
+    """
+
+    max_qubits: int = 30
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"kind": "simulate", "max_qubits": self.max_qubits}
+
+
+@dataclass
+class NoneReference:
+    """No reference distribution available.
+
+    Used for variational / parameterised circuits (QAOA, VQE) where the
+    output depends on optimised angles, or for circuits whose output is
+    essentially random (random circuits beyond simulation limits).
+    """
+
+    reason: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {"kind": "none"}
+        if self.reason:
+            d["reason"] = self.reason
+        return d
+
+
+ReferenceKind = SparseReference | UniformReference | SimulateReference | NoneReference
+
+
+@dataclass
+class ObjectiveSpec:
+    """Semantic objective — the *answer* the circuit is meant to find.
+
+    Complements the reference distribution: even when the distribution has
+    many non-zero entries, the objective pins down what "correct" means.
+
+    Examples of ``type`` values and their ``value`` payloads:
+
+    * ``"marked_states"``   - list of target bitstrings (Grover)
+    * ``"hidden_string"``   - the secret bitstring (BV)
+    * ``"balanced_or_constant"`` - ``"balanced"`` or ``"constant"`` (DJ)
+    * ``"phase"``           - estimated phase as a float (QPE)
+    """
+
+    type: str
+    value: Any = None
+
+    def to_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {"type": self.type}
+        if self.value is not None:
+            d["value"] = self.value
+        return d
+
+
+@dataclass
+class MetricApplicability:
+    """Whether a standard metric applies to this benchmark and its ideal value."""
+
+    applicable: bool
+    ideal: float | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {"applicable": self.applicable}
+        if self.ideal is not None:
+            d["ideal"] = self.ideal
+        return d
+
+
+@dataclass
+class ReferenceSpec:
+    """Complete reference specification for one benchmark instance.
+
+    Fields
+    ------
+    circuit:
+        Benchmark name (matches the registry key, e.g. ``"ghz"``).
+    n_qubits:
+        Total number of qubits in the circuit as returned by ``create_circuit``.
+    measured_qubits:
+        Indices (in circuit qubit order) of the qubits that contribute to the
+        classical output string.  Ancilla omitted.
+    bit_order:
+        ``"qiskit-little-endian"``
+    reference:
+        Compact description of the ideal probability distribution.
+    objective:
+        Optional semantic answer.
+    metrics:
+        Map from metric name to applicability and ideal value.
+        Standard keys: ``"hellinger_fidelity"``, ``"tvd"``,
+        ``"success_probability"``, ``"linear_xeb"``.
+    """
+
+    circuit: str
+    n_qubits: int
+    measured_qubits: list[int]
+    bit_order: str
+    reference: ReferenceKind
+    objective: ObjectiveSpec | None = None
+    metrics: dict[str, MetricApplicability] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a JSON-serialisable dict representation."""
+        d: dict[str, Any] = {
+            "circuit": self.circuit,
+            "n_qubits": self.n_qubits,
+            "measured_qubits": self.measured_qubits,
+            "bit_order": self.bit_order,
+            "reference": self.reference.to_dict(),
+        }
+        if self.objective is not None:
+            d["objective"] = self.objective.to_dict()
+        if self.metrics:
+            d["metrics"] = {k: v.to_dict() for k, v in self.metrics.items()}
+        return d