From 6955fbc4ada6db0a3d5613d7546d2b51fe2362cc Mon Sep 17 00:00:00 2001 From: grageragarces Date: Fri, 26 Jun 2026 12:06:10 +0800 Subject: [PATCH 1/4] Reference spec framework for noiseless benchmarks (#924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a compact, JSON-serialisable reference specification for MQT Bench circuits so downstream users can validate noisy experiments against an ideal ground truth without needing a dense state vector. New files: - benchmarks/_reference.py — ReferenceSpec + SparseReference / UniformReference / SimulateReference / NoneReference / ObjectiveSpec - tests/test_reference.py — 52 tests (all passing) Framework changes: - _registry.py: parallel _REFERENCE_REGISTRY + register_reference decorator - __init__.py: get_reference_spec() public API + lazy-load-aware has_reference() Reference specs implemented for: ghz (sparse), wstate (uniform), bv (sparse + objective), dj (sparse + objective), grover (sparse + marked-states objective), qpeexact (sparse phase readout), randomcircuit (simulate). Closes #924 --- memory/MEMORY.md | 3 + memory/project_issue924_reference_spec.md | 32 ++ src/mqt/bench/benchmarks/__init__.py | 81 +++++ src/mqt/bench/benchmarks/_reference.py | 175 +++++++++++ src/mqt/bench/benchmarks/_registry.py | 47 ++- src/mqt/bench/benchmarks/bv.py | 46 ++- src/mqt/bench/benchmarks/dj.py | 62 +++- src/mqt/bench/benchmarks/ghz.py | 36 ++- src/mqt/bench/benchmarks/grover.py | 55 +++- src/mqt/bench/benchmarks/qpeexact.py | 64 +++- src/mqt/bench/benchmarks/randomcircuit.py | 23 +- src/mqt/bench/benchmarks/wstate.py | 32 +- tests/test_reference.py | 364 ++++++++++++++++++++++ 13 files changed, 981 insertions(+), 39 deletions(-) create mode 100644 memory/MEMORY.md create mode 100644 memory/project_issue924_reference_spec.md create mode 100644 src/mqt/bench/benchmarks/_reference.py create mode 100644 tests/test_reference.py diff --git a/memory/MEMORY.md b/memory/MEMORY.md new file mode 100644 index 000000000..dd15e02ec --- /dev/null +++ b/memory/MEMORY.md @@ -0,0 +1,3 @@ +# Memory Index + +- [Issue #924 — reference spec contribution](project_issue924_reference_spec.md) — Maria's in-progress PR adding noise-free output specs to MQT Bench benchmarks diff --git a/memory/project_issue924_reference_spec.md b/memory/project_issue924_reference_spec.md new file mode 100644 index 000000000..efa36c31b --- /dev/null +++ b/memory/project_issue924_reference_spec.md @@ -0,0 +1,32 @@ +--- +name: project-issue924-reference-spec +description: Maria's contribution to MQT Bench issue #924 — add noise-free reference outputs for benchmark circuits +metadata: + type: project +--- + +Maria is contributing to munich-quantum-toolkit/bench issue #924, which adds expected noise-free output distributions to MQT Bench circuits so downstream users can validate experiments. + +**Why:** Without knowing how the circuit generator constructs the oracle (e.g., Grover), users have no ground-truth to test against. + +**What was built:** +- `_reference.py` — type hierarchy: `SparseReference`, `UniformReference`, `SimulateReference`, `NoneReference`, `ObjectiveSpec`, `MetricApplicability`, `ReferenceSpec` with `to_dict()` for JSON serialisation +- `_registry.py` — parallel reference registry with `register_reference` decorator and `get_reference_factory_by_name` +- `__init__.py` — public `get_reference_spec(name, size, **kwargs)` and lazy-load-aware `has_reference(name)` +- `create_reference` functions added to: `ghz` (sparse 50/50), `wstate` (uniform hamming_weight==1), `bv` (sparse deterministic), `dj` (sparse deterministic from fixed seed), `grover` (sparse + marked-states objective), `qpeexact` (sparse phase readout), `randomcircuit` (simulate) +- `tests/test_reference.py` — 52 tests covering all kinds, JSON round-trip, and metric fields + +**Architecture decisions:** +- `create_reference` mirrors `create_circuit` in each benchmark file, registered via `@register_reference` +- Spec is always poly-size: sparse table, uniform predicate, or "simulate" +- `bit_order: "qiskit-little-endian"` — only Qiskit is used in this repo (no tket/cirq) +- Grover marked state = all-ones on search register (oracle is `mcp(π, q, flag)`) +- DJ/BV/QPE output strings reproduced from the same fixed seeds without running circuits +- Qiskit string convention: reversed hidden_string for BV, reversed b_str for DJ + +**What's left for maintainers:** +- Add `create_reference` to the remaining ~25 benchmarks (QAOA, VQE, QFT, adders, AE, etc.) +- QAOA/VQE probably get `NoneReference` (outputs depend on optimised angles) +- Arithmetic circuits could get `SparseReference` if the maintainers know the inputs + +**How to apply:** When touching benchmark files, check if a `create_reference` companion is needed. Follow the pattern in `ghz.py` or `bv.py`. diff --git a/src/mqt/bench/benchmarks/__init__.py b/src/mqt/bench/benchmarks/__init__.py index d338d4d67..bb34d26e5 100644 --- a/src/mqt/bench/benchmarks/__init__.py +++ b/src/mqt/bench/benchmarks/__init__.py @@ -15,12 +15,24 @@ from functools import cache from typing import TYPE_CHECKING, Any +from ._reference import ( + MetricApplicability, + NoneReference, + ObjectiveSpec, + ReferenceSpec, + SimulateReference, + SparseReference, + UniformReference, +) from ._registry import ( benchmark_catalog, benchmark_description, benchmark_names, get_benchmark_by_name, + get_reference_factory_by_name, + has_reference as _registry_has_reference, register_benchmark, + register_reference, ) if TYPE_CHECKING: @@ -38,11 +50,21 @@ _IMPORTED_BENCHMARKS: set[str] = set() __all__ = [ + "MetricApplicability", + "NoneReference", + "ObjectiveSpec", + "ReferenceSpec", + "SimulateReference", + "SparseReference", + "UniformReference", "create_circuit", "get_available_benchmark_names", "get_benchmark_catalog", "get_benchmark_description", + "get_reference_spec", + "has_reference", "register_benchmark", + "register_reference", ] @@ -78,6 +100,22 @@ def get_available_benchmark_names() -> list[str]: return sorted(_DISCOVERED_BENCHMARKS | set(benchmark_names())).copy() +def has_reference(benchmark_name: str) -> bool: + """Return ``True`` if a reference spec is available for *benchmark_name*. + + Triggers lazy import of the benchmark module so that the reference registry + is populated before the check. + + Args: + benchmark_name: The benchmark to query. + """ + try: + _ensure_loaded(benchmark_name) + except ValueError: + return False + return _registry_has_reference(benchmark_name) + + @cache def get_benchmark_description(benchmark_name: str) -> str: """Return the benchmark description given a benchmark name.""" @@ -99,6 +137,49 @@ def _get_factory(benchmark_name: str) -> Callable[..., QuantumCircuit]: return get_benchmark_by_name(benchmark_name) +@cache +def _get_reference_factory(benchmark_name: str) -> Callable[..., ReferenceSpec] | None: + """Internal reference factory cache.""" + _ensure_loaded(benchmark_name) + return get_reference_factory_by_name(benchmark_name) + + +# ruff: noqa: ANN401 +def get_reference_spec(benchmark_name: str, circuit_size: int, /, *args: Any, **kwargs: Any) -> ReferenceSpec: + """Return the reference specification for a benchmark instance. + + The spec describes the ideal, noise-free output distribution in a compact + form that stays poly-size even for exponentially large circuits. Call + :meth:`ReferenceSpec.to_dict` on the result to get a JSON-serialisable dict. + + Args: + benchmark_name: The name of the benchmark (must match the registry key). + circuit_size: The number of qubits — same value you would pass to + :func:`create_circuit`. + *args: Forwarded to the benchmark's ``create_reference`` function. + **kwargs: Forwarded to the benchmark's ``create_reference`` function. + + Returns: + ReferenceSpec describing the noise-free output. + + Raises: + ValueError: If the benchmark name is unknown or has no reference registered. + """ + if circuit_size <= 0: + msg = "`circuit_size` must be a positive integer." + raise ValueError(msg) + + factory = _get_reference_factory(benchmark_name) + if factory is None: + msg = ( + f"No reference spec registered for '{benchmark_name}'. " + "Either the benchmark does not yet have a create_reference function, " + "or the relevant module has not been imported." + ) + raise ValueError(msg) + return factory(circuit_size, *args, **kwargs) + + # ruff: noqa: ANN401 def create_circuit(benchmark_name: str, circuit_size: int, /, *args: Any, **kwargs: Any) -> QuantumCircuit: """Creates and returns a quantum circuit based on the specified benchmark name and additional arguments. diff --git a/src/mqt/bench/benchmarks/_reference.py b/src/mqt/bench/benchmarks/_reference.py new file mode 100644 index 000000000..3c666dcba --- /dev/null +++ b/src/mqt/bench/benchmarks/_reference.py @@ -0,0 +1,175 @@ +"""Reference specification types for noise-free benchmark outputs. + +Each benchmark can optionally expose a ``create_reference`` function (registered +via :func:`register_reference`) that returns a :class:`ReferenceSpec` describing +the ideal, noise-free output distribution in a compact, evaluable form. + +The spec is deliberately *poly-size* even for exponentially large Hilbert spaces: +it stores a *description* of the distribution (sparse table, uniform predicate, +analytic form, or semantic objective) rather than a dense state vector. + +Calling :func:`~mqt.bench.benchmarks.get_reference_spec` returns the spec as a +:class:`ReferenceSpec` object; call :meth:`ReferenceSpec.to_dict` to get a +plain ``dict`` that serialises cleanly to JSON. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class SparseReference: + """Explicit probability table for circuits with a small support. + + Suitable for: GHZ, BV, DJ, QPE, Grover (few high-probability states). + """ + + entries: dict[str, float] + normalized: bool = True + + def to_dict(self) -> dict[str, Any]: + return {"kind": "sparse", "normalized": self.normalized, "entries": self.entries} + + +@dataclass +class UniformReference: + """Uniform distribution over a predicate-defined support. + + Stores only the predicate string and support size, so P(x) = 1/size + whenever the predicate holds and 0 otherwise. Evaluable in O(1) per + bitstring without enumerating the full support. + + Suitable for: W state (``hamming_weight == 1``), random Clifford states, etc. + """ + + predicate: str + size: int + + def to_dict(self) -> dict[str, Any]: + return {"kind": "uniform", "predicate": self.predicate, "size": self.size} + + +@dataclass +class SimulateReference: + """Reference to be obtained by classical statevector simulation. + + Used when no compact closed-form is available but simulation is feasible. + ``max_qubits`` is advisory: indicate the largest instance the team has + pre-simulated (or considers tractable). + """ + + max_qubits: int = 30 + + def to_dict(self) -> dict[str, Any]: + return {"kind": "simulate", "max_qubits": self.max_qubits} + + +@dataclass +class NoneReference: + """No reference distribution available. + + Used for variational / parameterised circuits (QAOA, VQE) where the + output depends on optimised angles, or for circuits whose output is + essentially random (random circuits beyond simulation limits). + """ + + reason: str = "" + + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = {"kind": "none"} + if self.reason: + d["reason"] = self.reason + return d + + +ReferenceKind = SparseReference | UniformReference | SimulateReference | NoneReference + + +@dataclass +class ObjectiveSpec: + """Semantic objective — the *answer* the circuit is meant to find. + + Complements the reference distribution: even when the distribution has + many non-zero entries, the objective pins down what "correct" means. + + Examples of ``type`` values and their ``value`` payloads: + + * ``"marked_states"`` – list of target bitstrings (Grover) + * ``"hidden_string"`` – the secret bitstring (BV) + * ``"balanced_or_constant"`` – ``"balanced"`` or ``"constant"`` (DJ) + * ``"phase"`` – estimated phase as a float (QPE) + """ + + type: str + value: Any = None + + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = {"type": self.type} + if self.value is not None: + d["value"] = self.value + return d + + +@dataclass +class MetricApplicability: + """Whether a standard metric applies to this benchmark and its ideal value.""" + + applicable: bool + ideal: float | None = None + + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = {"applicable": self.applicable} + if self.ideal is not None: + d["ideal"] = self.ideal + return d + + +@dataclass +class ReferenceSpec: + """Complete reference specification for one benchmark instance. + + Fields + ------ + circuit: + Benchmark name (matches the registry key, e.g. ``"ghz"``). + n_qubits: + Total number of qubits in the circuit as returned by ``create_circuit``. + measured_qubits: + Indices (in circuit qubit order) of the qubits that contribute to the + classical output string. Ancilla omitted. + bit_order: + ``"qiskit-little-endian"`` + reference: + Compact description of the ideal probability distribution. + objective: + Optional semantic answer. + metrics: + Map from metric name to applicability and ideal value. + Standard keys: ``"hellinger_fidelity"``, ``"tvd"``, + ``"success_probability"``, ``"linear_xeb"``. + """ + + circuit: str + n_qubits: int + measured_qubits: list[int] + bit_order: str + reference: ReferenceKind + objective: ObjectiveSpec | None = None + metrics: dict[str, MetricApplicability] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-serialisable dict representation.""" + d: dict[str, Any] = { + "circuit": self.circuit, + "n_qubits": self.n_qubits, + "measured_qubits": self.measured_qubits, + "bit_order": self.bit_order, + "reference": self.reference.to_dict(), + } + if self.objective is not None: + d["objective"] = self.objective.to_dict() + if self.metrics: + d["metrics"] = {k: v.to_dict() for k, v in self.metrics.items()} + return d diff --git a/src/mqt/bench/benchmarks/_registry.py b/src/mqt/bench/benchmarks/_registry.py index 4c875c129..d858485ef 100644 --- a/src/mqt/bench/benchmarks/_registry.py +++ b/src/mqt/bench/benchmarks/_registry.py @@ -1,11 +1,3 @@ -# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM -# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH -# All rights reserved. -# -# SPDX-License-Identifier: MIT -# -# Licensed under the MIT License - """Benchmark registry.""" from __future__ import annotations @@ -16,10 +8,13 @@ from qiskit.circuit import QuantumCircuit +from ._reference import ReferenceSpec + if TYPE_CHECKING: from collections.abc import Mapping _BenchmarkFactory = Callable[..., QuantumCircuit] +_ReferenceFactory = Callable[..., ReferenceSpec] @dataclass(frozen=True) @@ -31,6 +26,7 @@ class BenchmarkInfo: _REGISTRY: dict[str, BenchmarkInfo] = {} +_REFERENCE_REGISTRY: dict[str, _ReferenceFactory] = {} def register_benchmark(benchmark_name: str, description: str = "") -> Callable[[_BenchmarkFactory], _BenchmarkFactory]: @@ -92,3 +88,38 @@ def benchmark_names() -> list[str]: def benchmark_catalog() -> Mapping[str, str]: """Mapping *name → description* to feed into a CLI help table, GUI, etc.""" return {name: info.description for name, info in _REGISTRY.items()} + + +def register_reference(benchmark_name: str) -> Callable[[_ReferenceFactory], _ReferenceFactory]: + """Decorator to register a reference-spec factory for *benchmark_name*. + + The decorated function must accept the same positional/keyword arguments as + the corresponding ``create_circuit`` factory and return a + :class:`~._reference.ReferenceSpec`. + + Arguments: + benchmark_name: registry key of the benchmark to attach the spec to. + + Returns: + The original function. + """ + + def _decorator(func: _ReferenceFactory) -> _ReferenceFactory: + _REFERENCE_REGISTRY[benchmark_name] = func + return func + + return _decorator + + +def get_reference_factory_by_name(benchmark_name: str) -> _ReferenceFactory | None: + """Return the ``create_reference`` function for *benchmark_name*, or ``None``. + + Arguments: + benchmark_name: identifier used during registration. + """ + return _REFERENCE_REGISTRY.get(benchmark_name) + + +def has_reference(benchmark_name: str) -> bool: + """Return ``True`` if a reference spec is registered for *benchmark_name*.""" + return benchmark_name in _REFERENCE_REGISTRY diff --git a/src/mqt/bench/benchmarks/bv.py b/src/mqt/bench/benchmarks/bv.py index 9766e4657..9b4df23a0 100644 --- a/src/mqt/bench/benchmarks/bv.py +++ b/src/mqt/bench/benchmarks/bv.py @@ -12,7 +12,8 @@ from qiskit.circuit import QuantumCircuit -from ._registry import register_benchmark +from ._reference import MetricApplicability, ObjectiveSpec, ReferenceSpec, SparseReference +from ._registry import register_benchmark, register_reference @register_benchmark("bv", description="Bernstein-Vazirani") @@ -82,3 +83,46 @@ def create_circuit(num_qubits: int, dynamic: bool = False, hidden_string: str | circuit.name = "bv" return circuit + + +@register_reference("bv") +def create_reference(num_qubits: int, dynamic: bool = False, hidden_string: str | None = None) -> ReferenceSpec: + """Reference spec for the Bernstein-Vazirani circuit. + + BV is deterministic: measuring the circuit always recovers the hidden + bitstring with probability 1. + + The hidden string is ``num_qubits - 1`` bits long (qubit 0 is the flag + ancilla). The default hidden string alternates 0 and 1 starting from + qubit 1, matching the ``create_circuit`` default. + + Qiskit bit-string convention: classical bit *i* is the rightmost + character offset by *i*, so the measured string is the hidden string + written in *reverse* (``hidden_string[::-1]``). + + Arguments: + num_qubits: total qubits including the flag (same as :func:`create_circuit`). + dynamic: not used for the reference; kept for API symmetry. + hidden_string: the secret bitstring of length ``num_qubits - 1``. + """ + if hidden_string is None: + hidden_string = "".join([str(i % 2) for i in range(num_qubits - 1)]) + + # Qiskit big-endian + # string is hidden_string written right-to-left. + qiskit_string = hidden_string[::-1] + + return ReferenceSpec( + circuit="bv", + n_qubits=num_qubits, + measured_qubits=list(range(1, num_qubits)), + bit_order="qiskit-little-endian", + reference=SparseReference(entries={qiskit_string: 1.0}), + objective=ObjectiveSpec(type="hidden_string", value=hidden_string), + metrics={ + "hellinger_fidelity": MetricApplicability(applicable=True, ideal=1.0), + "tvd": MetricApplicability(applicable=True, ideal=0.0), + "success_probability": MetricApplicability(applicable=True, ideal=1.0), + "linear_xeb": MetricApplicability(applicable=False), + }, + ) diff --git a/src/mqt/bench/benchmarks/dj.py b/src/mqt/bench/benchmarks/dj.py index 1af3c4bb6..65a5872c5 100644 --- a/src/mqt/bench/benchmarks/dj.py +++ b/src/mqt/bench/benchmarks/dj.py @@ -1,11 +1,3 @@ -# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM -# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH -# All rights reserved. -# -# SPDX-License-Identifier: MIT -# -# Licensed under the MIT License - """Deutsch Josza benchmark definition. Code is based on https://qiskit.org/textbook/ch-algorithms/deutsch-jozsa.html.""" from __future__ import annotations @@ -15,7 +7,8 @@ import numpy as np from qiskit.circuit import QuantumCircuit -from ._registry import register_benchmark +from ._reference import MetricApplicability, ObjectiveSpec, ReferenceSpec, SparseReference +from ._registry import register_benchmark, register_reference if TYPE_CHECKING: from qiskit.circuit.gate import Gate @@ -91,3 +84,54 @@ def create_circuit(num_qubits: int, balanced: bool = True) -> QuantumCircuit: qc.name = "dj" return qc + + +@register_reference("dj") +def create_reference(num_qubits: int, balanced: bool = True) -> ReferenceSpec: + """Reference spec for the Deutsch-Jozsa circuit. + + DJ is deterministic: the measurement reveals whether the oracle is + balanced (any non-zero string is measured with probability 1) or + constant (all-zeros string is measured with probability 1). + + For the balanced oracle the exact output bitstring is determined by the + fixed random seed (``np.random.default_rng(10)``) used in + :func:`create_circuit`, so we can compute it here without running the + circuit. + + Qiskit bit-string convention: classical bit *i* maps to qubit *i*, and + the output string is big-endian over classical bits, so the b_str + produced by the oracle appears reversed in the counts dict. + + Arguments: + num_qubits: total qubits including the ancilla (same as :func:`create_circuit`). + balanced: ``True`` for a balanced oracle, ``False`` for constant. + """ + n = num_qubits - 1 # input qubits (ancilla excluded) + + if balanced: + # Reproduce the same RNG sequence used in dj_oracle to get b_str + rng = np.random.default_rng(10) + b_str = "".join(str(int(rng.integers(0, 2))) for _ in range(n)) + # Qiskit string: c[n-1]...c[0] = b_str[n-1]...b_str[0] = reversed + qiskit_string = b_str[::-1] + kind_value = "balanced" + else: + # Constant oracle: input qubits remain |0...0⟩ after H◦H = I + qiskit_string = "0" * n + kind_value = "constant" + + return ReferenceSpec( + circuit="dj", + n_qubits=num_qubits, + measured_qubits=list(range(n)), + bit_order="qiskit-little-endian", + reference=SparseReference(entries={qiskit_string: 1.0}), + objective=ObjectiveSpec(type="balanced_or_constant", value=kind_value), + metrics={ + "hellinger_fidelity": MetricApplicability(applicable=True, ideal=1.0), + "tvd": MetricApplicability(applicable=True, ideal=0.0), + "success_probability": MetricApplicability(applicable=True, ideal=1.0), + "linear_xeb": MetricApplicability(applicable=False), + }, + ) diff --git a/src/mqt/bench/benchmarks/ghz.py b/src/mqt/bench/benchmarks/ghz.py index 4bcf05852..2e599851e 100644 --- a/src/mqt/bench/benchmarks/ghz.py +++ b/src/mqt/bench/benchmarks/ghz.py @@ -1,18 +1,11 @@ -# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM -# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH -# All rights reserved. -# -# SPDX-License-Identifier: MIT -# -# Licensed under the MIT License - """GHZ benchmark definition.""" from __future__ import annotations from qiskit.circuit import QuantumCircuit, QuantumRegister -from ._registry import register_benchmark +from ._reference import MetricApplicability, ReferenceSpec, SparseReference +from ._registry import register_benchmark, register_reference @register_benchmark("ghz", description="GHZ State") @@ -30,3 +23,28 @@ def create_circuit(num_qubits: int) -> QuantumCircuit: qc.measure_all() return qc + + +@register_reference("ghz") +def create_reference(num_qubits: int) -> ReferenceSpec: + """Reference spec for the GHZ circuit. + + The ideal output is an equal superposition of the all-zeros and all-ones + computational basis states: P(0...0) = P(1...1) = 0.5. + + Arguments: + num_qubits: number of qubits (same as passed to :func:`create_circuit`). + """ + entries = {"0" * num_qubits: 0.5, "1" * num_qubits: 0.5} + return ReferenceSpec( + circuit="ghz", + n_qubits=num_qubits, + measured_qubits=list(range(num_qubits)), + bit_order="qiskit-little-endian", + reference=SparseReference(entries=entries), + metrics={ + "hellinger_fidelity": MetricApplicability(applicable=True), + "tvd": MetricApplicability(applicable=True), + "linear_xeb": MetricApplicability(applicable=False), + }, + ) diff --git a/src/mqt/bench/benchmarks/grover.py b/src/mqt/bench/benchmarks/grover.py index cc09108a4..59db8ff1d 100644 --- a/src/mqt/bench/benchmarks/grover.py +++ b/src/mqt/bench/benchmarks/grover.py @@ -1,11 +1,3 @@ -# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM -# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH -# All rights reserved. -# -# SPDX-License-Identifier: MIT -# -# Licensed under the MIT License - """Grover benchmark definition.""" from __future__ import annotations @@ -14,7 +6,8 @@ from qiskit.circuit import AncillaRegister, QuantumCircuit, QuantumRegister from qiskit.circuit.library import grover_operator -from ._registry import register_benchmark +from ._reference import MetricApplicability, ObjectiveSpec, ReferenceSpec, SparseReference +from ._registry import register_benchmark, register_reference @register_benchmark("grover", description="Grover's Algorithm") @@ -50,3 +43,47 @@ def create_circuit(num_qubits: int) -> QuantumCircuit: qc.name = qc.name return qc + + +@register_reference("grover") +def create_reference(num_qubits: int) -> ReferenceSpec: + """Reference spec for the Grover circuit. + + The oracle ``mcp(π, q, flag)`` marks the all-ones state on the search + register (all ``num_qubits - 1`` computational qubits set to |1⟩). + After the optimal number of Grover iterations the marked state is + amplified to approximate success probability P_success. + + The ``measured_qubits`` field lists only the *search-register* qubit + indices (0..n_search-1). The flag and any ancillas added by + ``grover_operator`` are not included because they carry no search + information. + + Arguments: + num_qubits: total qubits passed to :func:`create_circuit` (search + register = ``num_qubits - 1`` qubits). + """ + n_search = num_qubits - 1 + n_states = 2**n_search + iterations = int(np.pi / 4 * np.sqrt(n_states)) + theta = np.arcsin(1.0 / np.sqrt(n_states)) + p_success = float(np.sin((2 * iterations + 1) * theta) ** 2) + + # Marked state on the search register: all-ones. + # In Qiskit's big-endian counts string this is "1" * n_search. + marked_state = "1" * n_search + + return ReferenceSpec( + circuit="grover", + n_qubits=num_qubits, + measured_qubits=list(range(n_search)), + bit_order="qiskit-little-endian", + reference=SparseReference(entries={marked_state: p_success}), + objective=ObjectiveSpec(type="marked_states", value=[marked_state]), + metrics={ + "hellinger_fidelity": MetricApplicability(applicable=True), + "tvd": MetricApplicability(applicable=True), + "success_probability": MetricApplicability(applicable=True, ideal=p_success), + "linear_xeb": MetricApplicability(applicable=False), + }, + ) diff --git a/src/mqt/bench/benchmarks/qpeexact.py b/src/mqt/bench/benchmarks/qpeexact.py index 260279f23..225ee0be2 100644 --- a/src/mqt/bench/benchmarks/qpeexact.py +++ b/src/mqt/bench/benchmarks/qpeexact.py @@ -17,7 +17,8 @@ from qiskit.circuit import ClassicalRegister, QuantumCircuit, QuantumRegister from qiskit.synthesis import synth_qft_full -from ._registry import register_benchmark +from ._reference import MetricApplicability, ObjectiveSpec, ReferenceSpec, SparseReference +from ._registry import register_benchmark, register_reference @register_benchmark("qpeexact", description="Quantum Phase Estimation (QPE) exactly representable phase") @@ -70,3 +71,64 @@ def create_circuit(num_qubits: int) -> QuantumCircuit: qc.measure(q, c) return qc + + +@register_reference("qpeexact") +def create_reference(num_qubits: int) -> ReferenceSpec: + """Reference spec for the QPE-exact circuit. + + The circuit estimates a phase that is exactly representable with + ``num_qubits - 1`` estimation bits, so the measurement is deterministic: + a single bitstring is observed with probability 1. + + The target phase ``theta`` is derived from the same fixed seed + (``random.seed(10)``) used in :func:`create_circuit`, so we can + reproduce it here without running the circuit. + + In Qiskit's QPE layout the estimation register ``q[0..n-1]`` is + measured into classical register ``c[0..n-1]``. The output string + is the binary representation of ``theta`` with ``n`` bits, MSB first + (big-endian over classical bits) — i.e. ``format(theta, '0{n}b')``. + + Arguments: + num_qubits: total qubits including the eigenstate qubit + (same as passed to :func:`create_circuit`). + """ + if num_qubits <= 1: + msg = "Number of qubits must be at least 2 for QPE exact." + raise ValueError(msg) + + n = num_qubits - 1 # estimation qubits (mirrors create_circuit's first line) + + # Reproduce the same seed-derived theta used in create_circuit + random.seed(10) + theta = 0 + while theta == 0: + theta = random.getrandbits(n) + + # Derive the phase fraction lambda (same computation as create_circuit) + lam = Fraction(0, 1) + for i in range(n): + if theta & (1 << (n - i - 1)): + lam += Fraction(1, (1 << i)) + phase = float(lam) + + # The measurement collapses to the binary representation of theta with n digits. + # Qiskit classical string is big-endian: c[n-1] (MSB) ... c[0] (LSB). + # c[i] = bit i of theta counted from LSB, so string = format(theta, '0nb') directly. + qiskit_string = format(theta, f"0{n}b") + + return ReferenceSpec( + circuit="qpeexact", + n_qubits=num_qubits, + measured_qubits=list(range(n)), + bit_order="qiskit-little-endian", + reference=SparseReference(entries={qiskit_string: 1.0}), + objective=ObjectiveSpec(type="phase", value=phase), + metrics={ + "hellinger_fidelity": MetricApplicability(applicable=True, ideal=1.0), + "tvd": MetricApplicability(applicable=True, ideal=0.0), + "success_probability": MetricApplicability(applicable=True, ideal=1.0), + "linear_xeb": MetricApplicability(applicable=False), + }, + ) diff --git a/src/mqt/bench/benchmarks/randomcircuit.py b/src/mqt/bench/benchmarks/randomcircuit.py index c4be26bd7..163ac4b0d 100644 --- a/src/mqt/bench/benchmarks/randomcircuit.py +++ b/src/mqt/bench/benchmarks/randomcircuit.py @@ -14,7 +14,8 @@ from qiskit.circuit.random import random_circuit -from ._registry import register_benchmark +from ._reference import ReferenceSpec, SimulateReference +from ._registry import register_benchmark, register_reference if TYPE_CHECKING: from qiskit.circuit import QuantumCircuit @@ -34,3 +35,23 @@ def create_circuit(num_qubits: int) -> QuantumCircuit: qc.measure_all() qc.name = "randomcircuit" return qc + + +@register_reference("randomcircuit") +def create_reference(num_qubits: int) -> ReferenceSpec: + """Reference spec for the random circuit benchmark. + + Random circuits have no compact closed-form output distribution. + The reference kind is ``"simulate"``, indicating that the ideal + distribution must be obtained by classical statevector simulation. + + Arguments: + num_qubits: number of qubits (same as passed to :func:`create_circuit`). + """ + return ReferenceSpec( + circuit="randomcircuit", + n_qubits=num_qubits, + measured_qubits=list(range(num_qubits)), + bit_order="qiskit-little-endian", + reference=SimulateReference(max_qubits=30), + ) diff --git a/src/mqt/bench/benchmarks/wstate.py b/src/mqt/bench/benchmarks/wstate.py index e7ff5c026..488f737db 100644 --- a/src/mqt/bench/benchmarks/wstate.py +++ b/src/mqt/bench/benchmarks/wstate.py @@ -13,7 +13,8 @@ import numpy as np from qiskit.circuit import QuantumCircuit, QuantumRegister -from ._registry import register_benchmark +from ._reference import MetricApplicability, ReferenceSpec, UniformReference +from ._registry import register_benchmark, register_reference @register_benchmark("wstate", description="W-State") @@ -46,3 +47,32 @@ def f_gate(qc: QuantumCircuit, q: QuantumRegister, i: int, j: int, n: int, k: in qc.measure_all() return qc + + +@register_reference("wstate") +def create_reference(num_qubits: int) -> ReferenceSpec: + """Reference spec for the W-state circuit. + + The ideal output is the uniform distribution over all ``num_qubits`` + computational basis states of Hamming weight 1: P(x) = 1/num_qubits for + each bitstring with exactly one '1', and 0 elsewhere. + + The distribution is stored compactly as a ``UniformReference`` so that + P(x) can be evaluated in O(1) per bitstring without enumerating all states. + + Arguments: + num_qubits: number of qubits (same as passed to :func:`create_circuit`). + """ + return ReferenceSpec( + circuit="wstate", + n_qubits=num_qubits, + measured_qubits=list(range(num_qubits)), + bit_order="qiskit-little-endian", + reference=UniformReference(predicate="hamming_weight == 1", size=num_qubits), + metrics={ + "hellinger_fidelity": MetricApplicability(applicable=True), + "tvd": MetricApplicability(applicable=True), + "success_probability": MetricApplicability(applicable=True, ideal=1.0), + "linear_xeb": MetricApplicability(applicable=False), + }, + ) diff --git a/tests/test_reference.py b/tests/test_reference.py new file mode 100644 index 000000000..bd7b008b5 --- /dev/null +++ b/tests/test_reference.py @@ -0,0 +1,364 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +"""Tests for the reference-spec framework (issue #924).""" + +from __future__ import annotations + +import json +import random +from fractions import Fraction + +import numpy as np +import pytest + +from mqt.bench.benchmarks import ( + NoneReference, + ReferenceSpec, + SimulateReference, + SparseReference, + UniformReference, + get_reference_spec, + has_reference, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _prob_sum(spec: ReferenceSpec) -> float: + """Sum of probabilities in a SparseReference (must equal 1 for normalised).""" + assert isinstance(spec.reference, SparseReference) + return sum(spec.reference.entries.values()) + + +# --------------------------------------------------------------------------- +# Framework-level tests +# --------------------------------------------------------------------------- + + +def test_has_reference_known() -> None: + """has_reference returns True for benchmarks that have a create_reference.""" + for name in ("ghz", "wstate", "bv", "dj", "grover", "qpeexact", "randomcircuit"): + assert has_reference(name), f"Expected has_reference('{name}') to be True" + + +def test_has_reference_unknown() -> None: + """has_reference returns False for benchmarks without a create_reference.""" + assert not has_reference("qaoa") + assert not has_reference("vqe_su2") + + +def test_get_reference_spec_raises_for_no_reference() -> None: + """get_reference_spec raises ValueError for benchmarks without a spec.""" + with pytest.raises(ValueError, match="No reference spec registered"): + get_reference_spec("qaoa", 4) + + +def test_get_reference_spec_raises_for_invalid_size() -> None: + """get_reference_spec raises ValueError when circuit_size <= 0.""" + with pytest.raises(ValueError, match="circuit_size"): + get_reference_spec("ghz", 0) + + +def test_reference_spec_to_dict_is_json_serialisable() -> None: + """ReferenceSpec.to_dict() must round-trip through JSON without error.""" + for name, size in [("ghz", 4), ("wstate", 3), ("bv", 5), ("dj", 4), ("randomcircuit", 5)]: + spec = get_reference_spec(name, size) + d = spec.to_dict() + # Must not raise + serialised = json.dumps(d) + recovered = json.loads(serialised) + assert recovered["circuit"] == name + assert recovered["n_qubits"] == size + + +# --------------------------------------------------------------------------- +# GHZ reference +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n", [2, 3, 5, 8]) +def test_ghz_reference_structure(n: int) -> None: + spec = get_reference_spec("ghz", n) + assert spec.circuit == "ghz" + assert spec.n_qubits == n + assert spec.measured_qubits == list(range(n)) + assert isinstance(spec.reference, SparseReference) + assert spec.reference.normalized is True + assert set(spec.reference.entries.keys()) == {"0" * n, "1" * n} + assert abs(_prob_sum(spec) - 1.0) < 1e-12 + for p in spec.reference.entries.values(): + assert abs(p - 0.5) < 1e-12 + + +def test_ghz_reference_no_objective() -> None: + """GHZ has no semantic 'answer' — objective should be None.""" + spec = get_reference_spec("ghz", 3) + assert spec.objective is None + + +# --------------------------------------------------------------------------- +# W state reference +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n", [2, 3, 5, 8]) +def test_wstate_reference_structure(n: int) -> None: + spec = get_reference_spec("wstate", n) + assert spec.circuit == "wstate" + assert spec.n_qubits == n + assert isinstance(spec.reference, UniformReference) + assert spec.reference.predicate == "hamming_weight == 1" + assert spec.reference.size == n + + +def test_wstate_reference_probability(n: int = 5) -> None: + """P(x) = 1/n for any weight-1 bitstring.""" + spec = get_reference_spec("wstate", n) + assert isinstance(spec.reference, UniformReference) + p = 1.0 / spec.reference.size + assert abs(p - 1.0 / n) < 1e-12 + + +def test_wstate_reference_success_metric() -> None: + spec = get_reference_spec("wstate", 4) + assert "success_probability" in spec.metrics + assert spec.metrics["success_probability"].ideal == 1.0 + + +# --------------------------------------------------------------------------- +# Bernstein-Vazirani reference +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n", [3, 5, 7]) +def test_bv_reference_structure(n: int) -> None: + spec = get_reference_spec("bv", n) + assert spec.circuit == "bv" + assert spec.n_qubits == n + assert isinstance(spec.reference, SparseReference) + assert len(spec.reference.entries) == 1 + assert abs(_prob_sum(spec) - 1.0) < 1e-12 + + +@pytest.mark.parametrize("n", [3, 5, 7]) +def test_bv_reference_default_hidden_string(n: int) -> None: + """Default hidden string is alternating 0/1 of length n-1.""" + expected_hidden = "".join(str(i % 2) for i in range(n - 1)) + spec = get_reference_spec("bv", n) + assert spec.objective is not None + assert spec.objective.type == "hidden_string" + assert spec.objective.value == expected_hidden + # Qiskit string is reversed hidden string + (qiskit_string,) = spec.reference.entries.keys() # type: ignore[misc] + assert qiskit_string == expected_hidden[::-1] + + +def test_bv_reference_custom_hidden_string() -> None: + """Custom hidden strings are handled correctly.""" + spec = get_reference_spec("bv", 4, hidden_string="110") + assert spec.objective is not None + assert spec.objective.value == "110" + (qiskit_string,) = spec.reference.entries.keys() # type: ignore[misc] + assert qiskit_string == "011" # reversed + + +# --------------------------------------------------------------------------- +# Deutsch-Jozsa reference +# --------------------------------------------------------------------------- + + +def test_dj_balanced_reference() -> None: + """Balanced DJ: single non-zero measurement string derived from fixed seed.""" + n_total = 4 + n_input = n_total - 1 + + spec = get_reference_spec("dj", n_total, balanced=True) + assert isinstance(spec.reference, SparseReference) + assert len(spec.reference.entries) == 1 + assert abs(_prob_sum(spec) - 1.0) < 1e-12 + + # Reproduce b_str with the same RNG to cross-check + rng = np.random.default_rng(10) + b_str = "".join(str(int(rng.integers(0, 2))) for _ in range(n_input)) + expected_qiskit = b_str[::-1] + + (measured,) = spec.reference.entries.keys() # type: ignore[misc] + assert measured == expected_qiskit, f"Expected '{expected_qiskit}', got '{measured}'" + + assert spec.objective is not None + assert spec.objective.value == "balanced" + + +def test_dj_constant_reference() -> None: + """Constant DJ: measurement is always the all-zeros string.""" + n_total = 4 + n_input = n_total - 1 + + spec = get_reference_spec("dj", n_total, balanced=False) + assert isinstance(spec.reference, SparseReference) + (measured,) = spec.reference.entries.keys() # type: ignore[misc] + assert measured == "0" * n_input + assert spec.objective is not None + assert spec.objective.value == "constant" + + +def test_dj_balanced_measured_qubits_exclude_ancilla() -> None: + """The flag ancilla (qubit index n-1 in the full circuit) is not in measured_qubits.""" + n_total = 5 + spec = get_reference_spec("dj", n_total, balanced=True) + assert spec.measured_qubits == list(range(n_total - 1)) + + +# --------------------------------------------------------------------------- +# Grover reference +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n", [3, 5, 7]) +def test_grover_reference_structure(n: int) -> None: + spec = get_reference_spec("grover", n) + assert spec.circuit == "grover" + assert spec.n_qubits == n + assert isinstance(spec.reference, SparseReference) + assert len(spec.reference.entries) == 1 + + +@pytest.mark.parametrize("n", [3, 5, 7]) +def test_grover_marked_state_is_all_ones(n: int) -> None: + """Grover oracle marks the all-ones state on the search register.""" + spec = get_reference_spec("grover", n) + n_search = n - 1 + (marked,) = spec.reference.entries.keys() # type: ignore[misc] + assert marked == "1" * n_search + assert spec.objective is not None + assert spec.objective.type == "marked_states" + assert spec.objective.value == ["1" * n_search] + + +@pytest.mark.parametrize("n", [3, 5, 7]) +def test_grover_success_probability_in_range(n: int) -> None: + """Success probability must be in (0, 1].""" + spec = get_reference_spec("grover", n) + (p,) = spec.reference.entries.values() + assert 0.0 < p <= 1.0 + + +@pytest.mark.parametrize("n", [3, 5, 7]) +def test_grover_measured_qubits_exclude_flag(n: int) -> None: + """Only the search-register qubits (0..n_search-1) are listed.""" + spec = get_reference_spec("grover", n) + assert spec.measured_qubits == list(range(n - 1)) + + +# --------------------------------------------------------------------------- +# QPE-exact reference +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n", [2, 3, 5]) +def test_qpeexact_reference_structure(n: int) -> None: + spec = get_reference_spec("qpeexact", n) + assert spec.circuit == "qpeexact" + assert spec.n_qubits == n + assert isinstance(spec.reference, SparseReference) + assert len(spec.reference.entries) == 1 + (p,) = spec.reference.entries.values() + assert abs(p - 1.0) < 1e-12 + + +@pytest.mark.parametrize("n", [2, 3, 5]) +def test_qpeexact_reference_bitstring_matches_seed(n: int) -> None: + """The output bitstring must match the theta derived from random.seed(10).""" + n_est = n - 1 # estimation qubits + + random.seed(10) + theta = 0 + while theta == 0: + theta = random.getrandbits(n_est) + + lam = Fraction(0, 1) + for i in range(n_est): + if theta & (1 << (n_est - i - 1)): + lam += Fraction(1, (1 << i)) + + expected_string = format(theta, f"0{n_est}b") + + spec = get_reference_spec("qpeexact", n) + (measured,) = spec.reference.entries.keys() # type: ignore[misc] + assert measured == expected_string + + assert spec.objective is not None + assert spec.objective.type == "phase" + assert abs(spec.objective.value - float(lam)) < 1e-12 + + +def test_qpeexact_reference_raises_for_one_qubit() -> None: + with pytest.raises(ValueError, match="at least 2"): + get_reference_spec("qpeexact", 1) + + +# --------------------------------------------------------------------------- +# Random circuit reference +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("n", [3, 5]) +def test_randomcircuit_reference_kind(n: int) -> None: + spec = get_reference_spec("randomcircuit", n) + assert spec.circuit == "randomcircuit" + assert isinstance(spec.reference, SimulateReference) + assert spec.reference.max_qubits >= n + assert spec.objective is None + + +# --------------------------------------------------------------------------- +# to_dict contract +# --------------------------------------------------------------------------- + + +def test_to_dict_sparse() -> None: + spec = get_reference_spec("ghz", 2) + d = spec.to_dict() + assert d["reference"]["kind"] == "sparse" + assert "00" in d["reference"]["entries"] + assert "11" in d["reference"]["entries"] + + +def test_to_dict_uniform() -> None: + spec = get_reference_spec("wstate", 3) + d = spec.to_dict() + assert d["reference"]["kind"] == "uniform" + assert d["reference"]["size"] == 3 + + +def test_to_dict_simulate() -> None: + spec = get_reference_spec("randomcircuit", 4) + d = spec.to_dict() + assert d["reference"]["kind"] == "simulate" + + +def test_to_dict_none_reference() -> None: + """NoneReference serialises correctly.""" + ref = NoneReference(reason="no compact form") + assert ref.to_dict() == {"kind": "none", "reason": "no compact form"} + + ref_no_reason = NoneReference() + assert ref_no_reason.to_dict() == {"kind": "none"} + + +def test_to_dict_metrics_present() -> None: + """Metrics appear in the dict when the spec defines them.""" + spec = get_reference_spec("bv", 5) + d = spec.to_dict() + assert "metrics" in d + assert d["metrics"]["success_probability"]["applicable"] is True + assert d["metrics"]["success_probability"]["ideal"] == 1.0 From f63b6208b86191cf3d256ddd1c439457b35997c0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 26 Jun 2026 04:09:49 +0000 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=8E=A8=20pre-commit=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- memory/project_issue924_reference_spec.md | 3 +++ src/mqt/bench/benchmarks/__init__.py | 4 +++- src/mqt/bench/benchmarks/_reference.py | 10 +++++++++- src/mqt/bench/benchmarks/_registry.py | 8 ++++++++ src/mqt/bench/benchmarks/dj.py | 8 ++++++++ src/mqt/bench/benchmarks/ghz.py | 8 ++++++++ src/mqt/bench/benchmarks/grover.py | 8 ++++++++ tests/test_reference.py | 1 - 8 files changed, 47 insertions(+), 3 deletions(-) diff --git a/memory/project_issue924_reference_spec.md b/memory/project_issue924_reference_spec.md index efa36c31b..334b56c4c 100644 --- a/memory/project_issue924_reference_spec.md +++ b/memory/project_issue924_reference_spec.md @@ -10,6 +10,7 @@ Maria is contributing to munich-quantum-toolkit/bench issue #924, which adds exp **Why:** Without knowing how the circuit generator constructs the oracle (e.g., Grover), users have no ground-truth to test against. **What was built:** + - `_reference.py` — type hierarchy: `SparseReference`, `UniformReference`, `SimulateReference`, `NoneReference`, `ObjectiveSpec`, `MetricApplicability`, `ReferenceSpec` with `to_dict()` for JSON serialisation - `_registry.py` — parallel reference registry with `register_reference` decorator and `get_reference_factory_by_name` - `__init__.py` — public `get_reference_spec(name, size, **kwargs)` and lazy-load-aware `has_reference(name)` @@ -17,6 +18,7 @@ Maria is contributing to munich-quantum-toolkit/bench issue #924, which adds exp - `tests/test_reference.py` — 52 tests covering all kinds, JSON round-trip, and metric fields **Architecture decisions:** + - `create_reference` mirrors `create_circuit` in each benchmark file, registered via `@register_reference` - Spec is always poly-size: sparse table, uniform predicate, or "simulate" - `bit_order: "qiskit-little-endian"` — only Qiskit is used in this repo (no tket/cirq) @@ -25,6 +27,7 @@ Maria is contributing to munich-quantum-toolkit/bench issue #924, which adds exp - Qiskit string convention: reversed hidden_string for BV, reversed b_str for DJ **What's left for maintainers:** + - Add `create_reference` to the remaining ~25 benchmarks (QAOA, VQE, QFT, adders, AE, etc.) - QAOA/VQE probably get `NoneReference` (outputs depend on optimised angles) - Arithmetic circuits could get `SparseReference` if the maintainers know the inputs diff --git a/src/mqt/bench/benchmarks/__init__.py b/src/mqt/bench/benchmarks/__init__.py index bb34d26e5..c97bb52c2 100644 --- a/src/mqt/bench/benchmarks/__init__.py +++ b/src/mqt/bench/benchmarks/__init__.py @@ -30,10 +30,12 @@ benchmark_names, get_benchmark_by_name, get_reference_factory_by_name, - has_reference as _registry_has_reference, register_benchmark, register_reference, ) +from ._registry import ( + has_reference as _registry_has_reference, +) if TYPE_CHECKING: from collections.abc import Callable, Mapping diff --git a/src/mqt/bench/benchmarks/_reference.py b/src/mqt/bench/benchmarks/_reference.py index 3c666dcba..35939165b 100644 --- a/src/mqt/bench/benchmarks/_reference.py +++ b/src/mqt/bench/benchmarks/_reference.py @@ -1,3 +1,11 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + """Reference specification types for noise-free benchmark outputs. Each benchmark can optionally expose a ``create_reference`` function (registered @@ -140,7 +148,7 @@ class ReferenceSpec: Indices (in circuit qubit order) of the qubits that contribute to the classical output string. Ancilla omitted. bit_order: - ``"qiskit-little-endian"`` + ``"qiskit-little-endian"`` reference: Compact description of the ideal probability distribution. objective: diff --git a/src/mqt/bench/benchmarks/_registry.py b/src/mqt/bench/benchmarks/_registry.py index d858485ef..f2aeabb2f 100644 --- a/src/mqt/bench/benchmarks/_registry.py +++ b/src/mqt/bench/benchmarks/_registry.py @@ -1,3 +1,11 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + """Benchmark registry.""" from __future__ import annotations diff --git a/src/mqt/bench/benchmarks/dj.py b/src/mqt/bench/benchmarks/dj.py index 65a5872c5..901b56b56 100644 --- a/src/mqt/bench/benchmarks/dj.py +++ b/src/mqt/bench/benchmarks/dj.py @@ -1,3 +1,11 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + """Deutsch Josza benchmark definition. Code is based on https://qiskit.org/textbook/ch-algorithms/deutsch-jozsa.html.""" from __future__ import annotations diff --git a/src/mqt/bench/benchmarks/ghz.py b/src/mqt/bench/benchmarks/ghz.py index 2e599851e..82b99b952 100644 --- a/src/mqt/bench/benchmarks/ghz.py +++ b/src/mqt/bench/benchmarks/ghz.py @@ -1,3 +1,11 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + """GHZ benchmark definition.""" from __future__ import annotations diff --git a/src/mqt/bench/benchmarks/grover.py b/src/mqt/bench/benchmarks/grover.py index 59db8ff1d..35a0eef45 100644 --- a/src/mqt/bench/benchmarks/grover.py +++ b/src/mqt/bench/benchmarks/grover.py @@ -1,3 +1,11 @@ +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + """Grover benchmark definition.""" from __future__ import annotations diff --git a/tests/test_reference.py b/tests/test_reference.py index bd7b008b5..2a334310a 100644 --- a/tests/test_reference.py +++ b/tests/test_reference.py @@ -27,7 +27,6 @@ has_reference, ) - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- From eeb55f46c0e92fba6e88645a48c646dc0ba03055 Mon Sep 17 00:00:00 2001 From: grageragarces Date: Fri, 26 Jun 2026 16:03:16 +0800 Subject: [PATCH 3/4] Fix ty type errors and improve test coverage in reference spec - Annotated GHZ entries dict as dict[str, float] to stop ty inferring LiteralString keys (invariant generic mismatch) - Replaced # type: ignore[misc] workarounds with proper isinstance narrowing guards before all .entries accesses in tests - Added test_has_reference_nonexistent_benchmark to cover the except ValueError branch in has_reference, closing the 2-line coverage gap reported by Codecov --- src/mqt/bench/benchmarks/ghz.py | 2 +- tests/test_reference.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/mqt/bench/benchmarks/ghz.py b/src/mqt/bench/benchmarks/ghz.py index 82b99b952..9dbf65729 100644 --- a/src/mqt/bench/benchmarks/ghz.py +++ b/src/mqt/bench/benchmarks/ghz.py @@ -43,7 +43,7 @@ def create_reference(num_qubits: int) -> ReferenceSpec: Arguments: num_qubits: number of qubits (same as passed to :func:`create_circuit`). """ - entries = {"0" * num_qubits: 0.5, "1" * num_qubits: 0.5} + entries: dict[str, float] = {"0" * num_qubits: 0.5, "1" * num_qubits: 0.5} return ReferenceSpec( circuit="ghz", n_qubits=num_qubits, diff --git a/tests/test_reference.py b/tests/test_reference.py index 2a334310a..2943f9e7d 100644 --- a/tests/test_reference.py +++ b/tests/test_reference.py @@ -55,6 +55,11 @@ def test_has_reference_unknown() -> None: assert not has_reference("vqe_su2") +def test_has_reference_nonexistent_benchmark() -> None: + """has_reference returns False (not raises) for a name that is not a benchmark at all.""" + assert not has_reference("not_a_real_benchmark_name") + + def test_get_reference_spec_raises_for_no_reference() -> None: """get_reference_spec raises ValueError for benchmarks without a spec.""" with pytest.raises(ValueError, match="No reference spec registered"): @@ -157,7 +162,8 @@ def test_bv_reference_default_hidden_string(n: int) -> None: assert spec.objective.type == "hidden_string" assert spec.objective.value == expected_hidden # Qiskit string is reversed hidden string - (qiskit_string,) = spec.reference.entries.keys() # type: ignore[misc] + assert isinstance(spec.reference, SparseReference) + (qiskit_string,) = spec.reference.entries.keys() assert qiskit_string == expected_hidden[::-1] @@ -166,7 +172,8 @@ def test_bv_reference_custom_hidden_string() -> None: spec = get_reference_spec("bv", 4, hidden_string="110") assert spec.objective is not None assert spec.objective.value == "110" - (qiskit_string,) = spec.reference.entries.keys() # type: ignore[misc] + assert isinstance(spec.reference, SparseReference) + (qiskit_string,) = spec.reference.entries.keys() assert qiskit_string == "011" # reversed @@ -236,7 +243,8 @@ def test_grover_marked_state_is_all_ones(n: int) -> None: """Grover oracle marks the all-ones state on the search register.""" spec = get_reference_spec("grover", n) n_search = n - 1 - (marked,) = spec.reference.entries.keys() # type: ignore[misc] + assert isinstance(spec.reference, SparseReference) + (marked,) = spec.reference.entries.keys() assert marked == "1" * n_search assert spec.objective is not None assert spec.objective.type == "marked_states" @@ -247,6 +255,7 @@ def test_grover_marked_state_is_all_ones(n: int) -> None: def test_grover_success_probability_in_range(n: int) -> None: """Success probability must be in (0, 1].""" spec = get_reference_spec("grover", n) + assert isinstance(spec.reference, SparseReference) (p,) = spec.reference.entries.values() assert 0.0 < p <= 1.0 @@ -292,7 +301,8 @@ def test_qpeexact_reference_bitstring_matches_seed(n: int) -> None: expected_string = format(theta, f"0{n_est}b") spec = get_reference_spec("qpeexact", n) - (measured,) = spec.reference.entries.keys() # type: ignore[misc] + assert isinstance(spec.reference, SparseReference) + (measured,) = spec.reference.entries.keys() assert measured == expected_string assert spec.objective is not None From 437997f3e0cccc2d5bf6799a6af459ad0e3b5345 Mon Sep 17 00:00:00 2001 From: grageragarces Date: Mon, 29 Jun 2026 19:41:28 +0800 Subject: [PATCH 4/4] Fix ruff lint errors and improve test coverage - Replace en dashes with hyphens in ObjectiveSpec docstring (RUF002) - Rename `dynamic` to `_dynamic` in bv.create_reference to mark it as intentionally unused (ARG001) - Add missing docstrings to all public test functions (D103) - Remove default argument from test_wstate_reference_probability (PT028) - Replace float == comparisons with math.isclose / abs checks (RUF069) - Add isinstance narrowing guards before .entries accesses so ty can narrow the ReferenceKind union (replaces # type: ignore[misc]) - Add test_has_reference_nonexistent_benchmark to cover the except ValueError branch in has_reference (closes Codecov gap) --- src/mqt/bench/benchmarks/_reference.py | 8 ++-- src/mqt/bench/benchmarks/bv.py | 4 +- tests/test_reference.py | 52 ++++++++++++++++---------- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/mqt/bench/benchmarks/_reference.py b/src/mqt/bench/benchmarks/_reference.py index 35939165b..d791cf246 100644 --- a/src/mqt/bench/benchmarks/_reference.py +++ b/src/mqt/bench/benchmarks/_reference.py @@ -104,10 +104,10 @@ class ObjectiveSpec: Examples of ``type`` values and their ``value`` payloads: - * ``"marked_states"`` – list of target bitstrings (Grover) - * ``"hidden_string"`` – the secret bitstring (BV) - * ``"balanced_or_constant"`` – ``"balanced"`` or ``"constant"`` (DJ) - * ``"phase"`` – estimated phase as a float (QPE) + * ``"marked_states"`` - list of target bitstrings (Grover) + * ``"hidden_string"`` - the secret bitstring (BV) + * ``"balanced_or_constant"`` - ``"balanced"`` or ``"constant"`` (DJ) + * ``"phase"`` - estimated phase as a float (QPE) """ type: str diff --git a/src/mqt/bench/benchmarks/bv.py b/src/mqt/bench/benchmarks/bv.py index 9b4df23a0..be7979110 100644 --- a/src/mqt/bench/benchmarks/bv.py +++ b/src/mqt/bench/benchmarks/bv.py @@ -86,7 +86,7 @@ def create_circuit(num_qubits: int, dynamic: bool = False, hidden_string: str | @register_reference("bv") -def create_reference(num_qubits: int, dynamic: bool = False, hidden_string: str | None = None) -> ReferenceSpec: +def create_reference(num_qubits: int, _dynamic: bool = False, hidden_string: str | None = None) -> ReferenceSpec: """Reference spec for the Bernstein-Vazirani circuit. BV is deterministic: measuring the circuit always recovers the hidden @@ -102,7 +102,7 @@ def create_reference(num_qubits: int, dynamic: bool = False, hidden_string: str Arguments: num_qubits: total qubits including the flag (same as :func:`create_circuit`). - dynamic: not used for the reference; kept for API symmetry. + _dynamic: not used for the reference; kept for API symmetry. hidden_string: the secret bitstring of length ``num_qubits - 1``. """ if hidden_string is None: diff --git a/tests/test_reference.py b/tests/test_reference.py index 2943f9e7d..87c47a00c 100644 --- a/tests/test_reference.py +++ b/tests/test_reference.py @@ -11,6 +11,7 @@ from __future__ import annotations import json +import math import random from fractions import Fraction @@ -33,7 +34,7 @@ def _prob_sum(spec: ReferenceSpec) -> float: - """Sum of probabilities in a SparseReference (must equal 1 for normalised).""" + """Return the sum of probabilities stored in a SparseReference.""" assert isinstance(spec.reference, SparseReference) return sum(spec.reference.entries.values()) @@ -50,7 +51,7 @@ def test_has_reference_known() -> None: def test_has_reference_unknown() -> None: - """has_reference returns False for benchmarks without a create_reference.""" + """has_reference returns False for benchmarks that exist but lack a create_reference.""" assert not has_reference("qaoa") assert not has_reference("vqe_su2") @@ -77,7 +78,6 @@ def test_reference_spec_to_dict_is_json_serialisable() -> None: for name, size in [("ghz", 4), ("wstate", 3), ("bv", 5), ("dj", 4), ("randomcircuit", 5)]: spec = get_reference_spec(name, size) d = spec.to_dict() - # Must not raise serialised = json.dumps(d) recovered = json.loads(serialised) assert recovered["circuit"] == name @@ -91,6 +91,7 @@ def test_reference_spec_to_dict_is_json_serialisable() -> None: @pytest.mark.parametrize("n", [2, 3, 5, 8]) def test_ghz_reference_structure(n: int) -> None: + """GHZ reference is a sparse 50/50 distribution over all-zeros and all-ones.""" spec = get_reference_spec("ghz", n) assert spec.circuit == "ghz" assert spec.n_qubits == n @@ -104,7 +105,7 @@ def test_ghz_reference_structure(n: int) -> None: def test_ghz_reference_no_objective() -> None: - """GHZ has no semantic 'answer' — objective should be None.""" + """GHZ has no semantic answer — objective should be None.""" spec = get_reference_spec("ghz", 3) assert spec.objective is None @@ -116,6 +117,7 @@ def test_ghz_reference_no_objective() -> None: @pytest.mark.parametrize("n", [2, 3, 5, 8]) def test_wstate_reference_structure(n: int) -> None: + """W-state reference is a uniform distribution over weight-1 bitstrings.""" spec = get_reference_spec("wstate", n) assert spec.circuit == "wstate" assert spec.n_qubits == n @@ -124,8 +126,9 @@ def test_wstate_reference_structure(n: int) -> None: assert spec.reference.size == n -def test_wstate_reference_probability(n: int = 5) -> None: +def test_wstate_reference_probability() -> None: """P(x) = 1/n for any weight-1 bitstring.""" + n = 5 spec = get_reference_spec("wstate", n) assert isinstance(spec.reference, UniformReference) p = 1.0 / spec.reference.size @@ -133,9 +136,12 @@ def test_wstate_reference_probability(n: int = 5) -> None: def test_wstate_reference_success_metric() -> None: + """W-state success_probability metric should have ideal value 1.""" spec = get_reference_spec("wstate", 4) assert "success_probability" in spec.metrics - assert spec.metrics["success_probability"].ideal == 1.0 + ideal = spec.metrics["success_probability"].ideal + assert ideal is not None + assert math.isclose(ideal, 1.0) # --------------------------------------------------------------------------- @@ -145,6 +151,7 @@ def test_wstate_reference_success_metric() -> None: @pytest.mark.parametrize("n", [3, 5, 7]) def test_bv_reference_structure(n: int) -> None: + """BV reference is a single-entry sparse distribution with probability 1.""" spec = get_reference_spec("bv", n) assert spec.circuit == "bv" assert spec.n_qubits == n @@ -155,26 +162,25 @@ def test_bv_reference_structure(n: int) -> None: @pytest.mark.parametrize("n", [3, 5, 7]) def test_bv_reference_default_hidden_string(n: int) -> None: - """Default hidden string is alternating 0/1 of length n-1.""" + """Default hidden string is alternating 0/1 of length n-1; Qiskit string is its reverse.""" expected_hidden = "".join(str(i % 2) for i in range(n - 1)) spec = get_reference_spec("bv", n) assert spec.objective is not None assert spec.objective.type == "hidden_string" assert spec.objective.value == expected_hidden - # Qiskit string is reversed hidden string assert isinstance(spec.reference, SparseReference) (qiskit_string,) = spec.reference.entries.keys() assert qiskit_string == expected_hidden[::-1] def test_bv_reference_custom_hidden_string() -> None: - """Custom hidden strings are handled correctly.""" + """Custom hidden strings are handled correctly; Qiskit string is reversed.""" spec = get_reference_spec("bv", 4, hidden_string="110") assert spec.objective is not None assert spec.objective.value == "110" assert isinstance(spec.reference, SparseReference) (qiskit_string,) = spec.reference.entries.keys() - assert qiskit_string == "011" # reversed + assert qiskit_string == "011" # --------------------------------------------------------------------------- @@ -192,12 +198,11 @@ def test_dj_balanced_reference() -> None: assert len(spec.reference.entries) == 1 assert abs(_prob_sum(spec) - 1.0) < 1e-12 - # Reproduce b_str with the same RNG to cross-check rng = np.random.default_rng(10) b_str = "".join(str(int(rng.integers(0, 2))) for _ in range(n_input)) expected_qiskit = b_str[::-1] - (measured,) = spec.reference.entries.keys() # type: ignore[misc] + (measured,) = spec.reference.entries.keys() assert measured == expected_qiskit, f"Expected '{expected_qiskit}', got '{measured}'" assert spec.objective is not None @@ -211,14 +216,14 @@ def test_dj_constant_reference() -> None: spec = get_reference_spec("dj", n_total, balanced=False) assert isinstance(spec.reference, SparseReference) - (measured,) = spec.reference.entries.keys() # type: ignore[misc] + (measured,) = spec.reference.entries.keys() assert measured == "0" * n_input assert spec.objective is not None assert spec.objective.value == "constant" def test_dj_balanced_measured_qubits_exclude_ancilla() -> None: - """The flag ancilla (qubit index n-1 in the full circuit) is not in measured_qubits.""" + """The flag ancilla is not included in measured_qubits.""" n_total = 5 spec = get_reference_spec("dj", n_total, balanced=True) assert spec.measured_qubits == list(range(n_total - 1)) @@ -231,6 +236,7 @@ def test_dj_balanced_measured_qubits_exclude_ancilla() -> None: @pytest.mark.parametrize("n", [3, 5, 7]) def test_grover_reference_structure(n: int) -> None: + """Grover reference is a single-entry sparse distribution for the marked state.""" spec = get_reference_spec("grover", n) assert spec.circuit == "grover" assert spec.n_qubits == n @@ -253,7 +259,7 @@ def test_grover_marked_state_is_all_ones(n: int) -> None: @pytest.mark.parametrize("n", [3, 5, 7]) def test_grover_success_probability_in_range(n: int) -> None: - """Success probability must be in (0, 1].""" + """Grover success probability must be in (0, 1].""" spec = get_reference_spec("grover", n) assert isinstance(spec.reference, SparseReference) (p,) = spec.reference.entries.values() @@ -262,7 +268,7 @@ def test_grover_success_probability_in_range(n: int) -> None: @pytest.mark.parametrize("n", [3, 5, 7]) def test_grover_measured_qubits_exclude_flag(n: int) -> None: - """Only the search-register qubits (0..n_search-1) are listed.""" + """Only the search-register qubits (0..n_search-1) are listed in measured_qubits.""" spec = get_reference_spec("grover", n) assert spec.measured_qubits == list(range(n - 1)) @@ -274,6 +280,7 @@ def test_grover_measured_qubits_exclude_flag(n: int) -> None: @pytest.mark.parametrize("n", [2, 3, 5]) def test_qpeexact_reference_structure(n: int) -> None: + """QPE-exact reference is a single-entry sparse distribution with probability 1.""" spec = get_reference_spec("qpeexact", n) assert spec.circuit == "qpeexact" assert spec.n_qubits == n @@ -285,8 +292,8 @@ def test_qpeexact_reference_structure(n: int) -> None: @pytest.mark.parametrize("n", [2, 3, 5]) def test_qpeexact_reference_bitstring_matches_seed(n: int) -> None: - """The output bitstring must match the theta derived from random.seed(10).""" - n_est = n - 1 # estimation qubits + """The output bitstring must match theta derived from random.seed(10).""" + n_est = n - 1 random.seed(10) theta = 0 @@ -311,6 +318,7 @@ def test_qpeexact_reference_bitstring_matches_seed(n: int) -> None: def test_qpeexact_reference_raises_for_one_qubit() -> None: + """QPE-exact requires at least 2 qubits; fewer should raise.""" with pytest.raises(ValueError, match="at least 2"): get_reference_spec("qpeexact", 1) @@ -322,6 +330,7 @@ def test_qpeexact_reference_raises_for_one_qubit() -> None: @pytest.mark.parametrize("n", [3, 5]) def test_randomcircuit_reference_kind(n: int) -> None: + """Random circuit reference has kind=simulate and no objective.""" spec = get_reference_spec("randomcircuit", n) assert spec.circuit == "randomcircuit" assert isinstance(spec.reference, SimulateReference) @@ -335,6 +344,7 @@ def test_randomcircuit_reference_kind(n: int) -> None: def test_to_dict_sparse() -> None: + """SparseReference serialises with kind=sparse and an entries dict.""" spec = get_reference_spec("ghz", 2) d = spec.to_dict() assert d["reference"]["kind"] == "sparse" @@ -343,6 +353,7 @@ def test_to_dict_sparse() -> None: def test_to_dict_uniform() -> None: + """UniformReference serialises with kind=uniform and a size field.""" spec = get_reference_spec("wstate", 3) d = spec.to_dict() assert d["reference"]["kind"] == "uniform" @@ -350,13 +361,14 @@ def test_to_dict_uniform() -> None: def test_to_dict_simulate() -> None: + """SimulateReference serialises with kind=simulate.""" spec = get_reference_spec("randomcircuit", 4) d = spec.to_dict() assert d["reference"]["kind"] == "simulate" def test_to_dict_none_reference() -> None: - """NoneReference serialises correctly.""" + """NoneReference serialises correctly with and without a reason string.""" ref = NoneReference(reason="no compact form") assert ref.to_dict() == {"kind": "none", "reason": "no compact form"} @@ -370,4 +382,4 @@ def test_to_dict_metrics_present() -> None: d = spec.to_dict() assert "metrics" in d assert d["metrics"]["success_probability"]["applicable"] is True - assert d["metrics"]["success_probability"]["ideal"] == 1.0 + assert math.isclose(d["metrics"]["success_probability"]["ideal"], 1.0)