From 21c414c4f9ec392aaa0895bbf37738d5d91683ee Mon Sep 17 00:00:00 2001 From: flowerthrower Date: Thu, 27 Aug 2026 10:02:59 +0200 Subject: [PATCH 1/4] =?UTF-8?q?=E2=9C=A8=20Add=20configurable=20RL=20pass?= =?UTF-8?q?=20timeouts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: GPT-5.6 via Codex Signed-off-by: flowerthrower Assisted-by: GPT 5.6 via Codex --- docs/setup.md | 16 +++- src/mqt/predictor/qcompile.py | 12 ++- src/mqt/predictor/rl/predictor.py | 71 +++++++++----- src/mqt/predictor/rl/predictorenv.py | 65 ++++++++++++- tests/compilation/test_predictor_rl.py | 122 +++++++++++++++++++++++++ 5 files changed, 257 insertions(+), 29 deletions(-) diff --git a/docs/setup.md b/docs/setup.md index 1582cbb5e..63ee2bdea 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -45,9 +45,14 @@ from mqt.bench.targets import get_device device = get_device("ibm_falcon_27") rl_pred = RL_Predictor(device=device, figure_of_merit="expected_fidelity") -rl_pred.train_model(timesteps=100000) +rl_pred.train_model(timesteps=100000, pass_timeout=600) ``` +`pass_timeout` is optional and limits each compilation pass during training. If +it is omitted, pass execution is unbounded. Pass timeouts require POSIX signals +and execution on the main thread; unsupported environments emit a warning and +continue without a timeout. + Currently, the following figures of merit are supported: ```{code-cell} ipython3 @@ -121,9 +126,16 @@ from mqt.predictor import qcompile from mqt.bench import get_benchmark, BenchmarkLevel uncompiled_qc = get_benchmark("ghz", level=BenchmarkLevel.ALG, circuit_size=5) -compiled_qc, compilation_info, selected_device = qcompile(uncompiled_qc, figure_of_merit="expected_fidelity") +compiled_qc, compilation_info, selected_device = qcompile( + uncompiled_qc, + figure_of_merit="expected_fidelity", + pass_timeout=60, +) ``` +Inference has its own optional `pass_timeout`, so it can use a different limit +than training. + This returns: - the compiled quantum circuit, diff --git a/src/mqt/predictor/qcompile.py b/src/mqt/predictor/qcompile.py index a1f8240f0..6807ed6c5 100644 --- a/src/mqt/predictor/qcompile.py +++ b/src/mqt/predictor/qcompile.py @@ -27,6 +27,7 @@ def qcompile( qc: QuantumCircuit, figure_of_merit: figure_of_merit = "expected_fidelity", tracer_output_path: str | Path | None = None, + pass_timeout: float | None = None, ) -> tuple[QuantumCircuit, list[str], str]: """Compiles a given quantum circuit to a device with the highest predicted figure of merit. @@ -34,12 +35,21 @@ def qcompile( qc: The quantum circuit to be compiled. figure_of_merit: The figure of merit to be used for compilation. Defaults to "expected_fidelity". tracer_output_path: If provided, enables compiler tracing and exports the JSON log to this path/directory. + pass_timeout: Maximum duration in seconds for one compilation pass. + Defaults to None, which disables pass timeouts. Returns: A tuple containing the compiled quantum circuit, the compilation information, and the name of the device used for compilation. + + Raises: + ValueError: If ``pass_timeout`` is not positive. """ predicted_device = predict_device_for_figure_of_merit(qc, figure_of_merit=figure_of_merit) res = rl_compile( - qc, device=predicted_device, figure_of_merit=figure_of_merit, tracer_output_path=tracer_output_path + qc, + device=predicted_device, + figure_of_merit=figure_of_merit, + tracer_output_path=tracer_output_path, + pass_timeout=pass_timeout, ) return *res, predicted_device diff --git a/src/mqt/predictor/rl/predictor.py b/src/mqt/predictor/rl/predictor.py index 4ff261a9f..44e315c64 100644 --- a/src/mqt/predictor/rl/predictor.py +++ b/src/mqt/predictor/rl/predictor.py @@ -72,26 +72,32 @@ def compile_as_predicted( self, qc: QuantumCircuit | str, tracer_output_path: str | Path | None = None, + pass_timeout: float | None = None, ) -> tuple[QuantumCircuit, list[str]]: """Compiles a given quantum circuit such that the given figure of merit is maximized by using the respectively trained optimized compiler. Arguments: qc: The quantum circuit to be compiled or the path to a qasm file containing the quantum circuit. tracer_output_path: Optional temporary path to export the compilation trace for this specific run. + pass_timeout: Maximum duration in seconds for one compilation pass. + Defaults to None, which disables pass timeouts. Returns: A tuple containing the compiled quantum circuit and the compilation information. If compilation fails, False is returned. Raises: RuntimeError: If an error occurs during compilation. + ValueError: If ``pass_timeout`` is not positive. """ original_tracer_output_path = self.env.tracer_output_path - - # Temporarily override singleton if a new path is explicitly provided - if tracer_output_path is not None: - self.env.tracer_output_path = tracer_output_path + original_pass_timeout = self.env.pass_timeout try: + # Temporarily override singleton settings for this compilation. + if tracer_output_path is not None: + self.env.tracer_output_path = tracer_output_path + self.env.pass_timeout = pass_timeout + trained_rl_model = load_model(self.model_name) obs, _ = self.env.reset(qc, seed=0) @@ -114,8 +120,8 @@ def compile_as_predicted( raise RuntimeError(msg) finally: - # Restore original singleton path self.env.tracer_output_path = original_tracer_output_path + self.env.pass_timeout = original_pass_timeout def train_model( self, @@ -123,6 +129,7 @@ def train_model( verbose: int = 2, test: bool = False, seed: int | None = None, + pass_timeout: float | None = None, ) -> None: """Trains all models for the given reward functions and device. @@ -132,6 +139,11 @@ def train_model( test: Whether to train the model for testing purposes. Defaults to False. seed: The random seed to use for reproducible training. Set to None to use true randomness. Defaults to None. + pass_timeout: Maximum duration in seconds for one compilation pass. + Defaults to None, which disables pass timeouts. + + Raises: + ValueError: If ``pass_timeout`` is not positive. """ if seed is not None: set_random_seed(seed) @@ -148,22 +160,27 @@ def train_model( batch_size = 64 progress_bar = True - logger.debug("Start training for: " + self.figure_of_merit + " on " + self.device_name) - model = MaskablePPO( - MaskableMultiInputActorCriticPolicy, - self.env, - verbose=verbose, - tensorboard_log=f"./{self.model_name}", - gamma=0.98, - n_steps=n_steps, - batch_size=batch_size, - n_epochs=n_epochs, - seed=seed, - ) - # Training Loop: In each iteration, the agent collects n_steps steps (rollout), - # updates the policy for n_epochs, and then repeats the process until total_timesteps steps have been taken. - model.learn(total_timesteps=timesteps, progress_bar=progress_bar) - model.save(get_path_trained_model() / self.model_name) + original_pass_timeout = self.env.pass_timeout + self.env.pass_timeout = pass_timeout + try: + logger.debug("Start training for: " + self.figure_of_merit + " on " + self.device_name) + model = MaskablePPO( + MaskableMultiInputActorCriticPolicy, + self.env, + verbose=verbose, + tensorboard_log=f"./{self.model_name}", + gamma=0.98, + n_steps=n_steps, + batch_size=batch_size, + n_epochs=n_epochs, + seed=seed, + ) + # Training Loop: In each iteration, the agent collects n_steps steps (rollout), + # updates the policy for n_epochs, and then repeats the process until total_timesteps steps have been taken. + model.learn(total_timesteps=timesteps, progress_bar=progress_bar) + model.save(get_path_trained_model() / self.model_name) + finally: + self.env.pass_timeout = original_pass_timeout def load_model(model_name: str) -> MaskablePPO: @@ -194,6 +211,7 @@ def rl_compile( predictor_singleton: Predictor | None = None, tracer_output_path: str | Path | None = None, mdp: MDPPolicy = "v3", + pass_timeout: float | None = None, ) -> tuple[QuantumCircuit, list[str]]: """Compiles a given quantum circuit to a device optimizing for the given figure of merit. @@ -206,12 +224,15 @@ def rl_compile( mdp: The MDP transition policy used when constructing a predictor. ``v2`` is the original strategy and ``v3`` is the default. When ``predictor_singleton`` is provided, its configured policy is used instead. + pass_timeout: Maximum duration in seconds for one compilation pass. + Defaults to None, which disables pass timeouts. Returns: A tuple containing the compiled quantum circuit and the compilation information. If compilation fails, False is returned. Raises: - ValueError: If figure_of_merit or device is None and predictor_singleton is also None. + ValueError: If figure_of_merit or device is None and predictor_singleton is also None, + or if ``pass_timeout`` is not positive. """ if predictor_singleton is None: if figure_of_merit is None: @@ -226,6 +247,8 @@ def rl_compile( tracer_output_path=tracer_output_path, mdp=mdp, ) - return predictor.compile_as_predicted(qc) + return predictor.compile_as_predicted(qc, pass_timeout=pass_timeout) - return predictor_singleton.compile_as_predicted(qc, tracer_output_path=tracer_output_path) + return predictor_singleton.compile_as_predicted( + qc, tracer_output_path=tracer_output_path, pass_timeout=pass_timeout + ) diff --git a/src/mqt/predictor/rl/predictorenv.py b/src/mqt/predictor/rl/predictorenv.py index 7306f0ac3..fd850ad54 100644 --- a/src/mqt/predictor/rl/predictorenv.py +++ b/src/mqt/predictor/rl/predictorenv.py @@ -10,14 +10,20 @@ from __future__ import annotations +import contextlib import logging import re +import signal +import threading import time import warnings from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, get_args if TYPE_CHECKING: + from collections.abc import Iterator + from types import FrameType + from gymnasium.spaces import Space from qiskit.transpiler import Layout, Target @@ -60,6 +66,43 @@ MDP_POLICIES: frozenset[str] = frozenset(get_args(MDPPolicy)) +@contextlib.contextmanager +def _enforce_pass_timeout(pass_timeout: float | None) -> Iterator[None]: + if pass_timeout is None: + yield + return + + if not all(hasattr(signal, attribute) for attribute in ("SIGALRM", "ITIMER_REAL", "getitimer", "setitimer")): + warnings.warn("Pass timeouts are not supported on this platform.", RuntimeWarning, stacklevel=2) + yield + return + if threading.current_thread() is not threading.main_thread(): + warnings.warn("Pass timeouts are only supported on the main thread.", RuntimeWarning, stacklevel=2) + yield + return + + def timeout_handler(_signum: int, _frame: FrameType | None) -> None: + msg = f"Compilation pass exceeded the timeout of {pass_timeout:g} seconds." + raise TimeoutError(msg) + + previous_delay, previous_interval = signal.getitimer(signal.ITIMER_REAL) + if 0 < previous_delay <= pass_timeout: + yield + return + + previous_handler = signal.signal(signal.SIGALRM, timeout_handler) + start_time = time.monotonic() + try: + signal.setitimer(signal.ITIMER_REAL, pass_timeout) + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous_handler) + if previous_delay > 0: + remaining_delay = max(previous_delay - (time.monotonic() - start_time), 1e-6) + signal.setitimer(signal.ITIMER_REAL, remaining_delay, previous_interval) + + class PredictorEnv(Env): """Predictor environment for reinforcement learning.""" @@ -71,6 +114,7 @@ def __init__( max_steps: int | None = None, tracer_output_path: str | Path | None = None, mdp: MDPPolicy = "v3", + pass_timeout: float | None = None, ) -> None: """Initializes the PredictorEnv object. @@ -83,9 +127,12 @@ def __init__( mdp: The MDP transition policy. ``v2`` is the original MQT Predictor strategy. ``v3`` is the default and is permissive before layout while preserving the established compilation structure afterwards. + pass_timeout: Maximum duration in seconds for one compilation pass. + Defaults to None, which disables pass timeouts. Raises: - ValueError: If ``mdp`` is unsupported, if the reward function is + ValueError: If ``mdp`` is unsupported, ``pass_timeout`` is not positive, + if the reward function is "estimated_success_probability" and no calibration data is available for the device, or if the reward function is "estimated_hellinger_distance" and no trained model is available for @@ -100,6 +147,7 @@ def __init__( self.path_training_circuits = path_training_circuits or get_path_training_circuits() self.max_steps = max_steps self.mdp = mdp + self.pass_timeout = pass_timeout self.action_set = {} self.actions_synthesis_indices = [] @@ -201,6 +249,18 @@ def __init__( self.observation_space = Dict(spaces) self.filename = "" + @property + def pass_timeout(self) -> float | None: + """The current per-pass timeout in seconds.""" + return self._pass_timeout + + @pass_timeout.setter + def pass_timeout(self, pass_timeout: float | None) -> None: + if pass_timeout is not None and pass_timeout <= 0: + msg = "pass_timeout must be positive." + raise ValueError(msg) + self._pass_timeout = pass_timeout + def _collect_tracer_data( self, step_index: int, @@ -290,7 +350,8 @@ def step(self, action: int) -> tuple[dict[str, Any], float, bool, bool, dict[Any start_time = time.perf_counter() try: self.used_actions.append(action_name) - altered_qc = self.apply_action(action) + with _enforce_pass_timeout(self.pass_timeout): + altered_qc = self.apply_action(action) action_duration = time.perf_counter() - start_time except Exception as exc: # ruff:ignore[blind-except] action_duration = time.perf_counter() - start_time diff --git a/tests/compilation/test_predictor_rl.py b/tests/compilation/test_predictor_rl.py index c057431d6..74b5d3361 100644 --- a/tests/compilation/test_predictor_rl.py +++ b/tests/compilation/test_predictor_rl.py @@ -11,6 +11,9 @@ from __future__ import annotations import re +import signal +import time +from importlib import import_module from pathlib import Path from typing import TYPE_CHECKING, cast @@ -24,6 +27,7 @@ from qiskit.transpiler.passes import GatesInBasis from mqt.predictor.rl import Predictor, rl_compile +from mqt.predictor.rl import predictor as predictor_module from mqt.predictor.rl import predictorenv as predictorenv_module from mqt.predictor.rl.actions import ( CompilationOrigin, @@ -80,6 +84,124 @@ def test_predictor_env_rejects_unsupported_mdp() -> None: predictorenv_module.PredictorEnv(device=get_device("ibm_falcon_27"), mdp=invalid_mdp) +@pytest.mark.parametrize("pass_timeout", [0, -1]) +def test_predictor_env_rejects_nonpositive_pass_timeout(pass_timeout: float) -> None: + """Test that pass timeouts must be positive when enabled.""" + with pytest.raises(ValueError, match=re.escape("pass_timeout must be positive.")): + predictorenv_module.PredictorEnv(device=get_device("ibm_falcon_27"), pass_timeout=pass_timeout) + + +@pytest.mark.skipif(not hasattr(signal, "SIGALRM"), reason="SIGALRM is unavailable") +def test_predictor_env_truncates_timed_out_pass(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that a pass timeout truncates the current episode.""" + env = predictorenv_module.PredictorEnv( + device=get_device("ibm_falcon_27"), pass_timeout=0.01, intermediate_reward=False + ) + qc = QuantumCircuit(1) + env.reset(qc) + monkeypatch.setattr(env, "apply_action", lambda _action: time.sleep(1)) + + _, reward_val, terminated, truncated, info = env.step(env.actions_opt_indices[0]) + + assert reward_val == 0 + assert not terminated + assert truncated + assert info == { + "Truncated because of error": "TimeoutError: Compilation pass exceeded the timeout of 0.01 seconds." + } + + +def test_training_uses_temporary_pass_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that training applies and restores its pass timeout.""" + observed_timeouts: list[float | None] = [] + + class FakeModel: + """Minimal MaskablePPO replacement that records the environment timeout.""" + + def __init__( + self, + _policy: object, + env: predictorenv_module.PredictorEnv, + **_kwargs: object, + ) -> None: + self.env = env + + def learn(self, *, total_timesteps: int, progress_bar: bool) -> None: + assert total_timesteps == 2 + assert not progress_bar + observed_timeouts.append(self.env.pass_timeout) + + def save(self, _path: Path) -> None: + pass + + monkeypatch.setattr(predictor_module, "MaskablePPO", FakeModel) + predictor = Predictor(figure_of_merit="expected_fidelity", device=get_device("ibm_falcon_27")) + + predictor.train_model(timesteps=2, test=True, pass_timeout=2) + + assert observed_timeouts == [2] + assert predictor.env.pass_timeout is None + + +def test_inference_uses_temporary_pass_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that inference applies and restores its independent pass timeout.""" + observed_timeouts: list[float | None] = [] + qc = QuantumCircuit(1) + predictor = Predictor(figure_of_merit="expected_fidelity", device=get_device("ibm_falcon_27")) + + class FakeModel: + """Minimal trained model replacement.""" + + def predict(self, _obs: object, *, action_masks: object) -> tuple[int, None]: + assert action_masks == [] + return predictor.env.actions_opt_indices[0], None + + def fake_step(_action: int) -> tuple[dict[str, object], float, bool, bool, dict[str, object]]: + observed_timeouts.append(predictor.env.pass_timeout) + predictor.env.state = qc + return {}, 0, True, False, {} + + def fake_reset(_qc: QuantumCircuit | str, seed: int) -> tuple[dict[str, object], dict[str, object]]: + assert seed == 0 + predictor.env.error_occurred = False + return {}, {} + + monkeypatch.setattr(predictor_module, "load_model", lambda _model_name: FakeModel()) + monkeypatch.setattr(predictor_module, "get_action_masks", lambda _env: []) + monkeypatch.setattr(predictor.env, "reset", fake_reset) + monkeypatch.setattr(predictor.env, "step", fake_step) + + compiled_qc, _passes = predictor.compile_as_predicted(qc, pass_timeout=0.5) + + assert compiled_qc is qc + assert observed_timeouts == [0.5] + assert predictor.env.pass_timeout is None + + +def test_qcompile_forwards_inference_pass_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that the top-level inference API forwards its pass timeout.""" + qcompile_module = import_module("mqt.predictor.qcompile") + device = get_device("ibm_falcon_27") + qc = QuantumCircuit(1) + + def fake_predict_device(_qc: QuantumCircuit, figure_of_merit: str) -> Target: + assert figure_of_merit == "expected_fidelity" + return device + + def fake_rl_compile(circuit: QuantumCircuit, **kwargs: object) -> tuple[QuantumCircuit, list[str]]: + assert kwargs["pass_timeout"] == 3 + return circuit, [] + + monkeypatch.setattr(qcompile_module, "predict_device_for_figure_of_merit", fake_predict_device) + monkeypatch.setattr(qcompile_module, "rl_compile", fake_rl_compile) + + compiled_qc, compilation_info, selected_device = qcompile_module.qcompile(qc, pass_timeout=3) + + assert compiled_qc is qc + assert compilation_info == [] + assert selected_device is device + + @pytest.mark.parametrize( ("mdp", "expected_action_groups"), [ From 120c44ed596c629170c504cb50f028246cd3833e Mon Sep 17 00:00:00 2001 From: flowerthrower Date: Thu, 27 Aug 2026 10:04:32 +0200 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=93=9D=20Document=20configurable=20pa?= =?UTF-8?q?ss=20timeouts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: GPT-5.6 via Codex Signed-off-by: flowerthrower Assisted-by: GPT 5.6 via Codex --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 734be953a..e1e7fd6e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ releases may include breaking changes. ### Added +- ✨ Add opt-in per-pass timeouts for RL training and inference ([#778]) + ([**@flowerthrower**]) - ✨ Add Qiskit's `TrivialLayout`, `ElidePermutations`, `SabreSwap`, `BasicSwap`, `LookaheadSwap`, `RemoveIdentityEquivalent`, and `Optimize1qGatesSimpleCommutation` passes to the RL actions ([#785]) @@ -98,6 +100,7 @@ for previous changelogs._ +[#778]: https://github.com/munich-quantum-toolkit/predictor/pull/778 [#773]: https://github.com/munich-quantum-toolkit/predictor/pull/771 [#769]: https://github.com/munich-quantum-toolkit/predictor/pull/769 [#784]: https://github.com/munich-quantum-toolkit/predictor/pull/784 From 3e300aaf227b9eb8387351fe0e38df90c74792b2 Mon Sep 17 00:00:00 2001 From: flowerthrower Date: Thu, 27 Aug 2026 14:55:19 +0200 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=A7=AA=20Keep=20timeout=20coverage=20?= =?UTF-8?q?independent=20of=20reward=20shaping?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: flowerthrower --- tests/compilation/test_predictor_rl.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/compilation/test_predictor_rl.py b/tests/compilation/test_predictor_rl.py index 74b5d3361..326ef0e89 100644 --- a/tests/compilation/test_predictor_rl.py +++ b/tests/compilation/test_predictor_rl.py @@ -94,9 +94,7 @@ def test_predictor_env_rejects_nonpositive_pass_timeout(pass_timeout: float) -> @pytest.mark.skipif(not hasattr(signal, "SIGALRM"), reason="SIGALRM is unavailable") def test_predictor_env_truncates_timed_out_pass(monkeypatch: pytest.MonkeyPatch) -> None: """Test that a pass timeout truncates the current episode.""" - env = predictorenv_module.PredictorEnv( - device=get_device("ibm_falcon_27"), pass_timeout=0.01, intermediate_reward=False - ) + env = predictorenv_module.PredictorEnv(device=get_device("ibm_falcon_27"), pass_timeout=0.01) qc = QuantumCircuit(1) env.reset(qc) monkeypatch.setattr(env, "apply_action", lambda _action: time.sleep(1)) From a7cf25f1719268da6d4b8e6034c0abadf19653e9 Mon Sep 17 00:00:00 2001 From: flowerthrower Date: Thu, 27 Aug 2026 15:19:50 +0200 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=93=9D=20Attribute=20pass=20timeouts?= =?UTF-8?q?=20to=20PR=20#789?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: flowerthrower Assisted-by: GPT 5.6 via Codex --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1e7fd6e1..a84347857 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ releases may include breaking changes. ### Added -- ✨ Add opt-in per-pass timeouts for RL training and inference ([#778]) +- ✨ Add opt-in per-pass timeouts for RL training and inference ([#789]) ([**@flowerthrower**]) - ✨ Add Qiskit's `TrivialLayout`, `ElidePermutations`, `SabreSwap`, `BasicSwap`, `LookaheadSwap`, `RemoveIdentityEquivalent`, and @@ -100,7 +100,7 @@ for previous changelogs._ -[#778]: https://github.com/munich-quantum-toolkit/predictor/pull/778 +[#789]: https://github.com/munich-quantum-toolkit/predictor/pull/789 [#773]: https://github.com/munich-quantum-toolkit/predictor/pull/771 [#769]: https://github.com/munich-quantum-toolkit/predictor/pull/769 [#784]: https://github.com/munich-quantum-toolkit/predictor/pull/784