diff --git a/.github/workflows/code-quality.yaml b/.github/workflows/code-quality.yaml index 1110f98f..175a15b9 100644 --- a/.github/workflows/code-quality.yaml +++ b/.github/workflows/code-quality.yaml @@ -116,6 +116,11 @@ jobs: - name: Pyright run: uv run pyright + - name: Config schema freshness + run: | + uv run raitap config-schema -o /tmp/raitap.schema.json + git diff --exit-code --no-index -- src/raitap/schema/raitap.schema.json /tmp/raitap.schema.json + - name: Pytest on 3.13 (with coverage) run: uv run pytest -m "not e2e and not cuda" -v --cov=src/raitap --cov-report=term-missing --cov-report=html diff --git a/.gitignore b/.gitignore index b85682b0..eb2e7739 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,6 @@ contributor-configs/*/images/ /metrics/ /reports/ /transparency/ + +# Generated at docs build by docs/_ext/raitap_adapters.py (issue #301) +docs/modules/_generated_adapters.md diff --git a/contributor-configs/fasterrcnn-udacity/assessment.yaml b/contributor-configs/fasterrcnn-udacity/assessment.yaml index 7fd84ea7..3a501f7d 100644 --- a/contributor-configs/fasterrcnn-udacity/assessment.yaml +++ b/contributor-configs/fasterrcnn-udacity/assessment.yaml @@ -40,7 +40,7 @@ data: source: ${hydra:runtime.cwd}/contributor-configs/fasterrcnn-udacity/labels/udacity-boxes.json metrics: - _target_: DetectionMetrics + use: detection transparency: # One per-box Integrated Gradients run. ``call.target`` must be 0 — the @@ -48,7 +48,7 @@ transparency: # is rejected (see docs/modules/transparency/configuration.md "Detection # knobs"). ``max_boxes: 3`` keeps the K-loop small enough for CPU CI. detection_ig: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: 0 @@ -65,10 +65,10 @@ transparency: layout: NCHW show_sample_names: true visualisers: - - _target_: DetectionImageVisualiser + - use: detection_image reporting: - _target_: HTMLReporter + use: html filename: fasterrcnn_udacity_report # Force all 4 samples into the local-detail section (default cap is 3). sample_selection: [0, 1, 2, 3] diff --git a/contributor-configs/imagecorruptions-imagenet/assessment.yaml b/contributor-configs/imagecorruptions-imagenet/assessment.yaml index 659e8708..5fd98ff3 100644 --- a/contributor-configs/imagecorruptions-imagenet/assessment.yaml +++ b/contributor-configs/imagecorruptions-imagenet/assessment.yaml @@ -36,20 +36,20 @@ metrics: robustness: # Average-case: ImageNet-C common corruption at a fixed severity. gaussian_noise: - _target_: ImageCorruptionsAssessor + use: imagecorruptions algorithm: gaussian_noise # any of the 15 ImageNet-C corruptions constructor: severity: 3 # 1..5 visualisers: - - _target_: CorruptionAccuracyVisualiser + - use: corruption_accuracy # Worst-case: bounded PGD adversarial attack (torchattacks). pgd: - _target_: TorchattacksAssessor + use: torchattacks algorithm: PGD constructor: eps: 0.03 alpha: 0.008 steps: 10 visualisers: - - _target_: ImagePairVisualiser - - _target_: PerturbationHeatmapVisualiser + - use: image_pair + - use: perturbation_heatmap diff --git a/contributor-configs/lwise-ham10000/assessment.yaml b/contributor-configs/lwise-ham10000/assessment.yaml index a80866ba..5eb18722 100644 --- a/contributor-configs/lwise-ham10000/assessment.yaml +++ b/contributor-configs/lwise-ham10000/assessment.yaml @@ -30,12 +30,12 @@ data: column: label metrics: - _target_: MulticlassClassificationMetrics + use: multiclass_classification num_classes: 7 transparency: gradcam_localisation: - _target_: CaptumExplainer + use: captum algorithm: LayerGradCam constructor: layer_path: 1.layer4.2.conv3 @@ -49,7 +49,7 @@ transparency: layout: NCHW show_sample_names: true visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image constructor: method: heat_map sign: positive @@ -60,7 +60,7 @@ transparency: max_samples: 4 saliency_evidence: - _target_: CaptumExplainer + use: captum algorithm: Saliency call: target: auto_pred @@ -72,7 +72,7 @@ transparency: layout: NCHW show_sample_names: true visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image constructor: method: blended_heat_map sign: positive @@ -81,7 +81,7 @@ transparency: title: "Positive lesion evidence" call: max_samples: 4 - - _target_: CaptumImageVisualiser + - use: captum_image constructor: method: masked_image sign: positive @@ -92,7 +92,7 @@ transparency: max_samples: 4 occlusion_check: - _target_: CaptumExplainer + use: captum algorithm: Occlusion call: target: auto_pred @@ -106,7 +106,7 @@ transparency: layout: NCHW show_sample_names: true visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image constructor: method: heat_map sign: absolute_value @@ -117,7 +117,7 @@ transparency: max_samples: 4 integrated_evidence_detail: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: auto_pred @@ -130,7 +130,7 @@ transparency: layout: NCHW show_sample_names: true visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image constructor: method: blended_heat_map sign: absolute_value @@ -142,7 +142,7 @@ transparency: robustness: fgsm_linf_fast: - _target_: TorchattacksAssessor + use: torchattacks algorithm: FGSM constructor: eps: 0.03 @@ -153,15 +153,15 @@ robustness: layout: NCHW show_sample_names: true visualisers: - - _target_: ImagePairVisualiser + - use: image_pair constructor: max_samples: 4 - - _target_: PerturbationHeatmapVisualiser + - use: perturbation_heatmap constructor: max_samples: 4 pgd_linf_small: - _target_: TorchattacksAssessor + use: torchattacks algorithm: PGD constructor: eps: 0.03 @@ -174,25 +174,25 @@ robustness: layout: NCHW show_sample_names: true visualisers: - - _target_: ImagePairVisualiser + - use: image_pair constructor: max_samples: 4 - - _target_: PerturbationHeatmapVisualiser + - use: perturbation_heatmap constructor: max_samples: 4 marabou_linf: - _target_: MarabouAssessor + use: marabou algorithm: linf-box constructor: epsilon: 0.05 norm: Linf timeout_s: 300 visualisers: - - _target_: VerdictSummaryVisualiser + - use: verdict_summary reporting: - _target_: HTMLReporter + use: html filename: lwise_ham10000_report.pdf include_config: true include_metadata: true diff --git a/contributor-configs/lwise-ham10000/assessment_mlflow.yaml b/contributor-configs/lwise-ham10000/assessment_mlflow.yaml index 9a6358c0..e8c9d48d 100644 --- a/contributor-configs/lwise-ham10000/assessment_mlflow.yaml +++ b/contributor-configs/lwise-ham10000/assessment_mlflow.yaml @@ -3,7 +3,7 @@ defaults: - _self_ tracking: - _target_: MLFlowTracker + use: mlflow output_forwarding_url: http://127.0.0.1:5001 backend_store_uri: sqlite:///mlflow/mlflow.db default_artifact_root: ./mlflow/artifacts diff --git a/contributor-configs/marabou-mnist/assessment.yaml b/contributor-configs/marabou-mnist/assessment.yaml index 5a89aadd..1218ef67 100644 --- a/contributor-configs/marabou-mnist/assessment.yaml +++ b/contributor-configs/marabou-mnist/assessment.yaml @@ -39,11 +39,11 @@ model: robustness: marabou_linf: - _target_: MarabouAssessor + use: marabou algorithm: linf-box constructor: epsilon: 0.05 norm: Linf timeout_s: 300 visualisers: - - _target_: VerdictSummaryVisualiser + - use: verdict_summary diff --git a/contributor-configs/noisetunnel-smoothgrad/assessment.yaml b/contributor-configs/noisetunnel-smoothgrad/assessment.yaml index 44946d44..c02627d3 100644 --- a/contributor-configs/noisetunnel-smoothgrad/assessment.yaml +++ b/contributor-configs/noisetunnel-smoothgrad/assessment.yaml @@ -33,7 +33,7 @@ metrics: transparency: smoothgrad_over_saliency: - _target_: CaptumExplainer + use: captum algorithm: NoiseTunnel constructor: base_algorithm: Saliency @@ -49,7 +49,7 @@ transparency: layout: NCHW show_sample_names: true visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image constructor: method: blended_heat_map sign: absolute_value @@ -60,7 +60,7 @@ transparency: max_samples: 4 saliency_baseline: - _target_: CaptumExplainer + use: captum algorithm: Saliency call: target: auto_pred @@ -72,7 +72,7 @@ transparency: layout: NCHW show_sample_names: true visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image constructor: method: blended_heat_map sign: absolute_value @@ -83,7 +83,7 @@ transparency: max_samples: 4 reporting: - _target_: HTMLReporter + use: html filename: noisetunnel_smoothgrad_report include_config: true include_metadata: true diff --git a/contributor-configs/quantus-eval/assessment.yaml b/contributor-configs/quantus-eval/assessment.yaml index 2fce4d1b..d3a62f54 100644 --- a/contributor-configs/quantus-eval/assessment.yaml +++ b/contributor-configs/quantus-eval/assessment.yaml @@ -49,7 +49,7 @@ metrics: transparency: integrated_gradients: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: auto_pred @@ -60,7 +60,7 @@ transparency: layout: NCHW show_sample_names: true visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image constructor: method: blended_heat_map sign: absolute_value @@ -69,13 +69,13 @@ transparency: call: max_samples: 4 evaluation: - _target_: raitap.transparency.QuantusEvaluator + use: quantus metrics: [faithfulness_correlation, sparseness] constructor: faithfulness_correlation: {nr_runs: 10, subset_size: 32} reporting: - _target_: HTMLReporter + use: html filename: quantus_eval_report include_config: true include_metadata: true diff --git a/contributor-configs/text-classification-sst2/assessment.yaml b/contributor-configs/text-classification-sst2/assessment.yaml index 7e375146..0a22edb9 100644 --- a/contributor-configs/text-classification-sst2/assessment.yaml +++ b/contributor-configs/text-classification-sst2/assessment.yaml @@ -47,7 +47,7 @@ metrics: transparency: token_ig: - _target_: CaptumExplainer + use: captum algorithm: LayerIntegratedGradients constructor: # LayerIntegratedGradients attributes wrt. the embedding layer output. @@ -61,10 +61,10 @@ transparency: kind: text layout: TOKENS visualisers: - - _target_: CaptumTextVisualiser + - use: captum_text reporting: - _target_: HTMLReporter + use: html filename: text_classification_sst2_report include_config: true include_metadata: true diff --git a/contributor-configs/tree-xgboost-shap/assessment.yaml b/contributor-configs/tree-xgboost-shap/assessment.yaml index 8d4db3bb..2f424874 100644 --- a/contributor-configs/tree-xgboost-shap/assessment.yaml +++ b/contributor-configs/tree-xgboost-shap/assessment.yaml @@ -28,7 +28,7 @@ data: transparency: treeshap: - _target_: ShapExplainer + use: shap algorithm: TreeExplainer call: # Binary XGBoost SHAP returns stacked (B, F, 2); pick the positive class. @@ -39,12 +39,12 @@ transparency: layout: "(B,F)" show_sample_names: true visualisers: - - _target_: ShapBarVisualiser + - use: shap_bar constructor: feature_names: [f0, f1, f2, f3, f4, f5] reporting: - _target_: HTMLReporter + use: html filename: tree_xgboost_shap_report include_config: true include_metadata: true diff --git a/docs/_ext/raitap_adapters.py b/docs/_ext/raitap_adapters.py new file mode 100644 index 00000000..fdb88336 --- /dev/null +++ b/docs/_ext/raitap_adapters.py @@ -0,0 +1,155 @@ +"""Sphinx hook: generate one adapter-reference page from raitap's registry. + +Walks ``raitap._adapters._BUILDERS`` (populated by every ``@adapters.`` +decorator) and writes ``docs/modules/_generated_adapters.md``: one anchored +table per registry group (``use`` key | algorithms | extra | supported tasks). +Module doc pages pull individual group tables in with a MyST ``{include}`` +fence plus ``:start-after:``/``:end-before:`` markers (see ``_anchor`` below). + +Runs on ``builder-inited`` -- before any source file is read -- so the +generated page exists on disk by the time Sphinx parses the ``{include}`` +directives that reference it. + +Real adapter classes live behind heavy optional deps (torch, captum, shap, +foolbox, torchattacks, ...) that the docs build environment does not install +(see ``autodoc_mock_imports`` in ``conf.py``). This reuses that same mock list +via :func:`sphinx.ext.autodoc.mock.mock` so ``import raitap`` plus the +registry walk succeed without those deps -- exactly like autodoc's own +imports for ``automodule``/``autoclass``. + +Writing one single generated page (not per-symbol ``automodule``/``autoclass`` +directives scattered across pages) sidesteps the duplicate-object build +failure that hits a shared symbol re-exported from two packages under +``:imported-members:``. +""" + +from __future__ import annotations + +import importlib +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sphinx.application import Sphinx + +GENERATED_PAGE = "modules/_generated_adapters.md" + +# Canonical group display order; groups not listed here (future families, +# third-party plugin groups) are appended alphabetically after. +_GROUP_ORDER = ( + "transparency", + "robustness", + "metrics", + "reporting", + "tracking", + "data/inputs", + "data/labels", + "_unscoped", +) + +TABLE_HEADERS = ("`use` key", "Algorithms", "Extra", "Supported tasks") + + +def _anchor(group: str, edge: str) -> str: + return f"" + + +def _escape_cell(value: str) -> str: + return value.replace("|", r"\|") + + +def _format_names(names: list[str]) -> str: + if not names: + return "—" + return ", ".join(f"`{name}`" for name in sorted(names)) + + +def _adapter_row(*, registry_name: str, cls: type, extras: dict[str, str]) -> str: + algorithm_registry = getattr(cls, "algorithm_registry", None) + algorithms = _format_names(list(algorithm_registry)) if algorithm_registry else "—" + + supported_tasks = getattr(cls, "supported_tasks", None) + tasks = _format_names([str(task) for task in supported_tasks]) if supported_tasks else "—" + + extra = extras.get(cls.__name__, "") + extra_cell = f"`{extra}`" if extra else "—" + + cells = [f"`{registry_name}`", algorithms, extra_cell, tasks] + return "| " + " | ".join(_escape_cell(cell) for cell in cells) + " |" + + +def _group_table_lines( + *, + registry_names: list[str], + target_fqn: dict[str, str], + extras: dict[str, str], +) -> list[str]: + header_line = "| " + " | ".join(TABLE_HEADERS) + " |" + separator_line = "| " + " | ".join("---" for _ in TABLE_HEADERS) + " |" + lines = [header_line, separator_line] + for registry_name in sorted(registry_names): + fqn = target_fqn.get(registry_name) + if fqn is None: + lines.append(f"| `{registry_name}` | — | — | — |") + continue + module_name, _, cls_name = fqn.rpartition(".") + try: + module = importlib.import_module(module_name) + cls = getattr(module, cls_name) + except Exception: + # A third-party plugin adapter's own deps may genuinely be missing + # even under the mock list; degrade to an "unavailable" row + # instead of failing the whole doc build. + lines.append(f"| `{registry_name}` | — | — | — |") + continue + lines.append(_adapter_row(registry_name=registry_name, cls=cls, extras=extras)) + return lines + + +def _build_generated_page(mock_imports: list[str]) -> str: + from sphinx.ext.autodoc.mock import mock # pyright: ignore[reportPrivateImportUsage] + + with mock(mock_imports): + importlib.import_module("raitap") # side-effect import; discard the module object + from raitap._adapters import _BUILDERS, _TARGET_FQN, ADAPTER_EXTRAS + from raitap.configs import register_configs + + register_configs() # idempotent; imports every in-tree family + plugins + + groups = list(_BUILDERS) + ordered_groups = [g for g in _GROUP_ORDER if g in groups] + ordered_groups += sorted(g for g in groups if g not in _GROUP_ORDER) + + sections: list[str] = [ + "", + "", + ] + for group in ordered_groups: + sections.append(_anchor(group, "start")) + sections.extend( + _group_table_lines( + registry_names=list(_BUILDERS[group]), + target_fqn=_TARGET_FQN.get(group, {}), + extras=ADAPTER_EXTRAS, + ) + ) + sections.append(_anchor(group, "end")) + sections.append("") + return "\n".join(sections).rstrip() + "\n" + + +def generate_adapter_reference(app: Sphinx) -> None: + content = _build_generated_page(list(app.config.autodoc_mock_imports)) + target = Path(app.srcdir) / GENERATED_PAGE + target.parent.mkdir(parents=True, exist_ok=True) + if not target.exists() or target.read_text() != content: + target.write_text(content) + + +def setup(app: Sphinx) -> dict[str, bool]: + app.connect("builder-inited", generate_adapter_reference) + return { + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/docs/conf.py b/docs/conf.py index b1d5e8c4..e9a5c29b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,6 +18,7 @@ "config_options", "config_tabs", "install_tabs", + "raitap_adapters", "recipe", "visualiser_card", "myst_parser", diff --git a/docs/contributor/adding/adding-a-module.md b/docs/contributor/adding/adding-a-module.md index d05573e4..0964b30d 100644 --- a/docs/contributor/adding/adding-a-module.md +++ b/docs/contributor/adding/adding-a-module.md @@ -125,7 +125,7 @@ Add a dataclass for the per-adapter config + a field on `AppConfig`: ```python @dataclass class FairnessConfig: - _target_: str = MISSING + use: str = MISSING # Overridden by the fairness config-group YAML (fairness=demographic_parity / ...). # inside AppConfig: @@ -147,7 +147,7 @@ all in your module: **a. The phase work** (`fairness/phase.py`): mirror `transparency/phase.py` / `robustness/phase.py`. -Your `FairnessResult` must satisfy the **`AdapterResult`** contract (`raitap.pipeline.outputs`): `name` (config key), `adapter_target` (the `_target_`), `algorithm`, `semantics` (load-bearing: RAITAP's thesis requires every result to carry it), `run_dir`, and `visualisations` (a `list` it owns, populated by its own `_visualise()`). There is no parallel phase-level visualisation list. +Your `FairnessResult` must satisfy the **`AdapterResult`** contract (`raitap.pipeline.outputs`): `name` (config key), `adapter_target` (the resolved adapter class FQN), `algorithm`, `semantics` (load-bearing: RAITAP's thesis requires every result to carry it), `run_dir`, and `visualisations` (a `list` it owns, populated by its own `_visualise()`). There is no parallel phase-level visualisation list. Use the shared `run_adapters` helper. It runs the loop, calls each result's `_visualise()` (so you cannot forget the ownership contract), and collects the results: diff --git a/docs/contributor/adding/adding-an-adapter.md b/docs/contributor/adding/adding-an-adapter.md index 14de03de..4d8e64c8 100644 --- a/docs/contributor/adding/adding-an-adapter.md +++ b/docs/contributor/adding/adding-an-adapter.md @@ -38,7 +38,7 @@ if TYPE_CHECKING: @adapters.transparency( registry_name="superxai", # CLI `+transparency=superxai` / Python `from raitap.transparency import superxai` # extra="superxai", # uv extra name; defaults to `registry_name` (omit unless they differ, see metrics for an exception) - library="superxai-lib", # real PyPI package name; drives `self._lazy_import()` + import_name="superxai", # importable module name of the wrapped library; drives `self._lazy_import()` error_patterns={ # rewrite cryptic upstream errors at call sites r"some library footgun": "Do X instead.", }, @@ -77,12 +77,12 @@ class SuperXAIExplainer(AttributionOnlyExplainer): - **Base class.** `AttributionOnlyExplainer` provides batching, artefact persistence, and `explain()` orchestration. Other families use other bases (`EmpiricalAttackAssessor` for robustness, `BaseMetricComputer` for metrics, etc.); find them in files starting with `base_`. The concrete adapter is just a normal class; nothing flags it as abstract (the old `abstract=True` workaround was removed). - **Decorator.** `@adapters.transparency(...)` is the sole entry point for registration. `registry_name` is required and pyright-checked at the decoration site via `Required[str]`. Each family has its own facade attribute (`adapters.robustness`, `adapters.metrics`, `adapters.reporter`, `adapters.tracker`, `visualisers.transparency`, `visualisers.robustness`): pick the one matching your base class. -- **Registration kwargs.** `library` is the pip name powering `self._lazy_import()`: pass it when you wrap a third-party package (the usual case). `extra` is the uv extra surfaced in install hints and scanned by `raitap.deps.inference`; it **defaults to `registry_name`** so you only need to set it explicitly when they differ (e.g. `classification_metrics` + `detection_metrics` both share `extra="metrics"`). `error_patterns` and `suppress_warnings` are optional polish. +- **Registration kwargs.** `import_name` is the importable module name powering `self._lazy_import()`: pass it when you wrap a third-party package (the usual case). `extra` is the uv extra surfaced in install hints and scanned by `raitap.deps.inference`; it **defaults to `registry_name`** so you only need to set it explicitly when they differ (e.g. `classification_metrics` + `detection_metrics` both share `extra="metrics"`). `error_patterns` and `suppress_warnings` are optional polish. - **`algorithm_registry` (decorator kwarg).** Transparency and robustness only. Maps algorithm name to a per-algorithm semantics-hints value RAITAP tracks and reports on (`ExplainerAlgorithmSpec` for transparency, `AssessorAlgorithmSpec` for robustness). **Required**: pyright errors at the decoration site if you omit it. Missing or misnamed entries make algorithms unselectable. The decorator assigns it onto the class so `type(self).algorithm_registry` still works at runtime. - **`output_payload_kind` (decorator kwarg).** Transparency only. Tells the report renderer what artefact shape the explainer emits (`ATTRIBUTIONS`, `SALIENCY_MAP`, ...). Defaults to `ExplanationPayloadKind.ATTRIBUTIONS`; only pass it if your explainer emits something else. - **Backend compatibility.** Inherited `check_backend_compat` (from `AdapterMixin`) raises `BackendIncompatibilityError` when `algorithm.requires - backend.provides` is non-empty. You write zero gate code. Gradient-based algorithms declare `requires={Capability.AUTOGRAD}` on their `ExplainerAlgorithmSpec` / `AssessorAlgorithmSpec` entry; model-agnostic algorithms (SHAP KernelExplainer, Occlusion, FeatureAblation) leave `requires` at its default empty frozenset and run on any backend including ONNX. Override `check_backend_compat` only for a non-capability contract (Marabou uses it for per-call setup; auto-LiRPA calls `super()` then warns on XPU). Import: `from raitap.utils.errors import BackendIncompatibilityError` (also re-exported from `raitap.robustness` and `raitap.transparency`). See {doc}`../capabilities` for what each capability means. - **`super().__init__()`.** Cooperative parent init: the base class allocates buffers the framework reads later (e.g. `self.attributions = None`). Forgetting raises `AttributeError` deep inside `explain()`. Always call it first when overriding `__init__`. -- **`self._lazy_import()`.** Inherited from `AdapterMixin`. Imports `library` (or `f"{library}.{submodule}"` if you pass `submodule=`) at call time, keeping `import raitap` cheap and letting users install RAITAP without every wrapped library. Raises a clear install-hint `ImportError` if the library is missing. +- **`self._lazy_import()`.** Inherited from `AdapterMixin`. Imports `import_name` (or `f"{import_name}.{submodule}"` if you pass `submodule=`) at call time, keeping `import raitap` cheap and letting users install RAITAP without every wrapped library. Raises a clear install-hint `ImportError` if the library is missing. - **Backend libs (`torch`, `torchvision`, `onnxruntime`) need `lazy_import` too.** If your adapter file uses `torch.Tensor` / `torch.nn` / etc, do NOT add `import torch` at module top-level. Use the `from raitap.utils.lazy import lazy_import` pattern (see that module's docstring for the `TYPE_CHECKING` + `lazy_import("torch")` recipe). This preserves the bootstrap-from-zero promise: `raitap.deps.bootstrap._compose` walks every adapter `__init__` on a bare venv (no torch installed yet) to infer extras before it installs them. A top-level `import torch` breaks the whole bootstrap. The per-family `tests/test_partial_extras_safe.py` poisons `torch` in `sys.modules` to catch regressions immediately. - **`self._rethrow()`.** Inherited context manager. Catches exceptions from the wrapped library and rewrites known-cryptic ones using your `error_patterns` map. - **Abstract methods.** `AttributionOnlyExplainer` → `compute_attributions(...)`. `EmpiricalAttackAssessor` → `_default_invoke(self, ctx: AttackInvokeCtx) -> Tensor` (framework dispatches via `generate_adversarial`; adapters implement only `_default_invoke`). `BaseMetricComputer` → `compute() -> MetricResult`. Check the base file for exact signatures. @@ -133,7 +133,7 @@ If the library already has an adapter and you just want to expose a new algorith A handful of Hydra shapes can't be expressed via the registration decorators alone and live as direct `ConfigStore` writes in `src/raitap/configs/zen.py::register_zen_groups`: - **`# @package _global_` injections.** E.g. the `reporting=html` / `reporting=pdf` entries push both the `reporting` node **and** a `hydra.callbacks.reporting_sweep` block into the root config so multirun report aggregation auto-wires. The decorators always write under `package=""` or `"."`; they can't reach `_global_`. -- **`_target_: null` variants.** E.g. `reporting=disabled` carries `_target_: null` + `multirun_report: false`. The decorators always target a concrete class. +- **`use: null` variants.** E.g. `reporting=disabled` carries `use: null` + `multirun_report: false`. The decorators always target a concrete class. - **Custom hydra-zen `to_config=` or non-dataclass nodes.** Same escape hatch. -If your new adapter only needs to set `_target_` + optional kwargs on its schema (the 95% case), don't touch `zen.py`. Use the decorator, done. If you genuinely need one of the special shapes above, add a `cs.store(group=..., name=..., package=..., node=...)` block in `register_zen_groups` after the `store.add_to_hydra_store(...)` flush: that order lets your specialisation overwrite the decorator-generated entry. +If your new adapter only needs to set `use:` + optional kwargs on its schema (the 95% case), don't touch `zen.py`. Use the decorator, done. If you genuinely need one of the special shapes above, add a `cs.store(group=..., name=..., package=..., node=...)` block in `register_zen_groups` after the `store.add_to_hydra_store(...)` flush: that order lets your specialisation overwrite the decorator-generated entry. diff --git a/docs/contributor/adding/adding-an-algorithm.md b/docs/contributor/adding/adding-an-algorithm.md index 37c39a33..4ea5b40e 100644 --- a/docs/contributor/adding/adding-an-algorithm.md +++ b/docs/contributor/adding/adding-an-algorithm.md @@ -27,7 +27,7 @@ from raitap import adapters @adapters.transparency( registry_name="captum", - library="captum", + import_name="captum", algorithm_registry={ "IntegratedGradients": ExplainerAlgorithmSpec( {MethodFamily.GRADIENT}, @@ -52,7 +52,7 @@ from raitap import adapters @adapters.robustness( registry_name="torchattacks", - library="torchattacks", + import_name="torchattacks", algorithm_registry={ # ... existing entries ... "NewAttack": AssessorAlgorithmSpec( diff --git a/docs/contributor/adding/adding-an-input-parser.md b/docs/contributor/adding/adding-an-input-parser.md index dd815cd2..7f7485f8 100644 --- a/docs/contributor/adding/adding-an-input-parser.md +++ b/docs/contributor/adding/adding-an-input-parser.md @@ -56,14 +56,14 @@ In `src/raitap/configs/schema.py`, subclass `InputsConfig`: ```python @dataclass class PdfInputsConfig(InputsConfig): - _target_: str = "PdfInputParser" + use: str = "pdf" # add only the format fields this variant actually uses, e.g. a page # range. Never add a ``source`` field here — the source path is always # ``data.source``, passed into ``parse(source=...)`` at call time. ``` -`_target_` must match the class name from step 2 (resolved against -`raitap.data.input_parsers.` by the factory, see step 4). +`use` must match the `registry_name` from step 2 (resolved through the closed +registry the `@input_parser` decorator populates, see step 4). ## 2. Write the parser class @@ -135,8 +135,9 @@ data: ``` `create_input_parser` (`src/raitap/data/input_parsers/factory.py`) resolves -`_target_` against `raitap.data.input_parsers.` at call time, the same -dispatch `create_label_parser` uses for `data/labels`. +the `use` key against the closed registry (`raitap.configs.registry_resolve. +resolve_target_fqn`) at call time, the same dispatch `create_label_parser` +uses for `data/labels`. ## 5. Add tests diff --git a/docs/contributor/architecture.md b/docs/contributor/architecture.md index 1d16e5a4..445d8653 100644 --- a/docs/contributor/architecture.md +++ b/docs/contributor/architecture.md @@ -68,14 +68,15 @@ src/ │ ├── searchpath.py # RaitapSearchPathPlugin impl (appends pkg://raitap.configs) │ ├── utils.py # ConfigStore registration of `raitap_schema`; resolve_run_dir helpers │ ├── zen.py # `register_zen_groups()`: hydra-zen ConfigStore writes for shapes - │ │ # the decorator can't express (`_target_: null`, `# @package _global_`) - │ ├── adapter_factory.py # short-name → full-target resolution (HTMLReporter, CaptumExplainer, …) + │ │ # the decorator can't express (`use: null`, `# @package _global_`) + │ ├── registry_resolve.py # trusted `use: ` → class FQN resolver; rejects `_target_` + │ ├── adapter_factory.py # shared `use:`-key parsing/validation for transparency & robustness configs │ ├── demo.yaml # self-contained demo invoked by `raitap --demo` - │ ├── reporting/{html,pdf,disabled}.yaml - │ ├── transparency/{captum,shap}.yaml # `_target_`-only stubs, @package-nested per library - │ ├── robustness/{torchattacks,foolbox,marabou}.yaml - │ ├── metrics/classification.yaml - │ └── tracking/mlflow.yaml + │ └── hydra/launcher/ # bundled launcher presets (e.g. `default_submitit_slurm.yaml`) + │ # No per-adapter YAML stubs live here anymore: every `+=` + │ # preset (`+transparency=captum`, `+reporting=html`, `+tracking=mlflow`, ...) + │ # is generated by `_register_core`'s `store(...)` call at decoration time. + │ # `zen.py` only adds the handful of shapes decorators can't express. │ ├── deps/ # pre-pipeline dep inference + auto-sync (torch-free) │ │ # The "torch-free" promise is what unlocks bootstrap-from-zero @@ -84,7 +85,7 @@ src/ │ │ # backend-lib imports — see `raitap.utils.lazy` for the contract │ │ # and `deps/tests/test_bootstrap_from_zero.py` for the guard. │ ├── bootstrap.py # maybe_bootstrap(): top-level flow + case A/B/C/D dispatch - │ ├── inference.py # walks composed config, picks extras from `_target_` mapping + │ ├── inference.py # walks composed config, resolves each `use:` key to its uv extra │ ├── availability.py # reads raitap pyproject for declared extras + platform markers │ ├── conflicts.py # enforces tool.uv.conflicts groups (torch-cpu vs torch-cuda, …) │ ├── probe.py # host probe → cpu / cuda / xpu diff --git a/docs/contributor/logging.md b/docs/contributor/logging.md index 1ab97075..bf0b8bca 100644 --- a/docs/contributor/logging.md +++ b/docs/contributor/logging.md @@ -56,7 +56,7 @@ raise ValueError( ) ``` -For wrapped third-party calls (captum / shap / foolbox / torchattacks), use the adapter's `self._rethrow()` helper. It pulls `library`, the family group, and the `error_patterns` map straight from the adapter's `@adapters.(...)` decoration — no kwargs needed at the call site: +For wrapped third-party calls (captum / shap / foolbox / torchattacks), use the adapter's `self._rethrow()` helper. It pulls `import_name`, the family group, and the `error_patterns` map straight from the adapter's `@adapters.(...)` decoration — no kwargs needed at the call site: ```python # src/raitap/transparency/explainers/shap_explainer.py @@ -64,7 +64,7 @@ from raitap import adapters @adapters.transparency( registry_name="shap", - library="shap", + import_name="shap", error_patterns={ r"BackwardHookFunctionBackward is a view": ( "DeepExplainer can fail on PyTorch models that use SiLU " @@ -108,7 +108,7 @@ from raitap import adapters @adapters.transparency( registry_name="captum", - library="captum", + import_name="captum", suppress_warnings=[ # Captum emits this on every run when inputs don't already require # gradients. Auto-fixes the issue → pure noise. Scope module=captum @@ -125,7 +125,7 @@ Always scope `module=` to the wrapped library so unrelated UserWarnings with the ## Where the infrastructure lives - `src/raitap/utils/log.py` — `_RaitapLog` class + `raitap_log` singleton. Owns the thread-local diagnostic queue bridging `warnings.formatwarning` to the rich handler. -- `src/raitap/utils/diagnostics.py` — `Module` enum + frame-walking classifier + third-party library detection. The library set is auto-populated by `_register_core` from each `@register_*_adapter(..., library="...")` decoration and stored at `raitap._adapters.THIRD_PARTY_LIBS` (grouped by adapter family). +- `src/raitap/utils/diagnostics.py` — `Module` enum + frame-walking classifier + third-party library detection. The library set is auto-populated by `_register_core` from each `@register_*_adapter(..., import_name="...")` decoration and stored at `raitap._adapters.THIRD_PARTY_LIBS` (grouped by adapter family). - `src/raitap/utils/errors.py` — `RaitapError` / `AdapterError`, traceback-walking diagnostic resolver, `rethrow` context manager. - `src/raitap/utils/colour.py` — two-shade palette + Rich `Theme`. Edit here when adding/rebalancing colours. - `src/raitap/utils/console.py` — `RichHandler` subclass + `print_failure_panel`. Calls `logging.captureWarnings(True)` so external sinks see warnings. diff --git a/docs/contributor/modules/data.md b/docs/contributor/modules/data.md index b18093f2..db70bb66 100644 --- a/docs/contributor/modules/data.md +++ b/docs/contributor/modules/data.md @@ -82,7 +82,7 @@ referenceable by name in `data.source`. Registration lives in ```python @dataclass class MyFormatLabelsConfig(LabelsConfig): - _target_: str = "MyFormatLabelParser" + use: str = "my_format" source: str = MISSING id_strategy: IdStrategy = IdStrategy.auto # add only fields this variant uses diff --git a/docs/contributor/modules/metrics.md b/docs/contributor/modules/metrics.md index ba865403..eefbdf7c 100644 --- a/docs/contributor/modules/metrics.md +++ b/docs/contributor/modules/metrics.md @@ -8,7 +8,7 @@ myst: # Contributing to the metrics module -Metric computers wrap evaluation libraries (torchmetrics, faster-coco-eval, ...) behind a unified interface driven by Hydra `_target_` instantiation. +Metric computers wrap evaluation libraries (torchmetrics, faster-coco-eval, ...) behind a unified interface, selected via a `use: ` config key resolved to the adapter class through the trusted registry (`raitap.configs.registry_resolve`). ## `BaseMetricComputer` interface diff --git a/docs/contributor/modules/tracking.md b/docs/contributor/modules/tracking.md index 14ce61a3..b807d5cf 100644 --- a/docs/contributor/modules/tracking.md +++ b/docs/contributor/modules/tracking.md @@ -8,7 +8,7 @@ myst: # Contributing to the tracking module -Tracking is a backend plugin selected via the top-level `tracking._target_` field. The current built-in backend is `MLFlowTracker`. +Tracking is a backend plugin selected via the top-level `tracking.use` field. The current built-in backend is `MLFlowTracker` (`use: mlflow`). ## `BaseTracker` interface diff --git a/docs/contributor/modules/transparency.md b/docs/contributor/modules/transparency.md index f54299b5..dac65a72 100644 --- a/docs/contributor/modules/transparency.md +++ b/docs/contributor/modules/transparency.md @@ -306,9 +306,9 @@ library exceptions carry the raitap adapter-family context. `@transparency_evaluator` (`evaluators/registration.py`) registers with `family=None`, the same pattern `@visualisers.transparency` uses: no Hydra -config group, no schema dataclass, resolved purely by `_target_` nested under -`transparency..evaluation`. `extra` and `library` must be passed -explicitly (`extra="quantus"`, `library="quantus"`) since there is no family +config group, no schema dataclass, resolved purely by `use:` nested under +`transparency..evaluation`. `extra` and `import_name` must be passed +explicitly (`extra="quantus"`, `import_name="quantus"`) since there is no family default to fall back on. `QuantusEvaluator.__init__` takes `call` and `raitap` via `**kwargs` rather @@ -422,4 +422,4 @@ builds one `EvaluationContext` per `ExplanationResult`, and calls - `src/raitap/transparency/evaluation/step.py`: `grade_explanations`, the transparency-phase post-step. - `src/raitap/transparency/evaluation/evaluators/quantus_evaluator.py`: `QuantusEvaluator` + the metric `_REGISTRY`. -**Name resolution.** Bare class names in YAML `_target_` keys (e.g. `_target_: CaptumExplainer`) are resolved through the `@adapters.transparency` / `@visualisers.transparency` decorators and `raitap._adapters.lookup("transparency", name)`, not via the legacy class-kwarg path. To make a new class addressable by bare name, decorate it; that's the only requirement. +**Name resolution.** YAML `use: ` keys (e.g. `use: captum`) are resolved through the `@adapters.transparency` / `@visualisers.transparency` decorators, which populate the closed registry `raitap._adapters._TARGET_FQN`; `raitap.configs.registry_resolve.resolve_target_fqn` looks the key up against it. To make a new class addressable by `use:`, decorate it; that's the only requirement. diff --git a/docs/contributor/writing-a-plugin.md b/docs/contributor/writing-a-plugin.md index a864b188..38936391 100644 --- a/docs/contributor/writing-a-plugin.md +++ b/docs/contributor/writing-a-plugin.md @@ -15,6 +15,18 @@ library like any 1st party RAITAP adapter. In the following guide, we will imagine you want to make your "SuperXAI" library usable to RAITAP users seamlessly. +## Runnable reference + +`example-plugin/` in the RAITAP repo is a real, installable plugin following every step below. Copy +it as a starting point instead of typing the SuperXAI snippets from scratch: + +```bash +uv pip install ./example-plugin +python -c "from raitap.robustness import identity_attack; print(identity_attack)" +``` + +See `example-plugin/README.md` for install, discovery, and `use:` usage (YAML and Python). + ## Supported modules Plugins can register: @@ -66,7 +78,7 @@ if TYPE_CHECKING: @adapters.transparency( registry_name="superxai", # CLI `+transparency=superxai` / Python `from raitap.transparency import superxai` - library="superxai-lib", # real name of your PyPI package; drives `self._lazy_import()` (defaults to registry_name) + import_name="superxai", # importable module name of the wrapped library; drives `self._lazy_import()` error_patterns={ # rewrite cryptic upstream errors at call sites r"some library footgun": "Do X instead.", # nicer error messages to avoid deep stack traces in RAITAP }, @@ -102,7 +114,7 @@ class SuperXAIExplainer(AttributionOnlyExplainer): ) ``` -Decorator kwargs (`library`, `algorithm_registry`, `error_patterns`, +Decorator kwargs (`import_name`, `algorithm_registry`, `error_patterns`, `suppress_warnings`, ...) are documented in {doc}`adding/adding-an-adapter`. `AdapterDecoratorOptions` is exported for typing, in case you want additional custom logic on top of diff --git a/docs/modules/data/configuration.md b/docs/modules/data/configuration.md index d918c58a..6cdfaddb 100644 --- a/docs/modules/data/configuration.md +++ b/docs/modules/data/configuration.md @@ -122,7 +122,7 @@ data = DataConfig( ) ``` -**Label variants.** `data.labels` is a Hydra config-group: select the variant with `defaults: [data/labels: ]`, then set its fields under `data.labels:`. Each variant exposes only the fields it accepts — setting a foreign field is a load error. Prefer the `defaults` group: it validates fields at config-load with a clear error. Inlining `_target_` directly (e.g. `data.labels: {_target_: TabularLabelParser, bogus: 1}`) skips that load-time struct check, so a foreign field is not caught until the parser is built and then fails with a less obvious instantiation error. +**Label variants.** `data.labels` is a Hydra config-group: select the variant with `defaults: [data/labels: ]`, then set its fields under `data.labels:`. Each variant exposes only the fields it accepts — setting a foreign field is a load error. Prefer the `defaults` group: it validates fields at config-load with a clear error. Inlining `use` directly (e.g. `data.labels: {use: tabular, bogus: 1}`) skips that load-time struct check, so a foreign field is not caught until the parser is built and then fails with a less obvious instantiation error. ```yaml defaults: diff --git a/docs/modules/metrics/configuration.md b/docs/modules/metrics/configuration.md index c0092ce6..1803250c 100644 --- a/docs/modules/metrics/configuration.md +++ b/docs/modules/metrics/configuration.md @@ -8,7 +8,7 @@ myst: # Configuration -The `metrics` block scores model predictions. The `_target_` field is the +The `metrics` block scores model predictions. The `use` field is the discriminator: it selects one of four adapters, each with its own set of valid keys. @@ -22,7 +22,7 @@ keys. related summaries via torchmetrics). The previous unified `ClassificationMetrics` target with a `task: binary | multiclass | multilabel` -field has been removed. Pick the task-specific adapter directly via `_target_`; +field has been removed. Pick the task-specific adapter directly via `use`; each adapter only accepts the keys documented for its section below. See {doc}`frameworks-and-libraries` for the backend behaviour behind each @@ -34,9 +34,9 @@ adapter. :slug: binary :intro: Configures `BinaryClassificationMetrics` for two-class problems. -:option: _target_ -:allowed: "BinaryClassificationMetrics" -:default: "BinaryClassificationMetrics" +:option: use +:allowed: "binary_classification" +:default: "binary_classification" :description: Selects the binary-classification adapter. :option: ignore_index @@ -52,7 +52,7 @@ adapter. :yaml: metrics: - _target_: "BinaryClassificationMetrics" + use: binary_classification :cli: +metrics=binary_classification +metrics.threshold=0.6 @@ -69,9 +69,9 @@ metrics = binary_classification() :intro: Configures `MulticlassClassificationMetrics` for single-label multiclass problems. -:option: _target_ -:allowed: "MulticlassClassificationMetrics" -:default: "MulticlassClassificationMetrics" +:option: use +:allowed: "multiclass_classification" +:default: "multiclass_classification" :description: Selects the multiclass-classification adapter. :option: num_classes @@ -92,7 +92,7 @@ metrics = binary_classification() :yaml: metrics: - _target_: "MulticlassClassificationMetrics" + use: multiclass_classification num_classes: 7 :cli: +metrics=multiclass_classification +metrics.num_classes=7 @@ -110,9 +110,9 @@ metrics = multiclass_classification(num_classes=7) :intro: Configures `MultilabelClassificationMetrics` for multilabel problems (independent per-label decisions). -:option: _target_ -:allowed: "MultilabelClassificationMetrics" -:default: "MultilabelClassificationMetrics" +:option: use +:allowed: "multilabel_classification" +:default: "multilabel_classification" :description: Selects the multilabel-classification adapter. :option: num_labels @@ -138,7 +138,7 @@ metrics = multiclass_classification(num_classes=7) :yaml: metrics: - _target_: "MultilabelClassificationMetrics" + use: multilabel_classification num_labels: 5 :cli: +metrics=multilabel_classification +metrics.num_labels=5 @@ -157,9 +157,9 @@ metrics = multilabel_classification(num_labels=5) `MeanAveragePrecision`. The IoU-related knobs are grouped under the nested `iou:` block. -:option: _target_ -:allowed: "DetectionMetrics" -:default: "DetectionMetrics" +:option: use +:allowed: "detection" +:default: "detection" :description: Selects the object-detection adapter. :option: box_format @@ -216,7 +216,7 @@ metrics = multilabel_classification(num_labels=5) :yaml: metrics: - _target_: "DetectionMetrics" + use: detection iou: thresholds: [0.5, 0.75] class_metrics: true diff --git a/docs/modules/metrics/frameworks-and-libraries.md b/docs/modules/metrics/frameworks-and-libraries.md index a0363590..57d31502 100644 --- a/docs/modules/metrics/frameworks-and-libraries.md +++ b/docs/modules/metrics/frameworks-and-libraries.md @@ -16,6 +16,16 @@ The current metrics module relies on: To tweak specific options via the RAITAP config, you might need to refer to the underlying library's documentation. +## Adapter reference + +Every `use:` key registered for the metrics module, generated from the +adapter registry. + +```{include} ../_generated_adapters.md +:start-after: +:end-before: +``` + ## Classification metrics `BinaryClassificationMetrics`, `MulticlassClassificationMetrics`, and `MultilabelClassificationMetrics` are the task-specific adapters for classification models. They each wrap the following TorchMetrics classes (instantiated for the matching task): diff --git a/docs/modules/reporting/configuration.md b/docs/modules/reporting/configuration.md index 86021985..427fd7a8 100644 --- a/docs/modules/reporting/configuration.md +++ b/docs/modules/reporting/configuration.md @@ -9,11 +9,11 @@ myst: ```{config-page} :intro: This page describes how to configure the reporting module that generates reports from pipeline outputs. -:option: _target_ -:allowed: "HTMLReporter", "PDFReporter", null +:option: use +:allowed: "html", "pdf", null :default: null -:description: Hydra target for the reporting backend implementation. Set to null to disable reporting. - The default `reporting=html` config selects `HTMLReporter`; +:description: Selects the reporting backend implementation. Set to null to disable reporting. + The default `reporting=html` config selects `use: html` (`HTMLReporter`); use `reporting=pdf` for the PDF renderer. :option: filename @@ -98,7 +98,7 @@ myst: :yaml: reporting: - _target_: "HTMLReporter" + use: html filename: "experiment_report" sample_selection: - "ISIC_0024306.jpg" diff --git a/docs/modules/reporting/index.md b/docs/modules/reporting/index.md index 9af15dc6..2b076431 100644 --- a/docs/modules/reporting/index.md +++ b/docs/modules/reporting/index.md @@ -12,6 +12,16 @@ The reporting module generates compact reports from pipeline outputs. Reports summarize key metrics, transparency results, and robustness findings for review and sharing. +## Adapter reference + +Every `use:` key registered for the reporting module, generated from the +adapter registry. + +```{include} ../_generated_adapters.md +:start-after: +:end-before: +``` + ```{toctree} :maxdepth: 1 :caption: Reporting module documentation diff --git a/docs/modules/robustness/configuration.md b/docs/modules/robustness/configuration.md index 632e9106..f412dcde 100644 --- a/docs/modules/robustness/configuration.md +++ b/docs/modules/robustness/configuration.md @@ -14,12 +14,12 @@ myst: See {ref}`modules-robustness-configuration-examples` for the config shape. See {doc}`frameworks-and-libraries` for the backend behaviour behind - `_target_`, `algorithm`, and visualiser compatibility. + `use`, `algorithm`, and visualiser compatibility. -:option: _target_ -:allowed: "TorchattacksAssessor", "FoolboxAssessor", "ImageCorruptionsAssessor" +:option: use +:allowed: "torchattacks", "foolbox", "imagecorruptions" :default: null -:description: Hydra target for the assessor class. +:description: Selects the assessor implementation. :option: algorithm :allowed: See {doc}`frameworks-and-libraries` @@ -111,9 +111,9 @@ myst: :option: visualisers :allowed: list[dict] -:default: [ImagePairVisualiser] +:default: [{use: image_pair}] :description: Visualiser definitions. Each entry must include at least - `_target_`. Each visualiser can also define its own `constructor` and `call` + `use`. Each visualiser can also define its own `constructor` and `call` blocks. Visualisers declare which `AssessmentKind` (`empirical_attack` / `formal_verification` / `statistical_sampling`) they support; the factory rejects mismatches at parse time. @@ -121,16 +121,16 @@ myst: :yaml: robustness: pgd: - _target_: "TorchattacksAssessor" + use: torchattacks algorithm: "PGD" constructor: eps: 0.03 alpha: 0.0078 steps: 10 visualisers: - - _target_: "ImagePairVisualiser" + - use: image_pair linf_pgd: - _target_: "FoolboxAssessor" + use: foolbox algorithm: "LinfPGD" constructor: rel_stepsize: 0.025 @@ -138,10 +138,10 @@ robustness: call: eps: 0.03 visualisers: - - _target_: "ImagePairVisualiser" - - _target_: "PerturbationHeatmapVisualiser" + - use: image_pair + - use: perturbation_heatmap avg: - _target_: "ImageCorruptionsAssessor" + use: imagecorruptions algorithm: "gaussian_noise" # one of the 15 ImageNet-C corruptions constructor: severity: 3 # 1..5 @@ -149,7 +149,7 @@ robustness: ci_method: "wilson" # or clopper_pearson ci_level: 0.95 visualisers: - - _target_: "CorruptionAccuracyVisualiser" + - use: corruption_accuracy :cli: +robustness=torchattacks robustness.torchattacks.algorithm=PGD robustness.torchattacks.constructor.eps=0.05 diff --git a/docs/modules/robustness/frameworks-and-libraries.md b/docs/modules/robustness/frameworks-and-libraries.md index e5305156..ff251de6 100644 --- a/docs/modules/robustness/frameworks-and-libraries.md +++ b/docs/modules/robustness/frameworks-and-libraries.md @@ -8,6 +8,16 @@ myst: # Supported libraries +## Adapter reference + +Every `use:` key registered for the robustness module, generated from the +adapter registry. + +```{include} ../_generated_adapters.md +:start-after: +:end-before: +``` + ## `constructor`, `call`, and `raitap` keys Assessors support three config buckets: diff --git a/docs/modules/robustness/visualisers.md b/docs/modules/robustness/visualisers.md index 20e2950d..e97106ab 100644 --- a/docs/modules/robustness/visualisers.md +++ b/docs/modules/robustness/visualisers.md @@ -14,10 +14,10 @@ Each visualiser renders one figure per call. All of them are declared per-assess :yaml: robustness: marabou: - _target_: "MarabouAssessor" + use: marabou visualisers: - - _target_: "OutputBoundsCohortVisualiser" - - _target_: "OutputBoundsMarginHeatmapVisualiser" + - use: output_bounds_cohort + - use: output_bounds_margin_heatmap :python: from raitap.robustness import marabou, output_bounds_cohort, output_bounds_margin_heatmap diff --git a/docs/modules/tracking/configuration.md b/docs/modules/tracking/configuration.md index 31926e84..2c0ab7d5 100644 --- a/docs/modules/tracking/configuration.md +++ b/docs/modules/tracking/configuration.md @@ -9,10 +9,10 @@ myst: ```{config-page} :intro: This page describes how to configure the tracking module that forwards the pipeline's output to a tracking backend. -:option: _target_ -:allowed: "MLFlowTracker" -:default: "MLFlowTracker" -:description: Hydra target for the tracking backend implementation. +:option: use +:allowed: "mlflow" +:default: "mlflow" +:description: Selects the tracking backend implementation (`mlflow` -> `MLFlowTracker`). :option: output_forwarding_url :allowed: string, null @@ -53,7 +53,7 @@ myst: :yaml: tracking: - _target_: "MLFlowTracker" + use: mlflow output_forwarding_url: "http://127.0.0.1:5001" log_model: true diff --git a/docs/modules/tracking/frameworks-and-libraries.md b/docs/modules/tracking/frameworks-and-libraries.md index 7fc65760..e5c432b7 100644 --- a/docs/modules/tracking/frameworks-and-libraries.md +++ b/docs/modules/tracking/frameworks-and-libraries.md @@ -34,7 +34,7 @@ uv run raitap tracking stop ```{config-tabs} :yaml: tracking: - _target_: MLFlowTracker + use: mlflow output_forwarding_url: http://127.0.0.1:5001 log_model: true open_when_done: false diff --git a/docs/modules/transparency/configuration.md b/docs/modules/transparency/configuration.md index 7f51455b..2e55f923 100644 --- a/docs/modules/transparency/configuration.md +++ b/docs/modules/transparency/configuration.md @@ -13,7 +13,7 @@ myst: Inside the `transparency` key, you can configure one or more explainers. See {ref}`modules-transparency-configuration-examples` for the config shape. See {doc}`frameworks-and-libraries` for the backend behaviour behind - `_target_`, `algorithm`, and visualiser compatibility. + `use`, `algorithm`, and visualiser compatibility. RAITAP has two separate batch-size controls because predictions and explanations are different workload stages: @@ -25,10 +25,10 @@ myst: Occlusion often need a much smaller attribution batch size than the prediction batch size. -:option: _target_ -:allowed: "CaptumExplainer", "ShapExplainer" +:option: use +:allowed: "captum", "shap" :default: null -:description: Hydra target for the explainer class. +:description: Selects the explainer implementation. :option: algorithm :allowed: See {doc}`frameworks-and-libraries` @@ -130,7 +130,7 @@ myst: :allowed: list[dict] :default: [] :description: Visualiser definitions. Each entry must include at least - `_target_`. Each visualiser can also define its own `constructor` and `call` + `use`. Each visualiser can also define its own `constructor` and `call` blocks. Use `visualisers[].call.show_sample_names` for per-visualiser sample name overrides; use `raitap.show_sample_names` for the shared explainer-level default. @@ -141,7 +141,7 @@ myst: :description: Optional Quantus-backed explanation-quality grading for this explainer. Requires the `quantus` extra (`uv sync --extra quantus`); it is not pulled in by the `transparency` umbrella extra. Set - `evaluation._target_: raitap.transparency.QuantusEvaluator` to enable it. + `evaluation.use: quantus` to enable it. Grading runs as a post-step after the explainer produces its attributions; scores land on `TransparencyPhaseResult.evaluations`. See {doc}`output`. @@ -184,7 +184,7 @@ myst: :yaml: transparency: my_first_explainer: - _target_: "CaptumExplainer" + use: captum algorithm: "IntegratedGradients" call: target: 0 @@ -195,16 +195,16 @@ transparency: kind: image layout: NCHW visualisers: - - _target_: "CaptumImageVisualiser" + - use: captum_image call: max_samples: 1 evaluation: - _target_: "QuantusEvaluator" + use: quantus metrics: [faithfulness_correlation, sparseness] constructor: faithfulness_correlation: {nr_runs: 10, subset_size: 32} my_second_explainer: - _target_: "ShapExplainer" + use: shap algorithm: "KernelExplainer" call: target: 0 @@ -218,7 +218,7 @@ transparency: layout: "(B,F)" feature_names: [age, income, score] visualisers: - - _target_: "ShapBarVisualiser" + - use: shap_bar :cli: transparency.captum_ig.algorithm=GradientShap diff --git a/docs/modules/transparency/detection.md b/docs/modules/transparency/detection.md index 53fe14d4..b85a0af9 100644 --- a/docs/modules/transparency/detection.md +++ b/docs/modules/transparency/detection.md @@ -49,7 +49,7 @@ myst: :yaml: transparency: my_ig_explainer: - _target_: "CaptumExplainer" + use: captum algorithm: "IntegratedGradients" call: target: 0 @@ -59,7 +59,7 @@ transparency: max_boxes: 5 iou_threshold: 0.5 visualisers: - - _target_: "DetectionImageVisualiser" + - use: detection_image :cli: transparency.my_ig_explainer.raitap.detection.max_boxes=3 diff --git a/docs/modules/transparency/frameworks-and-libraries.md b/docs/modules/transparency/frameworks-and-libraries.md index b785127a..a0b2cbdc 100644 --- a/docs/modules/transparency/frameworks-and-libraries.md +++ b/docs/modules/transparency/frameworks-and-libraries.md @@ -8,6 +8,16 @@ myst: # Supported libraries +## Adapter reference + +Every `use:` key registered for the transparency module, generated from the +adapter registry. + +```{include} ../_generated_adapters.md +:start-after: +:end-before: +``` + ## `constructor`, `call`, and `raitap` keys Explainers support three config buckets: @@ -24,7 +34,7 @@ This keeps the boundary clear for users: `call` is what Captum or SHAP sees, whi :yaml: transparency: my_first_explainer: - _target_: "ShapExplainer" + use: shap algorithm: "GradientExplainer" constructor: local_smoothing: 0.0 @@ -35,7 +45,7 @@ transparency: source: imagenet_samples batch_size: 1 visualisers: - - _target_: "ShapImageVisualiser" + - use: shap_image call: max_samples: 1 @@ -101,7 +111,7 @@ documented in {doc}`../../contributor/modules/transparency`. :yaml: transparency: my_captum_explainer: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients constructor: {} call: @@ -147,7 +157,7 @@ wrapped method with `constructor.base_algorithm`; pass the noise knobs under :yaml: transparency: my_smoothgrad: - _target_: CaptumExplainer + use: captum algorithm: NoiseTunnel constructor: base_algorithm: IntegratedGradients @@ -203,7 +213,7 @@ validated. :yaml: transparency: my_shap_explainer: - _target_: ShapExplainer + use: shap algorithm: GradientExplainer constructor: {} call: @@ -253,15 +263,15 @@ Tabular SHAP values from `TreeExplainer` route to the tabular visualisers (`Shap :yaml: transparency: my_tree_explainer: - _target_: ShapExplainer + use: shap algorithm: TreeExplainer raitap: input_metadata: kind: tabular feature_names: [feature_a, feature_b, feature_c] visualisers: - - _target_: ShapBarVisualiser - - _target_: ShapBeeswarmVisualiser + - use: shap_bar + - use: shap_beeswarm :python: from raitap.transparency import shap, shap_bar, shap_beeswarm @@ -363,7 +373,7 @@ differently. :yaml: transparency: my_shap_explainer: - _target_: "ShapExplainer" + use: shap algorithm: "GradientExplainer" constructor: local_smoothing: 0.0 @@ -376,12 +386,12 @@ transparency: batch_size: 1 visualisers: # Minimal configuration - - _target_: "ShapImageVisualiser" + - use: shap_image constructor: max_samples: 1 # Full configuration with all options - - _target_: "ShapImageVisualiser" + - use: shap_image constructor: max_samples: 2 title: "Tumour attribution" diff --git a/docs/modules/transparency/visualisers.md b/docs/modules/transparency/visualisers.md index 29898d47..e9fe349e 100644 --- a/docs/modules/transparency/visualisers.md +++ b/docs/modules/transparency/visualisers.md @@ -14,11 +14,11 @@ Each visualiser renders one figure per call. All of them are declared per-explai :yaml: transparency: captum_ig: - _target_: "CaptumExplainer" + use: captum algorithm: "IntegratedGradients" visualisers: - - _target_: "CaptumImageVisualiser" - - _target_: "TabularBarChartVisualiser" + - use: captum_image + - use: tabular_bar_chart :python: from raitap.transparency import captum, captum_image, tabular_bar_chart @@ -207,14 +207,14 @@ the factory rejects mismatches at YAML parse time. See ```yaml transparency: my_layer_conductance: - _target_: CaptumExplainer + use: captum algorithm: LayerConductance constructor: layer_path: layer4.2.conv3 # dotted path to the target layer call: target: 0 visualisers: - - _target_: LayerActivationVisualiser + - use: layer_activation ``` ::::: @@ -228,13 +228,13 @@ transparency: ```yaml transparency: ig: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: 0 return_convergence_delta: true # emits the convergence_delta payload visualisers: - - _target_: StructuredPayloadSummaryVisualiser + - use: structured_payload_summary ``` ::::: @@ -276,7 +276,7 @@ transparency: ```yaml transparency: my_ig_explainer: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: 0 # required — wrapper exposes one scalar channel @@ -286,7 +286,7 @@ transparency: max_boxes: 5 # default; cap K per sample iou_threshold: 0.5 # default; used by reference_match target visualisers: - - _target_: DetectionImageVisualiser + - use: detection_image # all optional; omit for the default figure. Allowed values per kwarg # are in the table above. method: blended_heat_map diff --git a/docs/using-raitap/configuration/general.md b/docs/using-raitap/configuration/general.md index 0b718840..b0c21b4b 100644 --- a/docs/using-raitap/configuration/general.md +++ b/docs/using-raitap/configuration/general.md @@ -38,6 +38,32 @@ config = AppConfig() # includes the schema # ...your options on `config`, see below ``` +#### Editor autocomplete for `use:` keys + +Generate a JSON Schema from your installed RAITAP (including any plugins) and +point your editor at it for `use:` autocomplete + validation: + +```{install-tabs} +:uv: +uv run raitap config-schema -o raitap.schema.json + +:pip: +raitap config-schema -o raitap.schema.json +``` + +Add this line at the top of your config YAML (most editors with the YAML +Language Server extension pick it up automatically): + +```yaml +# yaml-language-server: $schema=./raitap.schema.json +defaults: + - raitap_schema + - _self_ +``` + +Re-run `config-schema` after installing or removing a plugin so the schema's +`use:` enums stay in sync. + Then, you can add your own options. You may find useful to refer to: - the {doc}`global-config-options` @@ -126,14 +152,14 @@ metrics: robustness: pgd: - _target_: TorchattacksAssessor + use: torchattacks algorithm: PGD constructor: eps: 0.03 alpha: 0.005 steps: 10 visualisers: - - _target_: ImagePairVisualiser + - use: image_pair :python: from raitap import AppConfig @@ -176,10 +202,10 @@ Or override all nested values at once: ```{install-tabs} :uv: -uv run raitap --config-name assessment "transparency.captum_saliency.visualisers=[{_target_: CaptumImageVisualiser, call: {show_sample_names: true}}]" +uv run raitap --config-name assessment "transparency.captum_saliency.visualisers=[{use: captum_image, call: {show_sample_names: true}}]" :pip: -raitap --config-name assessment "transparency.captum_saliency.visualisers=[{_target_: CaptumImageVisualiser, call: {show_sample_names: true}}]" +raitap --config-name assessment "transparency.captum_saliency.visualisers=[{use: captum_image, call: {show_sample_names: true}}]" ``` (composing-yaml-files)= @@ -203,7 +229,7 @@ defaults: experiment_name: "my-exp" hardware: cpu -# Bundled `transparency/shap.yaml` only sets `_target_: ShapExplainer` and +# Bundled `transparency/shap.yaml` only sets `use: shap` and # nests it under `transparency.shap`. The explainer's required fields # (`algorithm`, `call`, `visualisers`) still need to be supplied here: transparency: @@ -212,7 +238,7 @@ transparency: call: target: 0 visualisers: - - _target_: ShapImageVisualiser + - use: shap_image # Inline model + data — RAITAP does not ship `data=` or `model=` presets, so # define them in your own config (or reference your own group files). @@ -273,7 +299,7 @@ Hydra can execute multiple runs from a single command using `--multirun`. This is useful when you want to compare several presets or override values in one go. The bundled `+transparency=captum` / `+transparency=shap` stubs only set -`_target_` and nest under `transparency.captum` / `transparency.shap`; the +`use` and nest under `transparency.captum` / `transparency.shap`; the sweep below pairs each with the matching `algorithm` override. ```{install-tabs} diff --git a/docs/using-raitap/configuration/python-api.md b/docs/using-raitap/configuration/python-api.md index e701f7a2..423939bf 100644 --- a/docs/using-raitap/configuration/python-api.md +++ b/docs/using-raitap/configuration/python-api.md @@ -80,7 +80,7 @@ run(cfg, auto_install_deps=True) | YAML pattern | Python builder | | ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `_target_: CaptumExplainer` + `algorithm: IntegratedGradients` | `captum(algorithm="IntegratedGradients", ...)` (the `_target_` is baked in) | +| `use: captum` + `algorithm: IntegratedGradients` | `captum(algorithm="IntegratedGradients", ...)` (the `use` key is baked in) | | `defaults: [raitap_schema, _self_]` | Not needed — `AppConfig` already *is* the schema. The defaults entry is a Hydra-only construct. | | Group/name selection (`transparency: captum` + dict key in YAML) | Use the dict key on the Python side too: `transparency={"my_run": captum(algorithm=...)}`. | | List of visualisers | One builder per visualiser (`captum_image`, `image_pair`, …): flat constructor kwargs, optional `call={...}` for render-time options. `visualisers=[captum_image(max_samples=4, call={"show_sample_names": True})]`. | @@ -128,7 +128,7 @@ Library-forwarded kwargs (unchecked at schema time): | `outputs.transparency` | `list[ExplanationResult]` | one per explainer; `[]` if not run. | | `outputs.robustness` | `list[RobustnessResult]` | one per assessor; `[]` if not run. | -Every per-adapter result (`ExplanationResult`, `RobustnessResult`) shares one envelope — the `AdapterResult` contract: `.name` (the config key, e.g. `"ig"`), `.adapter_target` (the `_target_` class), `.algorithm`, `.semantics`, `.run_dir`, and `.visualisations` (the figures that result owns) — plus its own domain payload (`.attributions` / `.verdicts` / …). +Every per-adapter result (`ExplanationResult`, `RobustnessResult`) shares one envelope — the `AdapterResult` contract: `.name` (the config key, e.g. `"ig"`), `.adapter_target` (the resolved adapter class FQN), `.algorithm`, `.semantics`, `.run_dir`, and `.visualisations` (the figures that result owns) — plus its own domain payload (`.attributions` / `.verdicts` / …). **Mapping access** reaches the underlying `PhaseResult` wrapper for any (incl. future) phase: `outputs.get(name)`, `outputs[name]`, `name in outputs`. diff --git a/docs/using-raitap/examples/detection-fasterrcnn.md b/docs/using-raitap/examples/detection-fasterrcnn.md index 68880b9e..94943152 100644 --- a/docs/using-raitap/examples/detection-fasterrcnn.md +++ b/docs/using-raitap/examples/detection-fasterrcnn.md @@ -47,7 +47,7 @@ transparency: # ScalarDetectionWrapper exposes a single scalar channel per box, so # `auto_pred` is rejected. `max_boxes` caps the K-loop for CPU runs. detection_ig: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: 0 @@ -60,7 +60,7 @@ transparency: max_boxes: 3 iou_threshold: 0.5 visualisers: - - _target_: DetectionImageVisualiser + - use: detection_image :python: from raitap import AppConfig, Hardware, run diff --git a/docs/using-raitap/examples/imagenet-captum-ig-pgd.md b/docs/using-raitap/examples/imagenet-captum-ig-pgd.md index 12ee0d20..a532ca6a 100644 --- a/docs/using-raitap/examples/imagenet-captum-ig-pgd.md +++ b/docs/using-raitap/examples/imagenet-captum-ig-pgd.md @@ -42,7 +42,7 @@ metrics: transparency: default: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: 0 @@ -50,18 +50,18 @@ transparency: baseline: source: ./data/baseline visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image robustness: pgd: - _target_: TorchattacksAssessor + use: torchattacks algorithm: PGD constructor: eps: 0.03 alpha: 0.005 steps: 10 visualisers: - - _target_: ImagePairVisualiser + - use: image_pair :python: from raitap import AppConfig, Hardware, run diff --git a/docs/using-raitap/examples/kitchen-sink.md b/docs/using-raitap/examples/kitchen-sink.md index 2e8ea739..19720e51 100644 --- a/docs/using-raitap/examples/kitchen-sink.md +++ b/docs/using-raitap/examples/kitchen-sink.md @@ -38,7 +38,7 @@ data: transparency: captum_ig: - _target_: "CaptumExplainer" + use: captum algorithm: "IntegratedGradients" constructor: {} call: @@ -48,7 +48,7 @@ transparency: source: "./data/baselines" n_samples: 8 visualisers: - - _target_: "CaptumImageVisualiser" + - use: captum_image constructor: method: "blended_heat_map" sign: "all" @@ -59,7 +59,7 @@ transparency: max_samples: 4 show_sample_names: true shap_gradient: - _target_: "ShapExplainer" + use: shap algorithm: "GradientExplainer" constructor: local_smoothing: 0.0 @@ -73,24 +73,24 @@ transparency: batch_size: 1 progress_desc: "SHAP batches" visualisers: - - _target_: "ShapImageVisualiser" + - use: shap_image constructor: max_samples: 2 robustness: pgd: - _target_: "TorchattacksAssessor" + use: torchattacks algorithm: "PGD" constructor: eps: 0.03 alpha: 0.0078 steps: 10 visualisers: - - _target_: "ImagePairVisualiser" + - use: image_pair constructor: max_samples: 4 linf_pgd: - _target_: "FoolboxAssessor" + use: foolbox algorithm: "LinfPGD" constructor: rel_stepsize: 0.025 @@ -98,22 +98,22 @@ robustness: call: eps: 0.03 visualisers: - - _target_: "PerturbationHeatmapVisualiser" + - use: perturbation_heatmap metrics: - _target_: "MulticlassClassificationMetrics" + use: multiclass_classification num_classes: 7 average: "macro" ignore_index: null tracking: - _target_: "MLFlowTracker" + use: mlflow output_forwarding_url: "http://127.0.0.1:5001" log_model: false open_when_done: true reporting: - _target_: "HTMLReporter" + use: html filename: "report" multirun_report: true show_original_per_explainer: false diff --git a/docs/using-raitap/examples/multi-assessor.md b/docs/using-raitap/examples/multi-assessor.md index bdf8a74d..df389766 100644 --- a/docs/using-raitap/examples/multi-assessor.md +++ b/docs/using-raitap/examples/multi-assessor.md @@ -43,30 +43,30 @@ metrics: transparency: default: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: 0 visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image robustness: pgd: - _target_: TorchattacksAssessor + use: torchattacks algorithm: PGD constructor: eps: 0.03 alpha: 0.005 steps: 10 visualisers: - - _target_: ImagePairVisualiser + - use: image_pair fgsm: - _target_: TorchattacksAssessor + use: torchattacks algorithm: FGSM constructor: eps: 0.03 visualisers: - - _target_: ImagePairVisualiser + - use: image_pair :python: from raitap import AppConfig, Hardware, run diff --git a/docs/using-raitap/examples/multi-explainer.md b/docs/using-raitap/examples/multi-explainer.md index af7a7d97..6b03a137 100644 --- a/docs/using-raitap/examples/multi-explainer.md +++ b/docs/using-raitap/examples/multi-explainer.md @@ -42,19 +42,19 @@ metrics: transparency: ig: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: 0 visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image saliency: - _target_: CaptumExplainer + use: captum algorithm: Saliency call: target: 0 visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image :python: from raitap import AppConfig, Hardware, run diff --git a/docs/using-raitap/examples/multi-visualiser.md b/docs/using-raitap/examples/multi-visualiser.md index 146de4a6..3cb74233 100644 --- a/docs/using-raitap/examples/multi-visualiser.md +++ b/docs/using-raitap/examples/multi-visualiser.md @@ -42,17 +42,17 @@ metrics: transparency: default: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: 0 visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image constructor: method: blended_heat_map sign: all title: Integrated gradients (blended) - - _target_: CaptumImageVisualiser + - use: captum_image constructor: method: heat_map sign: absolute_value diff --git a/docs/using-raitap/job-launcher.md b/docs/using-raitap/job-launcher.md index 17144ff8..28127297 100644 --- a/docs/using-raitap/job-launcher.md +++ b/docs/using-raitap/job-launcher.md @@ -57,13 +57,13 @@ transparency: call: target: 0 visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image shap: algorithm: GradientExplainer call: target: 0 visualisers: - - _target_: ShapImageVisualiser + - use: shap_image data: source: my_dataset diff --git a/docs/using-raitap/using-plugins.md b/docs/using-raitap/using-plugins.md index a14c9431..86e06afa 100644 --- a/docs/using-raitap/using-plugins.md +++ b/docs/using-raitap/using-plugins.md @@ -33,14 +33,16 @@ That's it. Every installed plugin is discovered automatically. ## 2. Use it -Reference the adapter by its `registry_name` (here, `superxai`), exactly like a -built-in adapter. +Reference the adapter by its `registry_name` (here, `superxai`) via the same +`use:` selector as a built-in adapter. There is no `_target_` escape hatch: +config never carries a class path, so a plugin adapter is exactly as safe to +select as a first-party one. ```{config-tabs} :yaml: transparency: my_run: - _target_: "raitap_superxai.SuperXAIExplainer" # full import path — plugin classes live outside raitap.* + use: superxai algorithm: supertreeshap :python: @@ -49,11 +51,12 @@ from raitap.transparency import superxai transparency = {"my_run": superxai(algorithm="supertreeshap")} ``` -In YAML, use the plugin class's **full import path** (`raitap_superxai.SuperXAIExplainer`): -a bare `_target_: SuperXAIExplainer` only resolves for built-in adapters. The -Python form needs no path — `superxai` already carries it. The -`+transparency=superxai` CLI shorthand also works **if** the plugin ships a -matching preset. +`use: superxai` resolves through the same trusted registry as every built-in +key, populated by the plugin's `@adapters.transparency(registry_name="superxai", ...)` +decoration on discovery. See {doc}`../contributor/writing-a-plugin` for the +adapter-author side. The `+transparency=superxai` CLI shorthand also works: it +is generated automatically the moment the plugin registers, no extra config +file needed on the plugin side. ## If a plugin doesn't show up diff --git a/example-plugin/README.md b/example-plugin/README.md new file mode 100644 index 00000000..394058e3 --- /dev/null +++ b/example-plugin/README.md @@ -0,0 +1,74 @@ +# raitap-example-plugin + +A minimal, runnable RAITAP plugin. Copy this directory as a starting point for +your own plugin, or install it as-is to see plugin discovery work end to end. + +It registers one adapter, `identity_attack` — a robustness assessor that +returns the inputs unmodified. No real attack logic, no third-party +dependency beyond `raitap` itself; it exists to demonstrate the wiring, not +to test robustness. + +See `docs/contributor/writing-a-plugin.md` in the raitap repo for the full +guide this plugin follows. + +## Install + +Alongside a raitap project: + +```bash +uv add raitap raitap-example-plugin +# or +pip install raitap raitap-example-plugin +``` + +From this checkout (editable, for trying it locally): + +```bash +uv pip install -e ./example-plugin +``` + +## It's discovered automatically + +No registration step needed. RAITAP scans the `raitap.adapters` entry-point +group at config-load time and imports every installed plugin, which fires the +`@adapters.robustness(...)` decorator in `raitap_example_plugin/__init__.py`. +Confirm it resolved: + +```bash +python -c "from raitap.robustness import identity_attack; print(identity_attack)" +``` + +## Use it in a config + +YAML (`assessment.yaml`, see the fragment in this directory): + +```yaml +robustness: + identity_check: + use: identity_attack + algorithm: identity +``` + +Python: + +```python +from raitap import AppConfig +from raitap.robustness import identity_attack + +cfg = AppConfig( + ..., + robustness={ + "identity_check": identity_attack(algorithm="identity"), + }, +) +``` + +Both forms resolve `use: identity_attack` against the same trusted registry +a first-party adapter (e.g. `torchattacks`) resolves against — a plugin +adapter is indistinguishable from an in-tree one once installed. + +## Uninstall + +```bash +uv pip uninstall raitap-example-plugin +``` diff --git a/example-plugin/assessment.yaml b/example-plugin/assessment.yaml new file mode 100644 index 00000000..8dff7ba9 --- /dev/null +++ b/example-plugin/assessment.yaml @@ -0,0 +1,8 @@ +# Fragment: the `robustness:` block to add to your own assessment.yaml once +# `raitap-example-plugin` is installed. Not runnable standalone — it needs a +# `model:` / `data:` section like any other RAITAP assessment.yaml (see +# `example/assessment.yaml` in the raitap repo for a complete one). +robustness: + identity_check: + use: identity_attack + algorithm: identity diff --git a/example-plugin/pyproject.toml b/example-plugin/pyproject.toml new file mode 100644 index 00000000..e1b1190b --- /dev/null +++ b/example-plugin/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "raitap-example-plugin" +version = "0.0.1" +description = "Runnable reference plugin for RAITAP's entry-point adapter system." +requires-python = ">=3.11" +dependencies = ["raitap"] + +[project.entry-points."raitap.adapters"] +example = "raitap_example_plugin" + +[tool.hatch.build.targets.wheel] +packages = ["raitap_example_plugin"] diff --git a/example-plugin/raitap_example_plugin/__init__.py b/example-plugin/raitap_example_plugin/__init__.py new file mode 100644 index 00000000..fc9228be --- /dev/null +++ b/example-plugin/raitap_example_plugin/__init__.py @@ -0,0 +1,45 @@ +"""Runnable reference plugin for RAITAP's entry-point adapter system. + +Registers one trivial robustness assessor, ``identity_attack``, that returns +the inputs unmodified (no real attack logic, no third-party dependency) — it +exists purely to demonstrate the wiring: entry-point discovery, the +``@adapters.robustness`` decorator, and selecting the result with +``use: identity_attack`` in a RAITAP config. + +See ``README.md`` in this directory for install + usage instructions, and +``docs/contributor/writing-a-plugin.md`` in the RAITAP repo for the full guide. +""" + +from __future__ import annotations + +from raitap import adapters +from raitap.robustness.assessors.base_assessor import AttackInvokeCtx, EmpiricalAttackAssessor +from raitap.robustness.contracts import AssessmentKind, Objective, PerturbationNorm, ThreatModel +from raitap.robustness.semantics import AssessorAlgorithmSpec + + +@adapters.robustness( + registry_name="identity_attack", + algorithm_registry={ + "identity": AssessorAlgorithmSpec( + AssessmentKind.EMPIRICAL_ATTACK, + ThreatModel.WHITE_BOX, + Objective.UNTARGETED, + PerturbationNorm.LINF, + families=frozenset({"identity"}), + ), + }, +) +class IdentityAttackAssessor(EmpiricalAttackAssessor): + """No-op "attack": returns the inputs unchanged. + + Not a real robustness check — a minimal, dependency-free adapter body so + this plugin installs and runs with nothing beyond RAITAP itself. + """ + + def __init__(self, algorithm: str, **init_kwargs) -> None: + self.algorithm = algorithm + self.init_kwargs = init_kwargs + + def _default_invoke(self, ctx: AttackInvokeCtx): # noqa: ANN202 + return ctx.inputs diff --git a/example/assessment.yaml b/example/assessment.yaml index 13fb6c72..5e3b803b 100644 --- a/example/assessment.yaml +++ b/example/assessment.yaml @@ -25,20 +25,20 @@ metrics: transparency: default: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: 0 visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image robustness: pgd: - _target_: TorchattacksAssessor + use: torchattacks algorithm: PGD constructor: eps: 0.03 alpha: 0.005 steps: 10 visualisers: - - _target_: ImagePairVisualiser + - use: image_pair diff --git a/pyproject.toml b/pyproject.toml index 6dd022f5..6e449013 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -349,6 +349,7 @@ include = [ "src/raitap/configs/**/*.yaml", "src/raitap/configs/**/*.yml", "src/raitap/reporting/templates/**/*", + "src/raitap/schema/*.json", "src/hydra_plugins/**/*.py", ] exclude = ["src/raitap/**/test_*.py", "src/raitap/**/conftest.py", "src/raitap/tests/_fake_*plugin/**", "src/raitap/configs/zhaw/*.yaml", "src/raitap/configs/zhaw/*.yml"] diff --git a/src/raitap/_adapters.py b/src/raitap/_adapters.py index 6b231d41..c8e58cb4 100644 --- a/src/raitap/_adapters.py +++ b/src/raitap/_adapters.py @@ -6,9 +6,13 @@ ``@visualisers.transparency``). The decorator delegates to :func:`_register_core` which: -* generates the hydra-zen builder (``builds(...)``) -* registers it with Hydra's ``ConfigStore`` (when the family owns a top-level - config group such as ``transparency`` / ``robustness`` / ``metrics``) +* computes the adapter class's FQN and records it in :data:`_TARGET_FQN` + (``group -> registry_name -> FQN``) — the only trusted place a class FQN is + looked up from a ``use:`` config key (see :mod:`raitap.configs.registry_resolve`) +* builds a ``use:``-node dataclass (schema fields plus a ``use: `` + field, no ``_target_``) and registers it with Hydra's ``ConfigStore`` (when + the family owns a top-level config group such as ``transparency`` / + ``robustness`` / ``metrics``) * exposes the builder under ``raitap..`` (lazy ``__getattr__`` on each module looks it up in :data:`_BUILDERS`) * records the ``extra`` dependency for :mod:`raitap.deps.inference` @@ -22,7 +26,6 @@ import dataclasses import importlib -import inspect import os import pkgutil import re @@ -42,12 +45,12 @@ Unpack, ) -from hydra_zen import ZenStore, builds +from hydra_zen import ZenStore from raitap.types import TaskKind if TYPE_CHECKING: - from collections.abc import Iterator, Sequence + from collections.abc import Callable, Iterator, Sequence from types import ModuleType from raitap.models.base_backend import ModelBackend @@ -74,7 +77,21 @@ def __call__(self, ctx: CtxT, /) -> ResultT: ... store = ZenStore(overwrite_ok=True) # group -> name -> hydra-zen-generated dataclass builder -_BUILDERS: dict[str, dict[str, type]] = {} +# group -> name -> config-layer builder. Schema-backed families store a +# ``use:``-node dataclass *type*; visualisers ("_unscoped") store a callable +# factory (:func:`_make_visualiser_builder`) that folds flat kwargs into +# ``constructor``. Both are consumed by the lazy module ``__getattr__``. +_BUILDERS: dict[str, dict[str, Any]] = {} +# group -> registry_name -> adapter class FQN. The sole trusted seam a +# ``use: `` config key is resolved against +# (:func:`raitap.configs.registry_resolve.resolve_target_fqn`). Group +# ``"_unscoped"`` holds visualisers, which have no Hydra config group. +_TARGET_FQN: dict[str, dict[str, str]] = {} +# group -> FamilyConfig.package_style ("nested" | "flat"). The source of truth +# for whether a group's Hydra config holds multiple named entries +# (``cfg..``) or a single one (``cfg.``) — read by +# :mod:`raitap._config_schema` instead of hardcoding the group set there. +_GROUP_PACKAGE_STYLE: dict[str, str] = {} # adapter class name -> uv extra (consumed by raitap.deps.inference) ADAPTER_EXTRAS: dict[str, str] = {} # group -> set of wrapped third-party library names; used by @@ -108,7 +125,7 @@ class AdapterDecoratorOptions(TypedDict, total=False): registry_name: Required[str] extra: str - library: str + import_name: str # Raw regex strings → friendly messages. Compiled at registration by # ``_register_core`` (mirrors ``suppress_warnings``, which also takes raw # strings). Pass ``r"..."``; add inline flags like ``(?i)`` if needed. @@ -129,9 +146,10 @@ class AdapterMixin: registry_name: str | None = None extra: str | None = None - # Wrapped third-party library (pip name). Set by ``_register_core``; - # drives :meth:`_lazy_import` and :meth:`_rethrow`. - library: str | None = None + # Wrapped third-party library's import module name (not always the PyPI + # dist name). Set by ``_register_core``; drives :meth:`_lazy_import` and + # :meth:`_rethrow`. + import_name: str | None = None # Hydra config group ("transparency" / "robustness" / ...). Set by # ``_register_core`` and read by :meth:`_rethrow` to scope error chips. _adapter_group: str | None = None @@ -152,22 +170,22 @@ def _lazy_import(self, submodule: str | None = None) -> ModuleType: load a specific subpackage (e.g. ``"attr"`` for ``captum.attr``). """ cls = type(self) - if not cls.library: - raise RuntimeError(f"{cls.__name__} has no ``library`` declared") - target = f"{cls.library}.{submodule}" if submodule else cls.library + if not cls.import_name: + raise RuntimeError(f"{cls.__name__} has no ``import_name`` declared") + target = f"{cls.import_name}.{submodule}" if submodule else cls.import_name try: return importlib.import_module(target) except ModuleNotFoundError as exc: install_hint = f" (install with `uv sync --extra {cls.extra}`)" if cls.extra else "" raise ImportError( - f"{cls.__name__} requires the {cls.library!r} package{install_hint}." + f"{cls.__name__} requires the {cls.import_name!r} package{install_hint}." ) from exc @contextmanager def _rethrow(self, *, base_exc: type[BaseException] = Exception) -> Iterator[None]: """Wrap a third-party call so curated error patterns get rewritten. - Equivalent to ``rethrow(module=Module(), third_party_lib=, + Equivalent to ``rethrow(module=Module(), third_party_lib=, message_map=)`` but pulls all three from the adapter's own class declaration (set by the family decorator at registration time). """ @@ -177,7 +195,7 @@ def _rethrow(self, *, base_exc: type[BaseException] = Exception) -> Iterator[Non cls = type(self) with rethrow( module=Module(cls._adapter_group) if cls._adapter_group else Module.utils, - third_party_lib=cls.library, + third_party_lib=cls.import_name, message_map=cls.error_patterns or {}, base_exc=base_exc, ): @@ -216,42 +234,76 @@ def check_backend_compat(self, backend: ModelBackend | None) -> None: ) -def _build_schema_adapter(cls: type, schema: type) -> type: - """Pick a hydra-zen builder shape based on whether ``cls.__init__`` can - accept the schema's field kwargs. +def _class_fqn(cls: type) -> str: + """Fully-qualified ``module.ClassName`` for ``cls``. - * ``**kwargs`` in init (Captum, torchattacks, …) → ``builds(cls, builds_bases=schema)`` - lifts every schema field onto the resulting dataclass and forwards them - to the wrapped class at instantiate-time. - * Narrow init (``MLFlowTracker(config: AppConfig)``, ``HTMLReporter(config)``) - → schema-subclass with only ``_target_`` set; the wrapped class reads the - remaining fields off the composed config blob itself. + CI's ``pythonpath = ["src"]`` plus ``src/__init__.py`` makes ``src`` an + importable package too, so the same class can carry ``__module__ == + "src.raitap.…"`` when discovered by pytest before any ``raitap.*`` import + canonicalises it. Strip the prefix so downstream ``instantiate()`` resolves + the same module identity ``isinstance`` checks see. """ - sig = inspect.signature(cls.__init__) - has_var_kw = any(p.kind == p.VAR_KEYWORD for p in sig.parameters.values()) - init_params = {p for p in sig.parameters if p != "self"} - schema_fields = {f.name for f in dataclasses.fields(schema) if f.name != "_target_"} - accepts_schema = has_var_kw or schema_fields.issubset(init_params) - - if accepts_schema: - return builds(cls, builds_bases=(schema,)) - - # CI's ``pythonpath = ["src"]`` plus ``src/__init__.py`` makes ``src`` an - # importable package too, so the same class can carry ``__module__ == - # "src.raitap.…"`` when discovered by pytest before any ``raitap.*`` import - # canonicalises it. Strip the prefix so ``instantiate()`` resolves the - # same module identity ``isinstance`` checks see. module = cls.__module__ if module.startswith("src."): module = module[len("src.") :] - fqn = f"{module}.{cls.__name__}" + return f"{module}.{cls.__name__}" + + +def _use_node(schema: type, registry_name: str, cls_name: str) -> type: + """Build a config-layer dataclass: ``schema`` fields plus a ``use:`` field + defaulting to ``registry_name``. Never carries ``_target_`` — the real + class FQN lives only in :data:`_TARGET_FQN`, resolved by + :mod:`raitap.configs.registry_resolve` at instantiate-time. + """ return dataclasses.make_dataclass( - f"_{cls.__name__}Conf", - [("_target_", str, dataclasses.field(default=fqn))], + f"_{cls_name}UseConf", + [("use", str, dataclasses.field(default=registry_name))], bases=(schema,), ) +@dataclass +class _VisualiserUseBase: + """Minimal ``use:``-node schema for visualisers (family=None). + + Visualisers previously got a hydra-zen ``builds(cls, populate_full_signature=True, + zen_meta={"call": {}, "raitap": {}})`` builder; this is the equivalent + ``use:``-based shape: a bare ``use`` selector plus the three generic + pass-through blocks (``constructor`` / ``call`` / ``raitap``). + """ + + use: str = "" + constructor: dict[str, Any] = dataclasses.field(default_factory=dict) + call: dict[str, Any] = dataclasses.field(default_factory=dict) + raitap: dict[str, Any] = dataclasses.field(default_factory=dict) + + +def _make_visualiser_builder(registry_name: str) -> Callable[..., _VisualiserUseBase]: + """Programmatic visualiser builder returned by the lazy module ``__getattr__``. + + Accepts the visualiser's own ``__init__`` kwargs FLAT + (``captum_image(method="heat_map", sign="all")``) plus the reserved + ``constructor`` / ``call`` / ``raitap`` blocks, and returns a ``use:``-node. + Flat kwargs fold into ``constructor`` — this restores the pre-#301 + ergonomic (the old ``builds(populate_full_signature=True)`` surface) while + keeping the config layer free of ``_target_``. The YAML side mirrors this in + :func:`raitap.configs.adapter_factory.instantiate_visualisers`. + """ + + def build(**kwargs: Any) -> _VisualiserUseBase: + constructor = dict(kwargs.pop("constructor", None) or {}) + call = dict(kwargs.pop("call", None) or {}) + raitap = dict(kwargs.pop("raitap", None) or {}) + kwargs.pop("use", None) + constructor.update(kwargs) # remaining flat kwargs are __init__ args + return _VisualiserUseBase( + use=registry_name, constructor=constructor, call=call, raitap=raitap + ) + + build.__name__ = registry_name + return build + + def _register_core( cls: type, *, @@ -262,9 +314,10 @@ def _register_core( Sets identity attrs on ``cls``, installs warning filters, validates family-required class-body attributes (e.g. ``algorithm_registry`` when - ``family.has_algorithm_registry``), builds the hydra-zen builder - (``_build_schema_adapter`` for schema-backed families, signature-based - ``builds(...)`` for visualisers), and registers under + ``family.has_algorithm_registry``), records the class FQN in + :data:`_TARGET_FQN`, builds a ``use:``-node dataclass (:func:`_use_node`) + for schema-backed families, or a flat-kwarg visualiser builder + (:func:`_make_visualiser_builder`), and registers under ``_BUILDERS[family.group][registry_name]`` (or ``_BUILDERS["_unscoped"]`` when ``family is None``). """ @@ -279,7 +332,7 @@ def _register_core( extra = common.get("extra") if extra is None and family is not None: extra = registry_name - library = common.get("library") + import_name = common.get("import_name") error_patterns = common.get("error_patterns") suppress_warnings = common.get("suppress_warnings") schema_override = common.get("schema") @@ -287,8 +340,8 @@ def _register_core( cls.registry_name = registry_name if extra is not None: cls.extra = extra - if library is not None: - cls.library = library + if import_name is not None: + cls.import_name = import_name if error_patterns is not None: compiled: dict[re.Pattern[str], str] = {} for pattern, message in error_patterns.items(): @@ -307,7 +360,11 @@ def _register_core( try: if family is not None: cls._adapter_group = family.group - builder = _build_schema_adapter(cls, schema_override or family.schema) + fqn = _class_fqn(cls) + _TARGET_FQN.setdefault(family.group, {})[registry_name] = fqn + _GROUP_PACKAGE_STYLE[family.group] = family.package_style + schema = schema_override or family.schema + builder = _use_node(schema, registry_name, cls.__name__) # Hydra groups use ``/`` for nesting; OmegaConf packages use ``.``. # A nested group like ``data/labels`` must target package # ``data.labels`` so the composed node lands at ``cfg.data.labels``. @@ -317,15 +374,27 @@ def _register_core( if family.package_style == "nested" else package_base ) - store(builder, group=family.group, name=registry_name, package=package) + # ``to_config=lambda x: x`` stores ``builder`` verbatim. ZenStore's + # default ``to_config`` would otherwise run it through + # ``hydra_zen.builds()`` again (dataclass *types* get + # ``populate_full_signature=True, builds_bases=(target,)``), + # which stamps a fresh ``_target_`` pointing at ``builder`` itself + # — reopening exactly the arbitrary-``_target_`` surface this + # rename closes. + store( + builder, + group=family.group, + name=registry_name, + package=package, + to_config=lambda x: x, + ) _BUILDERS.setdefault(family.group, {})[registry_name] = builder else: - builder = builds( - cls, - populate_full_signature=True, - zen_meta={"call": {}, "raitap": {}}, + fqn = _class_fqn(cls) + _TARGET_FQN.setdefault("_unscoped", {})[registry_name] = fqn + _BUILDERS.setdefault("_unscoped", {})[registry_name] = _make_visualiser_builder( + registry_name ) - _BUILDERS.setdefault("_unscoped", {})[registry_name] = builder except (ModuleNotFoundError, TypeError): # Test fixtures defining inline classes without an importable qualname # — hydra-zen rejects them and we silently skip. @@ -333,8 +402,8 @@ def _register_core( if extra: ADAPTER_EXTRAS[cls.__name__] = extra - if library and family is not None: - THIRD_PARTY_LIBS.setdefault(family.group, set()).add(library) + if import_name and family is not None: + THIRD_PARTY_LIBS.setdefault(family.group, set()).add(import_name) return cls diff --git a/src/raitap/_config_schema.py b/src/raitap/_config_schema.py new file mode 100644 index 00000000..f6cd032c --- /dev/null +++ b/src/raitap/_config_schema.py @@ -0,0 +1,121 @@ +"""Generate a JSON Schema from the live adapter registry (refs #301). + +:func:`build_config_schema` walks :data:`raitap._adapters._BUILDERS` — the +same ``group -> registry_name -> use-node dataclass`` map each family +decorator populates at import time — and emits a JSON-Schema dict whose +per-group ``use`` property is an ``enum`` of that group's valid ``use:`` +selector values. A YAML editor pointed at the generated schema can then +autocomplete ``use:`` for ``transparency``, ``robustness``, ``metrics``, +``reporting``, ``tracking``, ``data.labels``, ``data.inputs`` and +visualiser entries. + +.. note:: + The ``tracking`` group is currently omitted from the generated schema: the + ``mlflow`` tracker registration is silently dropped during ``register_configs`` + (a circular import via ``raitap.configs`` that ``raitap.tracking.__init__`` + swallows). Pre-existing, tracked as a follow-up; the schema regains + ``tracking`` autocomplete once that registration is order-hardened. + +Nested-style families (``transparency``, ``robustness``) allow multiple +named entries per group (``cfg.transparency..use``), so their group +schema wraps the ``use`` enum in ``additionalProperties``. Flat-style +families (``reporting``, ``metrics``, ``tracking``, ``data/labels``, +``data/inputs``) hold a single config per group (``cfg..use``). +Visualisers have no Hydra config group of their own and are registered +under ``_BUILDERS["_unscoped"]``; they surface here as a top-level +``visualiser`` property. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from collections.abc import Iterable + +# Fallback only: used when a group is missing from +# :data:`raitap._adapters._GROUP_PACKAGE_STYLE` (e.g. no adapter of that +# family has registered yet, or the "_unscoped"/data groups, which carry no +# ``FamilyConfig``). The live map is the source of truth — see +# :func:`_group_is_nested`. +_NESTED_GROUPS_FALLBACK = frozenset({"transparency", "robustness"}) + + +def _group_is_nested(group: str) -> bool: + """True when ``group``'s Hydra package style is "nested" (multiple named + entries share the group, e.g. ``cfg.transparency..use``). + + Reads :data:`raitap._adapters._GROUP_PACKAGE_STYLE`, populated by each + family decorator from its own :class:`~raitap._adapters.FamilyConfig` + (the real source of truth), falling back to the historical hardcoded set + only when the group isn't recorded there (e.g. ``"_unscoped"``, ``data/*``, + or a family with no adapters registered yet). + """ + from raitap._adapters import _GROUP_PACKAGE_STYLE + + style = _GROUP_PACKAGE_STYLE.get(group) + if style is not None: + return style == "nested" + return group in _NESTED_GROUPS_FALLBACK + + +def _use_enum_schema(registry_names: Iterable[str]) -> dict[str, Any]: + return {"type": "string", "enum": sorted(registry_names)} + + +def _group_schema(registry_names: Iterable[str], *, nested: bool) -> dict[str, Any]: + use_schema = _use_enum_schema(registry_names) + if nested: + return { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": {"use": use_schema}, + }, + } + return {"type": "object", "properties": {"use": use_schema}} + + +def _set_nested_property( + properties: dict[str, Any], path: list[str], value: dict[str, Any] +) -> None: + """Place ``value`` at ``path`` in ``properties``, splitting on Hydra's + ``/``-nested group names (e.g. ``data/labels`` -> ``properties.data. + properties.labels``, matching the composed OmegaConf ``cfg.data.labels`` + package).""" + head, *rest = path + if not rest: + properties[head] = value + return + node = properties.setdefault(head, {"type": "object", "properties": {}}) + _set_nested_property(node["properties"], rest, value) + + +def build_config_schema() -> dict[str, Any]: + """Build a JSON Schema whose ``use`` fields enumerate the live adapter + registry. + + Triggers full family discovery (:func:`raitap.configs.register_configs`, + idempotent) before reading :data:`raitap._adapters._BUILDERS`, so the + schema reflects every in-tree and plugin adapter regardless of what the + caller already imported. + """ + from raitap._adapters import _BUILDERS + from raitap.configs import register_configs + + register_configs() + + properties: dict[str, Any] = {} + for group, entries in _BUILDERS.items(): + registry_names = entries.keys() + if group == "_unscoped": + properties["visualiser"] = _group_schema(registry_names, nested=False) + continue + schema = _group_schema(registry_names, nested=_group_is_nested(group)) + _set_nested_property(properties, group.split("/"), schema) + + return { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": properties, + } diff --git a/src/raitap/cli.py b/src/raitap/cli.py index b54f0b57..39be5220 100644 --- a/src/raitap/cli.py +++ b/src/raitap/cli.py @@ -7,8 +7,8 @@ Order of operations: -1. Handle non-Hydra subcommands (``raitap tracking stop``) — these do not - need the heavy deps. +1. Handle non-Hydra subcommands (``raitap tracking stop``, ``raitap + config-schema``) — these do not need the heavy deps. 2. Resolve ``--demo`` to the bundled :mod:`raitap.configs.demo` config. 3. :func:`raitap.deps.bootstrap.maybe_bootstrap` — re-exec via ``uv run`` when needed, exit otherwise. @@ -85,6 +85,47 @@ def _print_version() -> None: print("raitap (version unknown — not installed as a distribution)") +def _run_config_schema_command(argv: list[str]) -> None: + """Handle ``raitap config-schema [-o/--output PATH]``. + + Dumps :func:`raitap._config_schema.build_config_schema` (a JSON Schema + whose per-group ``use`` enums mirror the live adapter registry) after + :func:`raitap._adapters.discover_third_party_adapters` so installed + plugins are included. Deliberately cheap — no torch import anywhere on + this path — so it runs before the deps bootstrap and needs no extras. + """ + import argparse + import json + from pathlib import Path + + parser = argparse.ArgumentParser( + prog="raitap config-schema", + description="Dump the JSON Schema generated from the live adapter registry.", + ) + parser.add_argument( + "-o", + "--output", + type=Path, + default=None, + help="Write the schema to this file instead of stdout.", + ) + args = parser.parse_args(argv) + + from raitap._adapters import discover_third_party_adapters + from raitap._config_schema import build_config_schema + + discover_third_party_adapters() + schema = build_config_schema() + text = json.dumps(schema, indent=2, sort_keys=True) + "\n" + + output: Path | None = args.output + if output is None: + print(text, end="") + else: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(text) + + def main() -> None: if sys.argv[1:2] == ["tracking", "stop"]: import logging @@ -96,6 +137,10 @@ def main() -> None: run_stop_command() return + if sys.argv[1:2] == ["config-schema"]: + _run_config_schema_command(sys.argv[2:]) + return + if sys.argv[1:2] in (["--version"], ["-V"]): _print_version() return diff --git a/src/raitap/configs/__init__.py b/src/raitap/configs/__init__.py index 2758af9b..caf192c1 100644 --- a/src/raitap/configs/__init__.py +++ b/src/raitap/configs/__init__.py @@ -1,11 +1,10 @@ from .schema import AppConfig -from .utils import cfg_to_dict, register_configs, resolve_run_dir, resolve_target, set_output_root +from .utils import cfg_to_dict, register_configs, resolve_run_dir, set_output_root __all__ = [ "AppConfig", "cfg_to_dict", "register_configs", "resolve_run_dir", - "resolve_target", "set_output_root", ] diff --git a/src/raitap/configs/adapter_factory.py b/src/raitap/configs/adapter_factory.py index 94c6e002..c69899d1 100644 --- a/src/raitap/configs/adapter_factory.py +++ b/src/raitap/configs/adapter_factory.py @@ -3,7 +3,7 @@ Both :mod:`raitap.robustness` and :mod:`raitap.transparency` configure their adapters with the same YAML shape:: - _target_: raitap.. + use: algorithm: constructor: { ... } # __init__ kwargs call: { ... } # per-call library kwargs @@ -42,7 +42,8 @@ import torch from raitap import raitap_log -from raitap.configs.utils import cfg_to_dict, resolve_target +from raitap.configs.registry_resolve import reject_config_target, resolve_target_fqn +from raitap.configs.utils import cfg_to_dict from raitap.data.data import load_tensor_from_source from raitap.data.preprocessing import ( ResolvedPreprocessing, @@ -63,7 +64,10 @@ _DATA_SOURCE_KEYS = frozenset({"source", "n_samples"}) -_VISUALISER_ENTRY_KEYS = frozenset({"_target_", "constructor", "call"}) +# ``raitap`` is included because :class:`raitap._adapters._VisualiserUseBase` +# always carries a ``raitap`` field (default ``{}``); every builder-produced +# visualiser entry has the key even though visualisers don't consume it. +_VISUALISER_ENTRY_KEYS = frozenset({"use", "constructor", "call", "raitap"}) @dataclass(frozen=True) @@ -77,11 +81,10 @@ class AdapterSchema: e.g. ``"assessor"`` or ``"explainer"``. subdict_namespace: title-cased label used in subdict-type errors, e.g. ``"Robustness"``. - target_prefix: prefix prepended to bare ``_target_`` values - (matching :func:`raitap.configs.resolve_target`). - visualiser_prefix: prefix for visualiser ``_target_`` values. Often - equals ``target_prefix`` (transparency) or differs (robustness - uses a separate ``visualisers.`` namespace). + registry_group: the :data:`raitap._adapters._TARGET_FQN` group this + adapter's ``use:`` key is resolved against (``"transparency"`` / + ``"robustness"``). Visualisers are resolved separately under the + fixed ``"_unscoped"`` group. top_level_keys: allowed keys directly under the adapter config block. raitap_keys: allowed keys under the ``raitap:`` sub-block. removed_raitap_keys: keys that were valid in older versions but now @@ -93,8 +96,7 @@ class AdapterSchema: domain: str entity: str subdict_namespace: str - target_prefix: str - visualiser_prefix: str + registry_group: str top_level_keys: frozenset[str] raitap_keys: frozenset[str] top_level_error_hint: str @@ -106,8 +108,7 @@ class ParsedAdapterConfig: """Result of :func:`parse_adapter_config` — fully normalised + validated.""" raw: dict[str, Any] - target_path: str - resolved_target: str + use: str algorithm: Any constructor: dict[str, Any] call: dict[str, Any] @@ -178,12 +179,12 @@ def _validate_top_level_keys(raw: dict[str, Any], schema: AdapterSchema) -> None ) -def _validate_visualiser_entry_keys(entry: dict[str, Any], *, target_hint: str) -> None: +def _validate_visualiser_entry_keys(entry: dict[str, Any], *, use_hint: str) -> None: unknown = set(entry) - _VISUALISER_ENTRY_KEYS if unknown: sorted_unknown = ", ".join(sorted(unknown)) raise ValueError( - f"Unknown keys in visualiser config {target_hint!r}: {sorted_unknown}. " + f"Unknown keys in visualiser config {use_hint!r}: {sorted_unknown}. " "Use 'constructor' for __init__ kwargs and 'call' for visualise() kwargs." ) @@ -229,21 +230,20 @@ def _reject_misplaced_raitap_call_keys( def parse_adapter_config(adapter_config: Any, schema: AdapterSchema) -> ParsedAdapterConfig: """Parse + validate an adapter config block. Returns a frozen container.""" raw = raw_config_dict(adapter_config) + reject_config_target(raw) _validate_top_level_keys(raw, schema) - target_path = str(raw.get("_target_", "")) - resolved_target = resolve_target(target_path, schema.target_prefix) + use = str(raw.get("use", "")) constructor_plain = _subdict(raw.get("constructor"), label="constructor", schema=schema) call_plain = _subdict(raw.get("call"), label="call", schema=schema) raitap_plain = _subdict(raw.get("raitap"), label="raitap", schema=schema) - entity_name = resolved_target or target_path or "?" + entity_name = use or "?" _validate_raitap_keys(raitap_plain, entity_name=entity_name, schema=schema) _reject_misplaced_raitap_call_keys(call_plain, entity_name=entity_name, schema=schema) return ParsedAdapterConfig( raw=raw, - target_path=target_path, - resolved_target=resolved_target, + use=use, algorithm=raw.get("algorithm"), constructor=constructor_plain, call=call_plain, @@ -278,10 +278,11 @@ def instantiate_adapter( overridden so per-domain factories can wire in test doubles or monkey- patchable module-level bindings. """ + fqn = resolve_target_fqn(schema.registry_group, parsed.use) instantiate_cfg: dict[str, Any] = { **parsed.constructor, "algorithm": parsed.algorithm, - "_target_": parsed.resolved_target, + "_target_": fqn, } fn = instantiate_fn if instantiate_fn is not None else instantiate @@ -293,7 +294,7 @@ def instantiate_adapter( missing_key = getattr(error, "full_key", None) or "a required field" raise ValueError( f"Your {schema.entity} config is missing the required `{missing_key}` " - f"field for `{parsed.target_path}`. Set it in the YAML entry — e.g. " + f"field for `{parsed.use}`. Set it in the YAML entry — e.g. " f"`{missing_key}: ` — or pass it as a keyword argument when " f"building the config in Python." ) from error @@ -301,20 +302,19 @@ def instantiate_adapter( raitap_log.exception( "%s instantiation failed for target %r", schema.entity.capitalize(), - parsed.target_path, + parsed.use, ) raise ValueError( - f"Could not instantiate {schema.entity} {parsed.target_path!r}.\n" - f"{instantiate_error_hint}" + f"Could not instantiate {schema.entity} {parsed.use!r}.\n{instantiate_error_hint}" ) from error if not isinstance(adapter, protocol): raise ValueError( - f"Instantiated {schema.entity} {parsed.target_path!r} does not implement " + f"Instantiated {schema.entity} {parsed.use!r} does not implement " f"{protocol.__name__}. {type_error_hint}" ) - return cast("A", adapter), parsed.resolved_target + return cast("A", adapter), fqn def instantiate_visualisers( @@ -335,52 +335,35 @@ def instantiate_visualisers( for visualiser_config in raw.get("visualisers", []): entry = _visualiser_entry_to_dict(visualiser_config) - raw_target = str(entry.get("_target_", "")) - - # Three shapes accepted: - # 1. Hydra-zen builder with ``zen_meta`` (call/raitap as metadata): - # ``_target_`` is the zen-processing wrapper; the real class is - # under ``_zen_target``. Let hydra-zen's ``instantiate`` handle - # everything (target resolution, kwarg filtering, etc.). - # 2. YAML dict: ``{_target_, constructor: {...}, call: {...}}``. - # 3. Flat hydra-zen builder (no zen_meta): ``_target_`` is the real - # class FQN, every other key is a constructor kwarg. - if raw_target == "hydra_zen.funcs.zen_processing": - visualiser_target = str(entry.get("_zen_target", "")) - call_plain = _subdict( - entry.get("call"), - label=f"visualiser call ({visualiser_target})", - schema=schema, - ) - instantiate_cfg: dict[str, Any] = entry + reject_config_target(entry) + use = str(entry.get("use", "")) + # Flat form: ``- use: captum_image`` with ``method: heat_map`` at the entry + # top level (no ``constructor:`` sub-block) lifts those kwargs into the + # constructor — mirrors the programmatic ``captum_image(method="heat_map")`` + # builder. An explicit ``constructor:`` (or an all-reserved-keys entry) + # validates strictly. ``reject_config_target`` already ran above, so a + # nested ``_target_`` in a flat kwarg is still rejected. + if "constructor" in entry or set(entry).issubset(_VISUALISER_ENTRY_KEYS): + _validate_visualiser_entry_keys(entry, use_hint=use or "?") + constructor_source: Any = entry.get("constructor") else: - visualiser_target = raw_target - if "constructor" in entry or set(entry).issubset(_VISUALISER_ENTRY_KEYS): - _validate_visualiser_entry_keys(entry, target_hint=visualiser_target or "?") - constructor_source: Any = entry.get("constructor") - else: - constructor_source = { - k: v for k, v in entry.items() if k not in {"_target_", "call", "raitap"} - } - constructor_plain = _subdict( - constructor_source, - label=f"visualiser constructor ({visualiser_target})", - schema=schema, - ) - call_plain = _subdict( - entry.get("call"), - label=f"visualiser call ({visualiser_target})", - schema=schema, - ) - resolved_target = resolve_target(visualiser_target, schema.visualiser_prefix) - instantiate_cfg = {**constructor_plain, "_target_": resolved_target} + constructor_source = { + k: v for k, v in entry.items() if k not in {"use", "call", "raitap"} + } + constructor_plain = _subdict( + constructor_source, label=f"visualiser constructor ({use})", schema=schema + ) + call_plain = _subdict(entry.get("call"), label=f"visualiser call ({use})", schema=schema) + + fqn = resolve_target_fqn("_unscoped", use) + instantiate_cfg: dict[str, Any] = {**constructor_plain, "_target_": fqn} fn = instantiate_fn if instantiate_fn is not None else instantiate try: visualiser = fn(instantiate_cfg) except Exception as error: - raitap_log.exception("Visualiser instantiation failed for target %r", visualiser_target) - raise ValueError(f"Could not instantiate visualiser {visualiser_target!r}.") from error + raitap_log.exception("Visualiser instantiation failed for target %r", use) + raise ValueError(f"Could not instantiate visualiser {use!r}.") from error out.append(wrap(visualiser, call_plain)) diff --git a/src/raitap/configs/demo.yaml b/src/raitap/configs/demo.yaml index 41786842..365558a5 100644 --- a/src/raitap/configs/demo.yaml +++ b/src/raitap/configs/demo.yaml @@ -28,25 +28,25 @@ metrics: transparency: my_integrated_gradients_explainer: - _target_: CaptumExplainer + use: captum algorithm: IntegratedGradients call: target: 0 internal_batch_size: 8 # chunk IG steps so CPU peak RAM stays bounded visualisers: - - _target_: CaptumImageVisualiser + - use: captum_image robustness: my_pgd_attacker: - _target_: TorchattacksAssessor + use: torchattacks algorithm: PGD constructor: eps: 0.03 alpha: 0.005 steps: 10 visualisers: - - _target_: ImagePairVisualiser + - use: image_pair reporting: - _target_: HTMLReporter + use: html filename: demo_report diff --git a/src/raitap/configs/registry_resolve.py b/src/raitap/configs/registry_resolve.py new file mode 100644 index 00000000..95bf3c3d --- /dev/null +++ b/src/raitap/configs/registry_resolve.py @@ -0,0 +1,133 @@ +"""Single trusted seam: map a config ``use`` key to a vetted class FQN. + +Config never carries ``_target_``. Selection is a short ``use: `` +key; this module resolves it against the closed registry populated at adapter +registration (``raitap._adapters._TARGET_FQN``). Removing ``_target_`` from the +config layer is what closes the arbitrary-callable RCE surface (issue #301). +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from typing import Any + +from hydra.utils import instantiate as _default_instantiate + +from raitap import raitap_log + + +class UnsafeConfigTargetError(ValueError): + """A config carried a ``_target_`` key, which is no longer accepted.""" + + +def reject_config_target(cfg: Any) -> None: + """Reject a ``_target_`` key at any depth in ``cfg``. + + ``hydra.utils.instantiate`` recurses, so a nested ``_target_`` (e.g. inside a + ``constructor`` value) is as dangerous as a top-level one. Scan the whole tree. + """ + if isinstance(cfg, Mapping): + if "_target_" in cfg: + raise UnsafeConfigTargetError( + "`_target_` is no longer accepted in configs (arbitrary-callable " + "security surface). Select an implementation with `use: `." + ) + for value in cfg.values(): + reject_config_target(value) + elif isinstance(cfg, (list, tuple)): + for item in cfg: + reject_config_target(item) + + +def use_key_enabled(cfg: Any) -> bool: + """True when ``cfg`` (already ``cfg_to_dict``'d) carries a non-empty ``use``. + + Shared by :func:`raitap.metrics.factory.metrics_run_enabled` and + :func:`raitap.reporting.factory.reporting_enabled`, which both gate an + optional config block on "is `use` set to a non-empty string", after + rejecting a ``_target_``-carrying block loudly instead of silently reading + it as "not configured". Callers keep their own ``None``-config + short-circuit (this helper assumes a non-``None`` config was already + turned into a dict). + """ + reject_config_target(cfg) + use = cfg.get("use") + if use is None: + return False + return bool(str(use).strip()) + + +def resolve_target_fqn(group: str, use: str) -> str: + from raitap._adapters import _TARGET_FQN + + group_map = _TARGET_FQN.get(group, {}) + try: + return group_map[use] + except KeyError: + valid = ", ".join(sorted(group_map)) + error_group = "visualiser" if group == "_unscoped" else group + raise ValueError( + f"Unknown {error_group} key {use!r}. Valid keys: {valid or '(none registered)'}." + ) from None + + +def stamp_target_from_use(cfg: dict[str, Any], *, group: str) -> None: + """Reject ``_target_``, resolve ``cfg["use"]`` against ``group``, and stamp + the resolved FQN back onto ``cfg`` as ``_target_`` (popping ``use``). + + Mutates ``cfg`` in place. Shared by + :func:`raitap.metrics.factory.create_metric`, + :func:`raitap.transparency.evaluation.step.grade_explanations`, and + :func:`raitap.data._parser_factory.create_parser`, which all resolve a + ``use:``-keyed config to a vetted ``_target_`` before handing it to + ``hydra.utils.instantiate``. Callers keep their own post-instantiate + tails (return shape, error wording) — this helper only covers the shared + reject-resolve-stamp-pop core. Preserves the security ordering: ``cfg`` + is rejected for a smuggled ``_target_`` *before* ``use`` is resolved. + """ + reject_config_target(cfg) + use = str(cfg.get("use", "")) + cfg["_target_"] = resolve_target_fqn(group, use) + cfg.pop("use", None) + + +def instantiate_partial_from_use( + cfg: Any, + *, + group: str, + entity: str, + config: Any, + instantiate_fn: Callable[[dict[str, Any]], Any] | None = None, +) -> Any: + """Resolve ``cfg["use"]`` to a vetted FQN, partially instantiate it, and call + the result with ``config``. + + Shared by :func:`raitap.reporting.factory.create_report` and + :func:`raitap.tracking.base_tracker.BaseTracker.create_tracker`, which both + follow the same "reject ``_target_`` -> resolve ``use`` -> partial-instantiate + -> call with the app config" shape, differing only in the registry group and + the noun used in error messages. + + ``cfg`` is the already-``cfg_to_dict``'d sub-config (e.g. ``config.reporting``) + carrying the ``use`` key. ``group`` is the :data:`raitap._adapters._TARGET_FQN` + group to resolve ``use`` against (e.g. ``"reporting"``). ``entity`` names the + instantiated thing for error messages (e.g. ``"reporter"``). ``config`` is the + full :class:`~raitap.configs.schema.AppConfig` passed to the resolved class's + constructor. ``instantiate_fn`` defaults to :func:`hydra.utils.instantiate` + but can be overridden — mirrors :func:`raitap.configs.adapter_factory.instantiate_adapter`, + letting callers wire in test doubles against their own module-level binding. + """ + reject_config_target(cfg) + use = str(cfg.get("use", "")) + resolved_target = resolve_target_fqn(group, use) + + fn = instantiate_fn if instantiate_fn is not None else _default_instantiate + try: + klass = fn({"_target_": resolved_target, "_partial_": True}) + return klass(config) + except Exception as error: + raitap_log.exception("%s instantiation failed for target %r", entity.capitalize(), use) + raise ValueError( + f"Could not instantiate {entity} {use!r}.\n" + f"Check that `use` points to a registered {group} adapter." + ) from error diff --git a/src/raitap/configs/schema.py b/src/raitap/configs/schema.py index 91410227..86e55ccc 100644 --- a/src/raitap/configs/schema.py +++ b/src/raitap/configs/schema.py @@ -72,34 +72,34 @@ class ModelConfig: @dataclass class LabelsConfig: - _target_: str = MISSING + use: str = MISSING @dataclass class InputsConfig: - _target_: str = MISSING + use: str = MISSING @dataclass class TextCsvInputsConfig(InputsConfig): - _target_: str = "TextCsvInputParser" + use: str = "text_csv" text_column: str = MISSING @dataclass class TextJsonlInputsConfig(InputsConfig): - _target_: str = "TextJsonlInputParser" + use: str = "text_jsonl" text_field: str = "text" @dataclass class TextDirInputsConfig(InputsConfig): - _target_: str = "TextDirInputParser" + use: str = "text_dir" @dataclass class TabularLabelsConfig(LabelsConfig): - _target_: str = "TabularLabelParser" + use: str = "tabular" source: str = MISSING id_column: str | None = None column: str | None = None @@ -109,26 +109,26 @@ class TabularLabelsConfig(LabelsConfig): @dataclass class DirectoryLabelsConfig(LabelsConfig): - _target_: str = "DirectoryLabelParser" + use: str = "directory" @dataclass class CocoLabelsConfig(LabelsConfig): - _target_: str = "CocoLabelParser" + use: str = "coco" source: str = MISSING id_strategy: IdStrategy = IdStrategy.auto @dataclass class YoloLabelsConfig(LabelsConfig): - _target_: str = "YoloLabelParser" + use: str = "yolo" source: str = MISSING id_strategy: IdStrategy = IdStrategy.auto @dataclass class VocLabelsConfig(LabelsConfig): - _target_: str = "VocLabelParser" + use: str = "voc" source: str = MISSING id_strategy: IdStrategy = IdStrategy.auto class_names: list[str] | None = None @@ -136,7 +136,7 @@ class VocLabelsConfig(LabelsConfig): @dataclass class DetectionJsonLabelsConfig(LabelsConfig): - _target_: str = "DetectionJsonLabelParser" + use: str = "detection_json" source: str = MISSING id_strategy: IdStrategy = IdStrategy.auto @@ -178,26 +178,10 @@ class DataConfig: inputs: InputsConfig | None = None -@dataclass -class VisualiserConfig: - """Schema base for visualiser list entries. - - Used as ``builds_bases=`` for hydra-zen visualiser builders so they accept - a ``call=`` kwarg alongside their flat init kwargs. The adapter factory - accepts either this flat shape (``image_pair(max_samples=4, call={...})``) - or the historical YAML shape (``{"_target_": ..., "constructor": {...}, - "call": {...}}``). - """ - - _target_: str = MISSING - call: dict[str, Any] = field(default_factory=dict) - raitap: dict[str, Any] = field(default_factory=dict) - - @dataclass class EvaluationConfig: - # Hydra _target_: points to an evaluator (e.g. raitap.transparency.QuantusEvaluator). - _target_: str = MISSING + # Hydra use: selects an evaluator (e.g. "quantus" -> raitap.transparency.QuantusEvaluator). + use: str = MISSING # Names of the Quantus metrics to compute (e.g. "sparseness", "faithfulness_correlation"). metrics: list[str] = field(default_factory=list) # Constructor kwargs forwarded to the underlying Quantus metric classes. @@ -211,12 +195,12 @@ class EvaluationConfig: @dataclass class TransparencyConfig: - # Hydra _target_: points to an ExplainerAdapter - # (e.g. AttributionOnlyExplainer or FullExplainer subclass) + # Hydra use: selects an ExplainerAdapter registry key + # (e.g. "captum" -> AttributionOnlyExplainer or "shap" -> FullExplainer subclass) # Overridden by the transparency config-group YAML (transparency=captum / shap). # MISSING by default so omission fails validation loudly rather than # silently selecting a library. - _target_: str = MISSING + use: str = MISSING algorithm: str = MISSING # Constructor kwargs for the explainer / underlying library method (e.g. Captum # ``IntegratedGradients(model, **kwargs)``, SHAP ``GradientExplainer(model, data, **kwargs)``). @@ -234,21 +218,21 @@ class TransparencyConfig: # RAITAP-owned runtime options such as batch_size, progress bars, and # sample-name metadata. These keys are not forwarded to the explainability library. raitap: dict[str, Any] = field(default_factory=dict) - # Each entry needs at least ``_target_``; ``constructor`` / ``call`` are optional + # Each entry needs at least ``use``; ``constructor`` / ``call`` are optional # (same split as explainer). Default is minimal: Captum explainer + image visualiser. - visualisers: list[Any] = field(default_factory=lambda: [{"_target_": "CaptumImageVisualiser"}]) + visualisers: list[Any] = field(default_factory=lambda: [{"use": "captum_image"}]) # Optional Quantus-backed explanation-quality evaluation block. Left unset - # (``None``) by default; set ``evaluation._target_`` to enable it. + # (``None``) by default; set ``evaluation.use`` to enable it. evaluation: EvaluationConfig | None = None @dataclass class RobustnessConfig: - # Hydra _target_: points to a BaseAssessor subclass - # (e.g. EmpiricalAttackAssessor or FormalVerificationAssessor implementation). + # Hydra use: selects a BaseAssessor subclass registry key + # (e.g. "torchattacks" -> EmpiricalAttackAssessor or "marabou" -> FormalVerificationAssessor). # Overridden by the robustness config-group YAML # (robustness=torchattacks / foolbox / marabou). - _target_: str = MISSING + use: str = MISSING algorithm: str = MISSING # Constructor kwargs forwarded to the assessor's ``__init__``. For torchattacks # adapters this is where attack hyperparameters live (eps, alpha, steps), since @@ -262,14 +246,14 @@ class RobustnessConfig: # RAITAP-owned runtime options such as batch_size, progress bars, and # sample-name metadata. Not forwarded to the underlying library. raitap: dict[str, Any] = field(default_factory=dict) - # Each entry needs at least ``_target_``; ``constructor`` / ``call`` are optional. + # Each entry needs at least ``use``; ``constructor`` / ``call`` are optional. # Default is the empirical image-pair visualiser. - visualisers: list[Any] = field(default_factory=lambda: [{"_target_": "ImagePairVisualiser"}]) + visualisers: list[Any] = field(default_factory=lambda: [{"use": "image_pair"}]) @dataclass class MetricsConfig: - _target_: str = MISSING + use: str = MISSING @dataclass @@ -282,14 +266,14 @@ class IoUConfig: @dataclass class BinaryClassificationMetricsConfig(MetricsConfig): - _target_: str = "BinaryClassificationMetrics" + use: str = "binary_classification" ignore_index: int | None = None threshold: float = 0.5 @dataclass class MulticlassClassificationMetricsConfig(MetricsConfig): - _target_: str = "MulticlassClassificationMetrics" + use: str = "multiclass_classification" num_classes: int = MISSING average: ClassificationAverage = "macro" ignore_index: int | None = None @@ -297,7 +281,7 @@ class MulticlassClassificationMetricsConfig(MetricsConfig): @dataclass class MultilabelClassificationMetricsConfig(MetricsConfig): - _target_: str = "MultilabelClassificationMetrics" + use: str = "multilabel_classification" num_labels: int = MISSING average: ClassificationAverage = "macro" ignore_index: int | None = None @@ -306,7 +290,7 @@ class MultilabelClassificationMetricsConfig(MetricsConfig): @dataclass class DetectionMetricsConfig(MetricsConfig): - _target_: str = "DetectionMetrics" + use: str = "detection" box_format: BoxFormat = "xyxy" iou: IoUConfig = field(default_factory=IoUConfig) class_metrics: bool = False @@ -317,7 +301,7 @@ class DetectionMetricsConfig(MetricsConfig): @dataclass class TrackingConfig: - _target_: str = MISSING + use: str = MISSING output_forwarding_url: str | None = None backend_store_uri: str | None = None default_artifact_root: str | None = None @@ -330,7 +314,7 @@ class ReportingConfig: """Configuration for report generation.""" # ``None`` disables reporting entirely (used by ``reporting/disabled.yaml``). - _target_: str | None = MISSING + use: str | None = MISSING filename: str = "report" sample_selection: list[int | str] | None = None include_config: bool = True diff --git a/src/raitap/configs/tests/test_labels_schema.py b/src/raitap/configs/tests/test_labels_schema.py index 9db0c56f..30b5c4f1 100644 --- a/src/raitap/configs/tests/test_labels_schema.py +++ b/src/raitap/configs/tests/test_labels_schema.py @@ -11,12 +11,12 @@ def test_coco_config_has_no_tabular_fields() -> None: assert "id_column" not in names assert "column" not in names assert "encoding" not in names - assert {"_target_", "source", "id_strategy"} <= names + assert {"use", "source", "id_strategy"} <= names -def test_directory_config_has_only_target() -> None: +def test_directory_config_has_only_use() -> None: names = {f.name for f in dataclasses.fields(DirectoryLabelsConfig)} - assert names == {"_target_"} + assert names == {"use"} def test_labelformat_enum_is_gone() -> None: @@ -25,10 +25,12 @@ def test_labelformat_enum_is_gone() -> None: getattr(data_types, "LabelFormat") # noqa: B009 -# Ground truth (see task-2-report.md): composing ``+data/labels=directory`` onto -# the AppConfig schema lands the variant at ``cfg.data.labels`` with the FQN -# ``_target_`` that hydra-zen ``builds()`` injects. -_COMPOSED_TARGET = "raitap.data.label_parsers.directory.DirectoryLabelParser" +# Ground truth (issue #301): composing ``+data/labels=directory`` onto the +# AppConfig schema lands the variant at ``cfg.data.labels`` as a ``use: +# "directory"`` selector (config layer). The real class FQN never appears in +# config — it lives only in ``raitap._adapters._TARGET_FQN`` and is resolved by +# ``raitap.configs.registry_resolve.resolve_target_fqn`` at instantiate-time. +_DIRECTORY_LABEL_PARSER_FQN = "raitap.data.label_parsers.directory.DirectoryLabelParser" def _register_labels_group() -> None: @@ -38,7 +40,7 @@ def _register_labels_group() -> None: it sets up the AppConfig schema and every family's group nodes consistently. An earlier direct ``store.add_to_hydra_store(overwrite_ok=True)`` workaround flushed hydra-zen builders in isolation, clobbering other groups' short - ``_target_`` schema nodes and breaking later tests (e.g. reporting compose). + ``use`` schema nodes and breaking later tests (e.g. reporting compose). """ from raitap.configs import register_configs @@ -55,9 +57,10 @@ def test_directory_parser_group_lands_at_data_labels() -> None: GlobalHydra.instance().clear() with initialize(version_base=None, config_path=None): cfg = compose(config_name="raitap_schema", overrides=["+data/labels=directory"]) - # Assertion runs unconditionally (no swallowing). The composed value is the - # FQN hydra-zen stores, NOT the short dataclass default. - assert cfg.data.labels._target_ == _COMPOSED_TARGET + # Assertion runs unconditionally (no swallowing). The composed node carries + # a bare ``use:`` selector, never a class FQN (issue #301). + assert cfg.data.labels.use == "directory" + assert not hasattr(cfg.data.labels, "_target_") def test_directory_group_rejects_foreign_field() -> None: @@ -79,27 +82,35 @@ def test_directory_group_rejects_foreign_field() -> None: ) -def test_create_label_parser_handles_both_target_forms() -> None: - """``create_label_parser`` must instantiate for BOTH ``_target_`` shapes: +def test_directory_labels_use_resolves_to_directory_label_parser() -> None: + """The registration + resolver seam Task B1/A1 own: a ``use:`` selector + resolves to the real class FQN through the closed registry populated at + adapter registration — never through an arbitrary ``_target_`` in config. - * short bare name (``DirectoryLabelsConfig()`` dataclass default), resolved - against the ``raitap.data.label_parsers.`` prefix; - * the dotted FQN hydra-zen ``builds()`` stamps on the group-composed cfg. + Migrating ``create_label_parser`` itself to consume ``use:`` instead of + ``_target_`` is later (Phase D) work; this test scopes to the schema + + registry seam introduced here. """ _register_labels_group() + + from raitap.configs.registry_resolve import resolve_target_fqn from raitap.data.label_parsers.directory import DirectoryLabelParser - from raitap.data.label_parsers.factory import create_label_parser - short = create_label_parser(DirectoryLabelsConfig()) - assert isinstance(short, DirectoryLabelParser) + cfg = DirectoryLabelsConfig() + assert cfg.use == "directory" + assert not hasattr(cfg, "_target_") + + fqn = resolve_target_fqn("data/labels", cfg.use) + assert fqn == _DIRECTORY_LABEL_PARSER_FQN - fqn = create_label_parser({"_target_": _COMPOSED_TARGET}) - assert isinstance(fqn, DirectoryLabelParser) + module_path, _, class_name = fqn.rpartition(".") + resolved_cls = getattr(importlib.import_module(module_path), class_name) + assert resolved_cls is DirectoryLabelParser -def test_detection_json_config_has_exactly_target_source_id_strategy() -> None: +def test_detection_json_config_has_exactly_use_source_id_strategy() -> None: names = {f.name for f in dataclasses.fields(DetectionJsonLabelsConfig)} - assert names == {"_target_", "source", "id_strategy"} + assert names == {"use", "source", "id_strategy"} # --------------------------------------------------------------------------- @@ -113,7 +124,7 @@ def test_detection_json_config_has_exactly_target_source_id_strategy() -> None: # Fields that belong exclusively to the voc variant. _VOC_ONLY_FIELDS = {"class_names"} -# Variants that must have ONLY ``_target_`` (no source, no strategy, nothing). +# Variants that must have ONLY ``use`` (no source, no strategy, nothing). _TARGET_ONLY_VARIANTS: set[str] = {"directory"} # Variants that carry source + id_strategy but NO tabular fields and NO @@ -129,7 +140,7 @@ def test_no_cross_variant_field_leakage(registry_name: str) -> None: """Each label-parser builder dataclass must expose only its own fields. Specifically: - - ``directory`` has only ``_target_``. + - ``directory`` has only ``use``. - ``coco``/``yolo``/``detection_json`` have no tabular-only fields and no ``class_names``. - ``voc`` has ``class_names`` but no tabular-only fields. @@ -148,8 +159,8 @@ def test_no_cross_variant_field_leakage(registry_name: str) -> None: field_names = {f.name for f in dataclasses.fields(builder)} if registry_name in _TARGET_ONLY_VARIANTS: - assert field_names == {"_target_"}, ( - f"{registry_name!r} builder should have only '_target_' but got {field_names}" + assert field_names == {"use"}, ( + f"{registry_name!r} builder should have only 'use' but got {field_names}" ) if registry_name in _DETECTION_VARIANTS: diff --git a/src/raitap/configs/tests/test_registry_resolve.py b/src/raitap/configs/tests/test_registry_resolve.py new file mode 100644 index 00000000..a9c1825e --- /dev/null +++ b/src/raitap/configs/tests/test_registry_resolve.py @@ -0,0 +1,63 @@ +import pytest + +from raitap.configs import registry_resolve as rr + + +def test_reject_config_target_raises_on_target_key() -> None: + with pytest.raises(rr.UnsafeConfigTargetError, match="use:"): + rr.reject_config_target({"_target_": "os.system", "command": "x"}) + + +def test_reject_config_target_allows_clean_cfg() -> None: + rr.reject_config_target({"use": "captum", "algorithm": "IntegratedGradients"}) # no raise + + +def test_reject_config_target_raises_on_nested_target_in_dict() -> None: + cfg = { + "use": "captum", + "constructor": { + "baseline": {"_target_": "os.system", "_args_": ["id"]}, + }, + } + with pytest.raises(rr.UnsafeConfigTargetError, match="use:"): + rr.reject_config_target(cfg) + + +def test_reject_config_target_raises_on_target_nested_in_list() -> None: + cfg = { + "use": "captum", + "visualisers": [{"use": "clean"}, {"_target_": "os.system"}], + } + with pytest.raises(rr.UnsafeConfigTargetError, match="use:"): + rr.reject_config_target(cfg) + + +def test_reject_config_target_allows_clean_deeply_nested_cfg() -> None: + cfg = { + "use": "captum", + "constructor": { + "baseline": {"use": "zeros", "shape": [1, 2, 3]}, + }, + "visualisers": [{"use": "clean"}, {"use": "another", "options": {"a": 1}}], + } + rr.reject_config_target(cfg) # no raise + + +def test_resolve_target_fqn_returns_registered_fqn(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "raitap._adapters._TARGET_FQN", + { + "transparency": { + "captum": "raitap.transparency.explainers.captum_explainer.CaptumExplainer" + } + }, + ) + assert rr.resolve_target_fqn("transparency", "captum").endswith("CaptumExplainer") + + +def test_resolve_target_fqn_unknown_key_lists_valid(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "raitap._adapters._TARGET_FQN", {"transparency": {"captum": "x", "shap": "y"}} + ) + with pytest.raises(ValueError, match="captum, shap"): + rr.resolve_target_fqn("transparency", "capdum") diff --git a/src/raitap/configs/utils.py b/src/raitap/configs/utils.py index 1e7d7432..952a3ca9 100644 --- a/src/raitap/configs/utils.py +++ b/src/raitap/configs/utils.py @@ -48,12 +48,6 @@ def cfg_to_dict(cfg: Any) -> dict: return dict(cfg) -def resolve_target(target: str, prefix: str) -> str: - if not target: - return target - return target if "." in target else f"{prefix}{target}" - - def set_output_root(config: Any, output_root: str | Path) -> None: if isinstance(config, DictConfig): # OmegaConf structured configs reject keys outside the schema; flip diff --git a/src/raitap/configs/zen.py b/src/raitap/configs/zen.py index 05a893cf..48a66398 100644 --- a/src/raitap/configs/zen.py +++ b/src/raitap/configs/zen.py @@ -7,7 +7,7 @@ A handful of bundled Hydra group entries need shapes the decorators can't produce (``# @package _global_`` callback injection for reporting multirun, -``_target_: null`` for the ``disabled`` reporting variant). Those live +``use: null`` for the ``disabled`` reporting variant). Those live below as direct :class:`hydra.core.config_store.ConfigStore` writes. """ @@ -19,8 +19,6 @@ from raitap._adapters import store -_HTML_REPORTER_TARGET = "HTMLReporter" -_PDF_REPORTER_TARGET = "PDFReporter" _REPORTING_SWEEP_CALLBACK_TARGET = "raitap.reporting.hydra_callback.ReportingSweepCallback" @@ -72,13 +70,13 @@ def register_zen_groups() -> None: # Reporting variants that the mixin can't model: # - ``html`` / ``pdf`` need ``# @package _global_`` to inject the multirun # callback alongside the ``reporting`` node. - # - ``disabled`` carries ``_target_: null`` + ``multirun_report: false``. + # - ``disabled`` carries ``use: null`` + ``multirun_report: false``. cs.store( group="reporting", name="html", package="_global_", node={ - "reporting": {"_target_": _HTML_REPORTER_TARGET}, + "reporting": {"use": "html"}, "hydra": _callback_block(), }, ) @@ -87,7 +85,7 @@ def register_zen_groups() -> None: name="pdf", package="_global_", node={ - "reporting": {"_target_": _PDF_REPORTER_TARGET}, + "reporting": {"use": "pdf"}, "hydra": _callback_block(), }, ) @@ -97,7 +95,7 @@ def register_zen_groups() -> None: reporting_disabled_node = make_dataclass( "_ReportingDisabledNode", [ - ("_target_", str | None, field(default=None)), + ("use", str | None, field(default=None)), ("multirun_report", bool, field(default=False)), ], bases=(ReportingConfig,), diff --git a/src/raitap/data/_parser_factory.py b/src/raitap/data/_parser_factory.py index 4d9a552b..ada60c2c 100644 --- a/src/raitap/data/_parser_factory.py +++ b/src/raitap/data/_parser_factory.py @@ -1,4 +1,4 @@ -"""Shared _target_-resolve-and-instantiate core for data parser families.""" +"""Shared use-key-resolve-and-instantiate core for data parser families.""" from __future__ import annotations @@ -7,19 +7,21 @@ from hydra.utils import instantiate from raitap import raitap_log -from raitap.configs import cfg_to_dict, resolve_target +from raitap.configs import cfg_to_dict +from raitap.configs.registry_resolve import stamp_target_from_use -def create_parser(config: Any, *, prefix: str, kind: str) -> Any: - """Instantiate a parser from Hydra-style config (``_target_`` + kwargs).""" +def create_parser(config: Any, *, group: str, kind: str) -> Any: + """Instantiate a parser from a ``use:``-keyed config, resolved via the + trusted registry seam (``raitap.configs.registry_resolve``).""" cfg = cfg_to_dict(config) - target_path: str = cfg.get("_target_", "") - cfg["_target_"] = resolve_target(target_path, prefix) + stamp_target_from_use(cfg, group=group) + fqn = cfg["_target_"] try: return instantiate(cfg) except Exception as e: - raitap_log.exception("%s instantiation failed for target %r", kind, target_path) + raitap_log.exception("%s instantiation failed for target %r", kind, fqn) raise ValueError( - f"Could not instantiate {kind} {target_path!r}.\n" - f"Check that _target_ points to a valid {kind} implementation." + f"Could not instantiate {kind} {fqn!r}.\nCheck that `use` resolves to a " + f"valid {kind} implementation." ) from e diff --git a/src/raitap/data/input_parsers/factory.py b/src/raitap/data/input_parsers/factory.py index 0ff14c9c..5d34e356 100644 --- a/src/raitap/data/input_parsers/factory.py +++ b/src/raitap/data/input_parsers/factory.py @@ -9,4 +9,4 @@ def create_input_parser(inputs_config: Any) -> InputParser: - return create_parser(inputs_config, prefix="raitap.data.input_parsers.", kind="input parser") + return create_parser(inputs_config, group="data/inputs", kind="input parser") diff --git a/src/raitap/data/label_parsers/__init__.py b/src/raitap/data/label_parsers/__init__.py index 65a94173..d19f20ae 100644 --- a/src/raitap/data/label_parsers/__init__.py +++ b/src/raitap/data/label_parsers/__init__.py @@ -2,9 +2,12 @@ Importing this package fires the ``@label_parser`` decorator on every in-tree parser module, registering them with the hydra-zen store. Each -concrete parser is re-exported here so the short ``_target_`` form (a bare -class name resolved against ``raitap.data.label_parsers.``) instantiates, -mirroring how ``raitap.metrics`` re-exports its metric computers. +concrete parser is re-exported here for direct Python-side access (e.g. +``from raitap.data.label_parsers import CocoLabelParser``), mirroring how +``raitap.metrics`` re-exports its metric computers. YAML selection goes +through ``use: `` (e.g. ``use: coco``), resolved against the +trusted registry (:mod:`raitap.configs.registry_resolve`) independently of +this package's namespace. """ from __future__ import annotations diff --git a/src/raitap/data/label_parsers/factory.py b/src/raitap/data/label_parsers/factory.py index 9758055d..1575b91e 100644 --- a/src/raitap/data/label_parsers/factory.py +++ b/src/raitap/data/label_parsers/factory.py @@ -9,4 +9,4 @@ def create_label_parser(labels_config: Any) -> LabelParser: - return create_parser(labels_config, prefix="raitap.data.label_parsers.", kind="label parser") + return create_parser(labels_config, group="data/labels", kind="label parser") diff --git a/src/raitap/data/tests/test_input_parser_registry.py b/src/raitap/data/tests/test_input_parser_registry.py index e438cad1..e0f91fec 100644 --- a/src/raitap/data/tests/test_input_parser_registry.py +++ b/src/raitap/data/tests/test_input_parser_registry.py @@ -1,3 +1,5 @@ +import pytest + from raitap.configs.schema import InputsConfig from raitap.data.input_parsers.base import InputParser from raitap.data.input_parsers.registration import input_parser @@ -34,7 +36,7 @@ def _register_inputs_group() -> None: def test_integration_compose_data_inputs_dummy() -> None: - """Composing +data/inputs=dummy_compose lands cfg.data.inputs at the FQN. + """Composing +data/inputs=dummy_compose lands cfg.data.inputs.use, resolvable to the FQN. Regression for the Critical review finding: ``DataConfig`` was missing an ``inputs`` field, so composing any ``data/inputs`` variant raised @@ -43,11 +45,14 @@ def test_integration_compose_data_inputs_dummy() -> None: from hydra import compose, initialize from hydra.core.global_hydra import GlobalHydra + from raitap.configs.registry_resolve import resolve_target_fqn + _register_inputs_group() GlobalHydra.instance().clear() with initialize(version_base=None, config_path=None): cfg = compose(config_name="raitap_schema", overrides=["+data/inputs=dummy_compose"]) - assert cfg.data.inputs._target_ == _COMPOSED_TARGET + assert cfg.data.inputs.use == "dummy_compose" + assert resolve_target_fqn("data/inputs", cfg.data.inputs.use) == _COMPOSED_TARGET def test_decorated_parser_satisfies_protocol_and_registers() -> None: @@ -66,6 +71,14 @@ def parse(self, *, source: str) -> list[str]: def test_create_label_parser_still_works_after_factory_refactor() -> None: """Regression: the shared-core refactor of label_parsers/factory.py must - preserve existing label-parser instantiation behaviour.""" - parser = create_label_parser({"_target_": "TabularLabelParser", "source": "x"}) + preserve existing label-parser instantiation behaviour, now via ``use:``.""" + parser = create_label_parser({"use": "tabular", "source": "x"}) assert isinstance(parser, TabularLabelParser) + + +def test_create_label_parser_rejects_target_key() -> None: + """A ``_target_`` key in the config is rejected (issue #301 RCE surface).""" + from raitap.configs.registry_resolve import UnsafeConfigTargetError + + with pytest.raises(UnsafeConfigTargetError): + create_label_parser({"_target_": "TabularLabelParser", "source": "x"}) diff --git a/src/raitap/data/tests/test_label_parsers.py b/src/raitap/data/tests/test_label_parsers.py index b13ec783..be40e1ef 100644 --- a/src/raitap/data/tests/test_label_parsers.py +++ b/src/raitap/data/tests/test_label_parsers.py @@ -93,15 +93,18 @@ def _register_labels_group() -> None: def test_integration_compose_data_labels_directory() -> None: - """Composing +data/labels=directory lands cfg.data.labels._target_ at the FQN.""" + """Composing +data/labels=directory lands cfg.data.labels.use, resolvable to the FQN.""" from hydra import compose, initialize from hydra.core.global_hydra import GlobalHydra + from raitap.configs.registry_resolve import resolve_target_fqn + _register_labels_group() GlobalHydra.instance().clear() with initialize(version_base=None, config_path=None): cfg = compose(config_name="raitap_schema", overrides=["+data/labels=directory"]) - assert cfg.data.labels._target_ == _COMPOSED_TARGET + assert cfg.data.labels.use == "directory" + assert resolve_target_fqn("data/labels", cfg.data.labels.use) == _COMPOSED_TARGET # --- Task 4: TabularLabelParser --- diff --git a/src/raitap/deps/bootstrap.py b/src/raitap/deps/bootstrap.py index dbb763dc..4739e3ab 100644 --- a/src/raitap/deps/bootstrap.py +++ b/src/raitap/deps/bootstrap.py @@ -427,7 +427,7 @@ def _config_to_mapping(config: Any) -> Mapping[str, Any]: :mod:`raitap.deps.inference` walker can read. ``dataclasses.asdict`` recurses through nested dataclasses (the hydra-zen - builders return dataclass instances) and preserves the ``_target_`` field + builders return dataclass instances) and preserves the ``use`` field the walker keys on. """ import dataclasses @@ -460,7 +460,7 @@ def install_raitap_deps( Args: config: An ``AppConfig`` instance (or pre-converted ``Mapping``). - The walker reads ``_target_`` strings from ``transparency`` / + The walker reads ``use`` registry keys from ``transparency`` / ``robustness`` / ``metrics`` / ``reporting`` / ``tracking`` plus ``model.source`` to pick a backend extra. allow_project_edit: Consent to ``uv add`` modifying the caller's diff --git a/src/raitap/deps/inference.py b/src/raitap/deps/inference.py index 7508dbcb..b44ec9eb 100644 --- a/src/raitap/deps/inference.py +++ b/src/raitap/deps/inference.py @@ -2,15 +2,17 @@ The walker operates on a plain ``dict``/``Mapping`` — callers compose the Hydra config (or load YAML) and pass it in. Adapter blocks are recognised by -the presence of ``_target_``; bare class names and fully-qualified paths are -both accepted (the class name is taken from the last dotted segment). +the presence of a ``use: `` key, resolved to a class FQN via +:data:`raitap._adapters._TARGET_FQN` (bare class names and fully-qualified +paths are both accepted by the FQN->extra lookup; the class name is taken +from the last dotted segment). Outputs: - ``set[str]`` of extras (deduplicated; the CLI sorts at print/render time) - mapping from each extra name to a short human-readable origin phrase (used by :mod:`raitap.deps.conflicts` to build error messages) -Unknown adapter ``_target_`` values raise :class:`UnknownAdapterTargetError` +Unknown adapter ``use`` values raise :class:`UnknownAdapterTargetError` rather than emitting a possibly wrong command. """ @@ -26,7 +28,7 @@ class UnknownAdapterTargetError(RuntimeError): - """Raised when a ``_target_`` class is not in the adapter→extra map.""" + """Raised when a ``use`` registry key is not in the adapter→extra map.""" # ``ADAPTER_EXTRAS`` is populated by ``AdapterMixin.__init_subclass__`` once an @@ -39,7 +41,11 @@ class UnknownAdapterTargetError(RuntimeError): # Adding a new adapter therefore remains a single-file change — the AST scan # picks the new ``extra=`` kwarg up automatically. from raitap._adapters import ADAPTER_EXTRAS # noqa: E402 -from raitap.deps.static_scan import scan_adapter_extras, scan_backend_extras # noqa: E402 +from raitap.deps.static_scan import ( # noqa: E402 + scan_adapter_extras, + scan_adapter_registry, + scan_backend_extras, +) def _class_name(target: str) -> str: @@ -125,6 +131,40 @@ def _extra_for_target(target: str) -> str: ) +def _extra_for_use(group: str, use: str) -> str: + """Resolve a ``use: `` config key to its uv extra. + + Looks the key up in :data:`raitap._adapters._TARGET_FQN` (the sole trusted + seam a ``use`` key is resolved against at runtime — see + :mod:`raitap.configs.registry_resolve`) and reuses :func:`_extra_for_target` + on the resulting class FQN. ``_TARGET_FQN`` is only populated once the + matching adapter module has been imported though, so falls back to + :func:`raitap.deps.static_scan.scan_adapter_registry` (the import-free + ``(group, registry_name) -> extra`` scan) exactly like + :func:`_extra_for_target` falls back to ``scan_adapter_extras`` — this is + what keeps the partial-extras-venv bootstrap working before the very + libraries it is about to install are importable. + """ + from raitap._adapters import _TARGET_FQN + + fqn = _TARGET_FQN.get(group, {}).get(use) + if fqn is not None: + return _extra_for_target(fqn) + + scanned_group = scan_adapter_registry().get(group, {}) + extra = scanned_group.get(use) + if extra is not None: + return extra + + known = {**scanned_group, **_TARGET_FQN.get(group, {})} + match = difflib.get_close_matches(use, known, n=1) + suggestion = f" Did you mean '{match[0]}'?" if match else "" + known_list = ", ".join(sorted(known)) if known else "(none)" + raise UnknownAdapterTargetError( + f"Unknown {group} adapter key '{use}'.{suggestion}\nKnown {group} keys: {known_list}." + ) + + def _add(extras: dict[str, str], name: str, origin: str) -> None: if name not in extras: extras[name] = origin @@ -142,27 +182,29 @@ def _walk_section( for adapter_name, adapter_cfg in section.items(): if not isinstance(adapter_cfg, Mapping): continue - target = adapter_cfg.get("_target_") - if isinstance(target, str): + use = adapter_cfg.get("use") + if isinstance(use, str): _add( extras, - _extra_for_target(target), - f"{section_key}.{adapter_name}._target_={target}", + _extra_for_use(section_key, use), + f"{section_key}.{adapter_name}.use={use}", ) evaluation = adapter_cfg.get("evaluation") if isinstance(evaluation, Mapping): - eval_target = evaluation.get("_target_") - if isinstance(eval_target, str): + eval_use = evaluation.get("use") + if isinstance(eval_use, str): + # Evaluators register family=None, alongside visualisers, + # under the "_unscoped" group (no dedicated Hydra group). _add( extras, - _extra_for_target(eval_target), - f"{section_key}.{adapter_name}.evaluation._target_={eval_target}", + _extra_for_use("_unscoped", eval_use), + f"{section_key}.{adapter_name}.evaluation.use={eval_use}", ) return if isinstance(section, Mapping): - target = section.get("_target_") - if isinstance(target, str): - _add(extras, _extra_for_target(target), f"{section_key}._target_={target}") + use = section.get("use") + if isinstance(use, str): + _add(extras, _extra_for_use(section_key, use), f"{section_key}.use={use}") def _walk_launcher(extras: dict[str, str], cfg: Mapping[str, Any]) -> None: diff --git a/src/raitap/deps/static_scan.py b/src/raitap/deps/static_scan.py index e4323993..3daa26c3 100644 --- a/src/raitap/deps/static_scan.py +++ b/src/raitap/deps/static_scan.py @@ -27,9 +27,13 @@ import ast from functools import lru_cache from pathlib import Path +from typing import TYPE_CHECKING from raitap.types import ResolvedHardware +if TYPE_CHECKING: + from collections.abc import Iterator + # In-tree adapters decorate with the bare family decorator imported directly # from its ``registration`` module (e.g. ``@metrics_adapter(...)``). The public # ``@adapters.`` facade is for external plugin authors and is not used @@ -46,6 +50,20 @@ ) _VISUALISER_DECORATORS = frozenset({"transparency_visualiser", "robustness_visualiser"}) +# Family decorator -> the Hydra config group it registers under (mirrors each +# family's ``FamilyConfig(group=...)``). ``transparency_evaluator`` registers +# with ``family=None`` at runtime (like visualisers) so it lands in +# ``_TARGET_FQN["_unscoped"]`` — see +# :mod:`raitap.transparency.evaluation.evaluators.registration`. +_DECORATOR_GROUP: dict[str, str] = { + "transparency_adapter": "transparency", + "transparency_evaluator": "_unscoped", + "robustness_adapter": "robustness", + "metrics_adapter": "metrics", + "reporter": "reporting", + "tracker": "tracking", +} + def _str_set_literal(node: ast.expr | None) -> frozenset[str]: """Harvest a ``{"a", "b"}`` set literal of string constants, else empty.""" @@ -85,16 +103,37 @@ def _decorator_name(deco: ast.expr) -> str | None: return None -@lru_cache(maxsize=1) -def scan_adapter_extras() -> dict[str, str]: - """Return ``{class_name: extra}`` harvested from raitap's source tree.""" - import raitap +def _string_kwargs(call: ast.Call) -> dict[str, str]: + """Harvest the string-constant keyword arguments of a decorator call, e.g. + ``extra="captum"`` -> ``{"extra": "captum"}``. Non-string / non-constant + keyword values (and ``**kwargs`` spreads) are skipped.""" + return { + kw.arg: kw.value.value + for kw in call.keywords + if ( + kw.arg is not None + and isinstance(kw.value, ast.Constant) + and isinstance(kw.value.value, str) + ) + } - root = Path(raitap.__file__).resolve().parent - found: dict[str, str] = {} + +def _iter_decorated_classes(root: Path) -> Iterator[tuple[ast.ClassDef, ast.Call, str]]: + """Yield ``(class_node, decorator_call, decorator_name)`` for every + decorated class under ``root``, skipping ``tests`` directories and + unparseable files. + + Only a class's first decorator is considered, and only if it resolves to a + ``Call`` with a bare name (see :func:`_decorator_name`) — every + family/backend decorator in-tree decorates its class exactly once, so this + matches actual usage while staying import-free (no module in ``root`` is + imported, so this works even when the wrapped third-party library isn't + installed). + """ for path in root.rglob("*.py"): - # Tests can legitimately declare ``@_adapter(..., extra="…")`` — - # they are not real adapters and should not pollute the map. + # Tests can legitimately declare adapter/backend-shaped decorators for + # fixture purposes — they are not real adapters and must not pollute + # any map built on top of this generator. if "tests" in path.parts: continue try: @@ -106,29 +145,71 @@ def scan_adapter_extras() -> dict[str, str]: continue for deco in node.decorator_list: name = _decorator_name(deco) - if name not in _ADAPTER_DECORATORS and name not in _VISUALISER_DECORATORS: + if name is None: continue assert isinstance(deco, ast.Call) # narrowed by _decorator_name - kwargs = { - kw.arg: kw.value.value - for kw in deco.keywords - if ( - kw.arg is not None - and isinstance(kw.value, ast.Constant) - and isinstance(kw.value.value, str) - ) - } - # ``extra`` defaults to ``registry_name`` at runtime for every - # schema-backed adapter decorator; visualiser decorators don't - # get an auto-extra — they ship with their parent adapter's extra. - explicit_extra = kwargs.get("extra") - if explicit_extra: - found[node.name] = explicit_extra - elif name in _ADAPTER_DECORATORS: - registry_name = kwargs.get("registry_name") - if registry_name: - found[node.name] = registry_name + yield node, deco, name break + + +@lru_cache(maxsize=1) +def scan_adapter_extras() -> dict[str, str]: + """Return ``{class_name: extra}`` harvested from raitap's source tree.""" + import raitap + + root = Path(raitap.__file__).resolve().parent + found: dict[str, str] = {} + for class_node, deco, name in _iter_decorated_classes(root): + if name not in _ADAPTER_DECORATORS and name not in _VISUALISER_DECORATORS: + continue + kwargs = _string_kwargs(deco) + # ``extra`` defaults to ``registry_name`` at runtime for every + # schema-backed adapter decorator; visualiser decorators don't + # get an auto-extra — they ship with their parent adapter's extra. + explicit_extra = kwargs.get("extra") + if explicit_extra: + found[class_node.name] = explicit_extra + elif name in _ADAPTER_DECORATORS: + registry_name = kwargs.get("registry_name") + if registry_name: + found[class_node.name] = registry_name + return found + + +@lru_cache(maxsize=1) +def scan_adapter_registry() -> dict[str, dict[str, str]]: + """Return ``{group: {registry_name: extra}}`` harvested from raitap's + source tree — the import-free counterpart of + :data:`raitap._adapters._TARGET_FQN`. + + ``_TARGET_FQN`` maps ``group -> registry_name -> class FQN``, but it is + only populated once the adapter's module has been imported (the family + decorator fires at class-definition time). :mod:`raitap.deps.inference` + resolves a config's ``use: `` key to an extra *before* the + matching library is necessarily installed, so it cannot rely on + ``_TARGET_FQN`` alone — exactly why :func:`scan_adapter_extras` exists for + the legacy ``_target_`` lookup. This scanner mirrors that: the family + decorator name (e.g. ``robustness_adapter``) determines the group via + :data:`_DECORATOR_GROUP`, and ``extra`` defaults to ``registry_name`` the + same way :func:`scan_adapter_extras` and ``_register_core`` do — except for + the ``"_unscoped"`` group (``transparency_evaluator``, ``family=None`` at + runtime), where ``_register_core`` gives no auto-extra at all: only an + explicit ``extra=`` kwarg counts, else the extra is left empty. + """ + import raitap + + root = Path(raitap.__file__).resolve().parent + found: dict[str, dict[str, str]] = {} + for _class_node, deco, name in _iter_decorated_classes(root): + group = _DECORATOR_GROUP.get(name) + if group is None: + continue + kwargs = _string_kwargs(deco) + registry_name = kwargs.get("registry_name") + if not registry_name: + continue + default_extra = "" if group == "_unscoped" else registry_name + found.setdefault(group, {})[registry_name] = kwargs.get("extra", default_extra) return found @@ -149,30 +230,18 @@ def scan_backend_extras() -> dict[str, tuple[str, frozenset[ResolvedHardware]]]: root = Path(raitap.__file__).resolve().parent found: dict[str, tuple[str, frozenset[ResolvedHardware]]] = {} - for path in root.rglob("*.py"): - if "tests" in path.parts: + for _class_node, deco, name in _iter_decorated_classes(root): + if name != "register": continue - try: - tree = ast.parse(path.read_text(encoding="utf-8")) - except (SyntaxError, OSError, UnicodeDecodeError): + by_arg = {kw.arg: kw.value for kw in deco.keywords if kw.arg is not None} + # ``provides=`` disambiguates the backend @register from any other + # decorator that happens to be named ``register``. + if "provides" not in by_arg: continue - for node in ast.walk(tree): - if not isinstance(node, ast.ClassDef): - continue - for deco in node.decorator_list: - if _decorator_name(deco) != "register": - continue - assert isinstance(deco, ast.Call) # narrowed by _decorator_name - by_arg = {kw.arg: kw.value for kw in deco.keywords if kw.arg is not None} - # ``provides=`` disambiguates the backend @register from any other - # decorator that happens to be named ``register``. - if "provides" not in by_arg: - continue - extra_node = by_arg.get("extra") - if not (isinstance(extra_node, ast.Constant) and isinstance(extra_node.value, str)): - continue # file-backed backend without extra -> torch fallback - supported = _resolved_hardware_set_literal(by_arg.get("supported_hardware")) - for ext in _str_set_literal(by_arg.get("extensions")): - found[ext.lower()] = (extra_node.value, supported) - break + extra_node = by_arg.get("extra") + if not (isinstance(extra_node, ast.Constant) and isinstance(extra_node.value, str)): + continue # file-backed backend without extra -> torch fallback + supported = _resolved_hardware_set_literal(by_arg.get("supported_hardware")) + for ext in _str_set_literal(by_arg.get("extensions")): + found[ext.lower()] = (extra_node.value, supported) return found diff --git a/src/raitap/deps/tests/test_bootstrap.py b/src/raitap/deps/tests/test_bootstrap.py index ec362a50..ed4f4afc 100644 --- a/src/raitap/deps/tests/test_bootstrap.py +++ b/src/raitap/deps/tests/test_bootstrap.py @@ -25,8 +25,8 @@ def _fake_compose(monkeypatch: pytest.MonkeyPatch, cfg: dict[str, Any]) -> None: def _baseline_cfg() -> dict[str, Any]: return { "model": {"source": "x.pt"}, - "metrics": {"_target_": "MulticlassClassificationMetrics", "num_classes": 3}, - "reporting": {"_target_": "HTMLReporter", "filename": "r"}, + "metrics": {"use": "multiclass_classification", "num_classes": 3}, + "reporting": {"use": "html", "filename": "r"}, } diff --git a/src/raitap/deps/tests/test_e2e.py b/src/raitap/deps/tests/test_e2e.py index cf364c09..5debb445 100644 --- a/src/raitap/deps/tests/test_e2e.py +++ b/src/raitap/deps/tests/test_e2e.py @@ -21,23 +21,23 @@ "num_classes": 7, }, "data": {"name": "ham10000-presentation-balanced"}, - "metrics": {"_target_": "MulticlassClassificationMetrics", "num_classes": 7}, + "metrics": {"use": "multiclass_classification", "num_classes": 7}, "transparency": { - "gradcam": {"_target_": "CaptumExplainer", "algorithm": "LayerGradCam"}, - "saliency": {"_target_": "CaptumExplainer", "algorithm": "Saliency"}, + "gradcam": {"use": "captum", "algorithm": "LayerGradCam"}, + "saliency": {"use": "captum", "algorithm": "Saliency"}, }, "robustness": { - "fgsm": {"_target_": "TorchattacksAssessor", "algorithm": "FGSM"}, - "pgd": {"_target_": "TorchattacksAssessor", "algorithm": "PGD"}, - "marabou_linf": {"_target_": "MarabouAssessor", "algorithm": "linf-box"}, + "fgsm": {"use": "torchattacks", "algorithm": "FGSM"}, + "pgd": {"use": "torchattacks", "algorithm": "PGD"}, + "marabou_linf": {"use": "marabou", "algorithm": "linf-box"}, }, - "reporting": {"_target_": "HTMLReporter", "filename": "lwise_ham10000_report.pdf"}, + "reporting": {"use": "html", "filename": "lwise_ham10000_report.pdf"}, } _LWISE_ASSESSMENT_MLFLOW: dict = { **_LWISE_ASSESSMENT, - "tracking": {"_target_": "MLFlowTracker"}, + "tracking": {"use": "mlflow"}, } @@ -74,11 +74,11 @@ def test_lwise_ham10000_with_cuda_picks_torch_cuda() -> None: _MARABOU_MNIST_DEMO: dict = { "model": {"source": "mlp_mnist.onnx"}, "data": {"name": "mnist_samples"}, - "metrics": {"_target_": "MulticlassClassificationMetrics", "num_classes": 7}, + "metrics": {"use": "multiclass_classification", "num_classes": 7}, "robustness": { - "marabou_linf": {"_target_": "MarabouAssessor", "algorithm": "linf-box"}, + "marabou_linf": {"use": "marabou", "algorithm": "linf-box"}, }, - "reporting": {"_target_": "HTMLReporter"}, + "reporting": {"use": "html"}, } diff --git a/src/raitap/deps/tests/test_inference.py b/src/raitap/deps/tests/test_inference.py index 21def043..b47408f3 100644 --- a/src/raitap/deps/tests/test_inference.py +++ b/src/raitap/deps/tests/test_inference.py @@ -110,7 +110,7 @@ def test_captum_explainer_block_adds_extra() -> None: cfg = { "model": {"source": "x.pt"}, "transparency": { - "ig": {"_target_": "CaptumExplainer", "algorithm": "IntegratedGradients"}, + "ig": {"use": "captum", "algorithm": "IntegratedGradients"}, }, } extras, _ = infer_extras(cfg, hardware=ResolvedHardware.cpu) @@ -120,9 +120,7 @@ def test_captum_explainer_block_adds_extra() -> None: def test_shap_explainer_block_adds_extra() -> None: cfg = { "model": {"source": "x.pt"}, - "transparency": { - "shap_block": {"_target_": "ShapExplainer", "algorithm": "GradientExplainer"} - }, + "transparency": {"shap_block": {"use": "shap", "algorithm": "GradientExplainer"}}, } extras, _ = infer_extras(cfg, hardware=ResolvedHardware.cpu) assert "shap" in extras @@ -132,9 +130,9 @@ def test_robustness_extras() -> None: cfg = { "model": {"source": "x.pt"}, "robustness": { - "ta": {"_target_": "TorchattacksAssessor", "algorithm": "PGD"}, - "fb": {"_target_": "FoolboxAssessor", "algorithm": "LinfPGD"}, - "mb": {"_target_": "MarabouAssessor", "algorithm": "linf-box"}, + "ta": {"use": "torchattacks", "algorithm": "PGD"}, + "fb": {"use": "foolbox", "algorithm": "LinfPGD"}, + "mb": {"use": "marabou", "algorithm": "linf-box"}, }, } extras, _ = infer_extras(cfg, hardware=ResolvedHardware.cpu) @@ -144,7 +142,7 @@ def test_robustness_extras() -> None: def test_reporting_html_uses_html_extra() -> None: cfg = { "model": {"source": "x.pt"}, - "reporting": {"_target_": "HTMLReporter", "filename": "r"}, + "reporting": {"use": "html", "filename": "r"}, } extras, _ = infer_extras(cfg, hardware=ResolvedHardware.cpu) assert "html" in extras @@ -154,7 +152,7 @@ def test_reporting_html_uses_html_extra() -> None: def test_reporting_pdf_uses_pdf_extra() -> None: cfg = { "model": {"source": "x.pt"}, - "reporting": {"_target_": "PDFReporter", "filename": "r"}, + "reporting": {"use": "pdf", "filename": "r"}, } extras, _ = infer_extras(cfg, hardware=ResolvedHardware.cpu) assert "pdf" in extras @@ -162,7 +160,7 @@ def test_reporting_pdf_uses_pdf_extra() -> None: def test_reporting_disabled() -> None: - cfg = {"model": {"source": "x.pt"}, "reporting": {"_target_": None}} + cfg = {"model": {"source": "x.pt"}, "reporting": {"use": None}} extras, _ = infer_extras(cfg, hardware=ResolvedHardware.cpu) assert "pdf" not in extras and "html" not in extras @@ -170,7 +168,7 @@ def test_reporting_disabled() -> None: def test_tracking_mlflow() -> None: cfg = { "model": {"source": "x.pt"}, - "tracking": {"_target_": "MLFlowTracker"}, + "tracking": {"use": "mlflow"}, } extras, _ = infer_extras(cfg, hardware=ResolvedHardware.cpu) assert "mlflow" in extras @@ -179,44 +177,44 @@ def test_tracking_mlflow() -> None: def test_metrics_block_adds_extra() -> None: cfg = { "model": {"source": "x.pt"}, - "metrics": {"_target_": "MulticlassClassificationMetrics", "num_classes": 3}, + "metrics": {"use": "multiclass_classification", "num_classes": 3}, } extras, _ = infer_extras(cfg, hardware=ResolvedHardware.cpu) assert "metrics" in extras -def test_unknown_target_raises() -> None: +def test_unknown_use_raises() -> None: cfg = { "model": {"source": "x.pt"}, - "transparency": {"bad": {"_target_": "TotallyMadeUpAdapter"}}, + "transparency": {"bad": {"use": "totally_made_up"}}, } with pytest.raises(UnknownAdapterTargetError): infer_extras(cfg, hardware=ResolvedHardware.cpu) -def test_unknown_target_message_lists_known_adapters() -> None: +def test_unknown_use_message_lists_known_keys() -> None: cfg = { "model": {"source": "x.pt"}, - "transparency": {"bad": {"_target_": "djd"}}, + "transparency": {"bad": {"use": "djd"}}, } with pytest.raises(UnknownAdapterTargetError) as excinfo: infer_extras(cfg, hardware=ResolvedHardware.cpu) msg = str(excinfo.value) - # End-user-facing: names the bad target and enumerates valid ones. + # End-user-facing: names the bad key and enumerates valid ones. assert "djd" in msg - assert "Known adapters:" in msg - assert "CaptumExplainer" in msg + assert "transparency" in msg + assert "captum" in msg -def test_unknown_target_message_suggests_close_match() -> None: +def test_unknown_use_message_suggests_close_match() -> None: cfg = { "model": {"source": "x.pt"}, - # One-char typo of a real adapter -> difflib should suggest it. - "transparency": {"bad": {"_target_": "CaptumExplaner"}}, + # One-char typo of a real registry key -> difflib should suggest it. + "transparency": {"bad": {"use": "captm"}}, } with pytest.raises(UnknownAdapterTargetError) as excinfo: infer_extras(cfg, hardware=ResolvedHardware.cpu) - assert "Did you mean 'CaptumExplainer'?" in str(excinfo.value) + assert "Did you mean 'captum'?" in str(excinfo.value) def test_visualisers_do_not_contribute() -> None: @@ -224,8 +222,8 @@ def test_visualisers_do_not_contribute() -> None: "model": {"source": "x.pt"}, "transparency": { "ig": { - "_target_": "CaptumExplainer", - "visualisers": [{"_target_": "CaptumImageVisualiser"}], + "use": "captum", + "visualisers": [{"use": "captum_image"}], } }, } @@ -246,15 +244,17 @@ def test_launcher_extra() -> None: assert "launcher" in extras -def test_fully_qualified_target_still_resolves() -> None: - cfg = { - "model": {"source": "x.pt"}, - "transparency": { - "ig": {"_target_": "raitap.transparency.explainers.captum_explainer.CaptumExplainer"} - }, - } - extras, _ = infer_extras(cfg, hardware=ResolvedHardware.cpu) - assert "captum" in extras +def test_extra_for_target_accepts_fully_qualified_fqn() -> None: + # ``_extra_for_target`` still takes a bare class name or a fully-qualified + # path (it strips to the last dotted segment) — the walker feeds it the + # FQN resolved from the ``use`` registry key, not a raw config string, but + # the helper itself stays permissive. + from raitap.deps.inference import _extra_for_target + + assert ( + _extra_for_target("raitap.transparency.explainers.captum_explainer.CaptumExplainer") + == "captum" + ) def test_mapping_table_lists_all_known_targets() -> None: diff --git a/src/raitap/deps/tests/test_inference_evaluation.py b/src/raitap/deps/tests/test_inference_evaluation.py index e78a0297..54185527 100644 --- a/src/raitap/deps/tests/test_inference_evaluation.py +++ b/src/raitap/deps/tests/test_inference_evaluation.py @@ -1,4 +1,4 @@ -"""Nested transparency..evaluation._target_ must also infer an extra (#341).""" +"""Nested transparency..evaluation.use must also infer an extra (#341).""" from __future__ import annotations @@ -6,13 +6,13 @@ from raitap.types import ResolvedHardware -def test_nested_evaluation_target_infers_quantus() -> None: +def test_nested_evaluation_use_infers_quantus() -> None: cfg = { "model": {"source": "foo.pt"}, "transparency": { "ig": { - "_target_": "raitap.transparency.CaptumExplainer", - "evaluation": {"_target_": "raitap.transparency.QuantusEvaluator"}, + "use": "captum", + "evaluation": {"use": "quantus"}, } }, } diff --git a/src/raitap/deps/tests/test_static_scan.py b/src/raitap/deps/tests/test_static_scan.py index 9c673656..2035b699 100644 --- a/src/raitap/deps/tests/test_static_scan.py +++ b/src/raitap/deps/tests/test_static_scan.py @@ -16,8 +16,12 @@ from __future__ import annotations -from raitap._adapters import ADAPTER_EXTRAS -from raitap.deps.static_scan import scan_adapter_extras, scan_backend_extras +from raitap._adapters import _TARGET_FQN, ADAPTER_EXTRAS +from raitap.deps.static_scan import ( + scan_adapter_extras, + scan_adapter_registry, + scan_backend_extras, +) from raitap.types import ResolvedHardware @@ -57,6 +61,63 @@ def test_static_scan_finds_canonical_set() -> None: }, f"Scanner missed canonical adapters. Found: {sorted(scanned)!r}." +def test_runtime_target_fqn_subset_of_scan_adapter_registry() -> None: + # Same drift guard as ``test_runtime_extras_subset_of_static_scan``, but for + # the ``(group, registry_name) -> extra`` map ``_extra_for_use`` (#301) + # falls back on when ``_TARGET_FQN`` hasn't been populated yet (adapter + # module not imported in a partial-extras venv). + try: + from raitap.configs.zen import register_zen_groups + + register_zen_groups() + except Exception: # pragma: no cover — depends on venv state + pass + + # Only the groups deps inference actually walks — ``data/inputs`` and + # ``data/labels`` use a different family decorator, not covered by + # :data:`raitap.deps.static_scan._DECORATOR_GROUP`, and are irrelevant to + # :func:`raitap.deps.inference.infer_extras`. + relevant_groups = {"transparency", "robustness", "metrics", "reporting", "tracking"} + scanned = scan_adapter_registry() + for group, names in _TARGET_FQN.items(): + if group not in relevant_groups: + continue + for registry_name, fqn in names.items(): + # Sibling tests register stub adapters (e.g. ``_stub_metric``) into + # the runtime registry; the AST scanner intentionally skips ``tests`` + # dirs, so those never appear in ``scanned``. Guard only in-tree + # adapters — the drift this test protects against. + if ".tests." in fqn: + continue + assert registry_name in scanned.get(group, {}), ( + f"AST scanner missed {group}/{registry_name!r}." + ) + # "_unscoped" mixes evaluators (scanned) with visualisers (not scanned — + # the deps walker never reads a "visualisers" list's ``use``, see + # ``test_visualisers_do_not_contribute`` in test_inference.py); only guard + # the evaluator registered there. + if "quantus" in _TARGET_FQN.get("_unscoped", {}): + assert scanned["_unscoped"]["quantus"] == "quantus" + + +def test_scan_adapter_registry_finds_canonical_set() -> None: + """Scanner must always find the full set, regardless of import state.""" + scanned = scan_adapter_registry() + assert scanned["transparency"] == {"captum": "captum", "shap": "shap"} + assert scanned["reporting"] == {"html": "html", "pdf": "pdf"} + assert scanned["tracking"] == {"mlflow": "mlflow"} + assert scanned["_unscoped"]["quantus"] == "quantus" + assert {"auto_lirpa", "foolbox", "imagecorruptions", "marabou", "torchattacks"} <= set( + scanned["robustness"] + ) + assert { + "binary_classification", + "multiclass_classification", + "multilabel_classification", + "detection", + } <= set(scanned["metrics"]) + + def test_scan_backend_extras_maps_extension_to_extra_and_hardware() -> None: """Import-free scan of backend ``@register`` decorators: extension -> (extra, supported_hardware). Guards drift between the decorators and the diff --git a/src/raitap/metrics/factory.py b/src/raitap/metrics/factory.py index ab31a8fb..66b246f6 100644 --- a/src/raitap/metrics/factory.py +++ b/src/raitap/metrics/factory.py @@ -10,7 +10,8 @@ from hydra.utils import instantiate from raitap import raitap_log -from raitap.configs import cfg_to_dict, resolve_run_dir, resolve_target +from raitap.configs import cfg_to_dict, resolve_run_dir +from raitap.configs.registry_resolve import stamp_target_from_use, use_key_enabled from raitap.reporting.sections import Reportable, ReportGroup, ReportSection from raitap.reporting.staging import _copy_asset from raitap.tracking.base_tracker import BaseTracker, Trackable @@ -27,34 +28,29 @@ from .base_metric_computer import BaseMetricComputer, MetricResult, scalar_metrics_for_tracking from .visualizers import MetricsVisualizer -_METRICS_PREFIX = "raitap.metrics." - def metrics_run_enabled(config: AppConfig) -> bool: - """True when ``metrics`` is present and ``_target_`` is a non-empty string.""" + """True when ``metrics`` is present and ``use`` is a non-empty string.""" metrics_cfg = config.metrics if metrics_cfg is None: return False - target = getattr(metrics_cfg, "_target_", None) - if target is None: - return False - return bool(str(target).strip()) + return use_key_enabled(cfg_to_dict(metrics_cfg)) def create_metric(metrics_config: Any) -> tuple[BaseMetricComputer, str]: - """Instantiate a metric computer from Hydra-style config (``_target_`` + kwargs).""" + """Instantiate a metric computer from config (``use: `` + kwargs).""" metrics_cfg = cfg_to_dict(metrics_config) - target_path: str = metrics_cfg.get("_target_", "") - resolved_target = resolve_target(target_path, _METRICS_PREFIX) - metrics_cfg["_target_"] = resolved_target + use = str(metrics_cfg.get("use", "")) + stamp_target_from_use(metrics_cfg, group="metrics") + resolved_target = metrics_cfg["_target_"] try: metric = instantiate(metrics_cfg) except Exception as e: - raitap_log.exception("Metric instantiation failed for target %r", target_path) + raitap_log.exception("Metric instantiation failed for target %r", resolved_target) raise ValueError( - f"Could not instantiate metric {target_path!r}.\n" - "Check that _target_ points to a valid MetricComputer implementation." + f"Could not instantiate metric {use!r}.\n" + "Check that `use` points to a registered metrics adapter." ) from e return metric, resolved_target @@ -147,7 +143,9 @@ def __new__( "experiment_name": config.experiment_name, "target": resolved_target, "metric_config": { - k: to_json_serialisable(v) for k, v in metrics_cfg.items() if k != "_target_" + k: to_json_serialisable(v) + for k, v in metrics_cfg.items() + if k not in ("_target_", "use") }, } (run_dir / "metadata.json").write_text(json.dumps(metadata, indent=2), encoding="utf-8") diff --git a/src/raitap/metrics/tests/test_factory_evaluate.py b/src/raitap/metrics/tests/test_factory_evaluate.py index e11d9515..267307e9 100644 --- a/src/raitap/metrics/tests/test_factory_evaluate.py +++ b/src/raitap/metrics/tests/test_factory_evaluate.py @@ -11,6 +11,7 @@ from pathlib import Path from raitap.configs import set_output_root +from raitap.configs.registry_resolve import UnsafeConfigTargetError from raitap.configs.schema import ( AppConfig, MetricsConfig, @@ -18,18 +19,29 @@ ) from raitap.metrics import MetricsEvaluation, evaluate, metrics_run_enabled from raitap.metrics.base_metric_computer import BaseMetricComputer, MetricResult +from raitap.metrics.factory import create_metric -def test_metrics_run_enabled_respects_empty_target(tmp_path: Path) -> None: +def test_metrics_run_enabled_respects_empty_use(tmp_path: Path) -> None: cfg = AppConfig(experiment_name="t") set_output_root(cfg, tmp_path) assert not metrics_run_enabled(cfg) # metrics is None by default - cfg.metrics = MetricsConfig(_target_="") + cfg.metrics = MetricsConfig(use="") assert not metrics_run_enabled(cfg) - cfg.metrics = MetricsConfig(_target_="MulticlassClassificationMetrics") + cfg.metrics = MetricsConfig(use="multiclass_classification") assert metrics_run_enabled(cfg) +def test_metrics_run_enabled_rejects_config_target(tmp_path: Path) -> None: + """A `_target_`-carrying block must raise loudly, not read as "not configured".""" + cfg = AppConfig(experiment_name="t") + set_output_root(cfg, tmp_path) + cfg.metrics = {"_target_": "os.system"} # type: ignore[assignment] + + with pytest.raises(UnsafeConfigTargetError): + metrics_run_enabled(cfg) + + def _config(tmp_path: Path) -> AppConfig: cfg = AppConfig(experiment_name="test") set_output_root(cfg, tmp_path) @@ -56,18 +68,31 @@ def test_evaluate_writes_outputs(tmp_path: Path) -> None: metadata = json.loads((run_dir / "metadata.json").read_text(encoding="utf-8")) assert "accuracy" in metrics - assert metadata["target"] == "raitap.metrics.MulticlassClassificationMetrics" + expected_target = "raitap.metrics.classification_metrics.MulticlassClassificationMetrics" + assert metadata["target"] == expected_target + assert "use" not in metadata["metric_config"] + assert "_target_" not in metadata["metric_config"] -def test_evaluate_bad_target_raises(tmp_path: Path) -> None: +def test_evaluate_bad_use_raises(tmp_path: Path) -> None: cfg = _config(tmp_path) assert cfg.metrics is not None - cfg.metrics._target_ = "DoesNotExist" + cfg.metrics.use = "does_not_exist" - with pytest.raises(ValueError, match="Could not instantiate metric"): + with pytest.raises(ValueError, match="Unknown metrics key"): evaluate(cfg, torch.tensor([0]), torch.tensor([0])) +def test_create_metric_rejects_config_target() -> None: + with pytest.raises(UnsafeConfigTargetError): + create_metric({"_target_": "os.system", "use": "multiclass_classification"}) + + +def test_create_metric_rejects_unknown_use() -> None: + with pytest.raises(ValueError, match="Unknown metrics key"): + create_metric({"use": "does_not_exist"}) + + def test_evaluate_writes_under_metrics_subdirectory(tmp_path: Path) -> None: cfg = _config(tmp_path) out = evaluate(cfg, torch.tensor([0, 1, 2, 1]), torch.tensor([0, 1, 2, 0])) @@ -167,7 +192,7 @@ def test_evaluate_metrics_dispatches_detection_metrics_on_forward_output( result = evaluate_metrics(cfg, forward, labels) assert result is not None - assert result.resolved_target == "raitap.metrics.DetectionMetrics" + assert result.resolved_target == "raitap.metrics.detection_metrics.DetectionMetrics" def test_evaluate_metrics_skips_detection_without_labels(tmp_path: Path) -> None: diff --git a/src/raitap/pipeline/orchestrator.py b/src/raitap/pipeline/orchestrator.py index 36bbf5df..5f289f80 100644 --- a/src/raitap/pipeline/orchestrator.py +++ b/src/raitap/pipeline/orchestrator.py @@ -8,7 +8,8 @@ from typing import TYPE_CHECKING, cast from raitap import raitap_log -from raitap.configs import resolve_run_dir +from raitap.configs import cfg_to_dict, resolve_run_dir +from raitap.configs.registry_resolve import reject_config_target from raitap.data import Data from raitap.data.preprocessing import resolve_preprocessing from raitap.models import Model @@ -175,7 +176,13 @@ def _run_pipeline( raitap_log.warn(cast("str", reproducibility_caveat(repro))) tracking_config = config.tracking - if tracking_config is None or not getattr(tracking_config, "_target_", None): + if tracking_config is None: + return outputs + tracking_cfg = cfg_to_dict(tracking_config) + # Reject a `_target_`-carrying block loudly instead of silently reading it + # as "tracking disabled" (mirrors reporting_enabled's guard, issue #301). + reject_config_target(tracking_cfg) + if not tracking_cfg.get("use"): return outputs _log_run_to_tracker( config, model, data, outputs, report_generation, log_model=tracking_config.log_model diff --git a/src/raitap/reporting/builder.py b/src/raitap/reporting/builder.py index 61a97d69..8d16f9e4 100644 --- a/src/raitap/reporting/builder.py +++ b/src/raitap/reporting/builder.py @@ -77,10 +77,7 @@ def build_report(config: AppConfig, outputs: RunOutputs) -> BuiltReport: show_redundant_robustness_panels=bool( getattr(reporting_cfg, "show_redundant_robustness_panels", False) ), - explicit_selection=( - explicit_samples is not None - and _reporting_target(config) in {"PDFReporter", "raitap.reporting.PDFReporter"} - ), + explicit_selection=(explicit_samples is not None and _reporting_target(config) == "pdf"), ) sections: list[ReportSection] = [] @@ -219,14 +216,14 @@ def _manifest_filename(config: AppConfig) -> str: assert config.reporting is not None target = _reporting_target(config) filename = str(config.reporting.filename) - if target in {"HTMLReporter", "raitap.reporting.HTMLReporter"}: + if target == "html": return report_output_filename(filename, ".html") return report_output_filename(filename, ".pdf") def _reporting_target(config: AppConfig) -> str: reporting = config.reporting - return str(getattr(reporting, "_target_", "")) + return str(getattr(reporting, "use", "")) def _selected_sample_manifest_entry(sample: SelectedSample) -> dict[str, object]: diff --git a/src/raitap/reporting/factory.py b/src/raitap/reporting/factory.py index 993027ae..6a7a4753 100644 --- a/src/raitap/reporting/factory.py +++ b/src/raitap/reporting/factory.py @@ -7,7 +7,8 @@ from hydra.utils import instantiate from raitap import raitap_log -from raitap.configs import cfg_to_dict, resolve_target +from raitap.configs import cfg_to_dict +from raitap.configs.registry_resolve import instantiate_partial_from_use, use_key_enabled from raitap.tracking.base_tracker import BaseTracker, Trackable if TYPE_CHECKING: @@ -19,18 +20,11 @@ from .builder import BuiltReport -_REPORTING_PREFIX = "raitap.reporting." - - def reporting_enabled(config: AppConfig) -> bool: """Check if reporting is enabled in config.""" - reporting_cfg = config.reporting - if reporting_cfg is None: + if config.reporting is None: return False - target = getattr(reporting_cfg, "_target_", None) - if target is None: - return False - return bool(str(target).strip()) + return use_key_enabled(cfg_to_dict(config.reporting)) @dataclass @@ -60,18 +54,13 @@ def create_report( ) -> ReportGeneration: """Factory function to create and generate report.""" reporting_config = cfg_to_dict(config.reporting) - target_path = str(reporting_config.get("_target_", "")) - resolved_target = resolve_target(target_path, _REPORTING_PREFIX) - - try: - reporter_class = instantiate({"_target_": resolved_target, "_partial_": True}) - reporter = reporter_class(config) - except Exception as error: - raitap_log.exception("Reporter instantiation failed for target %r", target_path) - raise ValueError( - f"Could not instantiate reporter {target_path!r}.\n" - "Check that _target_ points to a valid BaseReporter implementation." - ) from error + reporter = instantiate_partial_from_use( + reporting_config, + group="reporting", + entity="reporter", + config=config, + instantiate_fn=instantiate, + ) report_path = reporter.generate(report.sections, report_dir=report.report_dir) manifest_path = report_path.parent / "report_manifest.json" diff --git a/src/raitap/reporting/tests/test_builder.py b/src/raitap/reporting/tests/test_builder.py index fc81e664..e845b731 100644 --- a/src/raitap/reporting/tests/test_builder.py +++ b/src/raitap/reporting/tests/test_builder.py @@ -376,7 +376,7 @@ def _make_robustness_result( def test_build_report_orders_sections_and_ranks_samples(tmp_path: Path) -> None: config = AppConfig(experiment_name="demo") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") metrics_image = _write_test_image(tmp_path / "metrics.png") explanation = ExplanationResult( @@ -482,7 +482,7 @@ def test_copy_asset_rejects_path_like_target_names(tmp_path: Path) -> None: def test_build_report_skips_global_section_for_local_only_outputs(tmp_path: Path) -> None: config = AppConfig(experiment_name="local_only") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") explanation = ExplanationResult( attributions=torch.rand(3, 1, 4, 4), @@ -545,7 +545,7 @@ def _repro_outputs(explanation: Any) -> RunOutputs: def test_build_report_prepends_reproducibility_banner_when_stochastic(tmp_path: Path) -> None: config = AppConfig(experiment_name="repro") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") outputs = _repro_outputs(_explanation_with_stochastic(tmp_path, seeding="global_rng")) @@ -559,7 +559,7 @@ def test_build_report_prepends_reproducibility_banner_when_stochastic(tmp_path: def test_build_report_no_banner_when_all_deterministic(tmp_path: Path) -> None: config = AppConfig(experiment_name="repro") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") outputs = _repro_outputs(_explanation_with_stochastic(tmp_path, seeding="deterministic")) @@ -571,7 +571,7 @@ def test_build_report_no_banner_when_all_deterministic(tmp_path: Path) -> None: def test_build_report_no_banner_when_global_rng_covered_by_seed(tmp_path: Path) -> None: config = AppConfig(experiment_name="repro") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") config.seed = 42 outputs = _repro_outputs(_explanation_with_stochastic(tmp_path, seeding="global_rng")) @@ -584,7 +584,7 @@ def test_build_report_no_banner_when_global_rng_covered_by_seed(tmp_path: Path) def test_build_report_banner_present_for_global_rng_when_seed_unset(tmp_path: Path) -> None: config = AppConfig(experiment_name="repro") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") config.seed = None outputs = _repro_outputs(_explanation_with_stochastic(tmp_path, seeding="global_rng")) @@ -599,7 +599,7 @@ def test_build_report_places_aggregated_visualisations_between_global_and_local( ) -> None: config = AppConfig(experiment_name="aggregated") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") metrics_image = _write_test_image(tmp_path / "metrics.png") explanation = ExplanationResult( @@ -646,7 +646,7 @@ def test_build_report_places_aggregated_visualisations_between_global_and_local( def test_build_report_local_assets_are_staged_and_closed(tmp_path: Path) -> None: config = AppConfig(experiment_name="local_assets") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") visualiser = _LocalImageVisualiser() explanation = ExplanationResult( @@ -701,7 +701,7 @@ def test_build_report_compact_local_thumbnail_titles_are_stripped( ) -> None: config = AppConfig(experiment_name="compact_thumbnail_titles") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") thumbnail_figures: list[Figure] = [] @@ -751,7 +751,7 @@ def test_build_report_compact_mode_omits_repeated_original_for_capable_visualise ) -> None: config = AppConfig(experiment_name="compact") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") compact_visualiser = _EmbeddedOriginalVisualiser() masked_visualiser = _MaskedLikeVisualiser() @@ -798,7 +798,7 @@ def test_build_report_local_explainer_group_includes_curated_transparency_rows( ) -> None: config = AppConfig(experiment_name="transparency_rows") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") heatmap_visualiser = _LocalImageVisualiser( title="Grad-CAM lesion localisation", @@ -900,7 +900,7 @@ def test_build_report_show_original_per_explainer_uses_verbose_local_layout( config = AppConfig(experiment_name="legacy_originals") set_output_root(config, tmp_path) config.reporting = ReportingConfig( - _target_="PDFReporter", + use="pdf", filename="report.pdf", show_original_per_explainer=True, ) @@ -943,7 +943,7 @@ def test_build_report_thumbnail_uses_first_compatible_explanation_in_order( ) -> None: config = AppConfig(experiment_name="thumbnail_fallback") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") image_visualiser = _EmbeddedOriginalVisualiser() tabular_visualiser = _EmbeddedOriginalVisualiser() @@ -1002,7 +1002,7 @@ def test_build_report_thumbnail_falls_back_to_later_explanation_after_runtime_er ) -> None: config = AppConfig(experiment_name="thumbnail_multi_fallback") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") original_visualise = InputThumbnailVisualiser.visualise call_count = {"n": 0} @@ -1066,7 +1066,7 @@ def test_build_report_thumbnail_failure_falls_back_for_that_sample_only( ) -> None: config = AppConfig(experiment_name="thumbnail_fallback") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") visualiser = _EmbeddedOriginalVisualiser() explanation = ExplanationResult( @@ -1105,7 +1105,7 @@ def test_build_report_thumbnail_runtime_failure_logs_traceback_and_falls_back( ) -> None: config = AppConfig(experiment_name="thumbnail_runtime_fallback") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") def _raise_runtime_error(*_args: Any, **_kwargs: Any) -> Figure: raise RuntimeError("thumbnail render failed") @@ -1156,7 +1156,7 @@ def test_build_report_thumbnail_programmer_error_is_not_swallowed( ) -> None: config = AppConfig(experiment_name="thumbnail_programmer_error") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") def _raise_type_error(*_args: Any, **_kwargs: Any) -> Figure: raise TypeError("programmer error") @@ -1230,7 +1230,7 @@ def test_build_report_html_explicit_selection_uses_compact_local_layout( tmp_path: Path, ) -> None: config, outputs = _explicit_selection_case(tmp_path) - config.reporting = ReportingConfig(_target_="HTMLReporter", filename="report") + config.reporting = ReportingConfig(use="html", filename="report") config.reporting.sample_selection = [ "case_gamma.png", "case_alpha.png", @@ -1363,7 +1363,7 @@ def test_report_sample_selection_entry_type_documents_supported_values() -> None def test_build_report_skips_local_groups_when_no_local_visualisations(tmp_path: Path) -> None: config = AppConfig(experiment_name="no_local") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") class _GlobalOnlyVisualiser(_LocalImageVisualiser): produces_scope = ExplanationScope.GLOBAL @@ -1440,7 +1440,7 @@ def test_build_report_compact_robustness_omits_non_owner_perturbation_panel( ) -> None: config = AppConfig(experiment_name="robustness_compact") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") result = _make_robustness_result( tmp_path, @@ -1477,7 +1477,7 @@ def test_build_report_compact_robustness_skips_redundant_single_facet_visualiser ) -> None: config = AppConfig(experiment_name="robustness_compact_skip") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") owner = _PerturbationRecordingVisualiser() redundant = _PerturbationRecordingVisualiser() @@ -1512,7 +1512,7 @@ def test_build_report_compact_robustness_propagates_visualiser_errors( ) -> None: config = AppConfig(experiment_name="robustness_compact_error") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") result = _make_robustness_result( tmp_path, @@ -1538,7 +1538,7 @@ def test_build_report_compact_robustness_renders_selected_samples_per_assessor( ) -> None: config = AppConfig(experiment_name="robustness_compact_samples") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") result = _make_robustness_result( tmp_path, @@ -1615,7 +1615,7 @@ def _record_stripped_titles(figure: Figure) -> None: def test_build_report_robustness_single_pair_keeps_all_panels(tmp_path: Path) -> None: config = AppConfig(experiment_name="robustness_single_pair") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") result = _make_robustness_result( tmp_path, @@ -1640,7 +1640,7 @@ def test_build_report_verbose_robustness_reuses_existing_visualisations_without_ config = AppConfig(experiment_name="robustness_legacy") set_output_root(config, tmp_path) config.reporting = ReportingConfig( - _target_="PDFReporter", + use="pdf", filename="report.pdf", show_redundant_robustness_panels=True, ) @@ -1678,7 +1678,7 @@ def test_build_report_robustness_redundant_single_facet_without_kwarg_support_is ) -> None: config = AppConfig(experiment_name="robustness_redundant_strict_visualiser") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") result = _make_robustness_result( tmp_path, @@ -1703,7 +1703,7 @@ def test_build_report_robustness_redundant_single_facet_without_kwarg_support_is def test_report_manifest_round_trip_preserves_relative_images(tmp_path: Path) -> None: config = AppConfig(experiment_name="demo") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") explanation = ExplanationResult( attributions=torch.rand(2, 1, 4, 4), @@ -1781,7 +1781,7 @@ def test_create_report_writes_manifest_next_to_generated_report( ) -> None: config = AppConfig(experiment_name="demo") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") built_report = ReportSection.from_groups( "Metrics", [ReportGroup(heading="Performance Metrics", table_rows=(("accuracy", "0.9"),))], @@ -1819,7 +1819,7 @@ def test_create_report_writes_html_archive_with_manifest_and_assets( ) -> None: config = AppConfig(experiment_name="demo") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="HTMLReporter", filename="report.html") + config.reporting = ReportingConfig(use="html", filename="report.html") built_report = ReportSection.from_groups( "Metrics", [ReportGroup(heading="Performance Metrics", table_rows=(("accuracy", "0.9"),))], @@ -1859,7 +1859,7 @@ def generate(self, sections: Any, *, report_dir: Path | None = None) -> Path: def test_create_report_does_not_archive_pdf_report(tmp_path: Path, monkeypatch: Any) -> None: config = AppConfig(experiment_name="demo") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") built_report = ReportSection.from_groups( "Metrics", [ReportGroup(heading="Performance Metrics", table_rows=(("accuracy", "0.9"),))], @@ -1890,6 +1890,64 @@ def generate(self, sections: Any, *, report_dir: Path | None = None) -> Path: assert not generated.report_path.with_suffix(".zip").exists() +def test_create_report_resolves_html_via_use_key(tmp_path: Path) -> None: + """``use: html`` resolves to the real ``HTMLReporter`` through the trusted + registry, without stubbing ``instantiate`` (issue #301).""" + config = AppConfig(experiment_name="demo") + set_output_root(config, tmp_path) + config.reporting = ReportingConfig(use="html", filename="report.html") + built_report = ReportSection.from_groups( + "Metrics", + [ReportGroup(heading="Performance Metrics", table_rows=(("accuracy", "0.9"),))], + ) + report = ReportManifest(kind="run", sections=(built_report,), filename="report.html") + built = BuiltReport( + report_dir=tmp_path / "builder-reports", + sections=(built_report,), + manifest=report, + ) + + generated = create_report(config, built) + + assert type(generated.reporter).__name__ == "HTMLReporter" + + +def test_create_report_rejects_target_key(tmp_path: Path) -> None: + """A config carrying ``_target_`` must be rejected before any resolution + is attempted — closes the arbitrary-callable RCE surface (issue #301).""" + from raitap.configs.registry_resolve import UnsafeConfigTargetError + + config = AppConfig(experiment_name="demo") + set_output_root(config, tmp_path) + config.reporting = {"_target_": "os.system", "filename": "report"} # type: ignore[assignment] + built_report = ReportSection.from_groups( + "Metrics", + [ReportGroup(heading="Performance Metrics", table_rows=(("accuracy", "0.9"),))], + ) + report = ReportManifest(kind="run", sections=(built_report,), filename="report") + built = BuiltReport( + report_dir=tmp_path / "builder-reports", + sections=(built_report,), + manifest=report, + ) + + with pytest.raises(UnsafeConfigTargetError): + create_report(config, built) + + +def test_reporting_enabled_rejects_target_key() -> None: + """The ``reporting_enabled`` guard must also reject a ``_target_`` block + loudly rather than silently reporting it as disabled — this guard gates + non-schema-checked callback configs (issue #301).""" + from raitap.configs.registry_resolve import UnsafeConfigTargetError + from raitap.reporting.factory import reporting_enabled + + config = AppConfig(experiment_name="demo") + config.reporting = {"_target_": "os.system", "command": "id"} # type: ignore[assignment] + with pytest.raises(UnsafeConfigTargetError): + reporting_enabled(config) + + @pytest.mark.parametrize( ("configured_filename", "expected_html_name", "expected_pdf_name"), [ @@ -1914,7 +1972,7 @@ def test_build_report_manifest_filename_matches_selected_reporter( html_config = AppConfig(experiment_name="demo") set_output_root(html_config, tmp_path / "html") html_config.reporting = ReportingConfig( - _target_="raitap.reporting.HTMLReporter", + use="html", filename=configured_filename, ) html_report = build_report(html_config, outputs) @@ -1924,7 +1982,7 @@ def test_build_report_manifest_filename_matches_selected_reporter( pdf_config = AppConfig(experiment_name="demo") set_output_root(pdf_config, tmp_path / "pdf") pdf_config.reporting = ReportingConfig( - _target_="raitap.reporting.PDFReporter", + use="pdf", filename=configured_filename, ) pdf_report = build_report(pdf_config, outputs) @@ -1953,7 +2011,7 @@ def _capture_report(cfg: Any, report: Any) -> Any: callback = ReportingSweepCallback() config = make_app_config( experiment_name="demo", - reporting={"_target_": "PDFReporter", "filename": "report.pdf"}, + reporting={"use": "pdf", "filename": "report.pdf"}, ) # ``hydra`` is a Hydra runtime node, not an ``AppConfig`` field; attach it # outside the struct schema, matching the shape Hydra hands callbacks. @@ -2002,7 +2060,7 @@ def test_build_merged_report_deduplicates_identical_metrics_only(tmp_path: Path) report = build_merged_report( AppConfig( experiment_name="demo", - reporting=ReportingConfig(_target_="PDFReporter", filename="report.pdf"), + reporting=ReportingConfig(use="pdf", filename="report.pdf"), ), sweep_dir=sweep_dir, child_manifests=child_manifests, @@ -2039,7 +2097,7 @@ def test_build_merged_report_preserves_present_section_order_with_aggregated( report = build_merged_report( AppConfig( experiment_name="demo", - reporting=ReportingConfig(_target_="PDFReporter", filename="report.pdf"), + reporting=ReportingConfig(use="pdf", filename="report.pdf"), ), sweep_dir=sweep_dir, child_manifests=child_manifests, @@ -2070,7 +2128,7 @@ def test_build_merged_report_keeps_empty_metrics_groups(tmp_path: Path) -> None: report = build_merged_report( AppConfig( experiment_name="demo", - reporting=ReportingConfig(_target_="PDFReporter", filename="report.pdf"), + reporting=ReportingConfig(use="pdf", filename="report.pdf"), ), sweep_dir=sweep_dir, child_manifests=child_manifests, @@ -2085,24 +2143,24 @@ def test_build_merged_report_keeps_empty_metrics_groups(tmp_path: Path) -> None: def test_reporting_configs_compose_multirun_report_controls() -> None: - """Bundled reporting presets resolve the right ``_target_`` and wire the + """Bundled reporting presets resolve the right ``use`` key and wire the multirun-aggregation Hydra callback. - The presets ship as minimal ``_target_``-only stubs (plus the + The presets ship as minimal ``use``-only stubs (plus the ``reporting_sweep`` callback for non-disabled presets). All other ReportingConfig fields come from the user's config or CLI overrides; this test only verifies what's actually shipped + the callback wiring. """ pdf_cfg = _compose_raitap_config(["+reporting=pdf"]) - assert pdf_cfg.reporting._target_ == "PDFReporter" + assert pdf_cfg.reporting.use == "pdf" assert pdf_cfg.hydra.callbacks.reporting_sweep._target_.endswith("ReportingSweepCallback") html_cfg = _compose_raitap_config(["+reporting=html"]) - assert html_cfg.reporting._target_ == "HTMLReporter" + assert html_cfg.reporting.use == "html" assert html_cfg.hydra.callbacks.reporting_sweep._target_.endswith("ReportingSweepCallback") disabled_cfg = _compose_raitap_config(["+reporting=disabled"]) - assert disabled_cfg.reporting._target_ is None + assert disabled_cfg.reporting.use is None assert disabled_cfg.reporting.multirun_report is False @@ -2125,7 +2183,7 @@ def _capture_report(*args: Any, **kwargs: Any) -> None: { "experiment_name": "demo", "reporting": { - "_target_": "PDFReporter", + "use": "pdf", "filename": "report.pdf", "multirun_report": False, }, @@ -2156,7 +2214,7 @@ def _capture_report(*args: Any, **kwargs: Any) -> None: config = OmegaConf.create( { "experiment_name": "demo", - "reporting": {"_target_": None, "multirun_report": False}, + "reporting": {"use": None, "multirun_report": False}, "hydra": {"sweep": {"dir": str(sweep_dir)}}, } ) @@ -2192,7 +2250,7 @@ def _explicit_selection_case( ] config = AppConfig(experiment_name="explicit_selection") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") explanation = ExplanationResult( attributions=torch.rand(len(ids), 1, 4, 4), inputs=torch.rand(len(ids), 1, 4, 4), @@ -2292,7 +2350,7 @@ def test_build_report_sampling_result_renders_without_error(tmp_path: Path) -> N config = AppConfig(experiment_name="sampling_test") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") semantics = RobustnessSemantics( assessment_kind=AssessmentKind.STATISTICAL_SAMPLING, @@ -2347,7 +2405,7 @@ def test_build_report_assessor_scope_figure_recorded_in_metadata(tmp_path: Path) config = AppConfig(experiment_name="sampling_scope") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") semantics = RobustnessSemantics( assessment_kind=AssessmentKind.STATISTICAL_SAMPLING, @@ -2533,7 +2591,7 @@ def test_build_report_attaches_baseline_image_once_per_explanation(tmp_path: Pat config = AppConfig(experiment_name="bl") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") run_dir = tmp_path / "transparency" / "exp" run_dir.mkdir(parents=True) @@ -2816,7 +2874,7 @@ def test_build_report_handles_reporting_block_without_sample_selection( # Build a struct-mode reporting block carrying every ReportingConfig field # EXCEPT ``sample_selection`` — isolates the failure to the line-51 read. reporting_dict = OmegaConf.to_container( - OmegaConf.structured(ReportingConfig(_target_="PDFReporter", filename="report.pdf")), + OmegaConf.structured(ReportingConfig(use="pdf", filename="report.pdf")), resolve=True, ) assert isinstance(reporting_dict, dict) diff --git a/src/raitap/reporting/tests/test_formal_section.py b/src/raitap/reporting/tests/test_formal_section.py index 9394faa9..74e846ac 100644 --- a/src/raitap/reporting/tests/test_formal_section.py +++ b/src/raitap/reporting/tests/test_formal_section.py @@ -76,7 +76,7 @@ def _formal_result(run_dir: Path) -> RobustnessResult: def test_build_report_renders_robustness_certification_section(tmp_path: Path) -> None: config = AppConfig(experiment_name="marabou_test") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") run_dir = tmp_path / "robustness" / "marabou_linf" run_dir.mkdir(parents=True, exist_ok=True) @@ -104,7 +104,7 @@ def test_build_report_includes_per_class_bound_rows_for_formal_results( ) -> None: config = AppConfig(experiment_name="marabou_test") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") run_dir = tmp_path / "robustness" / "marabou_linf" run_dir.mkdir(parents=True, exist_ok=True) @@ -146,7 +146,7 @@ def test_build_report_excludes_rows_with_only_lower_bound(tmp_path: Path) -> Non """A row with all-NaN upper must NOT count toward output_bounds_samples.""" config = AppConfig(experiment_name="marabou_test") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") run_dir = tmp_path / "robustness" / "marabou_linf" run_dir.mkdir(parents=True, exist_ok=True) @@ -187,7 +187,7 @@ def test_build_report_skips_bound_rows_for_empirical_attack_results( ) -> None: config = AppConfig(experiment_name="ea_test") set_output_root(config, tmp_path) - config.reporting = ReportingConfig(_target_="PDFReporter", filename="report.pdf") + config.reporting = ReportingConfig(use="pdf", filename="report.pdf") run_dir = tmp_path / "robustness" / "ea" run_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/raitap/reporting/tests/test_html_reporter.py b/src/raitap/reporting/tests/test_html_reporter.py index 9e2c9616..9ff00b7a 100644 --- a/src/raitap/reporting/tests/test_html_reporter.py +++ b/src/raitap/reporting/tests/test_html_reporter.py @@ -276,7 +276,7 @@ def test_html_reporter_omits_reproducibility_banner_for_deterministic_run(tmp_pa def _config(*, filename: str = "report.pdf") -> AppConfig: config = AppConfig(experiment_name="html-report-test") config.reporting = ReportingConfig( - _target_="raitap.reporting.HTMLReporter", + use="html", filename=filename, ) return config diff --git a/src/raitap/reporting/tests/test_pdf_reporter.py b/src/raitap/reporting/tests/test_pdf_reporter.py index 09993997..a38afe16 100644 --- a/src/raitap/reporting/tests/test_pdf_reporter.py +++ b/src/raitap/reporting/tests/test_pdf_reporter.py @@ -29,7 +29,7 @@ def mock_config(tmp_path: Path) -> AppConfig: cfg = AppConfig(experiment_name="test_experiment") set_output_root(cfg, tmp_path) cfg.reporting = ReportingConfig( - _target_="PDFReporter", + use="pdf", filename="test.pdf", ) return cfg diff --git a/src/raitap/reporting/tests/test_registration.py b/src/raitap/reporting/tests/test_registration.py index e6b0271d..de2f470c 100644 --- a/src/raitap/reporting/tests/test_registration.py +++ b/src/raitap/reporting/tests/test_registration.py @@ -19,7 +19,7 @@ def test_reporter_registers_under_reporting_group() -> None: @adapters.reporter( registry_name="_stub_reporter", extra="_stub_extra", - library="_stub_lib", + import_name="_stub_lib", ) class _StubReporter(BaseReporter): def __init__(self, config: object = None) -> None: diff --git a/src/raitap/robustness/__init__.py b/src/raitap/robustness/__init__.py index 8158f9ee..90cb90dd 100644 --- a/src/raitap/robustness/__init__.py +++ b/src/raitap/robustness/__init__.py @@ -10,10 +10,10 @@ Public Surface -------------- -Assessor classes (``_target_`` values; live under ``raitap.robustness.assessors.``): +Assessor classes (selected via ``use:`` values; live under ``raitap.robustness.assessors.``): TorchattacksAssessor, FoolboxAssessor -Visualiser classes (``_target_`` values; live under ``raitap.robustness.visualisers.``): +Visualiser classes (selected via ``use:`` values; live under ``raitap.robustness.visualisers.``): ImagePairVisualiser, PerturbationHeatmapVisualiser Module layout (for contributors): diff --git a/src/raitap/robustness/assessors/auto_lirpa_assessor.py b/src/raitap/robustness/assessors/auto_lirpa_assessor.py index 2d8b10c0..4ae64954 100644 --- a/src/raitap/robustness/assessors/auto_lirpa_assessor.py +++ b/src/raitap/robustness/assessors/auto_lirpa_assessor.py @@ -84,7 +84,7 @@ @robustness_adapter( registry_name="auto_lirpa", extra="auto-lirpa", - library="auto_LiRPA", + import_name="auto_LiRPA", error_patterns=_AUTO_LIRPA_ERROR_MESSAGES, # auto-LiRPA uses ``torch.jit.script`` internally; torch 2.x emits a # DeprecationWarning we can't fix upstream. Silence — not actionable. diff --git a/src/raitap/robustness/assessors/foolbox_assessor.py b/src/raitap/robustness/assessors/foolbox_assessor.py index 5c8aa11d..8d6ba93b 100644 --- a/src/raitap/robustness/assessors/foolbox_assessor.py +++ b/src/raitap/robustness/assessors/foolbox_assessor.py @@ -109,7 +109,7 @@ def _hint( @robustness_adapter( registry_name="foolbox", - library="foolbox", + import_name="foolbox", budget_kwarg_source="call_kwargs", error_patterns=_FOOLBOX_ERROR_MESSAGES, algorithm_registry={ diff --git a/src/raitap/robustness/assessors/imagecorruptions_assessor.py b/src/raitap/robustness/assessors/imagecorruptions_assessor.py index 29edb0e5..ba0c54b4 100644 --- a/src/raitap/robustness/assessors/imagecorruptions_assessor.py +++ b/src/raitap/robustness/assessors/imagecorruptions_assessor.py @@ -47,7 +47,7 @@ @robustness_adapter( registry_name="imagecorruptions", - library="imagecorruptions", + import_name="imagecorruptions", algorithm_registry={ name: AssessorAlgorithmSpec( AssessmentKind.STATISTICAL_SAMPLING, diff --git a/src/raitap/robustness/assessors/marabou_assessor.py b/src/raitap/robustness/assessors/marabou_assessor.py index 7e4e12e4..d46f1bdb 100644 --- a/src/raitap/robustness/assessors/marabou_assessor.py +++ b/src/raitap/robustness/assessors/marabou_assessor.py @@ -77,7 +77,7 @@ @robustness_adapter( registry_name="marabou", - library="maraboupy", + import_name="maraboupy", error_patterns=_MARABOUPY_ERROR_MESSAGES, algorithm_registry={ "linf-box": AssessorAlgorithmSpec( diff --git a/src/raitap/robustness/assessors/tests/test_registration.py b/src/raitap/robustness/assessors/tests/test_registration.py index 11ec497a..84e46d8c 100644 --- a/src/raitap/robustness/assessors/tests/test_registration.py +++ b/src/raitap/robustness/assessors/tests/test_registration.py @@ -20,7 +20,7 @@ def test_robustness_adapter_registers_under_robustness_group() -> None: @adapters.robustness( registry_name="_stub_attack", extra="_stub_extra", - library="_stub_lib", + import_name="_stub_lib", algorithm_registry={ "_stub_alg": AssessorAlgorithmSpec( AssessmentKind.EMPIRICAL_ATTACK, diff --git a/src/raitap/robustness/assessors/torchattacks_assessor.py b/src/raitap/robustness/assessors/torchattacks_assessor.py index 2576d1e0..825322a9 100644 --- a/src/raitap/robustness/assessors/torchattacks_assessor.py +++ b/src/raitap/robustness/assessors/torchattacks_assessor.py @@ -44,7 +44,7 @@ def _jsma_invoker(ctx: AttackInvokeCtx) -> torch.Tensor: @robustness_adapter( registry_name="torchattacks", - library="torchattacks", + import_name="torchattacks", algorithm_registry={ "FGSM": AssessorAlgorithmSpec( AssessmentKind.EMPIRICAL_ATTACK, diff --git a/src/raitap/robustness/factory.py b/src/raitap/robustness/factory.py index 02c33d5a..fa04b50f 100644 --- a/src/raitap/robustness/factory.py +++ b/src/raitap/robustness/factory.py @@ -33,18 +33,12 @@ from .results import RobustnessResult -_ROBUSTNESS_PREFIX = "raitap.robustness.assessors." -_VISUALISER_PREFIX = "raitap.robustness.visualisers." - _SCHEMA = AdapterSchema( domain="robustness", entity="assessor", subdict_namespace="Robustness", - target_prefix=_ROBUSTNESS_PREFIX, - visualiser_prefix=_VISUALISER_PREFIX, - top_level_keys=frozenset( - {"_target_", "algorithm", "constructor", "call", "raitap", "visualisers"} - ), + registry_group="robustness", + top_level_keys=frozenset({"use", "algorithm", "constructor", "call", "raitap", "visualisers"}), raitap_keys=frozenset( { "batch_size", @@ -192,7 +186,7 @@ def create_assessor(assessor_config: Any) -> tuple[AssessorAdapter, str]: protocol=AssessorAdapter, schema=_SCHEMA, instantiate_error_hint=( - "Check that _target_ points to a valid AssessorAdapter implementation " + "Check that use: points to a valid AssessorAdapter implementation " "(e.g. EmpiricalAttackAssessor or FormalVerificationAssessor subclass)." ), type_error_hint=( diff --git a/src/raitap/robustness/tests/test_assess_robustness_detection.py b/src/raitap/robustness/tests/test_assess_robustness_detection.py index 177fab9a..7d6a2acc 100644 --- a/src/raitap/robustness/tests/test_assess_robustness_detection.py +++ b/src/raitap/robustness/tests/test_assess_robustness_detection.py @@ -60,7 +60,7 @@ def test_assess_robustness_detection_returns_empty_and_skips_robustness_assessme # short-circuit — we need to reach the task-kind check. config.robustness = { "fgsm": RobustnessConfig( - _target_="EmpiricalAttackAssessor", + use="torchattacks", algorithm="FGSM", ) } diff --git a/src/raitap/robustness/tests/test_e2e_marabou_cli.py b/src/raitap/robustness/tests/test_e2e_marabou_cli.py index 7d1adaac..2ca208b6 100644 --- a/src/raitap/robustness/tests/test_e2e_marabou_cli.py +++ b/src/raitap/robustness/tests/test_e2e_marabou_cli.py @@ -122,7 +122,7 @@ def _write_tabular_config(target_path: Path, fixture_dir: Path) -> None: robustness: marabou_linf: - _target_: MarabouAssessor + use: marabou algorithm: linf-box constructor: epsilon: 0.001 @@ -151,7 +151,7 @@ def _write_image_config( fixture_str = fixture_dir.as_posix() reporting_block = ( "reporting:\n" - " _target_: HTMLReporter\n" + " use: html\n" " filename: report\n" " include_config: false\n" " include_metadata: false\n" @@ -192,7 +192,7 @@ def _write_image_config( robustness: marabou_linf: - _target_: MarabouAssessor + use: marabou algorithm: linf-box constructor: epsilon: 0.001 diff --git a/src/raitap/robustness/tests/test_e2e_real_data.py b/src/raitap/robustness/tests/test_e2e_real_data.py index 15ba1caa..eb421399 100644 --- a/src/raitap/robustness/tests/test_e2e_real_data.py +++ b/src/raitap/robustness/tests/test_e2e_real_data.py @@ -104,7 +104,7 @@ def test_torchattacks_pgdl2_handles_non_contiguous_input(tmp_path: Path) -> None tmp_path, OmegaConf.create( { - "_target_": "raitap.robustness.TorchattacksAssessor", + "use": "torchattacks", "algorithm": "PGDL2", "constructor": {"eps": 0.1, "alpha": 0.02, "steps": 3}, "visualisers": [], @@ -136,11 +136,11 @@ def test_image_pair_visualiser_diff_uses_diverging_cmap(tmp_path: Path) -> None: tmp_path, OmegaConf.create( { - "_target_": "raitap.robustness.TorchattacksAssessor", + "use": "torchattacks", "algorithm": "FGSM", "constructor": {"eps": 0.05}, "visualisers": [ - {"_target_": "raitap.robustness.ImagePairVisualiser"}, + {"use": "image_pair"}, ], } ), @@ -186,7 +186,7 @@ def test_assess_rejects_empty_batch(tmp_path: Path) -> None: tmp_path, OmegaConf.create( { - "_target_": "raitap.robustness.TorchattacksAssessor", + "use": "torchattacks", "algorithm": "FGSM", "constructor": {"eps": 0.05}, "visualisers": [], @@ -221,7 +221,7 @@ def test_pipeline_allows_robustness_only_runs(tmp_path: Path) -> None: robustness={ "pgd": OmegaConf.create( { - "_target_": "raitap.robustness.TorchattacksAssessor", + "use": "torchattacks", "algorithm": "FGSM", "constructor": {"eps": 0.05}, "visualisers": [], diff --git a/src/raitap/robustness/tests/test_e2e_robustness_matrix.py b/src/raitap/robustness/tests/test_e2e_robustness_matrix.py index 28cf4234..29a0e7e3 100644 --- a/src/raitap/robustness/tests/test_e2e_robustness_matrix.py +++ b/src/raitap/robustness/tests/test_e2e_robustness_matrix.py @@ -76,11 +76,7 @@ def tiny_model() -> torch.nn.Module: def _make_robustness_config(tmp_path: Path, family: str, case: AssessorMatrixCase) -> AppConfig: - target = ( - "raitap.robustness.TorchattacksAssessor" - if family == "torchattacks" - else "raitap.robustness.FoolboxAssessor" - ) + # ``family`` is already the registry ``use:`` key ("torchattacks" / "foolbox"). return cast( "AppConfig", SimpleNamespace( @@ -90,7 +86,7 @@ def _make_robustness_config(tmp_path: Path, family: str, case: AssessorMatrixCas robustness={ "matrix": OmegaConf.create( { - "_target_": target, + "use": family, "algorithm": case.algorithm, "constructor": dict(case.constructor_kwargs), "call": dict(case.call_kwargs), diff --git a/src/raitap/robustness/tests/test_e2e_statistical_sampling.py b/src/raitap/robustness/tests/test_e2e_statistical_sampling.py index 5fbff36b..d08efb21 100644 --- a/src/raitap/robustness/tests/test_e2e_statistical_sampling.py +++ b/src/raitap/robustness/tests/test_e2e_statistical_sampling.py @@ -67,12 +67,12 @@ def test_gaussian_noise_e2e(tmp_path: Path) -> None: robustness={ "avg": OmegaConf.create( { - "_target_": "raitap.robustness.ImageCorruptionsAssessor", + "use": "imagecorruptions", "algorithm": "gaussian_noise", "constructor": {"severity": 3}, "call": {}, "raitap": {"ci_method": "wilson", "ci_level": 0.95}, - "visualisers": [{"_target_": "CorruptionAccuracyVisualiser"}], + "visualisers": [{"use": "corruption_accuracy"}], } ) }, diff --git a/src/raitap/robustness/tests/test_factory.py b/src/raitap/robustness/tests/test_factory.py index c48e7971..94dcbff3 100644 --- a/src/raitap/robustness/tests/test_factory.py +++ b/src/raitap/robustness/tests/test_factory.py @@ -73,7 +73,7 @@ def _make_minimal_config(*, visualisers: list[Any] | None = None, tmp_path: Any robustness={ "pgd": OmegaConf.create( { - "_target_": "raitap.robustness.TorchattacksAssessor", + "use": "torchattacks", "algorithm": "PGD", "constructor": {"eps": 0.03, "alpha": 0.01, "steps": 1}, "call": {}, @@ -93,7 +93,7 @@ def _make_yaml_names_config(*, sample_names: list[str], tmp_path: Any = None) -> robustness={ "pgd": OmegaConf.create( { - "_target_": "raitap.robustness.TorchattacksAssessor", + "use": "torchattacks", "algorithm": "PGD", "constructor": {"eps": 0.03, "alpha": 0.01, "steps": 1}, "call": {}, @@ -109,7 +109,7 @@ def test_parse_validates_top_level_keys() -> None: with pytest.raises(ValueError, match="Unknown robustness assessor config keys"): _parse_assessor_config( { - "_target_": "TorchattacksAssessor", + "use": "torchattacks", "algorithm": "PGD", "wibble": True, # unknown key } @@ -121,7 +121,7 @@ def test_parse_rejects_misplaced_raitap_keys() -> None: with pytest.raises(ValueError) as excinfo: _parse_assessor_config( { - "_target_": "TorchattacksAssessor", + "use": "torchattacks", "algorithm": "PGD", "call": {"eps": 0.03, "batch_size": 8}, } @@ -131,6 +131,33 @@ def test_parse_rejects_misplaced_raitap_keys() -> None: assert "batch_size" in text +def test_parse_rejects_config_target_as_security_surface() -> None: + """``_target_`` in a config block is arbitrary-callable RCE surface (#301); + it must never reach ``hydra.utils.instantiate``.""" + from raitap.configs.registry_resolve import UnsafeConfigTargetError + + with pytest.raises(UnsafeConfigTargetError, match="_target_"): + _parse_assessor_config( + { + "use": "torchattacks", + "algorithm": "PGD", + "_target_": "os.system", + } + ) + + +def test_create_assessor_rejects_unknown_use_key() -> None: + from raitap.robustness.factory import create_assessor + + config = OmegaConf.create({"use": "does_not_exist", "algorithm": "PGD"}) + with pytest.raises( + ValueError, match=r"Unknown robustness key 'does_not_exist'\. Valid keys:" + ) as excinfo: + create_assessor(config) + assert "torchattacks" in str(excinfo.value) + assert "foolbox" in str(excinfo.value) + + def test_resolve_call_data_sources_passes_through_non_source_dicts() -> None: out = resolve_call_data_sources({"target_labels": [0, 1]}, log_label="robustness call") assert out == {"target_labels": [0, 1]} @@ -196,7 +223,7 @@ def assess(self, *_args: Any, **kwargs: Any) -> RobustnessResult: robustness={ "pgd": OmegaConf.create( { - "_target_": "TorchattacksAssessor", + "use": "torchattacks", "algorithm": "PGD", "call": { "target_labels": [0, 1], diff --git a/src/raitap/schema/raitap.schema.json b/src/raitap/schema/raitap.schema.json new file mode 100644 index 00000000..61ab5a12 --- /dev/null +++ b/src/raitap/schema/raitap.schema.json @@ -0,0 +1,131 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "data": { + "properties": { + "inputs": { + "properties": { + "use": { + "enum": [ + "text_csv", + "text_dir", + "text_jsonl" + ], + "type": "string" + } + }, + "type": "object" + }, + "labels": { + "properties": { + "use": { + "enum": [ + "coco", + "detection_json", + "directory", + "tabular", + "voc", + "yolo" + ], + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "metrics": { + "properties": { + "use": { + "enum": [ + "binary_classification", + "detection", + "multiclass_classification", + "multilabel_classification" + ], + "type": "string" + } + }, + "type": "object" + }, + "reporting": { + "properties": { + "use": { + "enum": [ + "html", + "pdf" + ], + "type": "string" + } + }, + "type": "object" + }, + "robustness": { + "additionalProperties": { + "properties": { + "use": { + "enum": [ + "auto_lirpa", + "foolbox", + "imagecorruptions", + "marabou", + "torchattacks" + ], + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "transparency": { + "additionalProperties": { + "properties": { + "use": { + "enum": [ + "captum", + "shap" + ], + "type": "string" + } + }, + "type": "object" + }, + "type": "object" + }, + "visualiser": { + "properties": { + "use": { + "enum": [ + "captum_image", + "captum_text", + "captum_time_series", + "corruption_accuracy", + "detection_image", + "image_pair", + "input_thumbnail", + "layer_activation", + "output_bounds_cohort", + "output_bounds_margin_heatmap", + "output_bounds_pinned", + "output_bounds_width_heatmap", + "perturbation_heatmap", + "quantus", + "shap_bar", + "shap_beeswarm", + "shap_force", + "shap_image", + "shap_waterfall", + "structured_payload_summary", + "tabular_bar_chart", + "verdict_summary" + ], + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" +} diff --git a/src/raitap/tests/_fake_plugin/raitap_fakeplugin/__init__.py b/src/raitap/tests/_fake_plugin/raitap_fakeplugin/__init__.py index 033457b3..d5d7dd49 100644 --- a/src/raitap/tests/_fake_plugin/raitap_fakeplugin/__init__.py +++ b/src/raitap/tests/_fake_plugin/raitap_fakeplugin/__init__.py @@ -17,5 +17,9 @@ }, ) class FakeAttackAssessor(EmpiricalAttackAssessor): + def __init__(self, algorithm: str, **init_kwargs): + self.algorithm = algorithm + self.init_kwargs = init_kwargs + def _default_invoke(self, ctx: AttackInvokeCtx): # noqa: ANN202 return ctx.inputs diff --git a/src/raitap/tests/test_adapter_registration_core.py b/src/raitap/tests/test_adapter_registration_core.py index 38f9b7d0..8d71468a 100644 --- a/src/raitap/tests/test_adapter_registration_core.py +++ b/src/raitap/tests/test_adapter_registration_core.py @@ -49,7 +49,7 @@ def __init__(self, max_samples: int = 4): def test_register_core_with_family_populates_builders_and_extras() -> None: """_register_core should run the same mechanics as __init_subclass__: hydra-zen - builder in _BUILDERS, ADAPTER_EXTRAS entry, library tracked in THIRD_PARTY_LIBS.""" + builder in _BUILDERS, ADAPTER_EXTRAS entry, import_name tracked in THIRD_PARTY_LIBS.""" from raitap._adapters import ( _BUILDERS, ADAPTER_EXTRAS, @@ -68,7 +68,7 @@ def test_register_core_with_family_populates_builders_and_extras() -> None: family=fc, registry_name="dummy", extra="dummy-extra", - library="dummy-lib", + import_name="dummy-lib", ) assert "_test_family" in _BUILDERS assert "dummy" in _BUILDERS["_test_family"] @@ -87,3 +87,13 @@ def test_register_core_without_family_uses_unscoped_pool() -> None: registry_name="dummy_visualiser", ) assert "dummy_visualiser" in _BUILDERS["_unscoped"] + + +def test_registration_records_target_fqn_and_use_node() -> None: + import raitap.transparency # noqa: F401 — fire discovery # pyright: ignore[reportUnusedImport] + from raitap._adapters import _BUILDERS, _TARGET_FQN + + assert _TARGET_FQN["transparency"]["captum"].endswith("CaptumExplainer") + node = _BUILDERS["transparency"]["captum"] + assert node.use == "captum" + assert not hasattr(node, "_target_") diff --git a/src/raitap/tests/test_api.py b/src/raitap/tests/test_api.py index 4f2d412c..cfc1e026 100644 --- a/src/raitap/tests/test_api.py +++ b/src/raitap/tests/test_api.py @@ -65,18 +65,18 @@ def _demo_app_config() -> AppConfig: metrics=MulticlassClassificationMetricsConfig(num_classes=1000), transparency={ "default": TransparencyConfig( - _target_="CaptumExplainer", + use="captum", algorithm="IntegratedGradients", call={"target": 0}, - visualisers=[{"_target_": "CaptumImageVisualiser"}], + visualisers=[{"use": "captum_image"}], ) }, robustness={ "pgd": RobustnessConfig( - _target_="TorchattacksAssessor", + use="torchattacks", algorithm="PGD", constructor={"eps": 0.03, "alpha": 0.005, "steps": 10}, - visualisers=[{"_target_": "ImagePairVisualiser"}], + visualisers=[{"use": "image_pair"}], ) }, # ``reporting`` left ``None`` so the smoke test stays fast and writes @@ -93,7 +93,8 @@ def test_api_surface_exports_what_docs_will_reference() -> None: # Hydra-zen builders are exposed under ``raitap.api`` as dataclass *types* # (calling them with kwargs yields a config instance the orchestrator can - # instantiate). Each builder must expose ``_target_`` so Hydra can resolve it. + # instantiate). Each builder must expose ``use`` so the registry resolver + # can pick the concrete adapter. for name, builder in [ ("captum", captum), ("shap", shap), @@ -104,18 +105,19 @@ def test_api_surface_exports_what_docs_will_reference() -> None: assert isinstance(builder, type), f"{name} should be a dataclass type" assert dataclasses.is_dataclass(builder), f"{name} should be a dataclass" field_names = {f.name for f in dataclasses.fields(builder)} - assert "_target_" in field_names, f"{name} should carry a ``_target_`` field" + assert "use" in field_names, f"{name} should carry a ``use`` field" # ``instantiate`` re-exported so users don't have to pip install hydra-zen. assert callable(instantiate) def test_classification_metrics_builder_instantiates_round_trip() -> None: - """Sanity: the builder produces a config that Hydra can instantiate.""" + """Sanity: the builder produces a config the metrics factory can resolve.""" from raitap.metrics.classification_metrics import MulticlassClassificationMetrics + from raitap.metrics.factory import create_metric cfg = classification_metrics(num_classes=3) - instance = instantiate(cfg) + instance, _resolved_target = create_metric(cfg) assert isinstance(instance, MulticlassClassificationMetrics) @@ -131,7 +133,7 @@ def test_explainer_builders_accept_schema_fields() -> None: captum_cfg = captum( algorithm="IntegratedGradients", call={"target": 0}, - visualisers=[{"_target_": "CaptumImageVisualiser"}], + visualisers=[{"use": "captum_image"}], ) assert captum_cfg.algorithm == "IntegratedGradients" assert captum_cfg.call == {"target": 0} @@ -150,7 +152,7 @@ def test_assessor_builders_accept_schema_fields() -> None: torchattacks_cfg = torchattacks( algorithm="PGD", constructor={"eps": 0.03, "alpha": 0.005, "steps": 10}, - visualisers=[{"_target_": "ImagePairVisualiser"}], + visualisers=[{"use": "image_pair"}], ) assert torchattacks_cfg.algorithm == "PGD" assert torchattacks_cfg.constructor == {"eps": 0.03, "alpha": 0.005, "steps": 10} @@ -274,7 +276,7 @@ def test_run_parity_with_yaml_demo(_demo_run: RunOutputs) -> None: outputs as the YAML path is enough to catch drift in either direction. """ with initialize_config_dir(version_base="1.3", config_dir=str(_configs_dir())): - yaml_cfg = compose(config_name="demo", overrides=["reporting._target_=null"]) + yaml_cfg = compose(config_name="demo", overrides=["reporting.use=null"]) yaml_outputs = run(cast("AppConfig", yaml_cfg), verbose=False) diff --git a/src/raitap/tests/test_config_schema.py b/src/raitap/tests/test_config_schema.py new file mode 100644 index 00000000..ebf118a9 --- /dev/null +++ b/src/raitap/tests/test_config_schema.py @@ -0,0 +1,43 @@ +"""Tests for :mod:`raitap._config_schema` (JSON Schema generated from the +live adapter registry, refs #301).""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +def test_schema_lists_transparency_use_enum() -> None: + import raitap.transparency # noqa: F401 — fire discovery # pyright: ignore[reportUnusedImport] + from raitap._config_schema import build_config_schema + + schema = build_config_schema() + enum = schema["properties"]["transparency"]["additionalProperties"]["properties"]["use"]["enum"] + assert "captum" in enum and "shap" in enum + + +def test_committed_schema_matches_fresh_regen(tmp_path: Path) -> None: + """The committed ``src/raitap/schema/raitap.schema.json`` must byte-match a + fresh ``raitap config-schema`` regen (this is CI's freshness check, run + locally too). + + Regenerates in a subprocess — a fresh interpreter, like a real ``raitap + config-schema`` invocation — rather than calling :func:`build_config_schema` + in-process. The family-registration import order is process-global + (:mod:`raitap._adapters`'s ``_BUILDERS``), so whatever another test in this + same pytest session already imported would otherwise leak in and make this + comparison depend on test order. + """ + committed_path = Path(__file__).resolve().parents[1] / "schema" / "raitap.schema.json" + output_path = tmp_path / "raitap.schema.json" + + subprocess.run( + [sys.executable, "-m", "raitap.cli", "config-schema", "-o", str(output_path)], + check=True, + ) + + assert committed_path.read_text() == output_path.read_text(), ( + f"{committed_path} is stale — regenerate with " + "`uv run raitap config-schema -o src/raitap/schema/raitap.schema.json`" + ) diff --git a/src/raitap/tests/test_e2e_detection.py b/src/raitap/tests/test_e2e_detection.py index cfacfbf2..e8054115 100644 --- a/src/raitap/tests/test_e2e_detection.py +++ b/src/raitap/tests/test_e2e_detection.py @@ -128,7 +128,7 @@ def test_detection_pipeline_e2e_via_fasterrcnn_mobilenet(tmp_path: Path) -> None ) transparency_cfg = TransparencyConfig( - _target_="CaptumExplainer", + use="captum", algorithm="IntegratedGradients", # ``n_steps=4`` + ``internal_batch_size=1`` keep IG memory bounded so # the test fits comfortably on the GitHub-hosted runner (~7 GB RAM). @@ -142,7 +142,7 @@ def test_detection_pipeline_e2e_via_fasterrcnn_mobilenet(tmp_path: Path) -> None }, "batch_size": 1, }, - visualisers=[{"_target_": "DetectionImageVisualiser"}], + visualisers=[{"use": "detection_image"}], ) config = AppConfig( @@ -163,7 +163,7 @@ def test_detection_pipeline_e2e_via_fasterrcnn_mobilenet(tmp_path: Path) -> None assert outputs.metrics is not None metrics_evaluation = cast("MetricsEvaluation", outputs.phase_results["metrics"]) - assert metrics_evaluation.resolved_target == "raitap.metrics.DetectionMetrics" + assert metrics_evaluation.resolved_target == "raitap.metrics.detection_metrics.DetectionMetrics" # At least one detection should pass score_threshold=0.5 in dashcam frames # with a COCO-pretrained Faster R-CNN. diff --git a/src/raitap/tests/test_register_adapter_typing.py b/src/raitap/tests/test_register_adapter_typing.py index 718e5feb..809ef5a1 100644 --- a/src/raitap/tests/test_register_adapter_typing.py +++ b/src/raitap/tests/test_register_adapter_typing.py @@ -45,8 +45,8 @@ def _pyright_errors(source: str, tmp_path: Path) -> list[str]: def test_missing_registry_name_is_pyright_error_for_every_family_decorator( tmp_path: Path, ) -> None: - _tr_call = "adapters.transparency(algorithm_registry={}, extra='x', library='x')" - _ro_call = "adapters.robustness(algorithm_registry={}, extra='x', library='x')" + _tr_call = "adapters.transparency(algorithm_registry={}, extra='x', import_name='x')" + _ro_call = "adapters.robustness(algorithm_registry={}, extra='x', import_name='x')" for decorator_import, decorator_call, expected in [ ("from raitap import adapters", _tr_call, "registry_name"), ("from raitap import adapters", _ro_call, "registry_name"), diff --git a/src/raitap/tests/test_run_main.py b/src/raitap/tests/test_run_main.py index a40f6107..b66fa5f4 100644 --- a/src/raitap/tests/test_run_main.py +++ b/src/raitap/tests/test_run_main.py @@ -364,8 +364,7 @@ def fake_run(config: object) -> None: cfg = cast("AppConfig", captured["config"]) assert cfg.model.source == "vit_b_32" assert cfg.metrics is not None - expected = "raitap.metrics.classification_metrics.MulticlassClassificationMetrics" - assert cfg.metrics._target_ == expected + assert cfg.metrics.use == "multiclass_classification" assert cfg.transparency @@ -440,8 +439,7 @@ def fake_run(config: object) -> None: assert cfg.hardware == "cpu" assert cfg.model.source == "vit_b_32" assert cfg.metrics is not None - expected = "raitap.metrics.classification_metrics.MulticlassClassificationMetrics" - assert cfg.metrics._target_ == expected + assert cfg.metrics.use == "multiclass_classification" assert cfg.transparency @@ -592,7 +590,7 @@ def test_run_invalid_report_sample_selection_fails_before_pipeline_work( monkeypatch.setattr(BaseTracker, "create_tracker", tracker_factory) config = _run_config( - reporting={"_target_": "PDFReporter", "sample_selection": ["missing.png"]}, + reporting={"use": "pdf", "sample_selection": ["missing.png"]}, tracking=None, ) @@ -637,7 +635,7 @@ def __exit__(self, exc_type: object, exc: object, tb: object) -> bool: monkeypatch.setattr(BaseTracker, "create_tracker", lambda _cfg: _TrackerContext()) config = _run_config( - tracking={"_target_": "MLFlowTracker", "log_model": True}, + tracking={"use": "mlflow", "log_model": True}, ) run_pipeline._run_pipeline(config) # type: ignore[arg-type] @@ -680,7 +678,7 @@ def __exit__(self, exc_type: object, exc: object, tb: object) -> bool: monkeypatch.setattr(BaseTracker, "create_tracker", lambda _cfg: _TrackerContext()) config = _run_config( - tracking={"_target_": "MLFlowTracker", "log_model": False}, + tracking={"use": "mlflow", "log_model": False}, ) run_pipeline._run_pipeline(config) # type: ignore[arg-type] @@ -721,7 +719,7 @@ def __exit__(self, exc_type: object, exc: object, tb: object) -> bool: monkeypatch.setattr(BaseTracker, "create_tracker", lambda _cfg: _TrackerContext()) config = _run_config( - tracking={"_target_": "MLFlowTracker", "log_model": False}, + tracking={"use": "mlflow", "log_model": False}, ) run_pipeline._run_pipeline(config) # type: ignore[arg-type] @@ -731,7 +729,7 @@ def __exit__(self, exc_type: object, exc: object, tb: object) -> bool: assert vis2.log_calls == [True] -def test_run_with_tracking_config_but_no_target_skips_tracking(monkeypatch: MonkeyPatch) -> None: +def test_run_with_tracking_config_but_no_use_skips_tracking(monkeypatch: MonkeyPatch) -> None: model = SimpleNamespace( backend=_BackendStub(torch.nn.Identity()), log=MagicMock(), @@ -751,22 +749,67 @@ def test_run_with_tracking_config_but_no_target_skips_tracking(monkeypatch: Monk monkeypatch.setattr(run_pipeline, "print_summary", lambda _cfg, _model: None) monkeypatch.setattr(BaseTracker, "create_tracker", tracker_factory) - # tracking config exists but _target_ is None or empty - config = _run_config(tracking={"_target_": ""}) + # tracking config exists but `use` is empty + config = _run_config(tracking={"use": ""}) result = run_pipeline._run_pipeline(config) # type: ignore[arg-type] assert result is fake_output tracker_factory.assert_not_called() +def test_run_with_tracking_config_target_raises(monkeypatch: MonkeyPatch) -> None: + """A `_target_`-carrying tracking block must raise loudly, not be silently + read as "tracking disabled" (issue #301). + + ``make_app_config``'s struct-mode ``TrackingConfig`` schema rejects an + undeclared ``_target_`` key at construction time, before the orchestrator + guard would ever see it — so this uses a plain (unvalidated) ``AppConfig`` + dataclass instance, mirroring how a non-schema-checked source (e.g. a + hand-built dict) would reach the guard. + """ + from raitap.configs.registry_resolve import UnsafeConfigTargetError + from raitap.configs.schema import AppConfig, DataConfig, ModelConfig + + model = SimpleNamespace( + backend=_BackendStub(torch.nn.Identity()), + log=MagicMock(), + ) + data = SimpleNamespace(tensor=torch.randn(2, 3)) + fake_output = _fake_run_outputs( + explanations=[], + visualisations=[], + metrics=None, + forward_output=_fo(torch.tensor([0, 0])), + ) + tracker_factory = MagicMock() + + monkeypatch.setattr(run_pipeline, "Model", lambda _cfg, **_kwargs: model) + monkeypatch.setattr(run_pipeline, "Data", lambda _cfg, **_kwargs: data) + monkeypatch.setattr(run_pipeline, "run_phases", lambda _c, _m, _d, **_kwargs: fake_output) + monkeypatch.setattr(run_pipeline, "print_summary", lambda _cfg, _model: None) + monkeypatch.setattr(BaseTracker, "create_tracker", tracker_factory) + + config = AppConfig( + experiment_name="test", + model=ModelConfig(source="resnet50"), + data=DataConfig(preprocessing=None), + ) + config.tracking = {"_target_": "os.system", "log_model": False} # type: ignore[assignment] + + with pytest.raises(UnsafeConfigTargetError): + run_pipeline._run_pipeline(config) # type: ignore[arg-type] + + tracker_factory.assert_not_called() + + def test_run_phases_raises_if_no_phase_configured() -> None: model = SimpleNamespace(backend=_BackendStub(torch.nn.Identity())) data = SimpleNamespace(tensor=torch.randn(2, 3), sample_ids=None, labels=None) - # No _target_ on metrics, empty transparency/robustness -> no phase configured. + # No `use` on metrics, empty transparency/robustness -> no phase configured. config = SimpleNamespace( transparency={}, robustness={}, - metrics=SimpleNamespace(_target_=None, num_classes=None), + metrics=SimpleNamespace(use=None, num_classes=None), ) with pytest.raises(ValueError, match="No assessment phase configured"): @@ -781,12 +824,12 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: model = SimpleNamespace(backend=_BackendStub(_Net())) data = SimpleNamespace(tensor=torch.randn(2, 4), sample_ids=None, labels=None) - # A real (non-empty) _target_ is exactly what metrics_run_enabled checks; no + # A real (non-empty) `use` is exactly what metrics_run_enabled checks; no # transparency/robustness configured -> genuine metrics-only run. config = make_app_config( transparency={}, robustness={}, - metrics={"_target_": "MulticlassClassificationMetrics"}, + metrics={"use": "multiclass_classification"}, ) # ``num_classes`` deliberately absent here (unlike the typed subclass, where # it is mandatory): exercises the auto-infer branch in @@ -817,7 +860,7 @@ def _fake_metrics(cfg: object, preds: torch.Tensor, targs: torch.Tensor) -> obje config = make_app_config( transparency={"one": {}}, - metrics={"_target_": "MulticlassClassificationMetrics"}, + metrics={"use": "multiclass_classification"}, ) OmegaConf.set_struct(config.metrics, False) _patch_prepare_explainer( diff --git a/src/raitap/tests/test_third_party_discovery.py b/src/raitap/tests/test_third_party_discovery.py index 34d0716e..6492e722 100644 --- a/src/raitap/tests/test_third_party_discovery.py +++ b/src/raitap/tests/test_third_party_discovery.py @@ -9,6 +9,9 @@ _FAKE = Path(__file__).parent / "_fake_plugin" _FAKE_BROKEN = Path(__file__).parent / "_fake_broken_plugin" +# The user-facing reference plugin (docs/contributor/writing-a-plugin.md +# points here), vs. ``_FAKE`` which is a hidden test-only fixture. +_EXAMPLE_PLUGIN = Path(__file__).parents[3] / "example-plugin" @pytest.fixture(scope="module") @@ -18,6 +21,13 @@ def _installed_fake_plugin(): # noqa: ANN202 subprocess.run(["uv", "pip", "uninstall", "raitap-fakeplugin"], check=True) +@pytest.fixture(scope="module") +def _installed_example_plugin(): # noqa: ANN202 + subprocess.run(["uv", "pip", "install", str(_EXAMPLE_PLUGIN)], check=True) + yield + subprocess.run(["uv", "pip", "uninstall", "raitap-example-plugin"], check=True) + + @pytest.fixture def _installed_broken_plugin(): # noqa: ANN202 subprocess.run(["uv", "pip", "install", str(_FAKE_BROKEN)], check=True) @@ -32,6 +42,59 @@ def test_plugin_adapter_registered(_installed_fake_plugin) -> None: # noqa: ANN assert "fakeattack" in adapters_mod._BUILDERS["robustness"] +def test_plugin_use_key_resolves(_installed_fake_plugin) -> None: # noqa: ANN001 + """The #301 ``use:`` seam resolves a plugin-registered adapter exactly like + an in-tree one — the trusted registry (:data:`raitap._adapters._TARGET_FQN`) + is populated by the same decorator regardless of where the class lives.""" + import raitap._adapters as adapters_mod + from raitap.configs.registry_resolve import resolve_target_fqn + + adapters_mod.discover_third_party_adapters() + fqn = resolve_target_fqn("robustness", "fakeattack") + assert fqn.endswith("FakeAttackAssessor") + + +def test_config_schema_includes_installed_plugin(_installed_fake_plugin) -> None: # noqa: ANN001 + """A plugin's ``use:`` key shows up in the generated JSON Schema enum, so a + YAML editor pointed at it autocompletes plugin adapters like first-party + ones. :func:`build_config_schema` runs full discovery itself.""" + from raitap._config_schema import build_config_schema + + schema = build_config_schema() + enum = schema["properties"]["robustness"]["additionalProperties"]["properties"]["use"]["enum"] + assert "fakeattack" in enum + + +def test_plugin_adapter_runs_through_factory(_installed_fake_plugin) -> None: # noqa: ANN001 + """A plugin-registered assessor resolves and instantiates through the real + ``raitap.robustness.factory.create_assessor`` path — the same one the + pipeline uses for in-tree adapters — proving ``use:`` isn't just + registry-visible but actually runnable end-to-end.""" + import raitap._adapters as adapters_mod + from raitap.robustness.factory import create_assessor + + adapters_mod.discover_third_party_adapters() + assessor, target_fqn = create_assessor({"use": "fakeattack", "algorithm": "PGD"}) + + assert type(assessor).__name__ == "FakeAttackAssessor" + assert target_fqn.endswith("FakeAttackAssessor") + + +def test_example_plugin_end_to_end(_installed_example_plugin) -> None: # noqa: ANN001 + """CI proof for the user-facing ``example-plugin/`` (the runnable reference + linked from docs/contributor/writing-a-plugin.md): install it the way a + real consumer would, then drive its README's ``use: identity_attack`` + example through the real ``raitap.robustness.factory`` path.""" + import raitap._adapters as adapters_mod + from raitap.robustness.factory import create_assessor + + adapters_mod.discover_third_party_adapters() + assessor, target_fqn = create_assessor({"use": "identity_attack", "algorithm": "identity"}) + + assert type(assessor).__name__ == "IdentityAttackAssessor" + assert target_fqn == "raitap_example_plugin.IdentityAttackAssessor" + + def test_disabled_env_skips(monkeypatch, _installed_fake_plugin) -> None: # noqa: ANN001 import raitap._adapters as adapters_mod diff --git a/src/raitap/tracking/base_tracker.py b/src/raitap/tracking/base_tracker.py index 51d9b146..012abfe2 100644 --- a/src/raitap/tracking/base_tracker.py +++ b/src/raitap/tracking/base_tracker.py @@ -7,9 +7,8 @@ from raitap import raitap_log from raitap._adapters import AdapterMixin -from raitap.configs import cfg_to_dict, resolve_target - -_TRACKING_PREFIX = "raitap.tracking." +from raitap.configs import cfg_to_dict +from raitap.configs.registry_resolve import instantiate_partial_from_use if TYPE_CHECKING: from pathlib import Path @@ -54,20 +53,13 @@ def stop_detached(cls, timeout: float = 5.0) -> tuple[int, int]: @staticmethod def create_tracker(config: AppConfig) -> BaseTracker: tracking_config = cfg_to_dict(config.tracking) - target_path = str(tracking_config.get("_target_", "")) - resolved_target = resolve_target(target_path, _TRACKING_PREFIX) - - try: - tracker_class = instantiate({"_target_": resolved_target, "_partial_": True}) - tracker = tracker_class(config) - except Exception as error: - raitap_log.exception("Tracker instantiation failed for target %r", target_path) - raise ValueError( - f"Could not instantiate tracker {target_path!r}.\n" - "Check that _target_ points to a valid TrackerProtocol implementation." - ) from error - - return tracker + return instantiate_partial_from_use( + tracking_config, + group="tracking", + entity="tracker", + config=config, + instantiate_fn=instantiate, + ) def __enter__(self): return self diff --git a/src/raitap/tracking/mlflow_tracker.py b/src/raitap/tracking/mlflow_tracker.py index faf32298..33535292 100644 --- a/src/raitap/tracking/mlflow_tracker.py +++ b/src/raitap/tracking/mlflow_tracker.py @@ -129,7 +129,7 @@ def _mlflow_summary_params(config_dict: dict[str, Any]) -> dict[str, str]: Walks the normalised config from :func:`~raitap.configs.factory_utils.cfg_to_dict` instead of hard-coding brittle ``.get`` chains. Transparency is a mapping of explainer name → explainer config; each explainer contributes - ``transparency..algorithm`` and ``transparency.._target_``. + ``transparency..algorithm`` and ``transparency..use``. """ out: dict[str, str] = {} @@ -166,7 +166,7 @@ def put(key: str, value: Any) -> None: continue prefix = f"transparency.{name}" put(f"{prefix}.algorithm", explainer.get("algorithm")) - put(f"{prefix}._target_", explainer.get("_target_")) + put(f"{prefix}.use", explainer.get("use")) return out diff --git a/src/raitap/tracking/smoke_test_mlflow.py b/src/raitap/tracking/smoke_test_mlflow.py index 797f09f4..f5650e3d 100644 --- a/src/raitap/tracking/smoke_test_mlflow.py +++ b/src/raitap/tracking/smoke_test_mlflow.py @@ -94,16 +94,17 @@ def main() -> int: ), transparency={ "smoke_captum": TransparencyConfig( - _target_="CaptumExplainer", + use="captum", algorithm="IntegratedGradients", call={"target": "auto_pred"}, - visualisers=[{"_target_": "CaptumImageVisualiser"}], + visualisers=[{"use": "captum_image"}], ) }, metrics=MulticlassClassificationMetricsConfig( num_classes=1000, ), tracking=TrackingConfig( + use="mlflow", output_forwarding_url=tracking_uri, log_model=args.log_model, ), diff --git a/src/raitap/tracking/tests/test_base_tracker.py b/src/raitap/tracking/tests/test_base_tracker.py index d23b1256..e251a6c6 100644 --- a/src/raitap/tracking/tests/test_base_tracker.py +++ b/src/raitap/tracking/tests/test_base_tracker.py @@ -13,15 +13,16 @@ from raitap.configs.schema import AppConfig +from raitap.configs.registry_resolve import UnsafeConfigTargetError from raitap.tracking import BaseTracker -def _make_config(tracker_target: str = "MLFlowTracker") -> AppConfig: +def _make_config(use: str = "mlflow") -> AppConfig: return cast( "AppConfig", SimpleNamespace( tracking=SimpleNamespace( - _target_=tracker_target, + use=use, output_forwarding_url="http://127.0.0.1:5000", log_model=False, open_when_done=False, @@ -34,7 +35,7 @@ def _make_config(tracker_target: str = "MLFlowTracker") -> AppConfig: class TestCreateTracker: def test_create_tracker_instantiates_from_config(self) -> None: - config = _make_config("raitap.tracking.MLFlowTracker") + config = _make_config("mlflow") with patch("raitap.tracking.base_tracker.instantiate") as mock_instantiate: mock_class = MagicMock() @@ -47,8 +48,8 @@ def test_create_tracker_instantiates_from_config(self) -> None: assert tracker is mock_instance mock_class.assert_called_once_with(config) - def test_create_tracker_resolves_short_target_names(self) -> None: - config = _make_config("MLFlowTracker") + def test_create_tracker_resolves_use_key_to_full_fqn(self) -> None: + config = _make_config("mlflow") with patch("raitap.tracking.base_tracker.instantiate") as mock_instantiate: mock_class = MagicMock() @@ -60,9 +61,10 @@ def test_create_tracker_resolves_short_target_names(self) -> None: call_args = mock_instantiate.call_args[0][0] assert "raitap.tracking." in call_args["_target_"] + assert call_args["_partial_"] is True def test_create_tracker_raises_on_invalid_target(self) -> None: - config = _make_config("NonExistentTracker") + config = _make_config("mlflow") with patch("raitap.tracking.base_tracker.instantiate") as mock_instantiate: mock_instantiate.side_effect = Exception("Cannot instantiate") @@ -70,6 +72,27 @@ def test_create_tracker_raises_on_invalid_target(self) -> None: with pytest.raises(ValueError, match="Could not instantiate tracker"): _ = BaseTracker.create_tracker(config) + def test_create_tracker_raises_on_unknown_use(self) -> None: + config = _make_config("does_not_exist") + + with pytest.raises(ValueError, match="Unknown tracking key"): + BaseTracker.create_tracker(config) + + def test_create_tracker_rejects_config_target(self) -> None: + """A `_target_`-carrying block must raise loudly instead of being + resolved as an arbitrary callable (issue #301).""" + config = cast( + "AppConfig", + SimpleNamespace( + tracking=SimpleNamespace(_target_="os.system", log_model=False), + experiment_name="test_experiment", + _output_root=".", + ), + ) + + with pytest.raises(UnsafeConfigTargetError): + BaseTracker.create_tracker(config) + class MockTracker(BaseTracker): """Concrete implementation for testing context manager.""" diff --git a/src/raitap/tracking/tests/test_mlflow_summary_params.py b/src/raitap/tracking/tests/test_mlflow_summary_params.py index 9d328165..abf39c95 100644 --- a/src/raitap/tracking/tests/test_mlflow_summary_params.py +++ b/src/raitap/tracking/tests/test_mlflow_summary_params.py @@ -18,12 +18,12 @@ def test_summary_params_includes_explainers_and_per_explainer_fields() -> None: "data": {"name": "isic2018", "source": "/data"}, "transparency": { "captum_ig": { - "_target_": "CaptumExplainer", + "use": "captum", "algorithm": "IntegratedGradients", "visualisers": [], }, "captum_saliency": { - "_target_": "CaptumExplainer", + "use": "captum", "algorithm": "Saliency", "visualisers": [], }, @@ -37,7 +37,7 @@ def test_summary_params_includes_explainers_and_per_explainer_fields() -> None: assert params["data.source"] == "/data" assert params["transparency.explainers"] == "captum_ig,captum_saliency" assert params["transparency.captum_ig.algorithm"] == "IntegratedGradients" - assert params["transparency.captum_ig._target_"] == "CaptumExplainer" + assert params["transparency.captum_ig.use"] == "captum" assert params["transparency.captum_saliency.algorithm"] == "Saliency" @@ -58,7 +58,7 @@ def test_summary_params_ignores_non_dict_explainer_entries() -> None: "experiment_name": "x", "model": {}, "data": {}, - "transparency": {"bad": "not-a-dict", "ok": {"algorithm": "IG", "_target_": "T"}}, + "transparency": {"bad": "not-a-dict", "ok": {"algorithm": "IG", "use": "captum"}}, } params = _mlflow_summary_params(cfg) diff --git a/src/raitap/tracking/tests/test_mlflow_tracker.py b/src/raitap/tracking/tests/test_mlflow_tracker.py index 2814f87b..62f47b49 100644 --- a/src/raitap/tracking/tests/test_mlflow_tracker.py +++ b/src/raitap/tracking/tests/test_mlflow_tracker.py @@ -36,7 +36,7 @@ def _make_config( "AppConfig", SimpleNamespace( tracking=SimpleNamespace( - _target_="MLFlowTracker", + use="mlflow", output_forwarding_url=url, backend_store_uri=backend_store_uri, default_artifact_root=default_artifact_root, diff --git a/src/raitap/transparency/README.md b/src/raitap/transparency/README.md index e6992aeb..00d9babc 100644 --- a/src/raitap/transparency/README.md +++ b/src/raitap/transparency/README.md @@ -24,7 +24,7 @@ result = explain(config, model, inputs, target=0) ``` CLI / Config └── explain() # factory.py — single entry point - ├── Hydra instantiate(_target_) + ├── registry resolve (use key -> vetted FQN) │ ├── CaptumExplainer / ShapExplainer │ └── CaptumImageVisualiser / ShapImageVisualiser / … └── outputs//