From f6ef2955dd2895da01891b1993a24b5721f358c3 Mon Sep 17 00:00:00 2001 From: Ulusha Date: Thu, 23 Jul 2026 17:46:39 +0500 Subject: [PATCH] fix(harbor): recognise multi-step tasks in the converter --- integrations/harbor.py | 42 +++++++++++++++++++++++++------ integrations/tests/conftest.py | 27 ++++++++++++++++++++ integrations/tests/test_harbor.py | 32 ++++++++++++++++++++++- 3 files changed, 92 insertions(+), 9 deletions(-) diff --git a/integrations/harbor.py b/integrations/harbor.py index 35b903f73..23a16c02f 100644 --- a/integrations/harbor.py +++ b/integrations/harbor.py @@ -132,8 +132,30 @@ def _slugify(name: str) -> str: return re.sub(r"-+", "-", normalized).strip("-") or "harbor" +def _read_task_config(task_dir: Path) -> dict[str, Any] | None: + try: + return tomllib.loads((task_dir / "task.toml").read_text("utf-8")) + except (OSError, tomllib.TOMLDecodeError): + return None + + +def _task_has_instruction(task_dir: Path, config: dict[str, Any]) -> bool: + """A task carries its instruction at the root (single-step) or under each + ``steps//`` directory declared by a ``[[steps]]`` array (multi-step). + A multi-step task has no root ``instruction.md``, so both forms count.""" + return (task_dir / "instruction.md").is_file() or bool(config.get("steps")) + + def _is_harbor_task(path: Path) -> bool: - return path.is_dir() and (path / "task.toml").exists() and (path / "instruction.md").exists() + if not path.is_dir() or not (path / "task.toml").exists(): + return False + config = _read_task_config(path) + if config is None: + # An unparseable task.toml still identifies a single-step task by its + # root instruction.md; a multi-step task can only be told apart via the + # config, so drop it when the config is unreadable. + return (path / "instruction.md").is_file() + return _task_has_instruction(path, config) def _hash_directory(path: Path) -> str: @@ -158,14 +180,18 @@ class _HarborTask: def _parse_task(task_dir: Path) -> _HarborTask | None: - if not (task_dir / "instruction.md").is_file(): - LOGGER.warning("failed to read instruction.md in %s", task_dir) - return None - try: - config: dict[str, Any] = tomllib.loads((task_dir / "task.toml").read_text("utf-8")) - except (OSError, tomllib.TOMLDecodeError): - LOGGER.warning("failed to parse task.toml in %s", task_dir) + config = _read_task_config(task_dir) + if config is None: + # Unparseable config degrades gracefully for a single-step task (kept + # with an empty config); a multi-step task needs the config to be read. + if not (task_dir / "instruction.md").is_file(): + LOGGER.warning("failed to parse task.toml in %s", task_dir) + return None + LOGGER.warning("failed to parse task.toml in %s; using empty config", task_dir) config = {} + elif not _task_has_instruction(task_dir, config): + LOGGER.warning("no instruction.md and no [[steps]] in %s", task_dir) + return None env_dir = task_dir / "environment" return _HarborTask( task_id=task_dir.name, diff --git a/integrations/tests/conftest.py b/integrations/tests/conftest.py index a7599026d..d50c86d90 100644 --- a/integrations/tests/conftest.py +++ b/integrations/tests/conftest.py @@ -76,6 +76,33 @@ def make_harbor_task( return task_dir +def make_multi_step_task( + parent: Path, + name: str, + steps: tuple[str, ...] = ("plan", "implement"), + dockerfile: str | None = _SIMPLE_DOCKERFILE, +) -> Path: + """Create a synthetic multi-step Harbor task: no root instruction.md; each + step under steps// carries its own instruction, declared by a + [[steps]] array in task.toml.""" + task_dir = parent / name + task_dir.mkdir(parents=True, exist_ok=True) + steps_toml = "".join(f'\n[[steps]]\nname = "{step}"\n' for step in steps) + (task_dir / "task.toml").write_text(_DEFAULT_TASK_TOML + steps_toml, encoding="utf-8") + if dockerfile is not None: + env_dir = task_dir / "environment" + env_dir.mkdir(exist_ok=True) + (env_dir / "Dockerfile").write_text(dockerfile, encoding="utf-8") + for step in steps: + step_dir = task_dir / "steps" / step + (step_dir / "tests").mkdir(parents=True, exist_ok=True) + (step_dir / "instruction.md").write_text(f"# {step}\n", encoding="utf-8") + (step_dir / "tests" / "test.sh").write_text( + '#!/bin/bash\necho "1.0" > /logs/verifier/reward.txt\n', encoding="utf-8" + ) + return task_dir + + @pytest.fixture() def single_task(tmp_path: Path) -> Path: """A single standalone Harbor task directory.""" diff --git a/integrations/tests/test_harbor.py b/integrations/tests/test_harbor.py index e25930925..29f9c0375 100644 --- a/integrations/tests/test_harbor.py +++ b/integrations/tests/test_harbor.py @@ -9,7 +9,7 @@ from integrations.harbor import detect, export, load -from .conftest import make_harbor_task +from .conftest import make_harbor_task, make_multi_step_task if TYPE_CHECKING: from pathlib import Path @@ -61,6 +61,36 @@ def test_load_rejects_dirs_without_harbor_tasks(tmp_path: Path) -> None: load(empty) +def test_detect_and_load_recognize_multi_step_tasks(tmp_path: Path) -> None: + # A multi-step task has no root instruction.md; its instructions live under + # steps//, declared by a [[steps]] array in task.toml. + task = make_multi_step_task(tmp_path, "multi") + + assert detect(task) + assert {row.id for row in load(task)} == {"multi"} + + +def test_load_keeps_multi_step_tasks_alongside_single_step(tmp_path: Path) -> None: + dataset = tmp_path / "bench" + dataset.mkdir() + make_harbor_task(dataset, "single") + make_multi_step_task(dataset, "multi") + + assert {row.id for row in load(dataset)} == {"single", "multi"} + + +def test_detect_rejects_task_toml_without_instruction_or_steps(tmp_path: Path) -> None: + # task.toml alone is not a task: it needs a root instruction.md (single-step) + # or a [[steps]] array (multi-step). + task = tmp_path / "bare" + task.mkdir() + (task / "task.toml").write_text('[metadata]\ncategory = "x"\n', encoding="utf-8") + + assert not detect(task) + with pytest.raises(ValueError, match="no Harbor tasks"): + load(task) + + def test_load_skips_unparseable_toml_but_keeps_the_rest(tmp_path: Path) -> None: dataset = tmp_path / "bench" dataset.mkdir()