-
-
Notifications
You must be signed in to change notification settings - Fork 1.4k
fix(train): the three commits that missed the merge — including a failed run reporting success #4481
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
fix(train): the three commits that missed the merge — including a failed run reporting success #4481
Changes from all commits
f68fdd2
1e877e5
6b1a792
a8e46fb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -153,7 +153,7 @@ def _v(value): | |
| # the difference between a caught typo and a wasted run. Resolved and | ||
| # printed WITHOUT loading the (heavy, optional) runner, so a preview | ||
| # never depends on the training deps being installed. | ||
| _print_resolved_config(config, overrides) | ||
| _print_resolved_config(config, overrides, remote_overrides) | ||
| return | ||
|
|
||
| if not dataset and not config: | ||
|
|
@@ -268,9 +268,14 @@ def _dispatch_remote(resolved, remote_overrides, config_path, dataset): | |
| workdir=block["workdir"]) | ||
|
|
||
| shipped = _write_shipped_config(resolved) | ||
| # A dataset named only in --config still has to be copied. Fall back to the | ||
| # resolved dataset when the positional argument is absent, but only when it | ||
| # points at a local file -- a HuggingFace id or a path already on the remote | ||
| # host is not something to ship. | ||
| ship_dataset = dataset or _local_dataset_in(resolved) | ||
| try: | ||
| run = runner.start(config_path=shipped, | ||
| dataset_path=Path(dataset) if dataset else None, | ||
| dataset_path=Path(ship_dataset) if ship_dataset else None, | ||
| expect_gpus=block["gpus"]) | ||
| except RemoteError as exc: | ||
| output.print_error(f"Could not start the run on {block['host']}", | ||
|
|
@@ -290,11 +295,52 @@ def _dispatch_remote(resolved, remote_overrides, config_path, dataset): | |
| runner.tail(run, on_line=typer.echo) | ||
| state = runner.status(run) | ||
| typer.echo(f"status: {state}") | ||
| if state == "failed": | ||
| # status() returns "failed (exit N)", not a bare "failed", so an equality | ||
| # check would print the failure and then exit 0 -- reporting success for a | ||
| # run that did not complete. | ||
| if state.startswith("failed"): | ||
| raise typer.Exit(1) | ||
| return True | ||
|
|
||
|
|
||
| def _local_dataset_in(resolved): | ||
| """The one local dataset file the resolved config names, or None. | ||
|
|
||
| The trainer accepts a dataset as a bare string or as the list-of-mappings | ||
| it normalises to -- `[{name: ...}]`, optionally with `data_files` -- and | ||
| loads any `name`/`data_files` that `os.path.exists` as a local file | ||
| (praisonai_train/train/llm/trainer.py:882). A remote run has to copy that | ||
| file, or the far side is handed a path that exists only on this machine. | ||
| Only the string form was covered before, so the canonical list form went | ||
| unshipped. | ||
|
|
||
| A HuggingFace id or a path already on the remote host is not a file here, | ||
| so it is left alone. Only the first local file is returned: the runner | ||
| ships a single positional dataset, which matches how a `--config` run is | ||
| launched. | ||
| """ | ||
| entries = resolved.get("dataset") | ||
| if isinstance(entries, str): | ||
| entries = [entries] | ||
| elif not isinstance(entries, list): | ||
| return None | ||
|
|
||
| for entry in entries: | ||
| if isinstance(entry, str): | ||
| candidate = entry | ||
| elif isinstance(entry, dict): | ||
| # `data_files` is the explicit local file; `name` doubles as a path | ||
| # when it is one, exactly as the trainer treats it. | ||
| candidate = entry.get("data_files") or entry.get("name") | ||
| if isinstance(candidate, (list, tuple)): | ||
| candidate = candidate[0] if candidate else None | ||
| else: | ||
| continue | ||
| if isinstance(candidate, str) and Path(candidate).is_file(): | ||
| return candidate | ||
|
Comment on lines
+328
to
+340
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🟠 Major | 🏗️ Heavy lift 🔎 Supported by static analysis🏁 Script executed: #!/bin/bash
set -euo pipefail
# Inspect the trainer contract for supported `data_files` shapes.
trainer="$(fd -t f '^trainer\.py$' src/praisonai-train | head -n 1)"
test -n "$trainer"
ast-grep outline "$trainer" --items all --view expanded
sed -n '840,930p' "$trainer"
# Locate all handling of data_files and dataset normalization.
rg -n -C 6 --glob '*.py' 'data_files|dataset.*normal|load_dataset' src/praisonai-train
# Inspect the remote dataset transfer contract and its regression coverage.
sed -n '306,368p' src/praisonai-train/praisonai_train/cli/commands/train.py
sed -n '112,165p' src/praisonai-train/tests/unit/test_remote_dispatch.pyRepository: MervinPraison/PraisonAI Length of output: 35503 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Read the repository conventions that cover the training CLI.
for f in /tmp/coderabbit-repo-knowledge/mervinpraison-praisonai-dcd467e7/*/*.md; do
case "$f" in
*train*|*cli*|*python*) printf '\n--- %s ---\n' "$f"; sed -n '1,160p' "$f" ;;
esac
done
# Inspect the complete remote dispatch path and the runner's dataset contract.
sed -n '220,370p' src/praisonai-train/praisonai_train/cli/commands/train.py
rg -n -C 10 'class RemoteRunner|def start|dataset_path|_write_shipped_config|_local_dataset_in' src/praisonai-train/praisonai_train src/praisonai-train/tests/unit/test_remote_dispatch.pyRepository: MervinPraison/PraisonAI Length of output: 28574 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Resolve how RemoteRunner.start injects the single shipped dataset into the
# remote command, and how the training CLI handles that argument versus config.
sed -n '301,350p' src/praisonai-train/praisonai_train/remote/runner.py
rg -n -C 12 --glob '*.py' 'dataset.*Argument|dataset.*Option|def train\(|_dispatch_remote\(' src/praisonai-train/praisonai_train/cli src/praisonai-train/tests/unitRepository: MervinPraison/PraisonAI Length of output: 21505 Ship every local file referenced by When Extend the remote transfer contract to ship and rewrite all local 🤖 Prompt for AI Agents
Comment on lines
+335
to
+340
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When a remote configuration contains multiple dataset entries or multiple Knowledge Base Used: Training and vision workflows |
||
| return None | ||
|
|
||
|
|
||
| def _write_shipped_config(resolved): | ||
| """The config to send, in a temp file. Returns its path. | ||
|
|
||
|
|
@@ -370,11 +416,29 @@ def _resolve_config(config_path, overrides): | |
| return resolved | ||
|
|
||
|
|
||
| def _print_resolved_config(config_path, overrides): | ||
| """Show the config the run would use: the file, then the flags on top.""" | ||
| def _print_resolved_config(config_path, overrides, remote_overrides=None): | ||
| """Show the config the run would use: the file, then the flags on top. | ||
|
|
||
| The remote block is resolved with the same precedence and validation as the | ||
| real dispatch, so the preview shows the host, interpreter, workdir and GPU | ||
| count the run would actually use -- and a bad remote setting is caught here | ||
| rather than after an hour of rented GPU. | ||
| """ | ||
| import yaml | ||
|
|
||
| from ..output.console import get_output_controller | ||
| from praisonai_train.remote import settings as remote_settings | ||
|
|
||
| resolved = _resolve_config(config_path, overrides) | ||
|
|
||
| try: | ||
| block = remote_settings.resolve(resolved, remote_overrides or {}) | ||
| except remote_settings.RemoteSettingsError as exc: | ||
| get_output_controller().print_error("Bad remote settings", remediation=str(exc)) | ||
| raise typer.Exit(1) from exc | ||
| if block: | ||
| resolved["remote"] = remote_settings.redact(block) | ||
|
|
||
| typer.echo(yaml.safe_dump(resolved, sort_keys=True, default_flow_style=False).rstrip()) | ||
| if overrides: | ||
| typer.echo(f"\n# {len(overrides)} value(s) came from flags: " | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When a remote config names a relative local dataset and the CLI runs outside the config directory,
_local_dataset_inchecks the path against the process working directory instead. The file is not shipped, so the remote trainer interprets the unavailable path as a dataset identifier and aborts before training.Knowledge Base Used: Training and vision workflows