From 173d3f0a459f3adf0a07c0fc8810aba1ec0bb490 Mon Sep 17 00:00:00 2001 From: Alex Filby Date: Mon, 27 Jul 2026 14:37:01 -0500 Subject: [PATCH 1/4] fix: use nanosecond experiment timestamps (#571) Reduce experiment directory collisions by using time_ns for generated and reset experiment IDs. Add focused coverage for both paths. Signed-off-by: Alex Filby --- nemo_run/run/experiment.py | 4 ++-- test/run/test_experiment.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/nemo_run/run/experiment.py b/nemo_run/run/experiment.py index ba430413..d963dd8e 100644 --- a/nemo_run/run/experiment.py +++ b/nemo_run/run/experiment.py @@ -333,7 +333,7 @@ def __init__( assert id, "Cannot reconstruct an experiment without id." self._title = title - self._id = id or f"{title}_{int(time.time())}" + self._id = id or f"{title}_{time.time_ns()}" self._enable_goodbye_message = enable_goodbye_message self._threadpool_workers = threadpool_workers self._skip_status_at_exit = skip_status_at_exit @@ -1068,7 +1068,7 @@ def reset(self) -> "Experiment": return self old_id, old_exp_dir, old_launched = self._id, self._exp_dir, self._launched - self._id = f"{self._title}_{int(time.time())}" + self._id = f"{self._title}_{time.time_ns()}" self._exp_dir = os.path.join(get_nemorun_home(), "experiments", self._title, self._id) self._launched = False self._live_progress = None diff --git a/test/run/test_experiment.py b/test/run/test_experiment.py index 4f160237..15977db0 100644 --- a/test/run/test_experiment.py +++ b/test/run/test_experiment.py @@ -82,6 +82,18 @@ def test_experiment_creation(temp_dir): assert isinstance(exp.executor, LocalExecutor) +def test_experiment_creation_uses_nanosecond_timestamp(temp_dir): + timestamp_ns = 1_753_041_987_123_456_789 + + with patch("nemo_run.run.experiment.time.time_ns", return_value=timestamp_ns): + exp = Experiment("test-exp") + + assert exp._id == f"test-exp_{timestamp_ns}" + assert exp._exp_dir == os.path.join( + temp_dir, "experiments", "test-exp", f"test-exp_{timestamp_ns}" + ) + + def test_experiment_with_custom_id(temp_dir): """Test creating an experiment with a custom id.""" exp = Experiment("test-exp", id="custom-id") @@ -345,6 +357,22 @@ def test_reset_not_run_experiment(temp_dir): assert reset_exp is exp # The implementation returns self now +def test_reset_uses_nanosecond_timestamp(temp_dir): + timestamp_ns = 1_753_041_987_987_654_321 + exp = Experiment("test-exp", id="test-exp_original") + exp._prepare() + Path(os.path.join(exp._exp_dir, Experiment._DONE_FILE)).touch() + + with patch("nemo_run.run.experiment.time.time_ns", return_value=timestamp_ns): + reset_exp = exp.reset() + + assert reset_exp is exp + assert exp._id == f"test-exp_{timestamp_ns}" + assert exp._exp_dir == os.path.join( + temp_dir, "experiments", "test-exp", f"test-exp_{timestamp_ns}" + ) + + @patch("nemo_run.run.experiment.get_runner") def test_experiment_from_id(mock_get_runner, temp_dir): """Test reconstructing an experiment from its ID.""" From e3935393a290aed1822af52139b4b8ee270fed1f Mon Sep 17 00:00:00 2001 From: svcnemo-autobot Date: Wed, 29 Jul 2026 00:32:40 +0200 Subject: [PATCH 2/4] fix(docker): AUT-1080 persist cancellation status (#570) * fix(docker): persist cancellation status Signed-off-by: svcnemo-autobot * test(docker): cover missing cancellation request Signed-off-by: svcnemo-autobot * fix(ci): skip release rehearsal for forks Signed-off-by: svcnemo-autobot --------- Signed-off-by: svcnemo-autobot --- .github/workflows/release.yml | 9 ++++-- .../run/torchx_backend/schedulers/docker.py | 5 +++- .../torchx_backend/schedulers/test_docker.py | 28 +++++++++++++++++++ 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2305ca21..2d9b2329 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,12 @@ concurrency: jobs: release: - if: '!cancelled()' + if: | + !cancelled() + && ( + github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository + ) uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_release_library.yml@v1.4.3 with: release-ref: ${{ inputs.release-ref || github.sha }} @@ -82,7 +87,7 @@ jobs: release-summary: needs: [release] - if: ${{ !cancelled() }} + if: ${{ !cancelled() && needs.release.result != 'skipped' }} runs-on: ubuntu-latest steps: - name: Result diff --git a/nemo_run/run/torchx_backend/schedulers/docker.py b/nemo_run/run/torchx_backend/schedulers/docker.py index 4f68920c..b162cfe1 100644 --- a/nemo_run/run/torchx_backend/schedulers/docker.py +++ b/nemo_run/run/torchx_backend/schedulers/docker.py @@ -276,8 +276,11 @@ def local_logs(container: DockerContainer): def _cancel_existing(self, app_id: str) -> None: req = DockerJobRequest.load(app_id=app_id) if not req: - return None + return + status = json.dumps({"id": app_id, "exit_code": "1"}) + "\n" for container in req.containers: + status_file = Path(req.executor.job_dir) / f"status_{container.name}.out" + status_file.write_text(status) container.delete(client=self._docker_client, id=req.id) def close(self) -> None: diff --git a/test/run/torchx_backend/schedulers/test_docker.py b/test/run/torchx_backend/schedulers/test_docker.py index 551d8a60..e555d4ae 100644 --- a/test/run/torchx_backend/schedulers/test_docker.py +++ b/test/run/torchx_backend/schedulers/test_docker.py @@ -84,6 +84,34 @@ def test_docker_scheduler_methods(docker_scheduler): assert hasattr(docker_scheduler, "close") +def test_cancel_existing_without_request_is_noop(): + docker_scheduler = object.__new__(PersistentDockerScheduler) + docker_scheduler._DockerWorkspaceMixin__docker_client = mock.Mock() + docker_scheduler._scheduled_reqs = [] + + with mock.patch.object(DockerJobRequest, "load", return_value=None): + docker_scheduler._cancel_existing("app-id") + + +def test_cancel_existing_persists_terminal_status(tmp_path): + docker_scheduler = object.__new__(PersistentDockerScheduler) + docker_scheduler._DockerWorkspaceMixin__docker_client = mock.Mock() + docker_scheduler._scheduled_reqs = [] + container = mock.Mock(name="container") + container.name = "task-1-0" + request = mock.Mock() + request.id = "app-id" + request.executor.job_dir = str(tmp_path) + request.containers = [container] + + with mock.patch.object(DockerJobRequest, "load", return_value=request): + docker_scheduler._cancel_existing("app-id") + + container.delete.assert_called_once_with(client=docker_scheduler._docker_client, id="app-id") + status = json.loads((tmp_path / "status_task-1-0.out").read_text()) + assert status == {"id": "app-id", "exit_code": "1"} + + def test_schedule(docker_scheduler, mock_app_def, docker_executor): with ( mock.patch.object(DockerExecutor, "package") as mock_package, From dfe0579df8b54eaabb0c43cd255665efa16b75f1 Mon Sep 17 00:00:00 2001 From: svcnemo-autobot Date: Thu, 30 Jul 2026 03:50:50 +0200 Subject: [PATCH 3/4] chore(ci): pin GitHub Actions to commit SHAs (#573) Signed-off-by: svcnemo-autobot --- .github/workflows/changelog-build.yml | 8 ++++---- .github/workflows/cherry-pick-release-commit.yml | 2 +- .github/workflows/close-inactive-issue-pr.yml | 2 +- .github/workflows/codeql.yml | 8 ++++---- .github/workflows/community-bot.yml | 2 +- .github/workflows/copyright-check.yml | 2 +- .github/workflows/detect-secrets.yml | 2 +- .github/workflows/install.yml | 4 ++-- .github/workflows/nightly.yml | 4 ++-- .github/workflows/release-docs.yml | 4 ++-- .github/workflows/release-freeze.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/ruff-format.yml | 4 ++-- .github/workflows/ruff-lint.yml | 4 ++-- .github/workflows/spelling.yml | 4 ++-- .github/workflows/test.yml | 6 +++--- 16 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/changelog-build.yml b/.github/workflows/changelog-build.yml index 8ae4344f..d2e609b4 100644 --- a/.github/workflows/changelog-build.yml +++ b/.github/workflows/changelog-build.yml @@ -22,14 +22,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout branch - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: ref: main fetch-depth: 0 - name: Build Changelog id: github_tag - uses: mikepenz/release-changelog-builder-action@v3.3.1 + uses: mikepenz/release-changelog-builder-action@000e44613cdb6c340ac98cb1582f99e8d3230058 # v3.3.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -80,7 +80,7 @@ jobs: run: cat CHANGELOG.md - name: Create or update label - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 with: script: | const labelName = '${{ inputs.release-branch }}'; @@ -112,7 +112,7 @@ jobs: } - name: Create Pull Request - uses: peter-evans/create-pull-request@v7 + uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7 with: commit-message: "beep boop: Update changelog" title: "Update changelog for `${{ inputs.release-branch }}`" diff --git a/.github/workflows/cherry-pick-release-commit.yml b/.github/workflows/cherry-pick-release-commit.yml index 32f06df5..c99145a1 100644 --- a/.github/workflows/cherry-pick-release-commit.yml +++ b/.github/workflows/cherry-pick-release-commit.yml @@ -7,7 +7,7 @@ on: jobs: cherry-pick: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cherry_pick.yml@v0.63.0 + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cherry_pick.yml@8ebb3aec4c246c944a25adc7b09107cbd6a76976 # v0.63.0 secrets: PAT: ${{ secrets.PAT }} SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }} diff --git a/.github/workflows/close-inactive-issue-pr.yml b/.github/workflows/close-inactive-issue-pr.yml index b81f8293..61c57c8f 100644 --- a/.github/workflows/close-inactive-issue-pr.yml +++ b/.github/workflows/close-inactive-issue-pr.yml @@ -5,4 +5,4 @@ on: jobs: close-issues: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_close_inactive_issue_pr.yml@v0.44.0 + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_close_inactive_issue_pr.yml@9e07489b8a6bc533c8792099b012c588f4430298 # v0.44.0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c0cd20e6..43c88d9a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -38,11 +38,11 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@b8d3b6e8af63cde30bdc382c0bc28114f4346c88 # v2 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -57,7 +57,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@b8d3b6e8af63cde30bdc382c0bc28114f4346c88 # v2 # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -70,6 +70,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@b8d3b6e8af63cde30bdc382c0bc28114f4346c88 # v2 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/community-bot.yml b/.github/workflows/community-bot.yml index fa004e28..4c507c19 100644 --- a/.github/workflows/community-bot.yml +++ b/.github/workflows/community-bot.yml @@ -8,7 +8,7 @@ on: jobs: community-bot: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_community_bot.yml@v0.54.4 + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_community_bot.yml@a8006e87bff5bce54336c3e1e8571563ac70d88e # v0.54.4 with: community_project_id: ${{ vars.COMMUNITY_PROJECT_ID }} secrets: diff --git a/.github/workflows/copyright-check.yml b/.github/workflows/copyright-check.yml index 55d95abd..6dd2e2ee 100644 --- a/.github/workflows/copyright-check.yml +++ b/.github/workflows/copyright-check.yml @@ -20,4 +20,4 @@ on: jobs: copyright-check: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_copyright_check.yml@v0.54.4 + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_copyright_check.yml@a8006e87bff5bce54336c3e1e8571563ac70d88e # v0.54.4 diff --git a/.github/workflows/detect-secrets.yml b/.github/workflows/detect-secrets.yml index d7aae1cd..fff5bcf2 100644 --- a/.github/workflows/detect-secrets.yml +++ b/.github/workflows/detect-secrets.yml @@ -18,4 +18,4 @@ on: jobs: secrets-detector: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_secrets-detector.yml@v0.74.0 + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_secrets-detector.yml@859ae21dd5588c87e3bde499ddab7a3025773e8d # v0.74.0 diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml index 740349bb..bd006e0a 100644 --- a/.github/workflows/install.yml +++ b/.github/workflows/install.yml @@ -16,10 +16,10 @@ jobs: os: [ubuntu-latest, macos-latest, ubuntu-22.04-arm] python: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Setup Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "${{ matrix.python }}" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 81850f76..f8416f22 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -17,10 +17,10 @@ jobs: runs-on: ubuntu-latest needs: [format, lint, test] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Create nightly tag - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 with: script: | // Delete existing nightly tag if it exists diff --git a/.github/workflows/release-docs.yml b/.github/workflows/release-docs.yml index 024ccdaf..821154c4 100644 --- a/.github/workflows/release-docs.yml +++ b/.github/workflows/release-docs.yml @@ -78,7 +78,7 @@ on: jobs: build-docs: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_docs.yml@v0.67.0 + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_docs.yml@3ab507cd035df3ae37cce8808ed3210ff6e7062b # v0.67.0 with: ref: ${{ inputs.github-ref }} @@ -86,7 +86,7 @@ jobs: runs-on: ubuntu-latest needs: [build-docs] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: repository: NVIDIA-NeMo/FW-CI-templates ref: v0.74.0 diff --git a/.github/workflows/release-freeze.yml b/.github/workflows/release-freeze.yml index 1996d062..b2b277ae 100644 --- a/.github/workflows/release-freeze.yml +++ b/.github/workflows/release-freeze.yml @@ -21,7 +21,7 @@ on: default: true jobs: code-freeze: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_code_freeze.yml@v1.4.2 + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_code_freeze.yml@bfdb5e35067fd8cd91ce21fca4eb1072ffd7ab8c # v1.4.2 with: library-name: NeMo Run python-package: nemo_run diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2d9b2329..ed4001ba 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,7 +68,7 @@ jobs: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_release_library.yml@v1.4.3 + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_release_library.yml@f7af92c4d1caea5a58577ce4ac5314f7de1bc5d4 # v1.4.3 with: release-ref: ${{ inputs.release-ref || github.sha }} python-package: nemo_run diff --git a/.github/workflows/ruff-format.yml b/.github/workflows/ruff-format.yml index 9369e6df..ca689060 100644 --- a/.github/workflows/ruff-format.yml +++ b/.github/workflows/ruff-format.yml @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3 with: version: "latest" diff --git a/.github/workflows/ruff-lint.yml b/.github/workflows/ruff-lint.yml index 1efe83ee..8a350d0c 100644 --- a/.github/workflows/ruff-lint.yml +++ b/.github/workflows/ruff-lint.yml @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3 with: version: "latest" diff --git a/.github/workflows/spelling.yml b/.github/workflows/spelling.yml index e782cdb5..856ed11b 100644 --- a/.github/workflows/spelling.yml +++ b/.github/workflows/spelling.yml @@ -11,8 +11,8 @@ jobs: name: "Spell check" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: crate-ci/typos@master + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - uses: crate-ci/typos@3bc303c295c081add5df4a4d52a1e117f2fb2dce # master with: files: . config: ./.github/workflows/config/typos.toml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b301f651..ddebfc2b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,10 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - name: Install the latest version of uv - uses: astral-sh/setup-uv@v3 + uses: astral-sh/setup-uv@caf0cab7a618c569241d31dcd442f54681755d39 # v3 with: version: "latest" @@ -48,7 +48,7 @@ jobs: uv run -- coverage report - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.xml From 60249d9f953dd7470e8bf3c43fad710cb1c1589e Mon Sep 17 00:00:00 2001 From: Andrew White Date: Fri, 31 Jul 2026 07:42:54 -0500 Subject: [PATCH 4/4] fix: publish-docs input is ignored when set to false Signed-off-by: Andrew White --- .github/workflows/release.yml | 2 +- tests/test_release_publish_docs.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/test_release_publish_docs.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ed4001ba..3768ebad 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,7 +80,7 @@ jobs: app-id: ${{ vars.BOT_ID }} packaging: setuptools gh-release-use-changelog-builder: ${{ inputs.generate-changelog || false }} - publish-docs: ${{ inputs.publish-docs || true }} + publish-docs: ${{ inputs.publish-docs == true }} docs-target-path: nemo/run restrict-to-admins: true secrets: inherit # pragma: allowlist secret diff --git a/tests/test_release_publish_docs.py b/tests/test_release_publish_docs.py new file mode 100644 index 00000000..4afb1315 --- /dev/null +++ b/tests/test_release_publish_docs.py @@ -0,0 +1,16 @@ +import re +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +class TestReleaseWorkflow(unittest.TestCase): + def test_publish_docs_respects_false_input(self): + workflow = REPO_ROOT / ".github" / "workflows" / "release.yml" + text = workflow.read_text() + match = re.search(r"^\s+publish-docs:\s*(.+)$", text, re.MULTILINE) + self.assertIsNotNone(match) + self.assertNotIn("|| true", match.group(1), + "publish-docs always evaluates to true; false input is ignored") +