From c5d8d7da0e20025cc2bed4cbe2a61ae35ebdfb1e Mon Sep 17 00:00:00 2001 From: Aayaan Naqvi Date: Fri, 7 Aug 2026 11:54:16 -0700 Subject: [PATCH 1/4] feat(cli): add beta clusters recreate command --- src/together/lib/cli/__init__.py | 4 ++ .../lib/cli/api/beta/clusters/recreate.py | 42 +++++++++++++++++++ tests/cli/test_beta_clusters.py | 33 +++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 src/together/lib/cli/api/beta/clusters/recreate.py diff --git a/src/together/lib/cli/__init__.py b/src/together/lib/cli/__init__.py index 3ded4e6f7..3230abe03 100644 --- a/src/together/lib/cli/__init__.py +++ b/src/together/lib/cli/__init__.py @@ -564,6 +564,10 @@ async def run_command() -> None: help_epilogue=BETA_CLUSTERS_UPDATE_HELP_EXAMPLES, ) clusters_app.command((f"{_CLI}.beta.clusters.delete:delete"), alias="-d", help="Delete a cluster") +clusters_app.command( + (f"{_CLI}.beta.clusters.recreate:recreate"), + help="Recreate a cluster, preserving its reservation", +) clusters_app.command((f"{_CLI}.beta.clusters.list_regions:list_regions"), help="List regions for deploying clusters") clusters_app.command( (f"{_CLI}.beta.clusters.get_credentials:get_credentials"), diff --git a/src/together/lib/cli/api/beta/clusters/recreate.py b/src/together/lib/cli/api/beta/clusters/recreate.py new file mode 100644 index 000000000..9f7fcc973 --- /dev/null +++ b/src/together/lib/cli/api/beta/clusters/recreate.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import httpx + +from together.lib.cli.utils.config import CLIConfigParameter +from together.lib.cli.utils._console import console +from together.lib.cli.components.loader import show_loading_status +from together.lib.cli.api.beta.clusters._util import print_clusters + + +async def recreate( + cluster_id: str, + *, + reason: str | None = None, + config: CLIConfigParameter, +) -> None: + """Tear down and rebuild a cluster while preserving its reservation.""" + + body = {"reason": reason} if reason else {} + + if config.json: + response = await config.client.post( + f"/compute/clusters/{cluster_id}:recreate", + cast_to=httpx.Response, + body=body, + ) + console.print_json(response.text) + return + + cluster = await show_loading_status("", config.client.beta.clusters.retrieve(cluster_id=cluster_id)) + print_clusters([cluster]) + resp = input(f"Clusters: Are you sure you want to recreate cluster {cluster.cluster_name}? [y/N] ").strip().lower() + if resp != "y" and resp != "yes": + return + response = await show_loading_status( + "Recreating cluster...", + config.client.post(f"/compute/clusters/{cluster_id}:recreate", cast_to=httpx.Response, body=body), + ) + intent = response.json() + console.print( + f"Recreate requested for {cluster.cluster_name} ({cluster_id}); intent {intent.get('id')} is {intent.get('status')}" + ) diff --git a/tests/cli/test_beta_clusters.py b/tests/cli/test_beta_clusters.py index d3a85a1ac..b744f1d44 100644 --- a/tests/cli/test_beta_clusters.py +++ b/tests/cli/test_beta_clusters.py @@ -1153,6 +1153,39 @@ def test_delete_confirm_yes(self, respx_mock: MockRouter, cli_runner: CliRunner) assert result.exit_code == 0 +class TestBetaClustersRecreate: + @pytest.mark.respx(base_url=base_url) + def test_recreate_json(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: + respx_mock.post("/compute/clusters/c-rec:recreate").mock( + return_value=httpx.Response(200, json={"id": "intent-1", "cluster_id": "c-rec", "status": "pending"}) + ) + result = cli_runner.invoke(["beta", "clusters", "recreate", "c-rec", "--json"]) + assert json.loads(result.output) == {"id": "intent-1", "cluster_id": "c-rec", "status": "pending"} + assert result.exit_code == 0 + + @pytest.mark.respx(base_url=base_url) + def test_recreate_confirm_yes_sends_reason(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: + c = _cluster_body("c1", "to-recreate") + respx_mock.get("/compute/clusters/c1").mock(return_value=httpx.Response(200, json=c)) + route = respx_mock.post("/compute/clusters/c1:recreate").mock( + return_value=httpx.Response(200, json={"id": "intent-1", "cluster_id": "c1", "status": "pending"}) + ) + result = cli_runner.invoke(["beta", "clusters", "recreate", "c1", "--reason", "maintenance"], input="y\n") + assert "Recreate requested" in result.output + assert "intent-1" in result.output + assert result.exit_code == 0 + sent = json.loads(route.calls.last.request.content) + assert sent == {"reason": "maintenance"} + + @pytest.mark.respx(base_url=base_url) + def test_recreate_confirm_no(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: + # No POST route is mocked: an unexpected recreate call would fail loudly. + c = _cluster_body("c1", "keep-me") + respx_mock.get("/compute/clusters/c1").mock(return_value=httpx.Response(200, json=c)) + result = cli_runner.invoke(["beta", "clusters", "recreate", "c1"], input="n\n") + assert result.exit_code == 0 + + class TestBetaClustersGetCredentials: @pytest.mark.respx(base_url=base_url) def test_get_credentials_stdout(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: From a8ce0b3bcb9abed2e353aef5e15e8f9f74f3c6f9 Mon Sep 17 00:00:00 2001 From: Aayaan Naqvi Date: Fri, 7 Aug 2026 13:08:40 -0700 Subject: [PATCH 2/4] feat(cli): support new spec on cluster recreate --- .../lib/cli/api/beta/clusters/recreate.py | 35 +++++++++++++++++-- tests/cli/test_beta_clusters.py | 12 +++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/together/lib/cli/api/beta/clusters/recreate.py b/src/together/lib/cli/api/beta/clusters/recreate.py index 9f7fcc973..27580e4c9 100644 --- a/src/together/lib/cli/api/beta/clusters/recreate.py +++ b/src/together/lib/cli/api/beta/clusters/recreate.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Literal + import httpx from together.lib.cli.utils.config import CLIConfigParameter @@ -12,11 +14,36 @@ async def recreate( cluster_id: str, *, reason: str | None = None, + cluster_type: Literal["KUBERNETES", "SLURM"] | None = None, + num_gpus: int | None = None, + num_reserved_gpus: int | None = None, + num_capacity_pool_gpus: int | None = None, + num_preemptible_gpus: int | None = None, config: CLIConfigParameter, ) -> None: - """Tear down and rebuild a cluster while preserving its reservation.""" + """Tear down and rebuild a cluster while preserving its reservation. + + Spec options (cluster type and GPU count targets) are applied as the + cluster's new spec; when none are given, the current spec is kept. + """ - body = {"reason": reason} if reason else {} + new_spec = { + key: value + for key, value in { + "cluster_type": cluster_type, + "num_gpus": num_gpus, + "num_reserved_gpus": num_reserved_gpus, + "num_capacity_pool_gpus": num_capacity_pool_gpus, + "num_preemptible_gpus": num_preemptible_gpus, + }.items() + if value is not None + } + + body: dict[str, object] = {} + if reason: + body["reason"] = reason + if new_spec: + body["new_spec"] = new_spec if config.json: response = await config.client.post( @@ -29,7 +56,9 @@ async def recreate( cluster = await show_loading_status("", config.client.beta.clusters.retrieve(cluster_id=cluster_id)) print_clusters([cluster]) - resp = input(f"Clusters: Are you sure you want to recreate cluster {cluster.cluster_name}? [y/N] ").strip().lower() + resp = ( + input(f"Clusters: Are you sure you want to recreate cluster {cluster.cluster_name}? [y/N] ").strip().lower() + ) if resp != "y" and resp != "yes": return response = await show_loading_status( diff --git a/tests/cli/test_beta_clusters.py b/tests/cli/test_beta_clusters.py index b744f1d44..6f835828c 100644 --- a/tests/cli/test_beta_clusters.py +++ b/tests/cli/test_beta_clusters.py @@ -1177,6 +1177,18 @@ def test_recreate_confirm_yes_sends_reason(self, respx_mock: MockRouter, cli_run sent = json.loads(route.calls.last.request.content) assert sent == {"reason": "maintenance"} + @pytest.mark.respx(base_url=base_url) + def test_recreate_with_new_spec(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: + route = respx_mock.post("/compute/clusters/c1:recreate").mock( + return_value=httpx.Response(200, json={"id": "intent-2", "cluster_id": "c1", "status": "pending"}) + ) + result = cli_runner.invoke( + ["beta", "clusters", "recreate", "c1", "--cluster-type", "SLURM", "--num-gpus", "16", "--json"] + ) + assert result.exit_code == 0 + sent = json.loads(route.calls.last.request.content) + assert sent == {"new_spec": {"cluster_type": "SLURM", "num_gpus": 16}} + @pytest.mark.respx(base_url=base_url) def test_recreate_confirm_no(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: # No POST route is mocked: an unexpected recreate call would fail loudly. From 9dfc9fb1fa6677107cad1865c97014eb60bc2701 Mon Sep 17 00:00:00 2001 From: Aayaan Naqvi Date: Fri, 7 Aug 2026 13:48:28 -0700 Subject: [PATCH 3/4] chore: format --- src/together/lib/cli/api/beta/clusters/recreate.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/together/lib/cli/api/beta/clusters/recreate.py b/src/together/lib/cli/api/beta/clusters/recreate.py index 27580e4c9..f79967ef7 100644 --- a/src/together/lib/cli/api/beta/clusters/recreate.py +++ b/src/together/lib/cli/api/beta/clusters/recreate.py @@ -56,9 +56,7 @@ async def recreate( cluster = await show_loading_status("", config.client.beta.clusters.retrieve(cluster_id=cluster_id)) print_clusters([cluster]) - resp = ( - input(f"Clusters: Are you sure you want to recreate cluster {cluster.cluster_name}? [y/N] ").strip().lower() - ) + resp = input(f"Clusters: Are you sure you want to recreate cluster {cluster.cluster_name}? [y/N] ").strip().lower() if resp != "y" and resp != "yes": return response = await show_loading_status( From b15b1a773bb8c2827b70d01352f2db789940d905 Mon Sep 17 00:00:00 2001 From: Aayaan Naqvi Date: Fri, 7 Aug 2026 14:41:47 -0700 Subject: [PATCH 4/4] feat(cli): add hidden --env flag for environment selection --- src/together/lib/cli/__init__.py | 13 ++++++++++++- tests/cli/test_beta_clusters.py | 9 +++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/together/lib/cli/__init__.py b/src/together/lib/cli/__init__.py index 3230abe03..bdd9655d8 100644 --- a/src/together/lib/cli/__init__.py +++ b/src/together/lib/cli/__init__.py @@ -3,7 +3,7 @@ import os import sys import inspect -from typing import Optional, Annotated, get_args, get_origin +from typing import Literal, Optional, Annotated, get_args, get_origin import httpx from cyclopts import App, Group, Parameter, CycloptsError, MissingArgumentError @@ -112,6 +112,13 @@ # stripped; reported separately via is_beta_command). _NO_AUTH_COMMANDS = frozenset({"clusters ssh"}) +# Internal environment shorthands for the hidden --env flag. An explicit +# --base-url wins; TOGETHER_BASE_URL only applies when neither flag is given. +_ENV_BASE_URLS = { + "qa": "https://api.qa.together.ai/v1", + "prod": "https://api.together.ai/v1", +} + async def _resolve_project_id(client: AsyncTogether) -> str: me = await client.whoami() @@ -205,6 +212,7 @@ async def launcher( *tokens: Annotated[str, Parameter(show=False, allow_leading_hyphen=True)], api_key: Annotated[Optional[str], Parameter(show=False)] = None, base_url: Annotated[Optional[str], Parameter(show=False)] = None, + env: Annotated[Optional[Literal["qa", "prod"]], Parameter(show=False)] = None, timeout: Annotated[Optional[int], Parameter(show=False)] = None, max_retries: Annotated[Optional[int], Parameter(show=False)] = None, debug: Annotated[Optional[bool], Parameter(show=False)] = False, @@ -244,6 +252,9 @@ async def launcher( # they stay keyless. no_auth_command = is_beta_command and parsed_command in _NO_AUTH_COMMANDS + if base_url is None and env is not None: + base_url = _ENV_BASE_URLS[env] + client = _create_client(api_key, base_url, timeout, max_retries, project_id, require_api_key=not no_auth_command) # Skip the project-resolution whoami() for out-of-band-auth commands: it is a diff --git a/tests/cli/test_beta_clusters.py b/tests/cli/test_beta_clusters.py index 6f835828c..d30181582 100644 --- a/tests/cli/test_beta_clusters.py +++ b/tests/cli/test_beta_clusters.py @@ -1189,6 +1189,15 @@ def test_recreate_with_new_spec(self, respx_mock: MockRouter, cli_runner: CliRun sent = json.loads(route.calls.last.request.content) assert sent == {"new_spec": {"cluster_type": "SLURM", "num_gpus": 16}} + @pytest.mark.respx(base_url="https://api.qa.together.ai/v1") + def test_recreate_env_flag_targets_qa(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: + respx_mock.post("/compute/clusters/c-qa:recreate").mock( + return_value=httpx.Response(200, json={"id": "intent-3", "cluster_id": "c-qa", "status": "pending"}) + ) + result = cli_runner.invoke(["--env", "qa", "beta", "clusters", "recreate", "c-qa", "--json"]) + assert result.exit_code == 0 + assert json.loads(result.output)["id"] == "intent-3" + @pytest.mark.respx(base_url=base_url) def test_recreate_confirm_no(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: # No POST route is mocked: an unexpected recreate call would fail loudly.