Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion src/together/lib/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import os
import sys
import inspect
from typing import Optional, Annotated, get_args, get_origin
from typing import Literal, Optional, Annotated, get_args, get_origin

import httpx
from cyclopts import App, Group, Parameter, CycloptsError, MissingArgumentError
Expand Down Expand Up @@ -112,6 +112,13 @@
# stripped; reported separately via is_beta_command).
_NO_AUTH_COMMANDS = frozenset({"clusters ssh"})

# Internal environment shorthands for the hidden --env flag. An explicit
# --base-url wins; TOGETHER_BASE_URL only applies when neither flag is given.
_ENV_BASE_URLS = {
"qa": "https://api.qa.together.ai/v1",
"prod": "https://api.together.ai/v1",
}


async def _resolve_project_id(client: AsyncTogether) -> str:
me = await client.whoami()
Expand Down Expand Up @@ -205,6 +212,7 @@ async def launcher(
*tokens: Annotated[str, Parameter(show=False, allow_leading_hyphen=True)],
api_key: Annotated[Optional[str], Parameter(show=False)] = None,
base_url: Annotated[Optional[str], Parameter(show=False)] = None,
env: Annotated[Optional[Literal["qa", "prod"]], Parameter(show=False)] = None,
timeout: Annotated[Optional[int], Parameter(show=False)] = None,
max_retries: Annotated[Optional[int], Parameter(show=False)] = None,
debug: Annotated[Optional[bool], Parameter(show=False)] = False,
Expand Down Expand Up @@ -244,6 +252,9 @@ async def launcher(
# they stay keyless.
no_auth_command = is_beta_command and parsed_command in _NO_AUTH_COMMANDS

if base_url is None and env is not None:
base_url = _ENV_BASE_URLS[env]

client = _create_client(api_key, base_url, timeout, max_retries, project_id, require_api_key=not no_auth_command)

# Skip the project-resolution whoami() for out-of-band-auth commands: it is a
Expand Down Expand Up @@ -564,6 +575,10 @@ async def run_command() -> None:
help_epilogue=BETA_CLUSTERS_UPDATE_HELP_EXAMPLES,
)
clusters_app.command((f"{_CLI}.beta.clusters.delete:delete"), alias="-d", help="Delete a cluster")
clusters_app.command(
(f"{_CLI}.beta.clusters.recreate:recreate"),
help="Recreate a cluster, preserving its reservation",
)
clusters_app.command((f"{_CLI}.beta.clusters.list_regions:list_regions"), help="List regions for deploying clusters")
clusters_app.command(
(f"{_CLI}.beta.clusters.get_credentials:get_credentials"),
Expand Down
69 changes: 69 additions & 0 deletions src/together/lib/cli/api/beta/clusters/recreate.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
from __future__ import annotations

from typing import Literal

import httpx

from together.lib.cli.utils.config import CLIConfigParameter
from together.lib.cli.utils._console import console
from together.lib.cli.components.loader import show_loading_status
from together.lib.cli.api.beta.clusters._util import print_clusters


async def recreate(
cluster_id: str,
*,
reason: str | None = None,
cluster_type: Literal["KUBERNETES", "SLURM"] | None = None,
num_gpus: int | None = None,
num_reserved_gpus: int | None = None,
num_capacity_pool_gpus: int | None = None,
num_preemptible_gpus: int | None = None,
config: CLIConfigParameter,
) -> None:
"""Tear down and rebuild a cluster while preserving its reservation.

Spec options (cluster type and GPU count targets) are applied as the
cluster's new spec; when none are given, the current spec is kept.
"""

new_spec = {
key: value
for key, value in {
"cluster_type": cluster_type,
"num_gpus": num_gpus,
"num_reserved_gpus": num_reserved_gpus,
"num_capacity_pool_gpus": num_capacity_pool_gpus,
"num_preemptible_gpus": num_preemptible_gpus,
}.items()
if value is not None
}

body: dict[str, object] = {}
if reason:
body["reason"] = reason
if new_spec:
body["new_spec"] = new_spec

if config.json:
response = await config.client.post(
f"/compute/clusters/{cluster_id}:recreate",
cast_to=httpx.Response,
body=body,
)
console.print_json(response.text)
return

cluster = await show_loading_status("", config.client.beta.clusters.retrieve(cluster_id=cluster_id))
print_clusters([cluster])
resp = input(f"Clusters: Are you sure you want to recreate cluster {cluster.cluster_name}? [y/N] ").strip().lower()
if resp != "y" and resp != "yes":
return
response = await show_loading_status(
"Recreating cluster...",
config.client.post(f"/compute/clusters/{cluster_id}:recreate", cast_to=httpx.Response, body=body),
)
intent = response.json()
console.print(
f"Recreate requested for {cluster.cluster_name} ({cluster_id}); intent {intent.get('id')} is {intent.get('status')}"
)
54 changes: 54 additions & 0 deletions tests/cli/test_beta_clusters.py
Original file line number Diff line number Diff line change
Expand Up @@ -1153,6 +1153,60 @@ def test_delete_confirm_yes(self, respx_mock: MockRouter, cli_runner: CliRunner)
assert result.exit_code == 0


class TestBetaClustersRecreate:
@pytest.mark.respx(base_url=base_url)
def test_recreate_json(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None:
respx_mock.post("/compute/clusters/c-rec:recreate").mock(
return_value=httpx.Response(200, json={"id": "intent-1", "cluster_id": "c-rec", "status": "pending"})
)
result = cli_runner.invoke(["beta", "clusters", "recreate", "c-rec", "--json"])
assert json.loads(result.output) == {"id": "intent-1", "cluster_id": "c-rec", "status": "pending"}
assert result.exit_code == 0

@pytest.mark.respx(base_url=base_url)
def test_recreate_confirm_yes_sends_reason(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None:
c = _cluster_body("c1", "to-recreate")
respx_mock.get("/compute/clusters/c1").mock(return_value=httpx.Response(200, json=c))
route = respx_mock.post("/compute/clusters/c1:recreate").mock(
return_value=httpx.Response(200, json={"id": "intent-1", "cluster_id": "c1", "status": "pending"})
)
result = cli_runner.invoke(["beta", "clusters", "recreate", "c1", "--reason", "maintenance"], input="y\n")
assert "Recreate requested" in result.output
assert "intent-1" in result.output
assert result.exit_code == 0
sent = json.loads(route.calls.last.request.content)
assert sent == {"reason": "maintenance"}

@pytest.mark.respx(base_url=base_url)
def test_recreate_with_new_spec(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None:
route = respx_mock.post("/compute/clusters/c1:recreate").mock(
return_value=httpx.Response(200, json={"id": "intent-2", "cluster_id": "c1", "status": "pending"})
)
result = cli_runner.invoke(
["beta", "clusters", "recreate", "c1", "--cluster-type", "SLURM", "--num-gpus", "16", "--json"]
)
assert result.exit_code == 0
sent = json.loads(route.calls.last.request.content)
assert sent == {"new_spec": {"cluster_type": "SLURM", "num_gpus": 16}}

@pytest.mark.respx(base_url="https://api.qa.together.ai/v1")
def test_recreate_env_flag_targets_qa(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None:
respx_mock.post("/compute/clusters/c-qa:recreate").mock(
return_value=httpx.Response(200, json={"id": "intent-3", "cluster_id": "c-qa", "status": "pending"})
)
result = cli_runner.invoke(["--env", "qa", "beta", "clusters", "recreate", "c-qa", "--json"])
assert result.exit_code == 0
assert json.loads(result.output)["id"] == "intent-3"

@pytest.mark.respx(base_url=base_url)
def test_recreate_confirm_no(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None:
# No POST route is mocked: an unexpected recreate call would fail loudly.
c = _cluster_body("c1", "keep-me")
respx_mock.get("/compute/clusters/c1").mock(return_value=httpx.Response(200, json=c))
result = cli_runner.invoke(["beta", "clusters", "recreate", "c1"], input="n\n")
assert result.exit_code == 0


class TestBetaClustersGetCredentials:
@pytest.mark.respx(base_url=base_url)
def test_get_credentials_stdout(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None:
Expand Down
Loading