From e6f3f5da2d740012f99c6531e1ca53bf6749fe15 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 8 Sep 2026 14:58:58 +0000 Subject: [PATCH 1/2] fix(cli): accept active sessions autoscaling metric Co-authored-by: Blaine Kasten --- .../endpoints/_utils/_build_autoscaling.py | 2 + .../lib/cli/api/beta/endpoints/deploy.py | 1 + tests/cli/test_beta_endpoints.py | 43 +++++++++++++++++++ tests/cli/test_beta_endpoints_update.py | 28 ++++++++++++ tests/cli/test_build_autoscaling.py | 10 +++++ 5 files changed, 84 insertions(+) diff --git a/src/together/lib/cli/api/beta/endpoints/_utils/_build_autoscaling.py b/src/together/lib/cli/api/beta/endpoints/_utils/_build_autoscaling.py index 5adb65fb4..d70068da0 100644 --- a/src/together/lib/cli/api/beta/endpoints/_utils/_build_autoscaling.py +++ b/src/together/lib/cli/api/beta/endpoints/_utils/_build_autoscaling.py @@ -19,6 +19,7 @@ ] ScalingMetricName = Literal[ + "active_sessions", "inflight_requests", "gpu_utilization", "token_utilization", @@ -33,6 +34,7 @@ # Fixed type per metric name (see examples/internal-team-guides/autoscaling.md). _METRIC_TYPES: dict[ScalingMetricName, MetricType] = { + "active_sessions": "METRIC_TARGET_TYPE_VALUE", "inflight_requests": "METRIC_TARGET_TYPE_AVERAGE_VALUE", "gpu_utilization": "METRIC_TARGET_TYPE_UTILIZATION", "token_utilization": "METRIC_TARGET_TYPE_UTILIZATION", diff --git a/src/together/lib/cli/api/beta/endpoints/deploy.py b/src/together/lib/cli/api/beta/endpoints/deploy.py index 5c26fb3ba..b859952c9 100644 --- a/src/together/lib/cli/api/beta/endpoints/deploy.py +++ b/src/together/lib/cli/api/beta/endpoints/deploy.py @@ -125,6 +125,7 @@ async def deploy( help=( """Autoscaling metric. Must be set with --scaling-target; --scaling-percentile is optional and only applies to latency metrics. +- active_sessions: Active sessions across the deployment. - inflight_requests: Concurrent in-flight requests per replica. - gpu_utilization: GPU compute utilization (%). - token_utilization: KV-cache utilization (%). diff --git a/tests/cli/test_beta_endpoints.py b/tests/cli/test_beta_endpoints.py index 8bf59b7f4..0445699da 100644 --- a/tests/cli/test_beta_endpoints.py +++ b/tests/cli/test_beta_endpoints.py @@ -377,6 +377,49 @@ def test_deploy_onto_existing_endpoint(self, respx_mock: MockRouter, cli_runner: assert result.exit_code == 0, result.output assert create_deployment_route.call_count == 1 + @pytest.mark.respx(base_url=base_url) + def test_deploy_accepts_active_sessions_scaling_metric( + self, respx_mock: MockRouter, cli_runner: CliRunner + ) -> None: + _mock_model_and_config(respx_mock) + respx_mock.get("/projects/proj/endpoints/ep_1").mock(return_value=httpx.Response(200, json=_endpoint_body())) + create_deployment_route = respx_mock.post("/projects/proj/endpoints/ep_1/deployments").mock( + return_value=httpx.Response(200, json=_deployment_body()) + ) + + result = cli_runner.invoke( + [ + "beta", + "endpoints", + "deploy", + "--project", + "proj", + "--endpoint", + "ep_1", + "--model", + "ml_1", + "--config", + "cr_1", + "--deployment-name", + "my-dep", + "--scaling-metric", + "active_sessions", + "--scaling-target", + "25", + "--json", + ] + ) + + assert result.exit_code == 0, result.output + deployment_body = json.loads(cast(Call, create_deployment_route.calls[0]).request.content.decode()) + assert deployment_body["autoscaling"]["scalingMetrics"] == [ + { + "name": "active_sessions", + "type": "METRIC_TARGET_TYPE_VALUE", + "target": 25.0, + } + ] + @pytest.mark.respx(base_url=base_url) def test_deploy_reuses_endpoint_when_name_already_exists( self, respx_mock: MockRouter, cli_runner: CliRunner diff --git a/tests/cli/test_beta_endpoints_update.py b/tests/cli/test_beta_endpoints_update.py index c7080f481..eaa858c33 100644 --- a/tests/cli/test_beta_endpoints_update.py +++ b/tests/cli/test_beta_endpoints_update.py @@ -169,6 +169,34 @@ def test_update_by_deployment_id(self, respx_mock: MockRouter, cli_runner: CliRu "autoscaling": {"minReplicas": 1, "maxReplicas": 2}, } + @pytest.mark.respx(base_url=base_url) + def test_update_accepts_active_sessions_scaling_metric( + self, respx_mock: MockRouter, cli_runner: CliRunner + ) -> None: + _mock_endpoint_list(respx_mock) + route = respx_mock.patch("/projects/proj/endpoints/ep_1/deployments/dep_control").mock( + return_value=httpx.Response(200, json=_deployment_body()) + ) + + result = cli_runner.invoke( + _update_args("dep_control", "--scaling-metric", "active_sessions", "--scaling-target", "25") + ) + + assert result.exit_code == 0, result.output + req = cast(Call, route.calls[0]).request + assert "updateMask=autoscaling" in str(req.url) + assert json.loads(req.content.decode()) == { + "autoscaling": { + "scalingMetrics": [ + { + "name": "active_sessions", + "type": "METRIC_TARGET_TYPE_VALUE", + "target": 25.0, + } + ] + }, + } + @pytest.mark.respx(base_url=base_url) def test_update_idle_deployment(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: _mock_endpoint_list(respx_mock) diff --git a/tests/cli/test_build_autoscaling.py b/tests/cli/test_build_autoscaling.py index 98b5f8c8a..18e309695 100644 --- a/tests/cli/test_build_autoscaling.py +++ b/tests/cli/test_build_autoscaling.py @@ -18,6 +18,16 @@ def test_build_scaling_metrics_utilization() -> None: ] +def test_build_scaling_metrics_active_sessions() -> None: + assert build_scaling_metrics(scaling_metric="active_sessions", scaling_target=25) == [ + { + "name": "active_sessions", + "type": "METRIC_TARGET_TYPE_VALUE", + "target": 25, + } + ] + + def test_build_scaling_metrics_latency_with_percentile() -> None: assert build_scaling_metrics( scaling_metric="ttft", From 8c799ec1a3309f9e82e6e20016c645676bf36690 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 10 Sep 2026 04:07:58 +0000 Subject: [PATCH 2/2] fix(cli): format active sessions autoscaling tests Co-authored-by: Blaine Kasten --- tests/cli/test_beta_endpoints.py | 4 +--- tests/cli/test_beta_endpoints_update.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/cli/test_beta_endpoints.py b/tests/cli/test_beta_endpoints.py index 0445699da..ea7f8d113 100644 --- a/tests/cli/test_beta_endpoints.py +++ b/tests/cli/test_beta_endpoints.py @@ -378,9 +378,7 @@ def test_deploy_onto_existing_endpoint(self, respx_mock: MockRouter, cli_runner: assert create_deployment_route.call_count == 1 @pytest.mark.respx(base_url=base_url) - def test_deploy_accepts_active_sessions_scaling_metric( - self, respx_mock: MockRouter, cli_runner: CliRunner - ) -> None: + def test_deploy_accepts_active_sessions_scaling_metric(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: _mock_model_and_config(respx_mock) respx_mock.get("/projects/proj/endpoints/ep_1").mock(return_value=httpx.Response(200, json=_endpoint_body())) create_deployment_route = respx_mock.post("/projects/proj/endpoints/ep_1/deployments").mock( diff --git a/tests/cli/test_beta_endpoints_update.py b/tests/cli/test_beta_endpoints_update.py index eaa858c33..8c8a3f2cc 100644 --- a/tests/cli/test_beta_endpoints_update.py +++ b/tests/cli/test_beta_endpoints_update.py @@ -170,9 +170,7 @@ def test_update_by_deployment_id(self, respx_mock: MockRouter, cli_runner: CliRu } @pytest.mark.respx(base_url=base_url) - def test_update_accepts_active_sessions_scaling_metric( - self, respx_mock: MockRouter, cli_runner: CliRunner - ) -> None: + def test_update_accepts_active_sessions_scaling_metric(self, respx_mock: MockRouter, cli_runner: CliRunner) -> None: _mock_endpoint_list(respx_mock) route = respx_mock.patch("/projects/proj/endpoints/ep_1/deployments/dep_control").mock( return_value=httpx.Response(200, json=_deployment_body())