Skip to content

Commit 7ca3154

Browse files
sid-rlsid-rl
committed
remove scenario builder smoketests
1 parent eee6b78 commit 7ca3154

2 files changed

Lines changed: 0 additions & 294 deletions

File tree

‎tests/smoketests/sdk/test_async_scenario.py‎

Lines changed: 0 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -2,30 +2,16 @@
22

33
from __future__ import annotations
44

5-
import datetime
6-
from typing import Optional
7-
85
import pytest
96

107
from runloop_api_client.sdk import AsyncRunloopSDK
11-
from runloop_api_client.sdk.async_scenario import AsyncScenario
128

139
pytestmark = [pytest.mark.smoketest]
1410

1511
TWO_MINUTE_TIMEOUT = 120
1612
FIVE_MINUTE_TIMEOUT = 300
1713

1814

19-
async def find_scenario_by_name_async(sdk_client: AsyncRunloopSDK, name: str) -> Optional[AsyncScenario]:
20-
"""Find an existing scenario by name, returns None if not found."""
21-
scenarios = await sdk_client.scenario.list(limit=100)
22-
for scenario in scenarios:
23-
info = await scenario.get_info()
24-
if info.name == name:
25-
return scenario
26-
return None
27-
28-
2915
class TestAsyncScenarioRetrieval:
3016
"""Test async scenario retrieval operations."""
3117

@@ -138,136 +124,3 @@ async def test_scenario_run_and_await_env_ready(self, async_sdk_client: AsyncRun
138124
await run.cancel()
139125
except Exception:
140126
pass
141-
142-
143-
class TestAsyncScenarioBuilder:
144-
"""Test async scenario builder operations.
145-
146-
These tests use fixed scenario names and reuse existing scenarios when possible
147-
to avoid accumulating test scenarios (since scenario deletion is not yet available).
148-
"""
149-
150-
# Metadata for identifying test scenarios that can be cleaned up later
151-
CLEANUP_METADATA = {"smoketest": "true", "sdk_smoketest": "true", "cleanup_safe": "true"}
152-
153-
# Fixed scenario names for reuse across test runs
154-
BASIC_SCENARIO_NAME = "sdk-smoketest-async-builder-basic"
155-
ENV_SCENARIO_NAME = "sdk-smoketest-async-builder-env"
156-
FLUENT_SCENARIO_NAME = "sdk-smoketest-async-builder-fluent"
157-
158-
@pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
159-
async def test_builder_create_scenario(self, async_sdk_client: AsyncRunloopSDK) -> None:
160-
"""Test creating a scenario with the async builder.
161-
162-
Uses a fixed name and reuses existing scenario if found.
163-
"""
164-
# Check if scenario already exists
165-
existing = await find_scenario_by_name_async(async_sdk_client, self.BASIC_SCENARIO_NAME)
166-
if existing:
167-
# Scenario exists, update metadata to exercise update path
168-
updated_metadata = {
169-
**self.CLEANUP_METADATA,
170-
"last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
171-
}
172-
await existing.update(metadata=updated_metadata)
173-
info = await existing.get_info()
174-
assert info.name == self.BASIC_SCENARIO_NAME
175-
assert info.input_context is not None
176-
return
177-
178-
# Create new scenario
179-
builder = async_sdk_client.scenario.builder(self.BASIC_SCENARIO_NAME)
180-
builder.with_problem_statement("This is a test scenario created by the async SDK builder smoke test.")
181-
builder.add_bash_scorer(
182-
"test-scorer",
183-
bash_script="echo 'score=1.0'",
184-
weight=1.0,
185-
)
186-
builder.with_metadata(self.CLEANUP_METADATA)
187-
188-
scenario = await builder.push()
189-
190-
assert scenario.id is not None
191-
info = await scenario.get_info()
192-
assert info.name == self.BASIC_SCENARIO_NAME
193-
assert info.input_context is not None
194-
195-
@pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
196-
async def test_builder_with_environment(self, async_sdk_client: AsyncRunloopSDK) -> None:
197-
"""Test creating a scenario with environment configuration.
198-
199-
Uses a fixed name and reuses existing scenario if found.
200-
"""
201-
# Check if scenario already exists
202-
existing = await find_scenario_by_name_async(async_sdk_client, self.ENV_SCENARIO_NAME)
203-
if existing:
204-
# Update metadata to exercise update path
205-
updated_metadata = {
206-
**self.CLEANUP_METADATA,
207-
"last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
208-
}
209-
await existing.update(metadata=updated_metadata)
210-
info = await existing.get_info()
211-
assert info.name == self.ENV_SCENARIO_NAME
212-
return
213-
214-
# Get a blueprint to use
215-
blueprints = await async_sdk_client.blueprint.list(limit=1)
216-
if not blueprints:
217-
pytest.skip("No blueprints available for environment test")
218-
219-
blueprint_id = blueprints[0].id
220-
221-
builder = (
222-
async_sdk_client.scenario.builder(self.ENV_SCENARIO_NAME)
223-
.from_blueprint_id(blueprint_id)
224-
.with_working_directory("/workspace")
225-
.with_problem_statement("Test scenario with environment configuration.")
226-
.add_command_scorer("check", command="echo 'score=1.0'", weight=1.0)
227-
.with_metadata(self.CLEANUP_METADATA)
228-
)
229-
230-
scenario = await builder.push()
231-
232-
info = await scenario.get_info()
233-
assert info.name == self.ENV_SCENARIO_NAME
234-
if info.environment:
235-
assert info.environment.blueprint_id == blueprint_id
236-
237-
@pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
238-
async def test_builder_fluent_api(self, async_sdk_client: AsyncRunloopSDK) -> None:
239-
"""Test async builder fluent API chaining.
240-
241-
Uses a fixed name and reuses existing scenario if found.
242-
"""
243-
# Check if scenario already exists
244-
existing = await find_scenario_by_name_async(async_sdk_client, self.FLUENT_SCENARIO_NAME)
245-
if existing:
246-
# Update metadata to exercise update path
247-
updated_metadata = {
248-
**self.CLEANUP_METADATA,
249-
"last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
250-
"test_type": "fluent_api",
251-
}
252-
await existing.update(metadata=updated_metadata)
253-
info = await existing.get_info()
254-
assert info.name == self.FLUENT_SCENARIO_NAME
255-
if info.scoring_contract and info.scoring_contract.scoring_function_parameters:
256-
assert len(info.scoring_contract.scoring_function_parameters) == 2
257-
return
258-
259-
# Create new scenario with fluent API
260-
scenario = await (
261-
async_sdk_client.scenario.builder(self.FLUENT_SCENARIO_NAME)
262-
.with_problem_statement("Async fluent API test scenario.")
263-
.with_additional_context({"hint": "This is a hint"})
264-
.add_bash_scorer("scorer1", bash_script="echo 'score=0.5'", weight=1.0)
265-
.add_command_scorer("scorer2", command="echo 'score=1.0'", weight=2.0)
266-
.with_metadata({**self.CLEANUP_METADATA, "test_type": "fluent_api"})
267-
.push()
268-
)
269-
270-
info = await scenario.get_info()
271-
assert info.name == self.FLUENT_SCENARIO_NAME
272-
if info.scoring_contract and info.scoring_contract.scoring_function_parameters:
273-
assert len(info.scoring_contract.scoring_function_parameters) == 2

‎tests/smoketests/sdk/test_scenario.py‎

Lines changed: 0 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -2,30 +2,16 @@
22

33
from __future__ import annotations
44

5-
import datetime
6-
from typing import Optional
7-
85
import pytest
96

107
from runloop_api_client.sdk import RunloopSDK
11-
from runloop_api_client.sdk.scenario import Scenario
128

139
pytestmark = [pytest.mark.smoketest]
1410

1511
TWO_MINUTE_TIMEOUT = 120
1612
FIVE_MINUTE_TIMEOUT = 300
1713

1814

19-
def find_scenario_by_name(sdk_client: RunloopSDK, name: str) -> Optional[Scenario]:
20-
"""Find an existing scenario by name, returns None if not found."""
21-
scenarios = sdk_client.scenario.list(limit=100)
22-
for scenario in scenarios:
23-
info = scenario.get_info()
24-
if info.name == name:
25-
return scenario
26-
return None
27-
28-
2915
class TestScenarioRetrieval:
3016
"""Test scenario retrieval operations."""
3117

@@ -138,136 +124,3 @@ def test_scenario_run_and_await_env_ready(self, sdk_client: RunloopSDK) -> None:
138124
run.cancel()
139125
except Exception:
140126
pass
141-
142-
143-
class TestScenarioBuilder:
144-
"""Test scenario builder operations.
145-
146-
These tests use fixed scenario names and reuse existing scenarios when possible
147-
to avoid accumulating test scenarios (since scenario deletion is not yet available).
148-
"""
149-
150-
# Metadata for identifying test scenarios that can be cleaned up later
151-
CLEANUP_METADATA = {"smoketest": "true", "sdk_smoketest": "true", "cleanup_safe": "true"}
152-
153-
# Fixed scenario names for reuse across test runs
154-
BASIC_SCENARIO_NAME = "sdk-smoketest-builder-basic"
155-
ENV_SCENARIO_NAME = "sdk-smoketest-builder-env"
156-
FLUENT_SCENARIO_NAME = "sdk-smoketest-builder-fluent"
157-
158-
@pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
159-
def test_builder_create_scenario(self, sdk_client: RunloopSDK) -> None:
160-
"""Test creating a scenario with the builder.
161-
162-
Uses a fixed name and reuses existing scenario if found.
163-
"""
164-
# Check if scenario already exists
165-
existing = find_scenario_by_name(sdk_client, self.BASIC_SCENARIO_NAME)
166-
if existing:
167-
# Scenario exists, update metadata to exercise update path
168-
updated_metadata = {
169-
**self.CLEANUP_METADATA,
170-
"last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
171-
}
172-
existing.update(metadata=updated_metadata)
173-
info = existing.get_info()
174-
assert info.name == self.BASIC_SCENARIO_NAME
175-
assert info.input_context is not None
176-
return
177-
178-
# Create new scenario
179-
builder = sdk_client.scenario.builder(self.BASIC_SCENARIO_NAME)
180-
builder.with_problem_statement("This is a test scenario created by the SDK builder smoke test.")
181-
builder.add_bash_scorer(
182-
"test-scorer",
183-
bash_script="echo 'score=1.0'",
184-
weight=1.0,
185-
)
186-
builder.with_metadata(self.CLEANUP_METADATA)
187-
188-
scenario = builder.push()
189-
190-
assert scenario.id is not None
191-
info = scenario.get_info()
192-
assert info.name == self.BASIC_SCENARIO_NAME
193-
assert info.input_context is not None
194-
195-
@pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
196-
def test_builder_with_environment(self, sdk_client: RunloopSDK) -> None:
197-
"""Test creating a scenario with environment configuration.
198-
199-
Uses a fixed name and reuses existing scenario if found.
200-
"""
201-
# Check if scenario already exists
202-
existing = find_scenario_by_name(sdk_client, self.ENV_SCENARIO_NAME)
203-
if existing:
204-
# Update metadata to exercise update path
205-
updated_metadata = {
206-
**self.CLEANUP_METADATA,
207-
"last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
208-
}
209-
existing.update(metadata=updated_metadata)
210-
info = existing.get_info()
211-
assert info.name == self.ENV_SCENARIO_NAME
212-
return
213-
214-
# Get a blueprint to use
215-
blueprints = sdk_client.blueprint.list(limit=1)
216-
if not blueprints:
217-
pytest.skip("No blueprints available for environment test")
218-
219-
blueprint_id = blueprints[0].id
220-
221-
builder = (
222-
sdk_client.scenario.builder(self.ENV_SCENARIO_NAME)
223-
.from_blueprint_id(blueprint_id)
224-
.with_working_directory("/workspace")
225-
.with_problem_statement("Test scenario with environment configuration.")
226-
.add_command_scorer("check", command="echo 'score=1.0'", weight=1.0)
227-
.with_metadata(self.CLEANUP_METADATA)
228-
)
229-
230-
scenario = builder.push()
231-
232-
info = scenario.get_info()
233-
assert info.name == self.ENV_SCENARIO_NAME
234-
if info.environment:
235-
assert info.environment.blueprint_id == blueprint_id
236-
237-
@pytest.mark.timeout(FIVE_MINUTE_TIMEOUT)
238-
def test_builder_fluent_api(self, sdk_client: RunloopSDK) -> None:
239-
"""Test builder fluent API chaining.
240-
241-
Uses a fixed name and reuses existing scenario if found.
242-
"""
243-
# Check if scenario already exists
244-
existing = find_scenario_by_name(sdk_client, self.FLUENT_SCENARIO_NAME)
245-
if existing:
246-
# Update metadata to exercise update path
247-
updated_metadata = {
248-
**self.CLEANUP_METADATA,
249-
"last_smoketest_run": datetime.datetime.now(datetime.timezone.utc).isoformat(),
250-
"test_type": "fluent_api",
251-
}
252-
existing.update(metadata=updated_metadata)
253-
info = existing.get_info()
254-
assert info.name == self.FLUENT_SCENARIO_NAME
255-
if info.scoring_contract and info.scoring_contract.scoring_function_parameters:
256-
assert len(info.scoring_contract.scoring_function_parameters) == 2
257-
return
258-
259-
# Create new scenario with fluent API
260-
scenario = (
261-
sdk_client.scenario.builder(self.FLUENT_SCENARIO_NAME)
262-
.with_problem_statement("Fluent API test scenario.")
263-
.with_additional_context({"hint": "This is a hint"})
264-
.add_bash_scorer("scorer1", bash_script="echo 'score=0.5'", weight=1.0)
265-
.add_command_scorer("scorer2", command="echo 'score=1.0'", weight=2.0)
266-
.with_metadata({**self.CLEANUP_METADATA, "test_type": "fluent_api"})
267-
.push()
268-
)
269-
270-
info = scenario.get_info()
271-
assert info.name == self.FLUENT_SCENARIO_NAME
272-
if info.scoring_contract and info.scoring_contract.scoring_function_parameters:
273-
assert len(info.scoring_contract.scoring_function_parameters) == 2

0 commit comments

Comments
 (0)