Skip to content

Commit d24b639

Browse files
stainless-app[bot]stainless-app[bot]
committed
feat(api): api update (#575)
1 parent 70ed3f7 commit d24b639

4 files changed

Lines changed: 147 additions & 45 deletions

File tree

‎.stats.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
configured_endpoints: 78
2-
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-1c6045460c43f65b30ffcef9f707e8d71dca568bf8d208347a6046a6f03ff239.yml
2+
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/runloop-ai%2Frunloop-4ac5f2286270f1f2c2b41e7f57fcc0527f602ce8d9383c790ed40ce21236e0cf.yml

‎src/runloop_api_client/types/scoring_function.py‎

Lines changed: 104 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,118 @@
11
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
22

3+
from typing import List, Union, Optional
4+
from typing_extensions import Literal, Annotated, TypeAlias
35

6+
from .._utils import PropertyInfo
47
from .._models import BaseModel
58

6-
__all__ = ["ScoringFunction"]
9+
__all__ = [
10+
"ScoringFunction",
11+
"Scorer",
12+
"ScorerAstGrepScoringFunction",
13+
"ScorerBashScriptScoringFunction",
14+
"ScorerCommandScoringFunction",
15+
"ScorerCustomScoringFunction",
16+
"ScorerPythonScriptScoringFunction",
17+
"ScorerTestBasedScoringFunction",
18+
"ScorerTestBasedScoringFunctionTestFile",
19+
]
20+
21+
22+
class ScorerAstGrepScoringFunction(BaseModel):
23+
pattern: str
24+
"""AST pattern to match."""
25+
26+
search_directory: str
27+
"""The path to search."""
28+
29+
type: Literal["ast_grep_scorer"]
30+
31+
lang: Optional[str] = None
32+
"""The language of the pattern."""
33+
34+
35+
class ScorerBashScriptScoringFunction(BaseModel):
36+
type: Literal["bash_script_scorer"]
37+
38+
bash_script: Optional[str] = None
39+
"""
40+
A single bash script that sets up the environment, scores, and prints the final
41+
score to standard out. Score should be a float between 0.0 and 1.0, and look
42+
like "score=[0.0..1.0].
43+
"""
44+
45+
46+
class ScorerCommandScoringFunction(BaseModel):
47+
type: Literal["command_scorer"]
48+
49+
command: Optional[str] = None
50+
"""The command to execute."""
51+
52+
53+
class ScorerCustomScoringFunction(BaseModel):
54+
type: Literal["custom_scorer"]
55+
56+
scorer_params: Optional[object] = None
57+
"""Additional JSON structured context to pass to the scoring function."""
58+
59+
60+
class ScorerPythonScriptScoringFunction(BaseModel):
61+
python_script: str
62+
"""Python script to be run.
63+
64+
The script should output the score to standard out as a float between 0.0 and
65+
1.0.
66+
"""
67+
68+
type: Literal["python_script_scorer"]
69+
70+
requirements_contents: Optional[str] = None
71+
"""Package dependencies to be installed.
72+
73+
The requirements should be a valid requirements.txt file.
74+
"""
75+
76+
77+
class ScorerTestBasedScoringFunctionTestFile(BaseModel):
78+
file_contents: Optional[str] = None
79+
"""Content of the test file"""
80+
81+
file_path: Optional[str] = None
82+
"""
83+
Path to write content of the test file, relative to your environment's working
84+
directory
85+
"""
86+
87+
88+
class ScorerTestBasedScoringFunction(BaseModel):
89+
type: Literal["test_based_scorer"]
90+
91+
test_command: Optional[str] = None
92+
"""The command to execute for running the tests"""
93+
94+
test_files: Optional[List[ScorerTestBasedScoringFunctionTestFile]] = None
95+
"""List of test files to create"""
96+
97+
98+
Scorer: TypeAlias = Annotated[
99+
Union[
100+
ScorerAstGrepScoringFunction,
101+
ScorerBashScriptScoringFunction,
102+
ScorerCommandScoringFunction,
103+
ScorerCustomScoringFunction,
104+
ScorerPythonScriptScoringFunction,
105+
ScorerTestBasedScoringFunction,
106+
],
107+
PropertyInfo(discriminator="type"),
108+
]
7109

8110

9111
class ScoringFunction(BaseModel):
10112
name: str
11113
"""Name of scoring function. Names must only contain [a-zA-Z0-9_-]."""
12114

13-
scoring_function: ScoringFunction
115+
scorer: Scorer
14116
"""The scoring function to use for evaluating this scenario.
15117
16118
The type field determines which built-in function to use.

‎src/runloop_api_client/types/scoring_function_param.py‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,18 @@
77

88
__all__ = [
99
"ScoringFunctionParam",
10-
"ScoringFunction",
11-
"ScoringFunctionAstGrepScoringFunction",
12-
"ScoringFunctionBashScriptScoringFunction",
13-
"ScoringFunctionCommandScoringFunction",
14-
"ScoringFunctionCustomScoringFunction",
15-
"ScoringFunctionPythonScriptScoringFunction",
16-
"ScoringFunctionTestBasedScoringFunction",
17-
"ScoringFunctionTestBasedScoringFunctionTestFile",
10+
"Scorer",
11+
"ScorerAstGrepScoringFunction",
12+
"ScorerBashScriptScoringFunction",
13+
"ScorerCommandScoringFunction",
14+
"ScorerCustomScoringFunction",
15+
"ScorerPythonScriptScoringFunction",
16+
"ScorerTestBasedScoringFunction",
17+
"ScorerTestBasedScoringFunctionTestFile",
1818
]
1919

2020

21-
class ScoringFunctionAstGrepScoringFunction(TypedDict, total=False):
21+
class ScorerAstGrepScoringFunction(TypedDict, total=False):
2222
pattern: Required[str]
2323
"""AST pattern to match."""
2424

@@ -31,7 +31,7 @@ class ScoringFunctionAstGrepScoringFunction(TypedDict, total=False):
3131
"""The language of the pattern."""
3232

3333

34-
class ScoringFunctionBashScriptScoringFunction(TypedDict, total=False):
34+
class ScorerBashScriptScoringFunction(TypedDict, total=False):
3535
type: Required[Literal["bash_script_scorer"]]
3636

3737
bash_script: str
@@ -42,21 +42,21 @@ class ScoringFunctionBashScriptScoringFunction(TypedDict, total=False):
4242
"""
4343

4444

45-
class ScoringFunctionCommandScoringFunction(TypedDict, total=False):
45+
class ScorerCommandScoringFunction(TypedDict, total=False):
4646
type: Required[Literal["command_scorer"]]
4747

4848
command: str
4949
"""The command to execute."""
5050

5151

52-
class ScoringFunctionCustomScoringFunction(TypedDict, total=False):
52+
class ScorerCustomScoringFunction(TypedDict, total=False):
5353
type: Required[Literal["custom_scorer"]]
5454

5555
scorer_params: Optional[object]
5656
"""Additional JSON structured context to pass to the scoring function."""
5757

5858

59-
class ScoringFunctionPythonScriptScoringFunction(TypedDict, total=False):
59+
class ScorerPythonScriptScoringFunction(TypedDict, total=False):
6060
python_script: Required[str]
6161
"""Python script to be run.
6262
@@ -73,7 +73,7 @@ class ScoringFunctionPythonScriptScoringFunction(TypedDict, total=False):
7373
"""
7474

7575

76-
class ScoringFunctionTestBasedScoringFunctionTestFile(TypedDict, total=False):
76+
class ScorerTestBasedScoringFunctionTestFile(TypedDict, total=False):
7777
file_contents: str
7878
"""Content of the test file"""
7979

@@ -84,31 +84,31 @@ class ScoringFunctionTestBasedScoringFunctionTestFile(TypedDict, total=False):
8484
"""
8585

8686

87-
class ScoringFunctionTestBasedScoringFunction(TypedDict, total=False):
87+
class ScorerTestBasedScoringFunction(TypedDict, total=False):
8888
type: Required[Literal["test_based_scorer"]]
8989

9090
test_command: str
9191
"""The command to execute for running the tests"""
9292

93-
test_files: Iterable[ScoringFunctionTestBasedScoringFunctionTestFile]
93+
test_files: Iterable[ScorerTestBasedScoringFunctionTestFile]
9494
"""List of test files to create"""
9595

9696

97-
ScoringFunction: TypeAlias = Union[
98-
ScoringFunctionAstGrepScoringFunction,
99-
ScoringFunctionBashScriptScoringFunction,
100-
ScoringFunctionCommandScoringFunction,
101-
ScoringFunctionCustomScoringFunction,
102-
ScoringFunctionPythonScriptScoringFunction,
103-
ScoringFunctionTestBasedScoringFunction,
97+
Scorer: TypeAlias = Union[
98+
ScorerAstGrepScoringFunction,
99+
ScorerBashScriptScoringFunction,
100+
ScorerCommandScoringFunction,
101+
ScorerCustomScoringFunction,
102+
ScorerPythonScriptScoringFunction,
103+
ScorerTestBasedScoringFunction,
104104
]
105105

106106

107107
class ScoringFunctionParam(TypedDict, total=False):
108108
name: Required[str]
109109
"""Name of scoring function. Names must only contain [a-zA-Z0-9_-]."""
110110

111-
scoring_function: Required[ScoringFunction]
111+
scorer: Required[Scorer]
112112
"""The scoring function to use for evaluating this scenario.
113113
114114
The type field determines which built-in function to use.

‎tests/api_resources/test_scenarios.py‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def test_method_create(self, client: Runloop) -> None:
3030
"scoring_function_parameters": [
3131
{
3232
"name": "name",
33-
"scoring_function": {
33+
"scorer": {
3434
"pattern": "pattern",
3535
"search_directory": "search_directory",
3636
"type": "ast_grep_scorer",
@@ -54,7 +54,7 @@ def test_method_create_with_all_params(self, client: Runloop) -> None:
5454
"scoring_function_parameters": [
5555
{
5656
"name": "name",
57-
"scoring_function": {
57+
"scorer": {
5858
"pattern": "pattern",
5959
"search_directory": "search_directory",
6060
"type": "ast_grep_scorer",
@@ -96,7 +96,7 @@ def test_raw_response_create(self, client: Runloop) -> None:
9696
"scoring_function_parameters": [
9797
{
9898
"name": "name",
99-
"scoring_function": {
99+
"scorer": {
100100
"pattern": "pattern",
101101
"search_directory": "search_directory",
102102
"type": "ast_grep_scorer",
@@ -121,7 +121,7 @@ def test_streaming_response_create(self, client: Runloop) -> None:
121121
"scoring_function_parameters": [
122122
{
123123
"name": "name",
124-
"scoring_function": {
124+
"scorer": {
125125
"pattern": "pattern",
126126
"search_directory": "search_directory",
127127
"type": "ast_grep_scorer",
@@ -187,7 +187,7 @@ def test_method_update(self, client: Runloop) -> None:
187187
"scoring_function_parameters": [
188188
{
189189
"name": "name",
190-
"scoring_function": {
190+
"scorer": {
191191
"pattern": "pattern",
192192
"search_directory": "search_directory",
193193
"type": "ast_grep_scorer",
@@ -212,7 +212,7 @@ def test_method_update_with_all_params(self, client: Runloop) -> None:
212212
"scoring_function_parameters": [
213213
{
214214
"name": "name",
215-
"scoring_function": {
215+
"scorer": {
216216
"pattern": "pattern",
217217
"search_directory": "search_directory",
218218
"type": "ast_grep_scorer",
@@ -255,7 +255,7 @@ def test_raw_response_update(self, client: Runloop) -> None:
255255
"scoring_function_parameters": [
256256
{
257257
"name": "name",
258-
"scoring_function": {
258+
"scorer": {
259259
"pattern": "pattern",
260260
"search_directory": "search_directory",
261261
"type": "ast_grep_scorer",
@@ -281,7 +281,7 @@ def test_streaming_response_update(self, client: Runloop) -> None:
281281
"scoring_function_parameters": [
282282
{
283283
"name": "name",
284-
"scoring_function": {
284+
"scorer": {
285285
"pattern": "pattern",
286286
"search_directory": "search_directory",
287287
"type": "ast_grep_scorer",
@@ -310,7 +310,7 @@ def test_path_params_update(self, client: Runloop) -> None:
310310
"scoring_function_parameters": [
311311
{
312312
"name": "name",
313-
"scoring_function": {
313+
"scorer": {
314314
"pattern": "pattern",
315315
"search_directory": "search_directory",
316316
"type": "ast_grep_scorer",
@@ -443,7 +443,7 @@ async def test_method_create(self, async_client: AsyncRunloop) -> None:
443443
"scoring_function_parameters": [
444444
{
445445
"name": "name",
446-
"scoring_function": {
446+
"scorer": {
447447
"pattern": "pattern",
448448
"search_directory": "search_directory",
449449
"type": "ast_grep_scorer",
@@ -467,7 +467,7 @@ async def test_method_create_with_all_params(self, async_client: AsyncRunloop) -
467467
"scoring_function_parameters": [
468468
{
469469
"name": "name",
470-
"scoring_function": {
470+
"scorer": {
471471
"pattern": "pattern",
472472
"search_directory": "search_directory",
473473
"type": "ast_grep_scorer",
@@ -509,7 +509,7 @@ async def test_raw_response_create(self, async_client: AsyncRunloop) -> None:
509509
"scoring_function_parameters": [
510510
{
511511
"name": "name",
512-
"scoring_function": {
512+
"scorer": {
513513
"pattern": "pattern",
514514
"search_directory": "search_directory",
515515
"type": "ast_grep_scorer",
@@ -534,7 +534,7 @@ async def test_streaming_response_create(self, async_client: AsyncRunloop) -> No
534534
"scoring_function_parameters": [
535535
{
536536
"name": "name",
537-
"scoring_function": {
537+
"scorer": {
538538
"pattern": "pattern",
539539
"search_directory": "search_directory",
540540
"type": "ast_grep_scorer",
@@ -600,7 +600,7 @@ async def test_method_update(self, async_client: AsyncRunloop) -> None:
600600
"scoring_function_parameters": [
601601
{
602602
"name": "name",
603-
"scoring_function": {
603+
"scorer": {
604604
"pattern": "pattern",
605605
"search_directory": "search_directory",
606606
"type": "ast_grep_scorer",
@@ -625,7 +625,7 @@ async def test_method_update_with_all_params(self, async_client: AsyncRunloop) -
625625
"scoring_function_parameters": [
626626
{
627627
"name": "name",
628-
"scoring_function": {
628+
"scorer": {
629629
"pattern": "pattern",
630630
"search_directory": "search_directory",
631631
"type": "ast_grep_scorer",
@@ -668,7 +668,7 @@ async def test_raw_response_update(self, async_client: AsyncRunloop) -> None:
668668
"scoring_function_parameters": [
669669
{
670670
"name": "name",
671-
"scoring_function": {
671+
"scorer": {
672672
"pattern": "pattern",
673673
"search_directory": "search_directory",
674674
"type": "ast_grep_scorer",
@@ -694,7 +694,7 @@ async def test_streaming_response_update(self, async_client: AsyncRunloop) -> No
694694
"scoring_function_parameters": [
695695
{
696696
"name": "name",
697-
"scoring_function": {
697+
"scorer": {
698698
"pattern": "pattern",
699699
"search_directory": "search_directory",
700700
"type": "ast_grep_scorer",
@@ -723,7 +723,7 @@ async def test_path_params_update(self, async_client: AsyncRunloop) -> None:
723723
"scoring_function_parameters": [
724724
{
725725
"name": "name",
726-
"scoring_function": {
726+
"scorer": {
727727
"pattern": "pattern",
728728
"search_directory": "search_directory",
729729
"type": "ast_grep_scorer",

0 commit comments

Comments
 (0)