Skip to content

Commit 9d689d4

Browse files
Bill LeoutsakosBill Leoutsakos
authored andcommitted
fix(billing): correct sandbox trace cost boundaries
1 parent 5a65b56 commit 9d689d4

6 files changed

Lines changed: 68 additions & 22 deletions

File tree

‎apps/sim/lib/execution/remote-sandbox/conformance.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2504,10 +2504,12 @@ describe('Pi sandbox lifetime', () => {
25042504
code: 'x',
25052505
language: CodeLanguage.Python,
25062506
timeoutMs: 7 * 24 * 60 * 60 * 1000,
2507+
meterUsage: true,
25072508
})
25082509

25092510
expect(result.error).toContain('E2B reached its 24-hour limit')
25102511
expect(result.error).toContain('workflow timeout may be longer')
2512+
expect(result.cost).toBeUndefined()
25112513
expect(mockRecordSandboxProviderLimit).toHaveBeenCalledWith({
25122514
provider: 'e2b',
25132515
operation: 'code',
@@ -2531,9 +2533,11 @@ describe('Pi sandbox lifetime', () => {
25312533
const result = await executeShellInSandbox({
25322534
code: 'sleep infinity',
25332535
timeoutMs: 7 * 24 * 60 * 60 * 1000,
2536+
meterUsage: true,
25342537
})
25352538

25362539
expect(result.error).toContain('E2B reached its 24-hour limit')
2540+
expect(result.cost).toBeUndefined()
25372541
expect(mockRecordSandboxProviderLimit).toHaveBeenCalledWith({
25382542
provider: 'e2b',
25392543
operation: 'command',

‎apps/sim/lib/execution/remote-sandbox/e2b.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,7 @@ class E2BSandboxHandle implements SandboxHandle {
372372
return { text: '', stdout: result.stdout, stderr: result.stderr, timedOut: true }
373373
}
374374
if (result.exitCode !== 0) {
375-
if (result.stderr === E2B_PROVIDER_LIMIT_ERROR) {
375+
if (result.providerFailure === 'provider_limit') {
376376
return {
377377
text: '',
378378
stdout: result.stdout,
@@ -382,6 +382,7 @@ class E2BSandboxHandle implements SandboxHandle {
382382
value: E2B_PROVIDER_LIMIT_ERROR,
383383
traceback: E2B_PROVIDER_LIMIT_ERROR,
384384
},
385+
providerFailure: result.providerFailure,
385386
}
386387
}
387388
return processCodeFailure(result)
@@ -541,7 +542,12 @@ class E2BSandboxHandle implements SandboxHandle {
541542
if (isNonRetryableExecutionError(error)) throw error
542543
if (reachedE2BProviderLimit(error, this.providerLimitAtMs, options.signal)) {
543544
recordSandboxProviderLimit({ provider: 'e2b', operation })
544-
return { stdout: '', stderr: E2B_PROVIDER_LIMIT_ERROR, exitCode: 1 }
545+
return {
546+
stdout: '',
547+
stderr: E2B_PROVIDER_LIMIT_ERROR,
548+
exitCode: 1,
549+
providerFailure: 'provider_limit',
550+
}
545551
}
546552
// The SDK throws on non-zero exit; callers want the streams, not a throw.
547553
const failure = error as {

‎apps/sim/lib/execution/remote-sandbox/index.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -864,13 +864,14 @@ async function executeInSandboxWithinBudget(
864864
sandboxId,
865865
hasTraceback: Boolean(execution.error.traceback),
866866
})
867-
billableResult = {
867+
const executionResult = {
868868
result: null,
869869
stdout: execution.error.traceback || errorMessage,
870870
error: errorMessage,
871871
sandboxId,
872872
}
873-
return billableResult
873+
if (execution.providerFailure !== 'provider_limit') billableResult = executionResult
874+
return executionResult
874875
}
875876

876877
// Distinct sources (final-expression text, stdout, stderr) join with '\n' so
@@ -1008,8 +1009,9 @@ async function executeShellInSandboxWithinBudget(
10081009
sandboxId,
10091010
exitCode: result.exitCode,
10101011
})
1011-
billableResult = { result: null, stdout, error: errorMessage, sandboxId }
1012-
return billableResult
1012+
const executionResult = { result: null, stdout, error: errorMessage, sandboxId }
1013+
if (result.providerFailure !== 'provider_limit') billableResult = executionResult
1014+
return executionResult
10131015
}
10141016

10151017
// Shell scripts have no wrapper: any __SIM_RESULT__ line is user-authored

‎apps/sim/lib/execution/remote-sandbox/types.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ export interface SandboxExecutionRequest {
7272
sandboxId?: string
7373
/** Cancels the provider sandbox when the caller's execution budget expires. */
7474
signal?: AbortSignal
75-
/** Adds the remote provider cost to a successful Function result. */
75+
/** Adds the remote provider cost to a completed, billable Function outcome. */
7676
meterUsage?: boolean
7777
}
7878

@@ -99,7 +99,7 @@ export interface SandboxShellExecutionRequest {
9999
sandboxId?: string
100100
/** Cancels the provider sandbox when the caller's execution budget expires. */
101101
signal?: AbortSignal
102-
/** Adds the remote provider cost to a successful Function result. */
102+
/** Adds the remote provider cost to a completed, billable Function outcome. */
103103
meterUsage?: boolean
104104
}
105105

@@ -138,6 +138,8 @@ export interface SandboxCommandResult {
138138
exitCode: number
139139
/** The provider stopped the command because its supplied execution budget elapsed. */
140140
timedOut?: boolean
141+
/** The provider ended execution for an infrastructure reason, not a user-process outcome. */
142+
providerFailure?: 'provider_limit'
141143
}
142144

143145
/**
@@ -161,6 +163,8 @@ export interface SandboxCodeResult {
161163
error?: SandboxCodeError
162164
/** The provider stopped the code runner because its supplied execution budget elapsed. */
163165
timedOut?: boolean
166+
/** The provider ended execution for an infrastructure reason, not a user-program outcome. */
167+
providerFailure?: 'provider_limit'
164168
}
165169

166170
export interface RunCommandOptions {

‎apps/sim/lib/function-execution/execute-request.test.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,7 @@ describe('Function execution request', () => {
328328
result: 'done',
329329
stdout: 'ok',
330330
sandboxId: 'sandbox-123',
331+
cost: { input: 0, output: 0, total: 0.00012345 },
331332
exportedFiles: { '/tmp/out.txt': 'owned by attacker' },
332333
})
333334
mockWriteWorkspaceFileByPath.mockRejectedValueOnce(
@@ -338,6 +339,8 @@ describe('Function execution request', () => {
338339
code: 'print("done")',
339340
language: 'python',
340341
workspaceId: 'workspace-victim',
342+
workflowId: 'workflow-1',
343+
executionId: 'execution-1',
341344
outputs: {
342345
files: [{ path: 'files/README.md', mode: 'overwrite', sandboxPath: '/tmp/out.txt' }],
343346
},
@@ -348,6 +351,7 @@ describe('Function execution request', () => {
348351

349352
expect(response.status).toBe(403)
350353
expect(data).toHaveProperty('error', 'Insufficient workspace permissions')
354+
expect(data.output.cost).toEqual({ input: 0, output: 0, total: 0.00012345 })
351355
expect(mockWriteWorkspaceFileByPath).toHaveBeenCalledTimes(1)
352356
})
353357

@@ -1543,6 +1547,7 @@ describe('Function execution request', () => {
15431547
result: 'done',
15441548
stdout: 'ok',
15451549
sandboxId: 'sandbox-123',
1550+
cost: { input: 0, output: 0, total: 0.00023456 },
15461551
exportedFiles: {
15471552
'/home/user/first.json': '{"first":true}',
15481553
'/home/user/second.json': '{"second":true}',
@@ -1556,6 +1561,8 @@ describe('Function execution request', () => {
15561561
code: 'print("done")',
15571562
language: 'python',
15581563
workspaceId: 'workspace-1',
1564+
workflowId: 'workflow-1',
1565+
executionId: 'execution-1',
15591566
outputs: {
15601567
files: [
15611568
{
@@ -1578,6 +1585,7 @@ describe('Function execution request', () => {
15781585
expect(response.status).toBe(400)
15791586
expect(data.success).toBe(false)
15801587
expect(data.error).toContain('Directory not yet created')
1588+
expect(data.output.cost).toEqual({ input: 0, output: 0, total: 0.00023456 })
15811589
expect(mockWriteWorkspaceFileByPath).not.toHaveBeenCalled()
15821590
})
15831591

‎apps/sim/lib/function-execution/execute-request.ts‎

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1438,10 +1438,20 @@ function exportFailure(
14381438
error: string,
14391439
status: number,
14401440
stdout: string,
1441-
executionTime: number
1441+
executionTime: number,
1442+
cost: FunctionExecutionCost | undefined
14421443
): NextResponse {
14431444
return NextResponse.json(
1444-
{ success: false, error, output: { result: null, stdout: cleanStdout(stdout), executionTime } },
1445+
{
1446+
success: false,
1447+
error,
1448+
output: {
1449+
result: null,
1450+
stdout: cleanStdout(stdout),
1451+
executionTime,
1452+
...(cost ? { cost } : {}),
1453+
},
1454+
},
14451455
{ status }
14461456
)
14471457
}
@@ -1490,7 +1500,8 @@ async function maybeExportSandboxFileToWorkspace(args: {
14901500
'outputSandboxPath requires outputPath. Set outputPath to the destination workspace file, e.g. "files/result.csv".',
14911501
400,
14921502
stdout,
1493-
executionTime
1503+
executionTime,
1504+
cost
14941505
)
14951506
}
14961507

@@ -1502,7 +1513,8 @@ async function maybeExportSandboxFileToWorkspace(args: {
15021513
'Workspace context required to save sandbox file to workspace',
15031514
400,
15041515
stdout,
1505-
executionTime
1516+
executionTime,
1517+
cost
15061518
)
15071519
}
15081520

@@ -1511,7 +1523,8 @@ async function maybeExportSandboxFileToWorkspace(args: {
15111523
`Sandbox file "${outputSandboxPath}" was not found or could not be read`,
15121524
500,
15131525
stdout,
1514-
executionTime
1526+
executionTime,
1527+
cost
15151528
)
15161529
}
15171530

@@ -1529,7 +1542,8 @@ async function maybeExportSandboxFileToWorkspace(args: {
15291542
`Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`,
15301543
400,
15311544
stdout,
1532-
executionTime
1545+
executionTime,
1546+
cost
15331547
)
15341548
}
15351549
const fileBuffer = isBinary
@@ -1612,7 +1626,8 @@ async function maybeExportSandboxFileToWorkspace(args: {
16121626
getErrorMessage(error, 'Failed to export sandbox file'),
16131627
workspaceFileExportErrorStatus(error),
16141628
stdout,
1615-
executionTime
1629+
executionTime,
1630+
cost
16161631
)
16171632
}
16181633
}
@@ -1636,7 +1651,8 @@ async function maybeExportSandboxFilesToWorkspace(args: {
16361651
`Too many sandbox output files requested (${sandboxFiles.length}). Maximum is ${MAX_SANDBOX_OUTPUT_FILES}.`,
16371652
400,
16381653
args.stdout,
1639-
args.executionTime
1654+
args.executionTime,
1655+
args.cost
16401656
)
16411657
}
16421658

@@ -1669,7 +1685,8 @@ async function maybeExportSandboxFilesToWorkspace(args: {
16691685
'Workspace context required to save sandbox files to workspace',
16701686
400,
16711687
args.stdout,
1672-
args.executionTime
1688+
args.executionTime,
1689+
args.cost
16731690
)
16741691
}
16751692

@@ -1683,7 +1700,8 @@ async function maybeExportSandboxFilesToWorkspace(args: {
16831700
`Sandbox file "${sandboxPath}" was not found or could not be read`,
16841701
500,
16851702
args.stdout,
1686-
args.executionTime
1703+
args.executionTime,
1704+
args.cost
16871705
)
16881706
}
16891707
const outputPath = file.formatPath ?? file.path
@@ -1700,7 +1718,8 @@ async function maybeExportSandboxFilesToWorkspace(args: {
17001718
`Sandbox output files exceed ${MAX_SANDBOX_OUTPUT_BYTES} bytes total`,
17011719
400,
17021720
args.stdout,
1703-
args.executionTime
1721+
args.executionTime,
1722+
args.cost
17041723
)
17051724
}
17061725
const scanBuffer = isBinary ? Buffer.from(content, 'base64') : Buffer.from(content, 'utf-8')
@@ -1749,7 +1768,8 @@ async function maybeExportSandboxFilesToWorkspace(args: {
17491768
getErrorMessage(error, 'Invalid sandbox output destination'),
17501769
workspaceFileExportErrorStatus(error),
17511770
args.stdout,
1752-
args.executionTime
1771+
args.executionTime,
1772+
args.cost
17531773
)
17541774
}
17551775
const duplicateDestination = validationPaths.find(
@@ -1760,7 +1780,8 @@ async function maybeExportSandboxFilesToWorkspace(args: {
17601780
`Duplicate sandbox output destination: ${duplicateDestination}`,
17611781
400,
17621782
args.stdout,
1763-
args.executionTime
1783+
args.executionTime,
1784+
args.cost
17641785
)
17651786
}
17661787

@@ -1816,7 +1837,8 @@ async function maybeExportSandboxFilesToWorkspace(args: {
18161837
getErrorMessage(error, 'Failed to export sandbox files'),
18171838
workspaceFileExportErrorStatus(error),
18181839
args.stdout,
1819-
args.executionTime
1840+
args.executionTime,
1841+
args.cost
18201842
)
18211843
}
18221844

0 commit comments

Comments
 (0)