Skip to content

Commit dfc0e35

Browse files
Bill LeoutsakosBill Leoutsakos
authored andcommitted
fix(billing): tighten sandbox completion boundaries
1 parent 9d689d4 commit dfc0e35

9 files changed

Lines changed: 341 additions & 62 deletions

File tree

‎apps/sim/lib/billing/sandbox-pricing.test.ts‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,14 @@ describe('sandbox pricing', () => {
2323
expect(priceSandboxUsage(pricing, 90_000, 60_000).durationMs).toBe(60_000)
2424
})
2525

26-
it('rejects a non-positive multiplier', () => {
27-
expect(() => createSandboxPricing('e2b', 0)).toThrow('finite positive')
26+
it('allows a zero multiplier and rejects invalid multipliers', () => {
27+
const freePricing = createSandboxPricing('e2b', 0)
28+
29+
expect(priceSandboxUsage(freePricing, 1000, 1000).billedCost).toBe(0)
30+
expect(() => createSandboxPricing('e2b', -1)).toThrow('finite nonnegative')
31+
expect(() => createSandboxPricing('e2b', Number.NaN)).toThrow('finite nonnegative')
32+
expect(() => createSandboxPricing('e2b', Number.POSITIVE_INFINITY)).toThrow(
33+
'finite nonnegative'
34+
)
2835
})
2936
})

‎apps/sim/lib/billing/sandbox-pricing.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ const E2B_CPU_USD_PER_VCPU_SECOND = 0.000014
1010
const E2B_MEMORY_USD_PER_GIB_SECOND = 0.0000045
1111
const DAYTONA_CPU_USD_PER_VCPU_SECOND = 0.0504 / 3600
1212
const DAYTONA_MEMORY_USD_PER_GIB_SECOND = 0.0162 / 3600
13+
/**
14+
* Sim prices the full provisioned disk at the marginal list rate; provider free allowances,
15+
* credits, and discounts are intentionally not subtracted.
16+
*/
1317
const DAYTONA_DISK_USD_PER_GIB_SECOND = 0.000108 / 3600
1418

1519
export interface SandboxPricing {
@@ -67,8 +71,8 @@ export function createSandboxPricing(
6771
provider: SandboxProviderId,
6872
multiplier = getCostMultiplier()
6973
): SandboxPricing {
70-
if (!Number.isFinite(multiplier) || multiplier <= 0) {
71-
throw new Error('Sandbox pricing multiplier must be a finite positive number')
74+
if (!Number.isFinite(multiplier) || multiplier < 0) {
75+
throw new Error('Sandbox pricing multiplier must be a finite nonnegative number')
7276
}
7377
const pricing = PRICING_BY_PROVIDER[provider]
7478
return {

‎apps/sim/lib/execution/remote-sandbox/conformance.test.ts‎

Lines changed: 164 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ import {
136136
MAX_SANDBOX_OUTPUT_BYTES,
137137
MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
138138
MAX_SANDBOX_STREAMED_OUTPUT_TAIL_BYTES,
139+
readTrustedSandboxOutputCost,
139140
} from '@/lib/execution/remote-sandbox/output-limits'
140141
import {
141142
PI_SANDBOX_MIN_LIFETIME_MS,
@@ -464,13 +465,19 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => {
464465
})
465466
}
466467

467-
await expect(
468-
executeInSandbox({ code: 'x', language: CodeLanguage.Python, timeoutMs: 1000 })
469-
).rejects.toMatchObject({
468+
const error = await executeInSandbox({
469+
code: 'x',
470+
language: CodeLanguage.Python,
471+
timeoutMs: 1000,
472+
meterUsage: true,
473+
}).catch((error: unknown) => error)
474+
475+
expect(error).toMatchObject({
470476
code: 'sandbox_output_limit_exceeded',
471477
outputKind: 'process',
472478
limitBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
473479
})
480+
expect(readTrustedSandboxOutputCost(error)).toBeUndefined()
474481
expect(provider === 'e2b' ? mockE2BKill : mockDelete).toHaveBeenCalledTimes(1)
475482
})
476483

@@ -1173,18 +1180,24 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => {
11731180
stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`)
11741181
stubOutputFileSizes(provider, MAX_SANDBOX_OUTPUT_BYTES + 1)
11751182

1176-
await expect(
1177-
executeInSandbox({
1178-
code: 'x',
1179-
language: CodeLanguage.Python,
1180-
timeoutMs: 1000,
1181-
outputSandboxPath: '/out/report.txt',
1182-
})
1183-
).rejects.toMatchObject({
1183+
const error = await executeInSandbox({
1184+
code: 'x',
1185+
language: CodeLanguage.Python,
1186+
timeoutMs: 1000,
1187+
outputSandboxPath: '/out/report.txt',
1188+
meterUsage: true,
1189+
}).catch((error: unknown) => error)
1190+
1191+
expect(error).toMatchObject({
11841192
code: 'sandbox_output_limit_exceeded',
11851193
attemptedBytes: MAX_SANDBOX_OUTPUT_BYTES + 1,
11861194
limitBytes: MAX_SANDBOX_OUTPUT_BYTES,
11871195
})
1196+
expect(readTrustedSandboxOutputCost(error)).toEqual({
1197+
input: 0,
1198+
output: 0,
1199+
total: expect.any(Number),
1200+
})
11881201

11891202
expect(provider === 'e2b' ? mockE2BFilesRead : mockDownloadFileStream).not.toHaveBeenCalled()
11901203
})
@@ -1237,15 +1250,96 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => {
12371250
mockGetFileDetails.mockResolvedValueOnce({ size: 1, isDir: false, mode: 'prw-r--r--' })
12381251
}
12391252

1240-
await expect(
1241-
executeInSandbox({
1242-
code: 'x',
1243-
language: CodeLanguage.Python,
1253+
const error = await executeInSandbox({
1254+
code: 'x',
1255+
language: CodeLanguage.Python,
1256+
timeoutMs: 1000,
1257+
outputSandboxPath: '/out/link.txt',
1258+
meterUsage: true,
1259+
}).catch((error: unknown) => error)
1260+
1261+
expect(error).toMatchObject({ code: 'sandbox_output_file_invalid' })
1262+
expect(readTrustedSandboxOutputCost(error)).toEqual({
1263+
input: 0,
1264+
output: 0,
1265+
total: expect.any(Number),
1266+
})
1267+
expect(provider === 'e2b' ? mockE2BFilesRead : mockDownloadFileStream).not.toHaveBeenCalled()
1268+
})
1269+
1270+
it.each(['oversized', 'non-regular'] as const)(
1271+
'retains metered shell cost for a completed execution with %s output',
1272+
async (failure) => {
1273+
stubShellCommand(provider, '', '', 0)
1274+
if (failure === 'oversized') {
1275+
stubOutputFileSizes(provider, MAX_SANDBOX_OUTPUT_BYTES + 1)
1276+
} else if (provider === 'e2b') {
1277+
mockE2BFilesGetInfo.mockResolvedValueOnce({ size: 1, type: 'symlink' })
1278+
} else {
1279+
mockGetFileDetails.mockResolvedValueOnce({ size: 1, isDir: false, mode: 'prw-r--r--' })
1280+
}
1281+
1282+
const error = await executeShellInSandbox({
1283+
code: 'echo done',
1284+
envs: {},
12441285
timeoutMs: 1000,
1245-
outputSandboxPath: '/out/link.txt',
1286+
outputSandboxPath: '/out/result.txt',
1287+
meterUsage: true,
1288+
}).catch((error: unknown) => error)
1289+
1290+
expect(error).toMatchObject({
1291+
code:
1292+
failure === 'oversized' ? 'sandbox_output_limit_exceeded' : 'sandbox_output_file_invalid',
12461293
})
1247-
).rejects.toMatchObject({ code: 'sandbox_output_file_invalid' })
1248-
expect(provider === 'e2b' ? mockE2BFilesRead : mockDownloadFileStream).not.toHaveBeenCalled()
1294+
expect(readTrustedSandboxOutputCost(error)).toEqual({
1295+
input: 0,
1296+
output: 0,
1297+
total: expect.any(Number),
1298+
})
1299+
}
1300+
)
1301+
1302+
it('does not attach cost to a generic provider failure during output collection', async () => {
1303+
stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`)
1304+
stubOutputFileSizes(provider, 1, 1)
1305+
const failure = new Error('provider file read failed')
1306+
if (provider === 'e2b') {
1307+
mockE2BFilesRead.mockRejectedValueOnce(failure)
1308+
} else {
1309+
mockDownloadFileStream.mockRejectedValueOnce(failure)
1310+
}
1311+
1312+
const error = await executeInSandbox({
1313+
code: 'x',
1314+
language: CodeLanguage.Python,
1315+
timeoutMs: 1000,
1316+
outputSandboxPath: '/out/result.txt',
1317+
meterUsage: true,
1318+
}).catch((error: unknown) => error)
1319+
1320+
expect(error).toBe(failure)
1321+
expect(readTrustedSandboxOutputCost(error)).toBeUndefined()
1322+
})
1323+
1324+
it('does not attach cost to a generic provider failure during output inspection', async () => {
1325+
stubCodeRun(provider, `${SIM_RESULT_PREFIX}null`)
1326+
const failure = new Error('provider file metadata failed')
1327+
if (provider === 'e2b') {
1328+
mockE2BFilesGetInfo.mockRejectedValueOnce(failure)
1329+
} else {
1330+
mockGetFileDetails.mockRejectedValueOnce(failure)
1331+
}
1332+
1333+
const error = await executeInSandbox({
1334+
code: 'x',
1335+
language: CodeLanguage.Python,
1336+
timeoutMs: 1000,
1337+
outputSandboxPath: '/out/result.txt',
1338+
meterUsage: true,
1339+
}).catch((error: unknown) => error)
1340+
1341+
expect(error).toBe(failure)
1342+
expect(readTrustedSandboxOutputCost(error)).toBeUndefined()
12491343
})
12501344

12511345
it('does not return code results when cancellation arrives during output collection', async () => {
@@ -1564,6 +1658,58 @@ describe('provider stream recovery', () => {
15641658
expect(mockGetSessionCommand).toHaveBeenCalledWith(expect.any(String), 'cmd_1')
15651659
})
15661660

1661+
it('fails an at-most-once Daytona run closed when final status has no exit code', async () => {
1662+
mockGetSessionCommand.mockResolvedValueOnce({})
1663+
const sandbox = await daytonaProvider.create('code')
1664+
1665+
await expect(
1666+
sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true })
1667+
).rejects.toMatchObject({
1668+
name: 'SandboxLaunchIndeterminateError',
1669+
retryable: false,
1670+
code: 'sandbox_launch_indeterminate',
1671+
})
1672+
})
1673+
1674+
it('fails an at-most-once Daytona run closed when its readiness handshake never completes', async () => {
1675+
mockGetSessionCommandLogs.mockResolvedValueOnce(undefined)
1676+
mockGetSessionCommand.mockResolvedValueOnce({ exitCode: 78 })
1677+
const sandbox = await daytonaProvider.create('code')
1678+
1679+
await expect(
1680+
sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true })
1681+
).rejects.toMatchObject({
1682+
name: 'SandboxLaunchIndeterminateError',
1683+
retryable: false,
1684+
code: 'sandbox_launch_indeterminate',
1685+
})
1686+
expect(mockSendSessionCommandInput).not.toHaveBeenCalled()
1687+
})
1688+
1689+
it('fails an at-most-once Daytona run closed when final status lookup fails', async () => {
1690+
mockGetSessionCommand.mockRejectedValueOnce(new Error('control plane unavailable'))
1691+
const sandbox = await daytonaProvider.create('code')
1692+
1693+
await expect(
1694+
sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true })
1695+
).rejects.toMatchObject({
1696+
name: 'SandboxLaunchIndeterminateError',
1697+
retryable: false,
1698+
code: 'sandbox_launch_indeterminate',
1699+
})
1700+
})
1701+
1702+
it('preserves a pre-dispatch Daytona failure for at-most-once runs', async () => {
1703+
const failure = new Error('session unavailable')
1704+
mockCreateSession.mockRejectedValueOnce(failure)
1705+
const sandbox = await daytonaProvider.create('code')
1706+
1707+
await expect(
1708+
sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true })
1709+
).rejects.toBe(failure)
1710+
expect(mockExecuteSessionCommand).not.toHaveBeenCalled()
1711+
})
1712+
15671713
it('keeps the original Daytona deadline while recovering a disconnected stream', async () => {
15681714
mockGetSessionCommandLogs
15691715
.mockRejectedValueOnce(new Error('stream disconnected'))

‎apps/sim/lib/execution/remote-sandbox/daytona.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,7 @@ class DaytonaSandboxHandle implements SandboxHandle {
301301
// must never have.
302302
const finalStdout = () => (retainStdout ? stdout : tailStreamedSandboxOutput(stdout))
303303
const finalStderr = () => (retainStderr ? stderr : tailStreamedSandboxOutput(stderr))
304+
let commandDispatched = false
304305
try {
305306
await this.sandbox.process.createSession(sessionId)
306307
sessionCreated = true
@@ -339,6 +340,7 @@ class DaytonaSandboxHandle implements SandboxHandle {
339340
if (typeof commandId !== 'string' || commandId.length === 0) {
340341
throw new SandboxLaunchIndeterminateError('Daytona')
341342
}
343+
commandDispatched = true
342344
// Accumulate the streamed chunks as well as forwarding them: callers read
343345
// markers out of stdout (the Pi cloud flow parses __BASE_SHA__/__CHANGED__)
344346
// and format failures from stderr, so returning empty strings here would
@@ -660,7 +662,14 @@ class DaytonaSandboxHandle implements SandboxHandle {
660662
}
661663

662664
const finished = await this.sandbox.process.getSessionCommand(sessionId, commandId)
663-
const exitCode = finished.exitCode ?? 0
665+
if (options.atMostOnce && !releaseRequested) {
666+
throw new SandboxLaunchIndeterminateError('Daytona')
667+
}
668+
const exitCode = finished.exitCode
669+
if (typeof exitCode !== 'number' || !Number.isFinite(exitCode)) {
670+
if (options.atMostOnce) throw new SandboxLaunchIndeterminateError('Daytona')
671+
return { stdout: finalStdout(), stderr: finalStderr(), exitCode: 0 }
672+
}
664673
return { stdout: finalStdout(), stderr: finalStderr(), exitCode }
665674
} catch (error) {
666675
if (isSandboxOutputLimitError(error)) {
@@ -681,6 +690,12 @@ class DaytonaSandboxHandle implements SandboxHandle {
681690
timedOut: true,
682691
}
683692
}
693+
if (options.atMostOnce) {
694+
if (commandDispatched) {
695+
throw new SandboxLaunchIndeterminateError('Daytona', { cause: error })
696+
}
697+
throw error
698+
}
684699
if (operation === 'code') throw error
685700
return { stdout: finalStdout(), stderr: finalStderr() || getErrorMessage(error), exitCode: 1 }
686701
} finally {

0 commit comments

Comments
 (0)