Skip to content

Commit 8b1e69f

Browse files
Vikhyath Mondreticlaude
andcommitted
test(billing): check the Pi sandbox charge against a real provider
The handler test mocks the backend and writes into the cost sink by hand, so it proves the wiring from a backend to the block's cost and nothing more — it would still pass if withPiSandbox never metered at all, which is precisely the bug that path had. Holds a real Pi sandbox open for a known interval and bounds the charge between what that interval must cost and what the whole session could justify. Verified to fail against the original unmetered call with "expected 0 to be greater than or equal to 0.00023", and to pass once the sink is threaded: 5.949s billed of a 6.141s session on E2B. The second case pins the other half of the contract — a caller that supplies no sink is not charged, which is what keeps mothership and other internal Pi sandboxes free. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 2beec08 commit 8b1e69f

1 file changed

Lines changed: 76 additions & 0 deletions

File tree

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/**
2+
* @vitest-environment node
3+
*
4+
* Checks that a Pi session's sandbox is actually metered against a real provider.
5+
*
6+
* The handler-level test mocks the backend and writes into the sink by hand, so
7+
* it proves the wiring from a backend to the block's cost and nothing else. It
8+
* would still pass if `withPiSandbox` never metered at all — which is exactly
9+
* the bug this path had. Only a real Pi sandbox shows that creation is metered,
10+
* that teardown reports, and that the amount tracks the session's real lifetime.
11+
*
12+
* Enable with `SANDBOX_BILLING_SMOKE=1`, against whichever provider
13+
* `SANDBOX_PROVIDER` selects. Needs that provider's Pi image configured
14+
* (`E2B_PI_TEMPLATE_ID` / `DAYTONA_PI_SNAPSHOT_ID`).
15+
*/
16+
import { describe, expect, it } from 'vitest'
17+
import { createSandboxPricing } from '@/lib/billing/sandbox-pricing'
18+
import { withPiSandbox } from '@/lib/execution/remote-sandbox'
19+
import { resolveProvider } from '@/lib/execution/remote-sandbox/provider'
20+
import type { SandboxCostSink } from '@/lib/execution/remote-sandbox/types'
21+
22+
const smokeEnabled = process.env.SANDBOX_BILLING_SMOKE === '1'
23+
const CASE_TIMEOUT_MS = 5 * 60_000
24+
25+
/** Long enough that provisioning jitter cannot dominate the measured session. */
26+
const SLEEP_SECONDS = 5
27+
/** Well under any provider ceiling, so the lifetime cap never clamps the charge. */
28+
const LIFETIME_MS = 10 * 60_000
29+
30+
describe.skipIf(!smokeEnabled)('pi sandbox billing smoke', () => {
31+
it(
32+
'bills the session a Pi sandbox was held for',
33+
async () => {
34+
const pricing = createSandboxPricing(resolveProvider().id)
35+
const usdPerBilledSecond =
36+
(pricing.resources.vcpu * pricing.rates.cpuUsdPerVcpuSecond +
37+
pricing.resources.memoryGiB * pricing.rates.memoryUsdPerGiBSecond +
38+
pricing.resources.diskGiB * pricing.rates.diskUsdPerGiBSecond) *
39+
pricing.multiplier
40+
41+
const sandboxCost: SandboxCostSink = { total: 0 }
42+
const wallClockStartedAtMs = Date.now()
43+
const exitCode = await withPiSandbox(
44+
{ lifetimeMs: LIFETIME_MS, cost: sandboxCost },
45+
async (runner) => {
46+
const result = await runner.run(`sleep ${SLEEP_SECONDS}; echo held`, {
47+
envs: {},
48+
timeoutMs: CASE_TIMEOUT_MS,
49+
})
50+
return result.exitCode
51+
}
52+
)
53+
const wallClockMs = Date.now() - wallClockStartedAtMs
54+
55+
expect(exitCode).toBe(0)
56+
expect(sandboxCost.total).toBeGreaterThanOrEqual(SLEEP_SECONDS * usdPerBilledSecond)
57+
expect(sandboxCost.total).toBeLessThanOrEqual((wallClockMs / 1000) * usdPerBilledSecond)
58+
},
59+
CASE_TIMEOUT_MS
60+
)
61+
62+
it(
63+
'bills nothing when no sink is supplied',
64+
async () => {
65+
// The mothership and any other internal caller must stay free, and the
66+
// absence of a sink is the whole mechanism keeping them that way.
67+
const held = await withPiSandbox({ lifetimeMs: LIFETIME_MS }, async (runner) => {
68+
const result = await runner.run('echo held', { envs: {}, timeoutMs: CASE_TIMEOUT_MS })
69+
return result.exitCode
70+
})
71+
72+
expect(held).toBe(0)
73+
},
74+
CASE_TIMEOUT_MS
75+
)
76+
})

0 commit comments

Comments
 (0)