Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions apps/web/e2e/fixture-console.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,23 @@ function adminRead(response, path, url) {
return error(response, 404, "Not found.");
}

// A node as Core details it: its last heartbeat and an hour of 60-second host buckets.
function nodeDetail(node) {
const total = 64 * 2 ** 30;
const minute = Math.floor(Date.now() / 60_000) * 60;
const points = Array.from({ length: 60 }, (_, index) => ({
start: new Date((minute - (60 - index) * 60) * 1000).toISOString(),
cpu_utilization_max: node.online ? 0.3 + 0.1 * Math.sin(index / 6) : null,
memory_used_bytes_max: node.online ? total - node.available_memory_bytes : null,
available_disk_bytes_min: node.available_disk_bytes,
}));
return {
...node,
host: { effective_cpu_cores: node.cpu_count, cpu_utilization: node.online ? 0.35 : null, total_memory_bytes: node.online ? total : null, available_memory_bytes: node.available_memory_bytes, available_disk_bytes: node.available_disk_bytes, observed_at: node.last_seen_at },
history: { resolution_seconds: 60, points },
};
}

async function sandboxRoute(request, response, path) {
if (path === "/deployment" && request.method === "POST") {
const input = await body(request);
Expand All @@ -216,6 +233,10 @@ async function sandboxRoute(request, response, path) {
}
let m;
if ((m = path.match(/^\/nodes\/([^/]+)\/allocations$/))) return send(response, 200, { data: state.allocations.filter((entry) => entry.node_id === m[1]) });
if ((m = path.match(/^\/nodes\/([^/]+)$/)) && request.method === "GET") {
const node = state.nodes.find((entry) => entry.id === m[1]);
return node ? send(response, 200, nodeDetail(node)) : error(response, 404, "No such node.");
}
if ((m = path.match(/^\/nodes\/([^/]+)$/)) && request.method === "DELETE") {
const index = state.nodes.findIndex((node) => node.id === m[1]);
if (index < 0) return error(response, 404, "No such node.");
Expand Down
4 changes: 3 additions & 1 deletion apps/web/e2e/monitoring.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ test("opens a node and a sandbox in dialogs from Sandbox metrics", async ({ page
await openConsole(page, request, "sandbox-metrics");
await page.getByRole("button", { name: "Show core-01" }).click();
const node = page.getByRole("dialog", { name: "core-01" });
await expect(node.getByLabel("Node figures")).toContainText("Free memory");
// The machine's own load, from its heartbeats, beside the sandboxes placed on it.
await expect(node.getByLabel("Node figures")).toContainText("35% of 16 cores");
await expect(node.getByRole("figure", { name: /^Host CPU/ })).toBeVisible();
await expect(node).toContainText("Hosted sandboxes on this node");
await node.getByRole("button", { name: "Close dialog" }).click();

Expand Down
10 changes: 9 additions & 1 deletion apps/web/src/features/fleet/fleet-queries.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { queryOptions } from "@tanstack/react-query";
import { SandboxAdminClient, type SandboxAllocation, type SandboxDeployment, type SandboxNode } from "@agents-core-web/agents-client";
import { SandboxAdminClient, type SandboxAllocation, type SandboxDeployment, type SandboxNode, type SandboxNodeHistoryRange } from "@agents-core-web/agents-client";

import { sandboxConsoleConfig } from "../sandbox/console-config";

Expand Down Expand Up @@ -47,3 +47,11 @@ export function fleetQuery(allocations: boolean) {
queryFn: ({ signal }) => loadFleet(allocations, signal),
});
}

/** One node's host observation and host history over a range; polled while shown. */
export function nodeDetailQuery(nodeId: string, range: SandboxNodeHistoryRange) {
return queryOptions({
queryKey: ["sandbox-node", nodeId, range],
queryFn: ({ signal }) => client().retrieveNode(nodeId, range, { signal }),
});
}
2 changes: 1 addition & 1 deletion apps/web/src/features/metrics/CoreMetricsPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ function ServiceMeta({ metrics }: { metrics: CoreMetrics }) {
tone={notOwner ? "danger" : statusTone[metrics.service.status]}
label={t("core.meta", {
status: notOwner ? t("core.notOwner") : t(`core.status.${metrics.service.status}`),
revision: metrics.service.revision ?? MISSING,
revision: metrics.service.revision?.slice(0, 12) ?? MISSING,
uptime: started === null ? MISSING : formatDuration(Math.max(0, now - started)),
})}
/>
Expand Down
67 changes: 67 additions & 0 deletions apps/web/src/features/metrics/NodeHostCharts.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import type { SandboxNodeDetail } from "@agents-core-web/agents-client";
import { useMemo } from "react";
import { useTranslation } from "react-i18next";

import { TimeSeriesChart } from "../../components/charts/TimeSeriesChart";
import { formatBucket, formatBytes } from "../../lib/format";

const GIB = 1024 ** 3;

/** A share (0–1) as a percentage that never rounds a small load down to "0%". */
export function formatShare(value: number, locale?: string): string {
const percent = value * 100;
return `${new Intl.NumberFormat(locale, { maximumFractionDigits: percent === 0 ? 0 : percent < 1 ? 2 : percent < 10 ? 1 : 0 }).format(percent)}%`;
}

/**
* A node machine's host history as the console's chart pair: the busy share of
* its CPU and its used memory against the total, the highest in each bucket.
* Offline and unmeasured buckets stay gaps, never zero.
*/
export function NodeHostCharts({ detail, height = 150 }: { detail: SandboxNodeDetail; height?: number }) {
const { t, i18n } = useTranslation("metrics");
const locale = i18n.resolvedLanguage;
const { resolution_seconds: resolution, points } = detail.history;
const bucket = formatBucket(resolution, locale);
const total = detail.host.total_memory_bytes;
const series = useMemo(() => ({
buckets: points.map((point) => Math.floor(Date.parse(point.start) / 1000)),
cpu: points.map((point) => (point.cpu_utilization_max === null ? null : point.cpu_utilization_max * 100)),
memory: points.map((point) => (point.memory_used_bytes_max === null ? null : point.memory_used_bytes_max / GIB)),
total: points.map(() => (total === null ? null : total / GIB)),
}), [points, total]);
const percent = (value: number) => formatShare(value / 100, locale);
return (
<div className="chart-grid">
<figure className="chart-panel">
<figcaption>{t("sandbox.nodeDialog.hostCpu", { bucket })}</figcaption>
<TimeSeriesChart
label={t("sandbox.nodeDialog.hostCpu", { bucket })}
kind="lines"
buckets={series.buckets}
bucketSeconds={resolution}
series={[{ id: "used", label: t("sandbox.nodeDialog.used"), color: "var(--series-1)", values: series.cpu }]}
formatValue={percent}
formatAxis={percent}
height={height}
/>
</figure>
<figure className="chart-panel">
<figcaption>{t("sandbox.nodeDialog.hostMemory", { bucket })}</figcaption>
<TimeSeriesChart
label={t("sandbox.nodeDialog.hostMemory", { bucket })}
kind="lines"
buckets={series.buckets}
bucketSeconds={resolution}
series={[
{ id: "used", label: t("sandbox.nodeDialog.used"), color: "var(--series-1)", values: series.memory },
{ id: "total", label: t("sandbox.nodeDialog.total"), color: "var(--ink-3)", values: series.total },
]}
formatValue={(value) => formatBytes(value * GIB)}
formatAxis={(value) => `${value.toFixed(value < 10 ? 1 : 0)} GiB`}
height={height}
/>
</figure>
</div>
);
}
29 changes: 27 additions & 2 deletions apps/web/src/features/metrics/SandboxMetricsPage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ import { loadRuntimeDurableSnapshot, RUNTIME_DURABLE_RANGES, type RuntimeDurable
import type { RuntimeDashboardSnapshot } from "../dashboard/runtime-snapshot";
import { RUNTIME_SNAPSHOT_REFRESH_MS } from "../dashboard/runtime-snapshot";
import { capacitySummary, nodeHealth, type NodeHealth } from "../fleet/fleet-model";
import { nodeDetailQuery } from "../fleet/fleet-queries";
import { fleetSnapshot, useSandboxFleet, type FleetState } from "../fleet/use-sandbox-fleet";
import { formatShare, NodeHostCharts } from "./NodeHostCharts";
import {
hostedRuntimeRows,
hostedRuntimeUsage,
Expand Down Expand Up @@ -383,9 +385,27 @@ function NodeDialog({ node, rows, load, range, onClose }: {
[load, node, sessionIds],
);
const history = useRuntimeHistory(snapshot, range);
// The machine itself: its last heartbeat and host history over the page's range.
const hostRange = RUNTIME_DURABLE_RANGES.find((entry) => entry.milliseconds === range)?.label ?? "1h";
const detail = useQuery({
...nodeDetailQuery(shown?.id ?? "", hostRange),
enabled: node !== null,
placeholderData: keepPreviousData,
refetchInterval: RUNTIME_SNAPSHOT_REFRESH_MS,
refetchIntervalInBackground: false,
});
const host = detail.data?.id === shown?.id ? detail.data?.host ?? null : null;
const now = Math.floor(Date.now() / 1000);
const online = shown?.online ?? false;
const seen = shown?.last_seen_at ? Date.parse(shown.last_seen_at) / 1000 : null;
const cores = host?.effective_cpu_cores ?? shown?.cpu_count ?? null;
const coresLabel = cores === null ? null : t("sandbox.cores", { value: formatInteger(cores, locale) });
const cpu = !online || coresLabel === null ? MISSING
: host?.cpu_utilization != null ? t("sandbox.nodeDialog.cpuOf", { percent: formatShare(host.cpu_utilization, locale), cores: coresLabel }) : coresLabel;
const memory = !online ? MISSING
: host?.total_memory_bytes != null && host.available_memory_bytes != null
? t("sandbox.nodeDialog.memoryOf", { used: formatBytes(host.total_memory_bytes - host.available_memory_bytes), total: formatBytes(host.total_memory_bytes) })
: formatBytes(shown?.available_memory_bytes ?? null);
return (
<Modal
open={node !== null}
Expand All @@ -400,12 +420,17 @@ function NodeDialog({ node, rows, load, range, onClose }: {
<div><dt>{t("sandbox.status")}</dt><dd><StatusDot tone={healthTone[nodeHealth(shown)]} label={t(`sandbox.health.${nodeHealth(shown)}`)} /></dd></div>
<div><dt>{t("sandbox.slots")}</dt><dd>{formatInteger(shown.active, locale)} / {formatInteger(shown.max_active, locale)}</dd></div>
<div><dt>{t("sandbox.nodeDialog.retainedSlots")}</dt><dd>{formatInteger(shown.retained, locale)} / {formatInteger(shown.max_retained, locale)}</dd></div>
<div><dt>{t("sandbox.cpus")}</dt><dd>{online && shown.cpu_count !== null ? t("sandbox.cores", { value: formatInteger(shown.cpu_count, locale) }) : MISSING}</dd></div>
<div><dt>{t("sandbox.freeMemory")}</dt><dd>{online ? formatBytes(shown.available_memory_bytes) : MISSING}</dd></div>
<div><dt>{t("sandbox.cpus")}</dt><dd>{cpu}</dd></div>
<div><dt>{t("sandbox.memory")}</dt><dd>{memory}</dd></div>
<div><dt>{t("sandbox.freeDiskColumn")}</dt><dd>{online ? formatBytes(shown.available_disk_bytes) : MISSING}</dd></div>
<div><dt>{t("sandbox.cleanupPending")}</dt><dd>{formatInteger(shown.cleanup_pending, locale)}</dd></div>
<div><dt>{t("sandbox.lastSeen")}</dt><dd title={seen === null ? undefined : formatDateTime(seen, locale)}>{formatRelative(seen, now, locale)}</dd></div>
</dl>
<Section headingId="node-host-heading" title={t("sandbox.nodeDialog.host")} help={t("sandbox.nodeDialog.hostHelp")}>
{detail.data?.id === shown.id ? <NodeHostCharts detail={detail.data} />
: detail.isError ? <p className="page-status">{t("sandbox.nodeDialog.hostFailed", { reason: detail.error instanceof Error ? detail.error.message : "" })}</p>
: <TableSkeleton label={t("sandbox.runtimeLoading")} rows={3} columns={4} />}
</Section>
<Section
headingId="node-runtimes-heading"
title={<>{t("sandbox.nodeDialog.runtimes")}<span className="section-meta">{t("sandbox.nodeDialog.runtimesMeta", { n: formatInteger(sessionIds.size, locale) })}</span></>}
Expand Down
11 changes: 10 additions & 1 deletion apps/web/src/i18n/locales/en/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,16 @@ export const metrics = {
openLabel: "Show {{name}}",
retainedSlots: "Retained / limit",
runtimes: "Hosted sandboxes on this node",
runtimesHelp: "CPU and memory of the Core-managed sandboxes placed on this node, over the page's range. Nodes report their free memory and disk with each heartbeat, not a history.",
runtimesHelp: "CPU and memory of the Core-managed sandboxes placed on this node, over the page's range.",
host: "Host",
hostHelp: "The whole machine as the node reports it with each heartbeat: the busy share of its CPU and its used memory, the highest in each interval. Intervals while the node was offline stay empty.",
hostCpu: "Host CPU / {{bucket}}",
hostMemory: "Host memory / {{bucket}}",
used: "Used",
total: "Total",
memoryOf: "{{used}} / {{total}}",
cpuOf: "{{percent}} of {{cores}}",
hostFailed: "Host history could not be read. {{reason}}",
runtimesMeta: "{{n}} sandboxes",
noRuntimes: "No hosted sandbox runs on this node.",
openNode: "Node details",
Expand Down
11 changes: 10 additions & 1 deletion apps/web/src/i18n/locales/zh-CN/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,16 @@ export const metrics = {
openLabel: "查看 {{name}}",
retainedSlots: "保留 / 上限",
runtimes: "这个节点上的托管沙箱",
runtimesHelp: "放在这个节点上的 Core 托管沙箱的 CPU 和内存,时间范围与页面一致。节点每次心跳只上报当前的可用内存和磁盘,没有历史。",
runtimesHelp: "放在这个节点上的 Core 托管沙箱的 CPU 和内存,时间范围与页面一致。",
host: "宿主机",
hostHelp: "节点每次心跳上报的整台机器的情况:CPU 忙碌占比和已用内存,取每个时间段内的最高值。节点离线的时间段留空。",
hostCpu: "宿主机 CPU / {{bucket}}",
hostMemory: "宿主机内存 / {{bucket}}",
used: "已用",
total: "总量",
memoryOf: "{{used}} / {{total}}",
cpuOf: "{{percent}}(共 {{cores}})",
hostFailed: "无法读取宿主机历史。{{reason}}",
runtimesMeta: "{{n}} 个沙箱",
noRuntimes: "这个节点上没有运行中的托管沙箱。",
openNode: "节点详情",
Expand Down
2 changes: 2 additions & 0 deletions apps/web/src/lib/format.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ export function formatBytes(value: number | null | undefined): string {
/** Durations in seconds, as a compact unit string (850 ms, 12.4 s, 4m 12s, 3h 5m). */
export function formatDuration(seconds: number | null | undefined): string {
if (seconds === null || seconds === undefined || !Number.isFinite(seconds) || seconds < 0) return MISSING;
// Sub-10 ms values keep decimals, so a fast database never reads "1 ms" on every tick.
if (seconds < 0.00995) return `${new Intl.NumberFormat("en", { maximumFractionDigits: seconds < 0.001 ? 2 : 1 }).format(seconds * 1000)} ms`;
if (seconds < 0.9995) return `${Math.round(seconds * 1000)} ms`;
if (seconds < 9.95) return `${seconds.toFixed(1)} s`;
const whole = Math.round(seconds);
Expand Down
1 change: 1 addition & 0 deletions docs/web/protocol-coverage.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ log (`GET /audit-log`) and Runtime history capabilities
| Deployment | `GET`, `POST`, `PUT /core/v1/sandbox/deployment` | Read the provider, Core origin, maintenance state, installation ID and specification; initialize the deployment with `resources`, the Docker or microsandbox `runtime` release, or the E2B account; change its settings with the expected generation |
| Maintenance | `PATCH /core/v1/sandbox/deployment/maintenance` | Enter or leave maintenance to change the provider |
| Nodes | `GET /core/v1/sandbox/nodes` | Nodes page; fleet on Overview; node capacity on Sandbox metrics |
| Node detail | `GET /core/v1/sandbox/nodes/{node_id}?range=1h\|6h\|24h` | Sandbox metrics node dialog: the host's CPU busy share and memory from its last heartbeat, and their history over the page's range |
| Allocations | `GET /core/v1/sandbox/nodes/{node_id}/allocations` | Nodes page; Sandbox metrics |
| Enrollment | `POST /core/v1/sandbox/enrollment-tokens` | **Add node**: a single-use token inside a command that verifies the installer checksum |
| Remove node | `DELETE /core/v1/sandbox/nodes/{node_id}` | Confirmed node removal; the row goes only after Core acknowledges the deletion |
Expand Down
25 changes: 25 additions & 0 deletions packages/agents-client/src/sandbox-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,28 @@ export interface SandboxNode {
cleanup_pending: number;
created_at: string;
}
/** The node machine's last heartbeat observation; unavailable measurements are null. */
export interface SandboxNodeHost {
effective_cpu_cores: number | null;
/** Busy share of the whole host's CPU between heartbeats, 0–1. */
cpu_utilization: number | null;
total_memory_bytes: number | null;
available_memory_bytes: number | null;
available_disk_bytes: number | null;
observed_at: string | null;
}
export interface SandboxNodeHostPoint {
start: string;
cpu_utilization_max: number | null;
memory_used_bytes_max: number | null;
available_disk_bytes_min: number | null;
}
export type SandboxNodeHistoryRange = "1h" | "6h" | "24h";
/** One node with its host observation and complete UTC buckets of host history. */
export interface SandboxNodeDetail extends SandboxNode {
host: SandboxNodeHost;
history: { resolution_seconds: number; points: SandboxNodeHostPoint[] };
}
export interface SandboxAllocation {
id: string;
node_id: string;
Expand Down Expand Up @@ -110,6 +132,9 @@ export class SandboxAdminClient extends OpenAIAgentsClient {
listNodes(options?: ReadOptions): Promise<{ data: SandboxNode[] }> {
return this.request("/nodes", { signal: options?.signal }, undefined, false);
}
retrieveNode(nodeId: string, range: SandboxNodeHistoryRange, options?: ReadOptions): Promise<SandboxNodeDetail> {
return this.request(`/nodes/${encodeURIComponent(nodeId)}?range=${range}`, { signal: options?.signal }, undefined, false);
}
listAllocations(nodeId: string, options?: ReadOptions): Promise<{ data: SandboxAllocation[] }> {
return this.request(`/nodes/${encodeURIComponent(nodeId)}/allocations`, { signal: options?.signal }, undefined, false);
}
Expand Down
Loading