From 6bfaacfc175f6a37420646859538f0d19cf06a78 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Fri, 25 Sep 2026 17:07:26 +0800 Subject: [PATCH 1/4] Sample bounded Core process CPU and resident memory metrics --- contracts/agents-api/core-metrics.md | 25 +++- docs/web/core-process-metrics-requirements.md | 28 +++- .../internal/coremetrics/process.go | 34 +++++ .../internal/coremetrics/process_files.go | 130 ++++++++++++++++++ .../coremetrics/process_files_test.go | 105 ++++++++++++++ .../internal/coremetrics/process_linux.go | 17 +++ .../coremetrics/process_linux_test.go | 35 +++++ .../internal/coremetrics/process_other.go | 9 ++ .../internal/coremetrics/process_test.go | 88 ++++++++++++ .../agents-api/internal/coremetrics/series.go | 6 +- .../internal/coremetrics/service.go | 9 +- .../agents-api/internal/coremetrics/types.go | 15 +- 12 files changed, 489 insertions(+), 12 deletions(-) create mode 100644 services/agents-api/internal/coremetrics/process.go create mode 100644 services/agents-api/internal/coremetrics/process_files.go create mode 100644 services/agents-api/internal/coremetrics/process_files_test.go create mode 100644 services/agents-api/internal/coremetrics/process_linux.go create mode 100644 services/agents-api/internal/coremetrics/process_linux_test.go create mode 100644 services/agents-api/internal/coremetrics/process_other.go create mode 100644 services/agents-api/internal/coremetrics/process_test.go diff --git a/contracts/agents-api/core-metrics.md b/contracts/agents-api/core-metrics.md index 4ac89bb19..e768b1d64 100644 --- a/contracts/agents-api/core-metrics.md +++ b/contracts/agents-api/core-metrics.md @@ -26,10 +26,11 @@ the current partial bucket is excluded from range aggregates and series. | 7d | 7200 seconds | 84 | Current gauges and range aggregates are intentionally different: current worker, -connection pool and process values are read when requested; queue gauges, database -size and deployment maintenance are sampled every 30 seconds with bounded I/O. +connection pool, Go heap and goroutine values are read when requested; process +CPU, RSS and limits, queue gauges, database size and deployment maintenance are +sampled every 30 seconds with bounded I/O. Samples older than 60 seconds are not reported as current. Queue, running and -pool series report the highest **observed** value in each bucket, not a claim +pool and process series report the highest **observed** value in each bucket, not a claim that all intermediate peaks were captured. Missing observations and the process's partial first bucket stay null. Successful periodic ping samples produce linear interpolated p50/p95; there is no request-triggered ping. @@ -79,6 +80,24 @@ present. one value does not turn it into zero. - `process.memory_bytes` is Go `runtime.MemStats.Alloc` (allocated heap bytes), not RSS or container memory. `goroutines` is `runtime.NumGoroutine()`. +- `process.cpu_cores` is the increase in this process's user plus system CPU + time divided by elapsed sampling time. Linux uses `getrusage(RUSAGE_SELF)`; + subprocess and whole-host CPU are excluded. The first interval is null. + Missing/invalid counters, counter resets, nonpositive elapsed time and gaps + longer than 60 seconds reset the baseline; they never manufacture a zero. +- `process.rss_bytes` is Linux `/proc/self/status` `VmRSS`, converted from KiB + to bytes. `cpu_limit_cores` is the process's cgroup v2 `cpu.max` quota/period, + or `GOMAXPROCS` when its quota is `max`. `memory_limit_bytes` is that cgroup's + finite `memory.max`; `max` is null. Membership and mount information resolve + the process's cgroup, including nested and subtree mounts. These are that + cgroup's configured limits, not whole-host metrics or ancestor-limit discovery. + Unreadable or malformed values remain null. Non-Linux builds report null CPU, + RSS and memory limit, with `GOMAXPROCS` as CPU capacity. +- `process.series` always contains the range's complete buckets with `start`, + `cpu_cores` and `rss_bytes`. Each measurement uses its own observed maximum; + missing samples and the partial first bucket stay null. These samples share + the existing bounded ring and restart gaps. Unsupported process measurements + do not by themselves change execution or database health. ## Background jobs diff --git a/docs/web/core-process-metrics-requirements.md b/docs/web/core-process-metrics-requirements.md index a4b0934fa..35a6f6710 100644 --- a/docs/web/core-process-metrics-requirements.md +++ b/docs/web/core-process-metrics-requirements.md @@ -1,15 +1,15 @@ # Core process CPU and memory: backend requirements -Status: requested by Core Web (Monitor › Core metrics, the Process section). The -page and the typed client (`packages/agents-client/src/core-metrics.ts`) read the -fields below; until Core serves them they show as missing ("—", "No data"). +Status: implemented by the Core backend for Monitor > Core metrics, Process. +The page and typed client (`packages/agents-client/src/core-metrics.ts`) consume +the fields below. Unavailable measurements show as missing ("—", "No data"). ## Why Core is one `agents-api` process. Operators size and alert on its CPU and resident memory, and today the response carries only the Go heap in use (`process.memory_bytes`, `runtime.MemStats.Alloc`) and the goroutine count, read -when requested, with no history. The heap is neither what the operating system +when requested. The heap is neither what the operating system charges the process nor what a container limit is compared against. ## Contract @@ -17,6 +17,8 @@ charges the process nor what a container limit is compared against. Reuse `GET /core/v1/admin/core-metrics?range=1h|6h|24h|7d`: add fields to `process`; change nothing else. `memory_bytes` and `goroutines` keep their meaning. Every figure Core cannot measure is `null`, never `0`. +Measured zero remains zero. New current values expire after 60 seconds without +a sample. Go heap and goroutine values continue to be read when requested. ```json "process": { @@ -38,5 +40,23 @@ Every figure Core cannot measure is `null`, never `0`. | `memory_limit_bytes` | The cgroup v2 `memory.max`; null when it is `max` or unreadable | | `series` | Per bucket of the response's range, the highest `cpu_cores` and `rss_bytes` observed, recorded by the existing 30-second sampler in the same in-process ring as the queue and database samples. Missing observations stay null; a restart does not backfill | +CPU uses Linux `getrusage(RUSAGE_SELF)`. Missing or invalid readings, counter +resets, nonpositive elapsed time and sampling gaps over 60 seconds reset its +baseline. The next valid interval supplies CPU again. RSS and limits are +independent readings, so a missing CPU interval does not hide measured memory. + +Linux resolves cgroup v2 membership using `/proc/self/cgroup` and +`/proc/self/mountinfo`; it reads the process's own cgroup rather than assuming +the mount root. Limits describe that cgroup's configuration; ancestor-limit +discovery is outside this contract. Unreadable and malformed values stay null. +All procfs and cgroup reads have byte bounds. Non-Linux builds return null for +CPU usage, RSS and memory limit, with `GOMAXPROCS` as the CPU capacity fallback. +Unsupported process measurements do not mark execution or database health as +degraded. + +All four existing ranges retain their bucket counts and exclude the active +partial bucket. The process's partial first bucket stays null. Series report +independent observed maxima, so they need not come from the same sample. + Not requested: whole-host CPU, memory or disk (Core may share its host; those belong to host monitoring), per-request CPU profiles, or garbage-collector detail. diff --git a/services/agents-api/internal/coremetrics/process.go b/services/agents-api/internal/coremetrics/process.go new file mode 100644 index 000000000..014a47c2e --- /dev/null +++ b/services/agents-api/internal/coremetrics/process.go @@ -0,0 +1,34 @@ +package coremetrics + +import ( + "math" + "time" +) + +type processReading struct { + cpuSeconds *float64 + cpuLimitCores *float64 + rssBytes *uint64 + memoryLimitBytes *uint64 +} + +// The existing sampling loop owns the baseline. Reads never advance it. +type processSampler struct { + at time.Time + cpuSeconds *float64 +} + +func (s *processSampler) sample(at time.Time, reading processReading) Process { + result := Process{CPULimitCores: reading.cpuLimitCores, RSSBytes: reading.rssBytes, MemoryLimitBytes: reading.memoryLimitBytes} + cpu := reading.cpuSeconds + if cpu != nil && (*cpu < 0 || math.IsNaN(*cpu) || math.IsInf(*cpu, 0)) { + cpu = nil + } + elapsed := at.Sub(s.at) + if cpu != nil && s.cpuSeconds != nil && !s.at.IsZero() && elapsed > 0 && elapsed <= 2*SampleInterval && *cpu >= *s.cpuSeconds { + result.CPUCores = ptr((*cpu - *s.cpuSeconds) / elapsed.Seconds()) + } + // Counter resets, missing observations and invalid gaps start a new interval. + s.at, s.cpuSeconds = at, cpu + return result +} diff --git a/services/agents-api/internal/coremetrics/process_files.go b/services/agents-api/internal/coremetrics/process_files.go new file mode 100644 index 000000000..389810f02 --- /dev/null +++ b/services/agents-api/internal/coremetrics/process_files.go @@ -0,0 +1,130 @@ +package coremetrics + +import ( + "io" + "math" + "os" + "path/filepath" + "strconv" + "strings" +) + +func readProcessFiles(procSelf string, cpus int) processReading { + reading := processReading{rssBytes: parseRSS(readBounded(filepath.Join(procSelf, "status"), 64<<10))} + group := cgroupDirectory(readBounded(filepath.Join(procSelf, "cgroup"), 64<<10), readBounded(filepath.Join(procSelf, "mountinfo"), 1<<20)) + if group != "" { + reading.cpuLimitCores = parseCPULimit(readBounded(filepath.Join(group, "cpu.max"), 4096), cpus) + reading.memoryLimitBytes = parseMemoryLimit(readBounded(filepath.Join(group, "memory.max"), 4096)) + } + return reading +} + +// procfs and cgroup values are small. A truncated or failed read is unknown. +func readBounded(path string, limit int64) string { + f, err := os.Open(path) + if err != nil { + return "" + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, limit+1)) + if err != nil || int64(len(data)) > limit { + return "" + } + return string(data) +} + +func parseRSS(status string) *uint64 { + for _, line := range strings.Split(status, "\n") { + fields := strings.Fields(line) + if len(fields) == 0 || fields[0] != "VmRSS:" { + continue + } + if len(fields) != 3 || fields[2] != "kB" { + return nil + } + value, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil || value > math.MaxUint64/1024 { + return nil + } + return ptr(value * 1024) + } + return nil +} + +func parseCPULimit(data string, cpus int) *float64 { + fields := strings.Fields(data) + if len(fields) != 2 { + return nil + } + period, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil || period == 0 { + return nil + } + if fields[0] == "max" { + if cpus > 0 { + return ptr(float64(cpus)) + } + return nil + } + quota, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil || quota == 0 { + return nil + } + return ptr(float64(quota) / float64(period)) +} + +func parseMemoryLimit(data string) *uint64 { + value, err := strconv.ParseUint(strings.TrimSpace(data), 10, 64) + if err != nil { + return nil + } + return &value +} + +// Resolve membership against the mounted cgroup v2 subtree. Do not assume that +// the process belongs to the root or that cgroupfs is mounted at /sys/fs/cgroup. +func cgroupDirectory(membership, mountinfo string) string { + group := "" + for _, line := range strings.Split(membership, "\n") { + if strings.HasPrefix(line, "0::") { + if group != "" { + return "" + } + group = strings.TrimPrefix(line, "0::") + } + } + if !canonicalAbsolute(group) { + return "" + } + decode := strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) + selected, rootLength := "", -1 + for _, line := range strings.Split(mountinfo, "\n") { + left, right, ok := strings.Cut(line, " - ") + fields, fs := strings.Fields(left), strings.Fields(right) + if !ok || len(fields) < 6 || len(fs) < 3 || fs[0] != "cgroup2" { + continue + } + root, point := decode.Replace(fields[3]), decode.Replace(fields[4]) + if !canonicalAbsolute(root) || !canonicalAbsolute(point) { + continue + } + relative := "" + switch { + case group == root: + case root == "/": + relative = strings.TrimPrefix(group, "/") + case strings.HasPrefix(group, root+"/"): + relative = strings.TrimPrefix(group, root+"/") + default: + continue + } + if len(root) > rootLength { + selected, rootLength = filepath.Join(point, relative), len(root) + } + } + return selected +} + +func canonicalAbsolute(path string) bool { + return filepath.IsAbs(path) && filepath.Clean(path) == path +} diff --git a/services/agents-api/internal/coremetrics/process_files_test.go b/services/agents-api/internal/coremetrics/process_files_test.go new file mode 100644 index 000000000..f6c67ba0b --- /dev/null +++ b/services/agents-api/internal/coremetrics/process_files_test.go @@ -0,0 +1,105 @@ +package coremetrics + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestProcessLinuxParsers(t *testing.T) { + for _, data := range []string{"", "VmRSS: x kB", "VmRSS: -1 kB", "VmRSS: 10 MB", "VmRSS: 18446744073709551615 kB", "VmRSS: 10 kB extra"} { + if parseRSS(data) != nil { + t.Fatal("invalid RSS accepted", data) + } + } + if got := parseRSS("Name: core\nVmRSS:\t123 kB\n"); got == nil || *got != 123*1024 { + t.Fatal(got) + } + if got := parseRSS("VmRSS: 0 kB"); got == nil || *got != 0 { + t.Fatal("measured zero RSS lost") + } + for _, data := range []string{"", "max", "0 100000", "1000 0", "-1 1000", "1.5 1000", "max nope", "1 1000 extra", "18446744073709551616 1000"} { + if parseCPULimit(data, 4) != nil { + t.Fatal("invalid CPU limit accepted", data) + } + } + if got := parseCPULimit("150000 100000\n", 4); got == nil || *got != 1.5 { + t.Fatal(got) + } + if got := parseCPULimit("max 100000\n", 4); got == nil || *got != 4 { + t.Fatal(got) + } + for _, data := range []string{"", "max", "-1", "2.5", "18446744073709551616", "1024 extra"} { + if parseMemoryLimit(data) != nil { + t.Fatal("invalid memory limit accepted", data) + } + } + if got := parseMemoryLimit("1073741824\n"); got == nil || *got != 1073741824 { + t.Fatal(got) + } +} + +func TestProcessCgroupDirectory(t *testing.T) { + for _, test := range []struct{ name, membership, mountinfo, want string }{ + {"nested", "0::/system.slice/core.service\n", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw\n", "/sys/fs/cgroup/system.slice/core.service"}, + {"namespace root", "0::/\n", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw\n", "/sys/fs/cgroup"}, + {"subtree", "0::/work/core\n", "1 0 0:1 /work /run/cgroup rw - cgroup2 cgroup rw\n", "/run/cgroup/core"}, + {"exact subtree", "0::/work\n", "1 0 0:1 /work /run/cgroup rw - cgroup2 cgroup rw\n", "/run/cgroup"}, + {"escaped mount", "0::/work/core\n", "1 0 0:1 /work /run/cgroup\\040space rw - cgroup2 cgroup rw\n", "/run/cgroup space/core"}, + {"unrelated subtree", "0::/workers/core\n", "1 0 0:1 /work /run/cgroup rw - cgroup2 cgroup rw\n", ""}, + {"v1", "1:cpu:/work\n", "1 0 0:1 / /sys/fs/cgroup rw - cgroup cgroup rw\n", ""}, + {"invalid membership", "0::/work/../core\n", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw\n", ""}, + {"invalid mount", "0::/work\n", "1 0 0:1 / relative rw - cgroup2 cgroup rw\n", ""}, + {"duplicate membership", "0::/one\n0::/two\n", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw\n", ""}, + {"specific mount", "0::/work/core\n", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw\n2 0 0:1 /work /run/cgroup rw - cgroup2 cgroup rw\n", "/run/cgroup/core"}, + } { + t.Run(test.name, func(t *testing.T) { + if got := cgroupDirectory(test.membership, test.mountinfo); got != test.want { + t.Fatalf("got %q, want %q", got, test.want) + } + }) + } +} + +func TestProcessLinuxFiles(t *testing.T) { + dir := t.TempDir() + group := filepath.Join(dir, "cgroup", "service") + if err := os.MkdirAll(group, 0700); err != nil { + t.Fatal(err) + } + write := func(path, data string) { + t.Helper() + if err := os.WriteFile(path, []byte(data), 0600); err != nil { + t.Fatal(err) + } + } + // proc and cgroup mounts have independent roots. + proc := filepath.Join(dir, "proc") + if err := os.Mkdir(proc, 0700); err != nil { + t.Fatal(err) + } + write(filepath.Join(proc, "status"), "VmRSS: 123 kB\n") + write(filepath.Join(proc, "cgroup"), "0::/tenant/service\n") + write(filepath.Join(proc, "mountinfo"), "1 0 0:1 /tenant "+filepath.Dir(group)+" rw - cgroup2 cgroup rw\n") + write(filepath.Join(group, "cpu.max"), "150000 100000\n") + write(filepath.Join(group, "memory.max"), "1073741824\n") + got := readProcessFiles(proc, 4) + if got.rssBytes == nil || *got.rssBytes != 123*1024 || got.cpuLimitCores == nil || *got.cpuLimitCores != 1.5 || got.memoryLimitBytes == nil || *got.memoryLimitBytes != 1073741824 { + t.Fatal(got) + } + write(filepath.Join(group, "cpu.max"), "max 100000\n") + write(filepath.Join(group, "memory.max"), "max\n") + got = readProcessFiles(proc, 4) + if got.cpuLimitCores == nil || *got.cpuLimitCores != 4 || got.memoryLimitBytes != nil { + t.Fatal(got) + } + if err := os.Remove(filepath.Join(group, "cpu.max")); err != nil { + t.Fatal(err) + } + write(filepath.Join(proc, "status"), strings.Repeat("x", 64<<10)+"\nVmRSS: 1 kB\n") + got = readProcessFiles(proc, 4) + if got.cpuLimitCores != nil || got.rssBytes != nil { + t.Fatal("missing or oversized reads must be unknown", got) + } +} diff --git a/services/agents-api/internal/coremetrics/process_linux.go b/services/agents-api/internal/coremetrics/process_linux.go new file mode 100644 index 000000000..4a443ea29 --- /dev/null +++ b/services/agents-api/internal/coremetrics/process_linux.go @@ -0,0 +1,17 @@ +//go:build linux + +package coremetrics + +import ( + "runtime" + "syscall" +) + +func readProcess() processReading { + reading := readProcessFiles("/proc/self", runtime.GOMAXPROCS(0)) + var usage syscall.Rusage + if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) == nil { + reading.cpuSeconds = ptr(float64(usage.Utime.Sec) + float64(usage.Utime.Usec)/1e6 + float64(usage.Stime.Sec) + float64(usage.Stime.Usec)/1e6) + } + return reading +} diff --git a/services/agents-api/internal/coremetrics/process_linux_test.go b/services/agents-api/internal/coremetrics/process_linux_test.go new file mode 100644 index 000000000..8eb846e4e --- /dev/null +++ b/services/agents-api/internal/coremetrics/process_linux_test.go @@ -0,0 +1,35 @@ +//go:build linux + +package coremetrics + +import ( + "encoding/json" + "runtime" + "testing" + "time" +) + +// This also runs against real procfs in the Linux package test, without models. +func TestProcessLinuxProbe(t *testing.T) { + var sampler processSampler + first := readProcess() + if first.cpuSeconds == nil || first.rssBytes == nil || *first.rssBytes == 0 { + t.Fatalf("live process measurements missing: %+v", first) + } + if got := sampler.sample(time.Now(), first); got.CPUCores != nil { + t.Fatal("first CPU interval must be unknown") + } + deadline := time.Now().Add(30 * time.Millisecond) + for time.Now().Before(deadline) { + runtime.Gosched() + } + got := sampler.sample(time.Now(), readProcess()) + if got.CPUCores == nil || *got.CPUCores <= 0 { + t.Fatal("live process CPU interval missing", got.CPUCores) + } + raw, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + t.Log(string(raw)) +} diff --git a/services/agents-api/internal/coremetrics/process_other.go b/services/agents-api/internal/coremetrics/process_other.go new file mode 100644 index 000000000..75f9c013b --- /dev/null +++ b/services/agents-api/internal/coremetrics/process_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package coremetrics + +import "runtime" + +func readProcess() processReading { + return processReading{cpuLimitCores: ptr(float64(runtime.GOMAXPROCS(0)))} +} diff --git a/services/agents-api/internal/coremetrics/process_test.go b/services/agents-api/internal/coremetrics/process_test.go new file mode 100644 index 000000000..3f13e3d95 --- /dev/null +++ b/services/agents-api/internal/coremetrics/process_test.go @@ -0,0 +1,88 @@ +package coremetrics + +import ( + "encoding/json" + "math" + "strings" + "testing" + "time" +) + +func TestProcessCPUIntervals(t *testing.T) { + start := time.Date(2026, 9, 25, 12, 0, 0, 0, time.UTC) + var sampler processSampler + for _, test := range []struct { + seconds int + cpu *float64 + want *float64 + }{ + {0, ptr(10.0), nil}, + {30, ptr(25.0), ptr(.5)}, + {60, ptr(25.0), ptr(0.0)}, + {90, ptr(1.0), nil}, // Counter reset. + {120, ptr(31.0), ptr(1.0)}, + {120, ptr(32.0), nil}, // No elapsed time. + {110, ptr(33.0), nil}, // Clock moved backwards. + {180, ptr(35.0), nil}, // Missing interval. + {210, nil, nil}, + {240, ptr(40.0), nil}, + {270, ptr(math.NaN()), nil}, + {300, ptr(50.0), nil}, + {330, ptr(math.Inf(1)), nil}, + {360, ptr(-1.0), nil}, + {390, ptr(60.0), nil}, + {420, ptr(120.0), ptr(2.0)}, + } { + got := sampler.sample(start.Add(time.Duration(test.seconds)*time.Second), processReading{cpuSeconds: test.cpu, rssBytes: ptr(uint64(1024)), memoryLimitBytes: ptr(uint64(4096)), cpuLimitCores: ptr(2.0)}) + if (got.CPUCores == nil) != (test.want == nil) || (test.want != nil && *got.CPUCores != *test.want) { + t.Fatalf("at %ds: cpu=%v, want=%v", test.seconds, got.CPUCores, test.want) + } + if *got.RSSBytes != 1024 || *got.MemoryLimitBytes != 4096 || *got.CPULimitCores != 2 { + t.Fatal("independent gauges lost", got) + } + } +} + +func TestProcessSeriesAndFreshness(t *testing.T) { + s, _, now := fixtureService(t) + end := now.Truncate(time.Minute) + s.started = end.Add(-8 * 24 * time.Hour) + s.record(Sample{At: end.Add(-30 * time.Second), Healthy: true, Process: Process{CPUCores: ptr(.8), RSSBytes: ptr(uint64(100))}}) + s.record(Sample{At: end.Add(-time.Second), Healthy: true, Process: Process{CPUCores: ptr(.3), RSSBytes: ptr(uint64(200))}}) + s.record(Sample{At: now, Healthy: true, Process: Process{CPUCores: ptr(9.0), RSSBytes: ptr(uint64(999)), CPULimitCores: ptr(2.0), MemoryLimitBytes: ptr(uint64(4096))}}) + for name, count := range map[string]int{"1h": 60, "6h": 72, "24h": 96, "7d": 84} { + view, err := s.Read(t.Context(), name) + if err != nil || len(view.Process.Series) != count { + t.Fatal(name, err, len(view.Process.Series)) + } + if *view.Process.CPUCores != 9 || *view.Process.RSSBytes != 999 || *view.Process.CPULimitCores != 2 || *view.Process.MemoryLimitBytes != 4096 || view.Process.MemoryBytes == nil || view.Process.Goroutines == nil { + t.Fatal("current or existing gauges lost", view.Process) + } + last := view.Process.Series[count-1] + if last.CPUCores == nil || last.RSSBytes == nil || *last.CPUCores != .8 || *last.RSSBytes != 200 || !last.Start.Equal(view.Range.End.Add(-time.Duration(view.Range.ResolutionSeconds)*time.Second)) { + t.Fatal("complete bucket maxima", last) + } + if view.Process.Series[0].CPUCores != nil || view.Process.Series[0].RSSBytes != nil { + t.Fatal("missing history became zero") + } + } + s.started = end.Add(-30 * time.Second) + view, _ := s.Read(t.Context(), "1h") + if view.Process.Series[59].CPUCores != nil || view.Process.Series[59].RSSBytes != nil { + t.Fatal("partial first bucket was filled") + } + s.now = func() time.Time { return now.Add(2*SampleInterval + time.Second) } + view, _ = s.Read(t.Context(), "1h") + if view.Process.CPUCores != nil || view.Process.RSSBytes != nil || view.Process.CPULimitCores != nil || view.Process.MemoryLimitBytes != nil { + t.Fatal("stale process values remained current") + } + raw, err := json.Marshal(view.Process) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{"cpu_cores", "rss_bytes", "cpu_limit_cores", "memory_limit_bytes"} { + if !strings.Contains(string(raw), `"`+key+`":null`) { + t.Fatal("unknown fields must be present as null", string(raw)) + } + } +} diff --git a/services/agents-api/internal/coremetrics/series.go b/services/agents-api/internal/coremetrics/series.go index ccb524686..cbf30fcda 100644 --- a/services/agents-api/internal/coremetrics/series.go +++ b/services/agents-api/internal/coremetrics/series.go @@ -13,12 +13,14 @@ func (s *Service) series(view *View) { count := int(view.Range.End.Sub(view.Range.Start) / step) view.Execution.Series = make([]ExecutionBucket, count) view.Database.Series = make([]DatabaseBucket, count) + view.Process.Series = make([]ProcessBucket, count) pings := make([][]float64, count) var allPings []float64 for i := 0; i < count; i++ { start := view.Range.Start.Add(time.Duration(i) * step) view.Execution.Series[i].Start = start view.Database.Series[i].Start = start + view.Process.Series[i].Start = start } for _, sample := range s.samples { if sample.At.Before(view.Range.Start) || !sample.At.Before(view.Range.End) || sample.At.IsZero() { @@ -34,6 +36,8 @@ func (s *Service) series(view *View) { maxInto(&view.Execution.Series[i].Queued, sample.Queued) maxInto(&view.Execution.Series[i].InProgress, sample.InProgress) maxInto(&view.Database.Series[i].PoolInUse, sample.PoolInUse) + maxInto(&view.Process.Series[i].CPUCores, sample.Process.CPUCores) + maxInto(&view.Process.Series[i].RSSBytes, sample.Process.RSSBytes) if sample.PingMS != nil { pings[i] = append(pings[i], *sample.PingMS) } @@ -54,7 +58,7 @@ func (s *Service) series(view *View) { view.Execution.Unavailable = &total } } -func maxInto(target **int64, value *int64) { +func maxInto[T int64 | uint64 | float64](target **T, value *T) { if value != nil && (*target == nil || **target < *value) { *target = ptr(*value) } diff --git a/services/agents-api/internal/coremetrics/service.go b/services/agents-api/internal/coremetrics/service.go index 646451d09..2c878d036 100644 --- a/services/agents-api/internal/coremetrics/service.go +++ b/services/agents-api/internal/coremetrics/service.go @@ -33,6 +33,7 @@ type Service struct { refusals [sampleCapacity]refusalSlot latest Sample jobs map[string]Job + process processSampler } func New(started time.Time, revision string, source Source) *Service { @@ -88,7 +89,9 @@ func (s *Service) Run(ctx context.Context) { probe, cancel := context.WithTimeout(ctx, 5*time.Second) sample := s.source.Sample(probe) cancel() - sample.At = s.now().UTC() + at := s.now() + sample.Process = s.process.sample(at, readProcess()) + sample.At = at.UTC() s.record(sample) select { case <-ctx.Done(): @@ -140,6 +143,7 @@ func (s *Service) Read(ctx context.Context, name string) (View, error) { view.Execution.QueuedTurns, view.Execution.WaitingForDaemon, view.Execution.InProgressTurns = latest.Queued, latest.WaitingForDaemon, latest.InProgress view.Execution.OldestQueuedSeconds = latest.OldestQueuedSeconds view.Database.SizeBytes = latest.DatabaseSize + view.Process = latest.Process if latest.Maintenance != nil && *latest.Maintenance && view.Service.Status == "running" { view.Service.Status = "maintenance" } @@ -175,7 +179,8 @@ func (s *Service) Read(ctx context.Context, name string) (View, error) { } var memory runtime.MemStats runtime.ReadMemStats(&memory) - view.Process = Process{MemoryBytes: ptr(memory.Alloc), Goroutines: ptr(int64(runtime.NumGoroutine()))} + view.Process.MemoryBytes = ptr(memory.Alloc) + view.Process.Goroutines = ptr(int64(runtime.NumGoroutine())) return view, nil } func ptr[T any](v T) *T { return &v } diff --git a/services/agents-api/internal/coremetrics/types.go b/services/agents-api/internal/coremetrics/types.go index 2a926604c..a0cfc19a7 100644 --- a/services/agents-api/internal/coremetrics/types.go +++ b/services/agents-api/internal/coremetrics/types.go @@ -64,8 +64,18 @@ type Job struct { Failed *int64 `json:"failed" extensions:"x-nullable"` } type Process struct { - MemoryBytes *uint64 `json:"memory_bytes" extensions:"x-nullable"` - Goroutines *int64 `json:"goroutines" extensions:"x-nullable"` + MemoryBytes *uint64 `json:"memory_bytes" extensions:"x-nullable"` + Goroutines *int64 `json:"goroutines" extensions:"x-nullable"` + CPUCores *float64 `json:"cpu_cores" extensions:"x-nullable"` + CPULimitCores *float64 `json:"cpu_limit_cores" extensions:"x-nullable"` + RSSBytes *uint64 `json:"rss_bytes" extensions:"x-nullable"` + MemoryLimitBytes *uint64 `json:"memory_limit_bytes" extensions:"x-nullable"` + Series []ProcessBucket `json:"series"` +} +type ProcessBucket struct { + Start time.Time `json:"start"` + CPUCores *float64 `json:"cpu_cores" extensions:"x-nullable"` + RSSBytes *uint64 `json:"rss_bytes" extensions:"x-nullable"` } type View struct { Object string `json:"object"` @@ -86,6 +96,7 @@ type Sample struct { PoolInUse, DatabaseSize *int64 Maintenance *bool Healthy bool + Process Process } type Live struct { SlotsInUse, SlotsTotal, ConnectedDaemons *int64 From 0681c3293285a1998d37e675b2dd41dbe1f633e4 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Fri, 25 Sep 2026 17:07:22 +0800 Subject: [PATCH 2/4] Observe host CPU utilization and memory in node heartbeats --- .../agents-api/internal/sandbox/node/agent.go | 13 +- .../sandbox/node/health_cgroup_linux.go | 157 ++++++++++ .../node/health_connection_linux_test.go | 53 ++++ .../internal/sandbox/node/health_cpu_linux.go | 68 +++++ .../sandbox/node/health_cpu_linux_test.go | 71 +++++ .../internal/sandbox/node/health_linux.go | 98 ++++++- .../sandbox/node/health_linux_test.go | 275 ++++++++++++++++++ .../internal/sandbox/node/health_other.go | 6 +- .../agents-api/internal/sandbox/node/wire.go | 3 + 9 files changed, 721 insertions(+), 23 deletions(-) create mode 100644 services/agents-api/internal/sandbox/node/health_cgroup_linux.go create mode 100644 services/agents-api/internal/sandbox/node/health_connection_linux_test.go create mode 100644 services/agents-api/internal/sandbox/node/health_cpu_linux.go create mode 100644 services/agents-api/internal/sandbox/node/health_cpu_linux_test.go create mode 100644 services/agents-api/internal/sandbox/node/health_linux_test.go diff --git a/services/agents-api/internal/sandbox/node/agent.go b/services/agents-api/internal/sandbox/node/agent.go index 3372e20a0..142f0903d 100644 --- a/services/agents-api/internal/sandbox/node/agent.go +++ b/services/agents-api/internal/sandbox/node/agent.go @@ -139,7 +139,7 @@ func Run(ctx context.Context, config AgentConfig) error { } } } -func (a *agent) health(ctx context.Context) (Health, error) { +func (a *agent) health(ctx context.Context, host *hostHealthSampler) (Health, error) { ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() h, e := a.config.Probe(ctx) @@ -155,11 +155,10 @@ func (a *agent) health(ctx context.Context) (Health, error) { } h.ObservedAt = time.Now().UTC() h.ActiveOperations = int(a.active.Load()) - fillHostHealth(&h, a.config.StateDirectory) + host.fill(&h, a.config.StateDirectory) return h, e } func (a *agent) connect(ctx context.Context) error { - health, _ := a.health(ctx) endpointURL, err := endpoint(a.config.CoreURL, "/core/v1/sandbox/node/connect") if err != nil { return err @@ -185,6 +184,8 @@ func (a *agent) connect(ctx context.Context) error { } defer conn.Close() conn.SetReadLimit(MaxFrameBytes) + host := new(hostHealthSampler) + health, _ := a.health(ctx, host) _ = conn.SetReadDeadline(time.Now().Add(15 * time.Second)) if err = writeFrame(conn, frame{Type: "hello", Identity: &a.config.Identity, Health: &health}); err != nil { return err @@ -216,7 +217,7 @@ func (a *agent) connect(ctx context.Context) error { defer cancel() detach := context.AfterFunc(connectionCtx, current.close) defer detach() - go a.heartbeats(connectionCtx, current) + go a.heartbeats(connectionCtx, current, host) sequence := uint64(0) for { _ = conn.SetReadDeadline(time.Now().Add(35 * time.Second)) @@ -247,7 +248,7 @@ func (a *agent) connect(ctx context.Context) error { } } } -func (a *agent) heartbeats(ctx context.Context, c *agentConnection) { +func (a *agent) heartbeats(ctx context.Context, c *agentConnection, host *hostHealthSampler) { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() for { @@ -257,7 +258,7 @@ func (a *agent) heartbeats(ctx context.Context, c *agentConnection) { case <-c.done: return case <-ticker.C: - h, _ := a.health(ctx) + h, _ := a.health(ctx, host) if c.write(frame{Type: "heartbeat", ConnectionID: c.id, Health: &h}) != nil { c.close() return diff --git a/services/agents-api/internal/sandbox/node/health_cgroup_linux.go b/services/agents-api/internal/sandbox/node/health_cgroup_linux.go new file mode 100644 index 000000000..f661189b6 --- /dev/null +++ b/services/agents-api/internal/sandbox/node/health_cgroup_linux.go @@ -0,0 +1,157 @@ +//go:build linux + +package node + +import ( + "math" + "os" + "path" + "slices" + "strconv" + "strings" +) + +type healthCgroup struct { + dir, mount string + v2 bool +} + +const healthMaxCgroupDepth = 128 + +// Resolve the process membership through mountinfo instead of assuming a fixed +// cgroup mount layout. A subtree-only mount hides ancestors, so it is unknown. +func findHealthCgroup(groups, mounts, controller string) (healthCgroup, bool) { + var group, unified string + found := false + for _, line := range strings.Split(strings.TrimSpace(groups), "\n") { + fields := strings.SplitN(line, ":", 3) + if len(fields) != 3 || !cleanHealthPath(fields[2]) { + return healthCgroup{}, false + } + id, err := strconv.ParseUint(fields[0], 10, 32) + if err != nil || (id == 0) != (fields[1] == "") { + return healthCgroup{}, false + } + if fields[0] == "0" && fields[1] == "" { + if unified != "" { + return healthCgroup{}, false + } + unified = fields[2] + } + if slices.Contains(strings.Split(fields[1], ","), controller) { + if found { + return healthCgroup{}, false + } + group, found = fields[2], true + } + } + v2 := !found && unified != "" + if v2 { + group = unified + } else if !found { + // A valid membership list without this controller has no such limit. + return healthCgroup{}, true + } + for _, line := range strings.Split(mounts, "\n") { + before, after, ok := strings.Cut(line, " - ") + left, right := strings.Fields(before), strings.Fields(after) + if !ok || len(left) < 6 || len(right) < 3 { + continue + } + if v2 && right[0] != "cgroup2" || !v2 && (right[0] != "cgroup" || !slices.Contains(strings.Split(right[2], ","), controller)) { + continue + } + root, mount := unescapeMountPath(left[3]), unescapeMountPath(left[4]) + if root != "/" || !cleanHealthPath(mount) { + continue + } + return healthCgroup{dir: path.Join(mount, group), mount: mount, v2: v2}, true + } + return healthCgroup{}, false +} + +func cleanHealthPath(p string) bool { + return strings.HasPrefix(p, "/") && path.Clean(p) == p && !strings.HasSuffix(p, " (deleted)") +} + +func unescapeMountPath(p string) string { + return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(p) +} + +func (g healthCgroup) visibleRoot(read func(string) ([]byte, error)) bool { + if g.v2 { + // cgroup.type exists only below the real root, including a namespace root. + _, err := read(path.Join(g.mount, "cgroup.type")) + return os.IsNotExist(err) + } + _, err := read(path.Join(g.mount, "release_agent")) + return err == nil +} + +func (g healthCgroup) cpuLimit(read func(string) ([]byte, error)) (float64, bool) { + limit := math.Inf(1) + if g.dir == "" { + return limit, true + } + if !g.visibleRoot(read) { + return 0, false + } + for dir, depth := g.dir, 0; depth < healthMaxCgroupDepth; dir, depth = path.Dir(dir), depth+1 { + var quota, period string + if g.v2 { + data, ok := g.v2Limit(read, dir, "cpu", "cpu.max") + if !ok { + return 0, false + } + fields := strings.Fields(data) + if len(fields) != 2 { + return 0, false + } + quota, period = fields[0], fields[1] + } else { + q, qe := read(path.Join(dir, "cpu.cfs_quota_us")) + p, pe := read(path.Join(dir, "cpu.cfs_period_us")) + if qe != nil || pe != nil { + return 0, false + } + quota, period = strings.TrimSpace(string(q)), strings.TrimSpace(string(p)) + } + p, err := strconv.ParseInt(period, 10, 64) + if err != nil || p <= 0 { + return 0, false + } + if !(g.v2 && quota == "max" || !g.v2 && quota == "-1") { + q, err := strconv.ParseInt(quota, 10, 64) + if err != nil || q <= 0 { + return 0, false + } + limit = min(limit, float64(q)/float64(p)) + } + if dir == g.mount { + return limit, true + } + } + return 0, false +} + +func (g healthCgroup) v2Limit(read func(string) ([]byte, error), dir, controller, file string) (string, bool) { + data, err := read(path.Join(dir, file)) + if err == nil { + // The real v2 root has no resource limit file; a namespace root can hide ancestors. + return strings.TrimSpace(string(data)), dir != g.mount + } + if !os.IsNotExist(err) { + return "", false + } + controllers, err := read(path.Join(dir, "cgroup.controllers")) + if err != nil { + return "", false + } + // Root has no quota/max file. At other levels absence is legitimate only + // when the controller is not enabled there. Still inspect every ancestor. + unlimited := "max" + if controller == "cpu" { + unlimited = "max 1" + } + return unlimited, dir == g.mount || !slices.Contains(strings.Fields(string(controllers)), controller) +} diff --git a/services/agents-api/internal/sandbox/node/health_connection_linux_test.go b/services/agents-api/internal/sandbox/node/health_connection_linux_test.go new file mode 100644 index 000000000..c2c8fefae --- /dev/null +++ b/services/agents-api/internal/sandbox/node/health_connection_linux_test.go @@ -0,0 +1,53 @@ +//go:build linux + +package node + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/google/uuid" + "github.com/gorilla/websocket" +) + +func TestHostHealthReconnectStartsFreshInterval(t *testing.T) { + hellos := make(chan Health, 2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.Close() + hello, err := readFrame(conn) + if err != nil || hello.Type != "hello" || hello.Health == nil { + t.Errorf("invalid hello: %+v, %v", hello, err) + return + } + hellos <- *hello.Health + if err := writeFrame(conn, frame{Type: "welcome", ConnectionID: uuid.NewString(), OwnerEpoch: 1}); err != nil { + t.Error(err) + } + })) + defer server.Close() + a := agent{config: AgentConfig{CoreURL: server.URL, StateDirectory: t.TempDir(), Identity: identity(), Probe: probe}, stored: StoredIdentity{OwnerEpoch: 1}} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + for i := 0; i < 2; i++ { + if err := a.connect(ctx); err == nil { + t.Fatal("server disconnect was not observed") + } + select { + case h := <-hellos: + if h.CPUUtilization != nil || h.TotalMemoryBytes == nil || h.ObservedAt.IsZero() { + t.Fatalf("connection %d inherited a baseline or omitted its observation: %+v", i, h) + } + case <-ctx.Done(): + t.Fatal("missing hello") + } + time.Sleep(100 * time.Millisecond) + } +} diff --git a/services/agents-api/internal/sandbox/node/health_cpu_linux.go b/services/agents-api/internal/sandbox/node/health_cpu_linux.go new file mode 100644 index 000000000..8b2168a06 --- /dev/null +++ b/services/agents-api/internal/sandbox/node/health_cpu_linux.go @@ -0,0 +1,68 @@ +//go:build linux + +package node + +import ( + "math" + "strconv" + "strings" +) + +// Each connection owns its baseline; missing or reset counters break the interval. +type hostCPUSampler struct { + previous [8]uint64 + fields int +} + +func (s *hostCPUSampler) sample(read func(string) ([]byte, error)) *float64 { + data, err := read("/proc/stat") + if err != nil { + s.fields = 0 + return nil + } + current, fields := parseHostCPU(string(data)) + previous, previousFields := s.previous, s.fields + s.previous, s.fields = current, fields + if fields == 0 || previousFields != fields { + return nil + } + var total, idle uint64 + for i, value := range current { + if value < previous[i] { + return nil + } + delta := value - previous[i] + if delta > math.MaxUint64-total { + return nil + } + total += delta + if i == 3 || i == 4 { // Idle and iowait are both non-busy time. + idle += delta + } + } + if total == 0 { + return nil + } + utilization := float64(total-idle) / float64(total) + return &utilization +} + +func parseHostCPU(data string) ([8]uint64, int) { + var counters [8]uint64 + line, _, _ := strings.Cut(data, "\n") + fields := strings.Fields(line) + if len(fields) < 5 || fields[0] != "cpu" { + return counters, 0 + } + for i, field := range fields[1:] { + value, err := strconv.ParseUint(field, 10, 64) + if err != nil { + return counters, 0 + } + // Guest and guest_nice are already included in user and nice. + if i < len(counters) { + counters[i] = value + } + } + return counters, min(len(fields)-1, len(counters)) +} diff --git a/services/agents-api/internal/sandbox/node/health_cpu_linux_test.go b/services/agents-api/internal/sandbox/node/health_cpu_linux_test.go new file mode 100644 index 000000000..a2fb8c3ea --- /dev/null +++ b/services/agents-api/internal/sandbox/node/health_cpu_linux_test.go @@ -0,0 +1,71 @@ +//go:build linux + +package node + +import ( + "math" + "os" + "testing" +) + +func TestHostCPUUtilization(t *testing.T) { + var sampler hostCPUSampler + sample := func(data string) *float64 { + return sampler.sample(func(string) ([]byte, error) { return []byte(data), nil }) + } + if sample("cpu 100 20 30 400 50 10 20 5 60 10\ncpu0 9999") != nil { + t.Fatal("first sample must be unknown") + } + // Busy increases 40, idle+iowait increases 60. Guest deltas must not be added. + if got := sample("cpu 120 20 40 450 60 15 25 5 80 10"); got == nil || math.Abs(*got-.4) > 1e-12 { + t.Fatalf("host busy ratio = %v, want .4", got) + } + if got := sample("cpu 120 20 40 460 60 15 25 5 80 10"); got == nil || *got != 0 { + t.Fatalf("idle host = %v, want known zero", got) + } + if got := sample("cpu 130 20 40 460 60 15 25 5 80 10"); got == nil || *got != 1 { + t.Fatalf("busy host = %v, want one", got) + } +} + +func TestHostCPUUnknownAndReset(t *testing.T) { + for _, tc := range []struct { + name, middle string + err error + }{ + {"read-error", "", os.ErrPermission}, + {"missing", "intr 100", nil}, + {"malformed", "cpu 20 bad 20 20", nil}, + {"negative", "cpu -1 20 20 20", nil}, + {"overflow", "cpu 18446744073709551616 20 20 20", nil}, + {"reset", "cpu 5 5 5 5", nil}, + {"unchanged", "cpu 10 10 10 10", nil}, + {"field-change", "cpu 20 20 20 20 20", nil}, + {"delta-overflow", "cpu 18446744073709551615 18446744073709551615 20 20", nil}, + } { + t.Run(tc.name, func(t *testing.T) { + var sampler hostCPUSampler + read := func(string) ([]byte, error) { return []byte("cpu 10 10 10 10"), nil } + sampler.sample(read) + if got := sampler.sample(func(string) ([]byte, error) { return []byte(tc.middle), tc.err }); got != nil { + t.Fatalf("invalid interval returned %g", *got) + } + if tc.err != nil || tc.name == "missing" || tc.name == "malformed" || tc.name == "negative" || tc.name == "overflow" { + if sampler.sample(read) != nil { + t.Fatal("missing evidence must break the interval") + } + } + }) + } +} + +func TestHostCPUCountersCannotDecreaseEvenIfTotalIncreases(t *testing.T) { + var sampler hostCPUSampler + sampler.sample(func(string) ([]byte, error) { return []byte("cpu 100 10 10 10 10"), nil }) + if got := sampler.sample(func(string) ([]byte, error) { return []byte("cpu 200 10 10 10 9"), nil }); got != nil { + t.Fatal("decreasing iowait counter was accepted") + } + if got := sampler.sample(func(string) ([]byte, error) { return []byte("cpu 210 10 10 10 9"), nil }); got == nil || *got != 1 { + t.Fatal("valid interval after reset did not recover") + } +} diff --git a/services/agents-api/internal/sandbox/node/health_linux.go b/services/agents-api/internal/sandbox/node/health_linux.go index 7c208bb24..c589c216b 100644 --- a/services/agents-api/internal/sandbox/node/health_linux.go +++ b/services/agents-api/internal/sandbox/node/health_linux.go @@ -3,34 +3,100 @@ package node import ( - "bufio" + "fmt" + "io" + "math" "os" "runtime" "strconv" "strings" "syscall" + + "golang.org/x/sys/unix" ) -func fillHostHealth(h *Health, dir string) { +const healthMaxFileBytes = 1024 * 1024 + +func readHealthFile(name string) ([]byte, error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, healthMaxFileBytes+1)) + if len(data) > healthMaxFileBytes { + return nil, fmt.Errorf("health observation exceeds size limit") + } + return data, err +} + +type hostHealthSampler struct { + cpu hostCPUSampler +} + +func (s *hostHealthSampler) fill(h *Health, dir string) { cpu := int64(runtime.NumCPU()) h.CPUCount = &cpu - if f, err := os.Open("/proc/meminfo"); err == nil { - defer f.Close() - scanner := bufio.NewScanner(f) - for scanner.Scan() { - fields := strings.Fields(scanner.Text()) - if len(fields) == 3 && fields[0] == "MemAvailable:" && fields[2] == "kB" { - if n, e := strconv.ParseInt(fields[1], 10, 64); e == nil && n >= 0 && n < (1<<63-1)/1024 { - n *= 1024 - h.AvailableMemoryBytes = &n - } - break - } - } + h.CPUUtilization = s.cpu.sample(readHealthFile) + // The kernel applies cpuset restrictions and online CPU changes to affinity. + // Use the process leader, not whichever Go thread happens to run this probe. + var affinity unix.CPUSet + cpus := 0 + if unix.SchedGetaffinity(os.Getpid(), &affinity) == nil { + cpus = affinity.Count() } + fillHostResources(h, readHealthFile, cpus) + h.AvailableDiskBytes = nil var disk syscall.Statfs_t - if syscall.Statfs(dir, &disk) == nil && disk.Bsize > 0 && disk.Bavail < uint64((1<<63-1)/disk.Bsize) { + if syscall.Statfs(dir, &disk) == nil && disk.Bsize > 0 && disk.Bavail <= uint64(math.MaxInt64/disk.Bsize) { available := int64(disk.Bavail) * disk.Bsize h.AvailableDiskBytes = &available } } + +func fillHostResources(h *Health, readFile func(string) ([]byte, error), cpus int) { + h.AvailableMemoryBytes, h.TotalMemoryBytes, h.EffectiveCPUCores = nil, nil, nil + if data, err := readFile("/proc/meminfo"); err == nil { + h.TotalMemoryBytes = meminfoBytes(string(data), "MemTotal:") + h.AvailableMemoryBytes = meminfoBytes(string(data), "MemAvailable:") + } + groups, err := readFile("/proc/self/cgroup") + if err != nil { + return + } + mounts, err := readFile("/proc/self/mountinfo") + if err != nil { + return + } + cpuGroup, cpuOK := findHealthCgroup(string(groups), string(mounts), "cpu") + if cpuOK && cpus > 0 { + if limit, ok := cpuGroup.cpuLimit(readFile); ok { + n := min(float64(cpus), limit) + h.EffectiveCPUCores = &n + } + } + // A migration during collection cannot establish one effective boundary. + if after, err := readFile("/proc/self/cgroup"); err != nil || string(after) != string(groups) { + h.EffectiveCPUCores = nil + } +} + +func meminfoBytes(data, name string) *int64 { + var result *int64 + for _, line := range strings.Split(data, "\n") { + fields := strings.Fields(line) + if len(fields) == 0 || fields[0] != name { + continue + } + if result != nil || len(fields) != 3 || fields[2] != "kB" { + return nil + } + n, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil || n < 0 || n > math.MaxInt64/1024 || name == "MemTotal:" && n == 0 { + return nil + } + n *= 1024 + result = &n + } + return result +} diff --git a/services/agents-api/internal/sandbox/node/health_linux_test.go b/services/agents-api/internal/sandbox/node/health_linux_test.go new file mode 100644 index 000000000..09ac37d05 --- /dev/null +++ b/services/agents-api/internal/sandbox/node/health_linux_test.go @@ -0,0 +1,275 @@ +//go:build linux + +package node + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" +) + +const gib int64 = 1024 * 1024 * 1024 + +func hostFixture(t *testing.T, files map[string]string) func(string) ([]byte, error) { + t.Helper() + root := t.TempDir() + for name, value := range files { + name = filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(name), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(name, []byte(value), 0600); err != nil { + t.Fatal(err) + } + } + return func(name string) ([]byte, error) { return os.ReadFile(filepath.Join(root, name)) } +} + +func v2HostFixture() map[string]string { + return map[string]string{ + "/proc/meminfo": "MemTotal: 8388608 kB\nMemAvailable: 3145728 kB\n", + "/proc/self/cgroup": "0::/parent/leaf\n", + "/proc/self/mountinfo": "35 25 0:30 / /sys/fs/cgroup rw - cgroup2 cgroup2 rw\n", + "/sys/fs/cgroup/cgroup.controllers": "cpu memory cpuset\n", + "/sys/fs/cgroup/parent/cpu.max": "150000 100000\n", + "/sys/fs/cgroup/parent/leaf/cpu.max": "max 100000\n", + } +} + +func TestHostResourcesV2(t *testing.T) { + for _, tc := range []struct { + name string + change func(map[string]string) + cpus int + want float64 + }{ + {"inherited", nil, 4, 1.5}, + {"leaf", func(f map[string]string) { f["/sys/fs/cgroup/parent/leaf/cpu.max"] = "25000 100000" }, 4, .25}, + {"affinity", nil, 1, 1}, + {"unlimited", func(f map[string]string) { f["/sys/fs/cgroup/parent/cpu.max"] = "max 100000" }, 4, 4}, + {"controller-disabled-at-leaf", func(f map[string]string) { + delete(f, "/sys/fs/cgroup/parent/leaf/cpu.max") + f["/sys/fs/cgroup/parent/leaf/cgroup.controllers"] = "" + }, 4, 1.5}, + {"root", func(f map[string]string) { f["/proc/self/cgroup"] = "0::/\n" }, 4, 4}, + } { + t.Run(tc.name, func(t *testing.T) { + files := v2HostFixture() + if tc.change != nil { + tc.change(files) + } + var h Health + fillHostResources(&h, hostFixture(t, files), tc.cpus) + if h.TotalMemoryBytes == nil || *h.TotalMemoryBytes != 8*gib || h.AvailableMemoryBytes == nil || *h.AvailableMemoryBytes != 3*gib { + t.Fatalf("host memory = %+v", h) + } + if h.EffectiveCPUCores == nil || *h.EffectiveCPUCores != tc.want { + t.Fatalf("effective CPU = %v, want %g", h.EffectiveCPUCores, tc.want) + } + }) + } +} + +func TestHostResourcesUnknownLimits(t *testing.T) { + for _, tc := range []struct { + name, file, value string + remove bool + }{ + {"membership-missing", "/proc/self/cgroup", "", true}, + {"membership-invalid", "/proc/self/cgroup", "0::/../bad", false}, + {"membership-invalid-id", "/proc/self/cgroup", "bad:pids:/parent/leaf", false}, + {"membership-duplicate", "/proc/self/cgroup", "0::/parent/leaf\n0::/", false}, + {"mount-missing", "/proc/self/mountinfo", "", true}, + {"hidden-ancestors", "/proc/self/mountinfo", "35 25 0:30 /parent /sys/fs/cgroup rw - cgroup2 cgroup2 rw", false}, + {"namespace-hidden-ancestors", "/sys/fs/cgroup/cgroup.type", "domain", false}, + {"cpu-missing", "/sys/fs/cgroup/parent/cpu.max", "", true}, + {"cpu-zero-period", "/sys/fs/cgroup/parent/cpu.max", "1000 0", false}, + {"cpu-negative-quota", "/sys/fs/cgroup/parent/cpu.max", "-1 100000", false}, + {"cpu-incomplete", "/sys/fs/cgroup/parent/cpu.max", "max", false}, + {"cpu-overflow", "/sys/fs/cgroup/parent/cpu.max", "99999999999999999999 100000", false}, + } { + t.Run(tc.name, func(t *testing.T) { + files := v2HostFixture() + if tc.remove { + delete(files, tc.file) + } else { + files[tc.file] = tc.value + } + stale := float64(100) + h := Health{EffectiveCPUCores: &stale} + fillHostResources(&h, hostFixture(t, files), 4) + if h.EffectiveCPUCores != nil { + t.Fatalf("unknown CPU limit accepted: %+v", h) + } + }) + } +} + +func TestHostResourcesV1AndHybrid(t *testing.T) { + files := map[string]string{ + "/proc/self/cgroup": "4:cpu,cpuacct:/parent/leaf\n0::/unified\n", + "/proc/self/mountinfo": "31 20 0:31 / /cgroup/cpu rw - cgroup cgroup rw,cpu,cpuacct\n", + "/cgroup/cpu/release_agent": "", + "/cgroup/cpu/cpu.cfs_quota_us": "-1", "/cgroup/cpu/cpu.cfs_period_us": "100000", + "/cgroup/cpu/parent/cpu.cfs_quota_us": "75000", "/cgroup/cpu/parent/cpu.cfs_period_us": "100000", + "/cgroup/cpu/parent/leaf/cpu.cfs_quota_us": "200000", "/cgroup/cpu/parent/leaf/cpu.cfs_period_us": "100000", + } + var h Health + fillHostResources(&h, hostFixture(t, files), 4) + if h.EffectiveCPUCores == nil || *h.EffectiveCPUCores != .75 { + t.Fatalf("v1 inherited limits: %+v", h) + } + files["/cgroup/cpu/parent/cpu.cfs_quota_us"], files["/cgroup/cpu/parent/leaf/cpu.cfs_quota_us"] = "-1", "-1" + fillHostResources(&h, hostFixture(t, files), 4) + if h.EffectiveCPUCores == nil || *h.EffectiveCPUCores != 4 { + t.Fatalf("v1 unlimited: %+v", h) + } +} + +func TestHostResourcesMissingAndChangingEvidence(t *testing.T) { + files := v2HostFixture() + delete(files, "/proc/meminfo") + var h Health + fillHostResources(&h, hostFixture(t, files), 0) + if h.TotalMemoryBytes != nil || h.AvailableMemoryBytes != nil || h.EffectiveCPUCores != nil { + t.Fatalf("missing base evidence: %+v", h) + } + read := hostFixture(t, v2HostFixture()) + for _, mode := range []string{"permission", "migration"} { + t.Run(mode, func(t *testing.T) { + calls := 0 + fillHostResources(&h, func(name string) ([]byte, error) { + if mode == "permission" && strings.HasSuffix(name, "cpu.max") { + return nil, os.ErrPermission + } + if name == "/proc/self/cgroup" { + calls++ + if mode == "migration" && calls > 1 { + return []byte("0::/new"), nil + } + } + return read(name) + }, 4) + if h.EffectiveCPUCores != nil { + t.Fatalf("unreliable evidence accepted: %+v", h) + } + }) + } +} + +func TestMeminfoBytes(t *testing.T) { + for _, value := range []string{"MemTotal: 0 kB", "MemTotal: -1 kB", "MemTotal: 9223372036854775807 kB", "MemTotal: 4 MB", "MemTotal: 4 kB\nMemTotal: 5 kB", "MemTotal: nope kB"} { + if meminfoBytes(value, "MemTotal:") != nil { + t.Fatalf("accepted %q", value) + } + } + if n := meminfoBytes("MemAvailable: 0 kB", "MemAvailable:"); n == nil || *n != 0 { + t.Fatal("zero available is valid") + } +} + +func TestHealthCgroupMountPaths(t *testing.T) { + g, ok := findHealthCgroup("0::/parent/leaf", `1 2 0:1 / /custom\040mount rw - cgroup2 cgroup2 rw`, "cpu") + if !ok || g.dir != "/custom mount/parent/leaf" || g.mount != "/custom mount" { + t.Fatalf("escaped mount: %+v %v", g, ok) + } + g, ok = findHealthCgroup("5:pids:/leaf", "", "cpu") + if !ok || g.dir != "" { + t.Fatalf("absent controller: %+v %v", g, ok) + } +} + +func TestHostHealthReadOnly(t *testing.T) { + dir := t.TempDir() + sampler := new(hostHealthSampler) + a := agent{config: AgentConfig{StateDirectory: dir, Probe: probe}} + before := time.Now().UTC() + h, err := a.health(context.Background(), sampler) + if err != nil || h.ObservedAt.Before(before) || h.ObservedAt.After(time.Now()) || h.ObservedAt.Location() != time.UTC { + t.Fatalf("invalid sample identity: %+v, %v", h, err) + } + if h.CPUUtilization != nil || h.TotalMemoryBytes == nil || *h.TotalMemoryBytes <= 0 || h.AvailableMemoryBytes == nil || *h.AvailableMemoryBytes < 0 || h.CPUCount == nil || *h.CPUCount <= 0 { + t.Fatalf("invalid initial host observation: %+v", h) + } + var disk syscall.Statfs_t + if err := syscall.Statfs(dir, &disk); err != nil { + t.Fatal(err) + } + if h.AvailableDiskBytes == nil || *h.AvailableDiskBytes < 0 || *h.AvailableDiskBytes > int64(disk.Blocks)*disk.Bsize { + t.Fatal("state filesystem free space unavailable or invalid") + } + if os.Getenv("PARSAR_NODE_HEALTH_PROBE") == "1" { + data, err := json.Marshal(h) + if err != nil { + t.Fatal(err) + } + t.Log(string(data)) + } + time.Sleep(100 * time.Millisecond) + h, err = a.health(context.Background(), sampler) + if err != nil || h.CPUUtilization == nil || *h.CPUUtilization < 0 || *h.CPUUtilization > 1 || !h.ObservedAt.After(before) { + t.Fatalf("invalid follow-up observation: %+v, %v", h, err) + } + if os.Getenv("PARSAR_NODE_HEALTH_PROBE") == "1" { + data, _ := json.Marshal(h) + t.Log(string(data)) + } + if fresh, _ := a.health(context.Background(), new(hostHealthSampler)); fresh.CPUUtilization != nil { + t.Fatal("new connection reused a CPU baseline") + } + sampler.fill(&h, filepath.Join(dir, "missing")) + if h.AvailableDiskBytes != nil { + t.Fatal("failed state filesystem must not use another disk") + } +} + +func TestHealthResourcesJSONNull(t *testing.T) { + data, err := json.Marshal(Health{}) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err = json.Unmarshal(data, &fields); err != nil { + t.Fatal(err) + } + for _, key := range []string{"total_memory_bytes", "cpu_utilization", "effective_cpu_cores"} { + if string(fields[key]) != "null" { + t.Fatalf("unknown %s = %s", key, fields[key]) + } + } +} + +func TestHealthReadBounds(t *testing.T) { + file := filepath.Join(t.TempDir(), "large") + if err := os.WriteFile(file, []byte(strings.Repeat("x", healthMaxFileBytes+1)), 0600); err != nil { + t.Fatal(err) + } + if _, err := readHealthFile(file); err == nil { + t.Fatal("oversized observation accepted") + } + g := healthCgroup{dir: "/cgroup" + strings.Repeat("/child", healthMaxCgroupDepth), mount: "/cgroup", v2: true} + reads := 0 + read := func(name string) ([]byte, error) { + reads++ + switch filepath.Base(name) { + case "cgroup.type": + return nil, os.ErrNotExist + case "cpu.max": + return []byte("max 100000"), nil + default: + t.Fatalf("unexpected read: %s", name) + return nil, os.ErrNotExist + } + } + if _, ok := g.cpuLimit(read); ok { + t.Fatal("incomplete CPU ancestry accepted") + } + if reads > (healthMaxCgroupDepth + 1) { + t.Fatalf("unbounded ancestry reads: %d", reads) + } +} diff --git a/services/agents-api/internal/sandbox/node/health_other.go b/services/agents-api/internal/sandbox/node/health_other.go index 48856dc1c..f25554c2a 100644 --- a/services/agents-api/internal/sandbox/node/health_other.go +++ b/services/agents-api/internal/sandbox/node/health_other.go @@ -2,4 +2,8 @@ package node -func fillHostHealth(*Health, string) {} +type hostHealthSampler struct{} + +func (*hostHealthSampler) fill(h *Health, _ string) { + h.CPUUtilization, h.TotalMemoryBytes, h.EffectiveCPUCores = nil, nil, nil +} diff --git a/services/agents-api/internal/sandbox/node/wire.go b/services/agents-api/internal/sandbox/node/wire.go index b221f74a3..0526e53b5 100644 --- a/services/agents-api/internal/sandbox/node/wire.go +++ b/services/agents-api/internal/sandbox/node/wire.go @@ -41,6 +41,9 @@ type Health struct { ObservedAt time.Time `json:"observed_at"` ActiveOperations int `json:"active_operations"` CPUCount *int64 `json:"cpu_count,omitempty"` + CPUUtilization *float64 `json:"cpu_utilization"` + TotalMemoryBytes *int64 `json:"total_memory_bytes"` + EffectiveCPUCores *float64 `json:"effective_cpu_cores"` AvailableMemoryBytes *int64 `json:"available_memory_bytes,omitempty"` AvailableDiskBytes *int64 `json:"available_disk_bytes,omitempty"` } From 8d853c19e7ed475a618a7e9e317368e795fd3a19 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Fri, 25 Sep 2026 17:09:46 +0800 Subject: [PATCH 3/4] Expose administrator node host history from existing runtime sampling --- CONTRIBUTING.md | 13 ++ contracts/agents-api/node-host-history.md | 69 ++++++++ .../agents-api/sandbox-manager.openapi.yaml | 164 +++++++++++++++++ docs/api/README.md | 3 + docs/api/web-management.md | 7 + services/agents-api/cmd/server/main.go | 7 +- .../agents-api/cmd/server/managed_nodes.go | 2 +- .../internal/api/sandbox_manager.go | 1 + .../internal/api/sandbox_node_detail.go | 41 +++++ .../internal/api/sandbox_node_detail_test.go | 30 ++++ .../internal/db/queries/node_host_history.sql | 35 ++++ .../internal/db/queries/runtime_nodes.sql | 3 +- .../agents-api/internal/db/sqlc/models.go | 8 + .../internal/db/sqlc/node_host_history.sql.go | 117 ++++++++++++ .../internal/db/sqlc/runtime_nodes.sql.go | 7 +- .../runtimehistory/postgresreader/reader.go | 3 +- .../internal/store/node_host_history.go | 132 ++++++++++++++ .../internal/store/node_host_history_test.go | 167 ++++++++++++++++++ .../internal/store/runtime_history.go | 7 +- .../internal/store/runtime_node_presence.go | 5 +- .../internal/store/runtime_node_types.go | 11 +- .../internal/store/runtime_nodes.go | 8 +- .../internal/store/runtime_observation.go | 2 +- .../internal/store/runtime_placements.go | 4 +- .../migrations/000071_node_host_history.sql | 13 ++ services/core-console/sandbox_admin.go | 2 +- services/core-console/sandbox_admin_test.go | 4 +- 27 files changed, 843 insertions(+), 22 deletions(-) create mode 100644 contracts/agents-api/node-host-history.md create mode 100644 services/agents-api/internal/api/sandbox_node_detail.go create mode 100644 services/agents-api/internal/api/sandbox_node_detail_test.go create mode 100644 services/agents-api/internal/db/queries/node_host_history.sql create mode 100644 services/agents-api/internal/db/sqlc/node_host_history.sql.go create mode 100644 services/agents-api/internal/store/node_host_history.go create mode 100644 services/agents-api/internal/store/node_host_history_test.go create mode 100644 services/agents-api/migrations/000071_node_host_history.sql diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1ac61ca68..fefcb808e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -218,6 +218,19 @@ response; never record request/response bodies or infer this count from every service or storage system is required. Keep the frontend response shape aligned with the paired console contract. +Core process CPU, RSS and cgroup limits are sampled by the existing 30-second +Core metrics loop; Go heap and goroutine reads retain their meaning. Process +series use the same bounded ring and complete buckets, with null first CPU +intervals and restart gaps. Do not substitute host usage for process usage. + +Administrator node detail adds host observations and history as documented in +[node-host-history.md](contracts/agents-api/node-host-history.md). Keep the node +list unchanged. Reuse authenticated heartbeat ownership, the Runtime sampling +sweep and PostgreSQL retention cleanup; node observations have their own table +because they do not belong to a Project, Session or Environment. History is +best-effort telemetry, never scheduling truth. No read-triggered sampling or +offline backfill is allowed. + ## Architecture boundaries The following execution rules are retained from the source contributor guide. diff --git a/contracts/agents-api/node-host-history.md b/contracts/agents-api/node-host-history.md new file mode 100644 index 000000000..3cd5c7fcd --- /dev/null +++ b/contracts/agents-api/node-host-history.md @@ -0,0 +1,69 @@ +# Administrator node host observations + +`GET /core/v1/sandbox/nodes/{node_id}?range=1h|6h|24h` requires deployment +administrator authentication. Core Web forwards it through its authenticated +server; Project, enrollment and node credentials do not authorize this read. +The omitted range defaults to `1h`. Invalid, repeated or unknown query parameters +return the existing invalid-input response; missing or removed nodes return 404. + +The response contains the existing node fields plus these two objects. The node +list response is unchanged. This endpoint does not alter enrollment, administrator +capacity approval, scheduling or any public Agents API contract. + +```json +{ + "host": { + "effective_cpu_cores": 4, + "cpu_utilization": 0.35, + "total_memory_bytes": 17179869184, + "available_memory_bytes": 8589934592, + "available_disk_bytes": 107374182400, + "observed_at": "2026-09-25T09:00:00Z" + }, + "history": { + "resolution_seconds": 60, + "points": [{ + "start": "2026-09-25T08:59:00Z", + "cpu_utilization_max": 0.4, + "memory_used_bytes_max": 8589934592, + "available_disk_bytes_min": 107374182400 + }] + } +} +``` + +Every unavailable measurement is `null`, including unobserved timestamps. `host` +is the last received observation; an offline node retains its original timestamp +and last-known measurements. Clients must use `online` and `observed_at` when +presenting freshness. Reading the endpoint never samples or writes history. + +On Linux, CPU utilization is the increase in aggregate busy `/proc/stat` ticks +divided by total ticks between heartbeats, across the whole visible host. Idle +and I/O-wait ticks are not busy; guest counters are not counted twice. The first +observation and a reset or unavailable baseline have null utilization. This is +not node-process CPU or the sum of sandbox utilization. Total memory and available +memory come from `MemTotal` and `MemAvailable`. Effective CPU capacity accounts for +observable process affinity and cgroup limits; it is null if those limits cannot +be established, rather than an invented scheduling limit. Available disk refers +to the node's existing state filesystem, not a sandbox quota. The node should run +on the machine being monitored; namespace visibility limits what it can observe. + +The existing Runtime sampling sweep copies fresh, authenticated node heartbeats +into `node_host_history_samples` in the same PostgreSQL database. It uses the +Runtime history cadence (30 seconds by default) and seven-day retention cleanup. +A node observation is keyed by node and its original timestamp, so repeated sweeps +do not manufacture additional observations. Disconnected, stale, future-dated or +removed observations are not copied. The sampler is best effort with a bounded +query timeout; failures do not change execution or admission authority. + +Ranges use complete UTC buckets: `1h` has 60-second buckets, `6h` has 300-second +buckets and `24h` has 900-second buckets. Each response includes every bucket in +the range. CPU and memory are maxima of recorded observations; used memory is +`total_memory_bytes - available_memory_bytes` from the same observation. Disk is +the minimum recorded available space. Missing measurements and offline buckets +stay null, independently for each metric. The active partial bucket is excluded. +No interpolation, offline backfill or whole-host process inventory is performed. + +History survives Core and node restarts until normal retention expiry. Core +process history remains a separate in-memory series and resets when Core restarts. +Neither series contains credentials, model data, paths or user content. diff --git a/contracts/agents-api/sandbox-manager.openapi.yaml b/contracts/agents-api/sandbox-manager.openapi.yaml index 7ce59abe6..60e3574ab 100644 --- a/contracts/agents-api/sandbox-manager.openapi.yaml +++ b/contracts/agents-api/sandbox-manager.openapi.yaml @@ -304,12 +304,39 @@ definitions: type: object coremetrics.Process: properties: + cpu_cores: + type: number + x-nullable: true + cpu_limit_cores: + type: number + x-nullable: true goroutines: type: integer x-nullable: true memory_bytes: type: integer x-nullable: true + memory_limit_bytes: + type: integer + x-nullable: true + rss_bytes: + type: integer + x-nullable: true + series: + items: + $ref: '#/definitions/coremetrics.ProcessBucket' + type: array + type: object + coremetrics.ProcessBucket: + properties: + cpu_cores: + type: number + x-nullable: true + rss_bytes: + type: integer + x-nullable: true + start: + type: string type: object coremetrics.Range: properties: @@ -516,6 +543,29 @@ definitions: state: type: string type: object + store.NodeHostHistory: + properties: + points: + items: + $ref: '#/definitions/store.NodeHostHistoryPoint' + type: array + resolution_seconds: + type: integer + type: object + store.NodeHostHistoryPoint: + properties: + available_disk_bytes_min: + type: integer + x-nullable: true + cpu_utilization_max: + type: number + x-nullable: true + memory_used_bytes_max: + type: integer + x-nullable: true + start: + type: string + type: object store.Project: properties: active_key_count: @@ -681,6 +731,51 @@ definitions: specification_digest: type: string type: object + store.RuntimeNodeDetail: + properties: + active: + type: integer + available_disk_bytes: + type: integer + available_memory_bytes: + type: integer + cleanup_pending: + type: integer + cpu_count: + type: integer + created_at: + type: string + diagnostic: + type: string + history: + $ref: '#/definitions/store.NodeHostHistory' + host: + $ref: '#/definitions/store.RuntimeNodeHost' + id: + type: string + last_seen_at: + type: string + max_active: + type: integer + max_retained: + type: integer + name: + type: string + online: + type: boolean + provider: + type: string + provider_ready: + type: boolean + reserved: + type: integer + retained: + type: integer + running: + type: integer + snapshots: + type: integer + type: object store.RuntimeNodeEnrollment: properties: backend_fingerprint: @@ -698,6 +793,27 @@ definitions: specification_digest: type: string type: object + store.RuntimeNodeHost: + properties: + available_disk_bytes: + type: integer + x-nullable: true + available_memory_bytes: + type: integer + x-nullable: true + cpu_utilization: + type: number + x-nullable: true + effective_cpu_cores: + type: number + x-nullable: true + observed_at: + type: string + x-nullable: true + total_memory_bytes: + type: integer + x-nullable: true + type: object store.RuntimeNodeIdentity: properties: deployment_generation: @@ -5859,6 +5975,54 @@ paths: summary: Remove a sandbox node with no retained resources tags: - Sandbox Manager + get: + description: Deployment administrator only. Complete UTC buckets. Missing host measurements and offline history are null; reads never sample or backfill. + parameters: + - description: Sandbox node UUID + in: path + name: node_id + required: true + type: string + - description: Time range (default 1h) + enum: + - 1h + - 6h + - 24h + in: query + name: range + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/store.RuntimeNodeDetail' + "400": + description: Bad Request + schema: + $ref: '#/definitions/v1.ErrorResponse' + "401": + description: Unauthorized + schema: + $ref: '#/definitions/v1.ErrorResponse' + "404": + description: Not Found + schema: + $ref: '#/definitions/v1.ErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/v1.ErrorResponse' + "503": + description: Service Unavailable + schema: + $ref: '#/definitions/v1.ErrorResponse' + security: + - DeploymentAdminAuth: [] + summary: Retrieve sandbox node and host history + tags: + - Sandbox Manager patch: consumes: - application/json diff --git a/docs/api/README.md b/docs/api/README.md index def125d84..eaa5c9821 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -42,3 +42,6 @@ Generated schemas do not establish complete compatibility or real execution support. The [coverage record](../../contracts/agents-api/README.md) identifies qualified workflows, native differences and unresolved behavior. Update the relevant contract and this index when adding or moving an API surface. + +Node host observations and retained charts are deployment-administrator reads; +see [node host history](../../contracts/agents-api/node-host-history.md). diff --git a/docs/api/web-management.md b/docs/api/web-management.md index 4ca733e2b..bd0f2e7f4 100644 --- a/docs/api/web-management.md +++ b/docs/api/web-management.md @@ -124,3 +124,10 @@ Agent execution access. See the [exact measurement contract](../../contracts/age for complete buckets, null values, units and process-local retention. Frontend implementation is maintained separately; this backend change does not modify Agent metrics or the public Agent API. + +## Node host history + +Deployment administrators can read a node and its host history through +`GET /core/v1/sandbox/nodes/{node_id}?range=1h|6h|24h`. See the +[node host history contract](../../contracts/agents-api/node-host-history.md) +for nullable observations, freshness and aggregation. The node list is unchanged. diff --git a/services/agents-api/cmd/server/main.go b/services/agents-api/cmd/server/main.go index 17ee15c17..c0920d4ee 100644 --- a/services/agents-api/cmd/server/main.go +++ b/services/agents-api/cmd/server/main.go @@ -244,7 +244,12 @@ func run() error { sampler, err := runtimeobs.NewSampler(observationResolver, observationService, worker, runtimeobs.SamplerOptions{ Interval: history.SampleInterval, Report: func(result runtimeobs.SweepResult) { - var sampleErr error + sampleCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + sampleErr := worker.CheckOwnership(sampleCtx) + if sampleErr == nil { + _, sampleErr = executionStore.SampleNodeHostHistory(sampleCtx) + } + cancel() if !result.Complete { sampleErr = errors.New("incomplete Runtime sampling sweep") } diff --git a/services/agents-api/cmd/server/managed_nodes.go b/services/agents-api/cmd/server/managed_nodes.go index a42b66a6f..10c629d06 100644 --- a/services/agents-api/cmd/server/managed_nodes.go +++ b/services/agents-api/cmd/server/managed_nodes.go @@ -78,7 +78,7 @@ func configureManagedNodes(s *store.Store, owner func(context.Context) error) (* if err := owner(ctx); err != nil { return err } - return s.HeartbeatRuntimeNode(ctx, n.NodeID, connection, epoch, store.RuntimeNodeHealth{ProviderReady: health.ProviderReady, Diagnostic: health.Diagnostic, CPUCount: health.CPUCount, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes}) + return s.HeartbeatRuntimeNode(ctx, n.NodeID, connection, epoch, store.RuntimeNodeHealth{Host: &store.RuntimeNodeHost{EffectiveCPUCores: health.EffectiveCPUCores, CPUUtilization: health.CPUUtilization, TotalMemoryBytes: health.TotalMemoryBytes, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes, ObservedAt: &health.ObservedAt}, ProviderReady: health.ProviderReady, Diagnostic: health.Diagnostic, CPUCount: health.CPUCount, AvailableMemoryBytes: health.AvailableMemoryBytes, AvailableDiskBytes: health.AvailableDiskBytes}) }, }) result.setup = &managedSetup{store: s, hub: result.hub, installationID: setupID} diff --git a/services/agents-api/internal/api/sandbox_manager.go b/services/agents-api/internal/api/sandbox_manager.go index ffb6cf27d..9c87c6875 100644 --- a/services/agents-api/internal/api/sandbox_manager.go +++ b/services/agents-api/internal/api/sandbox_manager.go @@ -48,6 +48,7 @@ func (h *Handler) registerSandboxManagerRoutes(r chi.Router) { r.Put("/deployment", h.updateSandboxDeployment) r.Patch("/deployment/maintenance", h.setSandboxMaintenance) r.Get("/nodes", h.sandboxNodes) + r.Get("/nodes/{node_id}", h.sandboxNodeDetail) r.Patch("/nodes/{node_id}", h.updateSandboxNode) r.Delete("/nodes/{node_id}", h.removeSandboxNode) r.Get("/nodes/{node_id}/allocations", h.sandboxAllocations) diff --git a/services/agents-api/internal/api/sandbox_node_detail.go b/services/agents-api/internal/api/sandbox_node_detail.go new file mode 100644 index 000000000..c84df767c --- /dev/null +++ b/services/agents-api/internal/api/sandbox_node_detail.go @@ -0,0 +1,41 @@ +package api + +import ( + "context" + "net/http" + "net/url" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/go-chi/chi/v5" +) + +// @Summary Retrieve sandbox node and host history +// @Description Deployment administrator only. Complete UTC buckets. Missing host measurements and offline history are null; reads never sample or backfill. +// @Tags Sandbox Manager +// @Produce json +// @Security DeploymentAdminAuth +// @Param node_id path string true "Sandbox node UUID" +// @Param range query string false "Time range (default 1h)" Enums(1h,6h,24h) +// @Success 200 {object} store.RuntimeNodeDetail +// @Failure 400,401,404,500,503 {object} v1.ErrorResponse +// @Router /core/v1/sandbox/nodes/{node_id} [get] +func (h *Handler) sandboxNodeDetail(w http.ResponseWriter, r *http.Request) { + values, err := url.ParseQuery(r.URL.RawQuery) + if err != nil || len(values) > 1 || len(values) == 1 && len(values["range"]) != 1 { + writeStoreError(w, r, store.ErrInvalidInput) + return + } + name := "1h" + if v, ok := values["range"]; ok { + name = v[0] + } + ctx, cancel := context.WithTimeout(r.Context(), 3*time.Second) + defer cancel() + value, err := h.sandboxStore.GetRuntimeNodeDetail(ctx, chi.URLParam(r, "node_id"), name) + if err != nil { + writeStoreError(w, r, err) + return + } + writeJSON(w, http.StatusOK, value) +} diff --git a/services/agents-api/internal/api/sandbox_node_detail_test.go b/services/agents-api/internal/api/sandbox_node_detail_test.go new file mode 100644 index 000000000..bcad20c2b --- /dev/null +++ b/services/agents-api/internal/api/sandbox_node_detail_test.go @@ -0,0 +1,30 @@ +package api + +import ( + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "testing" +) + +func TestSandboxNodeDetailValidationAndAuthentication(t *testing.T) { + auth, _ := NewAuthenticator([]APIKey{callerBinding()}) + admin, _ := NewDeploymentAuthenticator([]string{device.HashCredential("administrator")}) + h, err := NewHandler(&recordingStore{}, auth, "codex", WithSandboxManager(&store.Store{}, admin)) + if err != nil { + t.Fatal(err) + } + path := "/core/v1/sandbox/nodes/11111111-1111-4111-8111-111111111111" + for _, token := range []string{"", "caller", "enrollment", "node"} { + if w := projectKeyHTTP(h, "GET", path, token, ""); w.Code != 401 { + t.Fatal(token, w.Code) + } + } + for _, query := range []string{"?range=", "?range=7d", "?range=1h&range=6h", "?tenant_id=x", "?range=1h;bad=1", "?range=%xx"} { + if w := projectKeyHTTP(h, "GET", path+query, "administrator", ""); w.Code != 400 { + t.Fatal(query, w.Code) + } + } + if w := projectKeyHTTP(h, "GET", "/core/v1/sandbox/nodes/invalid", "administrator", ""); w.Code != 400 { + t.Fatal(w.Code) + } +} diff --git a/services/agents-api/internal/db/queries/node_host_history.sql b/services/agents-api/internal/db/queries/node_host_history.sql new file mode 100644 index 000000000..2893ce697 --- /dev/null +++ b/services/agents-api/internal/db/queries/node_host_history.sql @@ -0,0 +1,35 @@ +-- name: SampleNodeHostHistory :execrows +INSERT INTO node_host_history_samples (node_id, observed_at, cpu_utilization, memory_used_bytes, available_disk_bytes) +SELECT n.id, (n.health->'host'->>'observed_at')::timestamptz, + (n.health->'host'->>'cpu_utilization')::double precision, + CASE WHEN (n.health->'host'->>'total_memory_bytes')::bigint >= (n.health->'host'->>'available_memory_bytes')::bigint + THEN (n.health->'host'->>'total_memory_bytes')::bigint - (n.health->'host'->>'available_memory_bytes')::bigint END, + (n.health->'host'->>'available_disk_bytes')::bigint +FROM runtime_nodes n CROSS JOIN runtime_deployment d +WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id + AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch + AND n.last_seen_at > clock_timestamp()-interval '45 seconds' + AND (n.health->'host'->>'observed_at')::timestamptz > clock_timestamp()-interval '45 seconds' + AND (n.health->'host'->>'observed_at')::timestamptz <= clock_timestamp() +ON CONFLICT (node_id, observed_at) DO NOTHING; + +-- name: ListNodeHostHistory :many +SELECT date_bin(sqlc.arg(bucket_width)::interval, observed_at, '1970-01-01T00:00:00Z'::timestamptz)::timestamptz AS start, + count(cpu_utilization)::bigint AS cpu_samples, + COALESCE(max(cpu_utilization), 0)::double precision AS cpu_utilization_max, + count(memory_used_bytes)::bigint AS memory_samples, + COALESCE(max(memory_used_bytes), 0)::bigint AS memory_used_bytes_max, + count(available_disk_bytes)::bigint AS disk_samples, + COALESCE(min(available_disk_bytes), 0)::bigint AS available_disk_bytes_min +FROM node_host_history_samples +WHERE node_id=sqlc.arg(node_id) AND observed_at >= sqlc.arg(start_at)::timestamptz AND observed_at < sqlc.arg(end_at)::timestamptz +GROUP BY 1 ORDER BY 1; + +-- name: PruneNodeHostHistory :execrows +WITH expired AS ( + SELECT node_id, observed_at FROM node_host_history_samples + WHERE observed_at < sqlc.arg(before_at)::timestamptz + ORDER BY observed_at LIMIT 256 FOR UPDATE SKIP LOCKED +) +DELETE FROM node_host_history_samples h USING expired e +WHERE h.node_id=e.node_id AND h.observed_at=e.observed_at; diff --git a/services/agents-api/internal/db/queries/runtime_nodes.sql b/services/agents-api/internal/db/queries/runtime_nodes.sql index 06773291a..ec0fa27df 100644 --- a/services/agents-api/internal/db/queries/runtime_nodes.sql +++ b/services/agents-api/internal/db/queries/runtime_nodes.sql @@ -21,7 +21,8 @@ SELECT n.*, (n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch AND (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='running' AND a.compute_phase IN('running','disabled'))::bigint AS running, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'snapshot' IS NOT NULL AND a.compute_state->'snapshot'<>'null'::jsonb)::bigint AS snapshots FROM runtime_nodes n CROSS JOIN runtime_deployment d -WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id ORDER BY n.id; +WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id +AND (sqlc.narg(node_id)::uuid IS NULL OR n.id=sqlc.narg(node_id)::uuid) ORDER BY n.id; -- name: UpdateRuntimeNode :one UPDATE runtime_nodes SET name=$2,max_active=$3,max_retained=$4 WHERE id=$1 AND removed_at IS NULL RETURNING *; diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index 5a555b545..7b3087e92 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -167,6 +167,14 @@ type InitialEnvironmentFile struct { Contents []byte `json:"contents"` } +type NodeHostHistorySample struct { + NodeID pgtype.UUID `json:"node_id"` + ObservedAt pgtype.Timestamptz `json:"observed_at"` + CpuUtilization pgtype.Float8 `json:"cpu_utilization"` + MemoryUsedBytes pgtype.Int8 `json:"memory_used_bytes"` + AvailableDiskBytes pgtype.Int8 `json:"available_disk_bytes"` +} + type Project struct { ID pgtype.UUID `json:"id"` Name string `json:"name"` diff --git a/services/agents-api/internal/db/sqlc/node_host_history.sql.go b/services/agents-api/internal/db/sqlc/node_host_history.sql.go new file mode 100644 index 000000000..caeb62635 --- /dev/null +++ b/services/agents-api/internal/db/sqlc/node_host_history.sql.go @@ -0,0 +1,117 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: node_host_history.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const listNodeHostHistory = `-- name: ListNodeHostHistory :many +SELECT date_bin($1::interval, observed_at, '1970-01-01T00:00:00Z'::timestamptz)::timestamptz AS start, + count(cpu_utilization)::bigint AS cpu_samples, + COALESCE(max(cpu_utilization), 0)::double precision AS cpu_utilization_max, + count(memory_used_bytes)::bigint AS memory_samples, + COALESCE(max(memory_used_bytes), 0)::bigint AS memory_used_bytes_max, + count(available_disk_bytes)::bigint AS disk_samples, + COALESCE(min(available_disk_bytes), 0)::bigint AS available_disk_bytes_min +FROM node_host_history_samples +WHERE node_id=$2 AND observed_at >= $3::timestamptz AND observed_at < $4::timestamptz +GROUP BY 1 ORDER BY 1 +` + +type ListNodeHostHistoryParams struct { + BucketWidth pgtype.Interval `json:"bucket_width"` + NodeID pgtype.UUID `json:"node_id"` + StartAt pgtype.Timestamptz `json:"start_at"` + EndAt pgtype.Timestamptz `json:"end_at"` +} + +type ListNodeHostHistoryRow struct { + Start pgtype.Timestamptz `json:"start"` + CpuSamples int64 `json:"cpu_samples"` + CpuUtilizationMax float64 `json:"cpu_utilization_max"` + MemorySamples int64 `json:"memory_samples"` + MemoryUsedBytesMax int64 `json:"memory_used_bytes_max"` + DiskSamples int64 `json:"disk_samples"` + AvailableDiskBytesMin int64 `json:"available_disk_bytes_min"` +} + +func (q *Queries) ListNodeHostHistory(ctx context.Context, arg ListNodeHostHistoryParams) ([]ListNodeHostHistoryRow, error) { + rows, err := q.db.Query(ctx, listNodeHostHistory, + arg.BucketWidth, + arg.NodeID, + arg.StartAt, + arg.EndAt, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ListNodeHostHistoryRow{} + for rows.Next() { + var i ListNodeHostHistoryRow + if err := rows.Scan( + &i.Start, + &i.CpuSamples, + &i.CpuUtilizationMax, + &i.MemorySamples, + &i.MemoryUsedBytesMax, + &i.DiskSamples, + &i.AvailableDiskBytesMin, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const pruneNodeHostHistory = `-- name: PruneNodeHostHistory :execrows +WITH expired AS ( + SELECT node_id, observed_at FROM node_host_history_samples + WHERE observed_at < $1::timestamptz + ORDER BY observed_at LIMIT 256 FOR UPDATE SKIP LOCKED +) +DELETE FROM node_host_history_samples h USING expired e +WHERE h.node_id=e.node_id AND h.observed_at=e.observed_at +` + +func (q *Queries) PruneNodeHostHistory(ctx context.Context, beforeAt pgtype.Timestamptz) (int64, error) { + result, err := q.db.Exec(ctx, pruneNodeHostHistory, beforeAt) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const sampleNodeHostHistory = `-- name: SampleNodeHostHistory :execrows +INSERT INTO node_host_history_samples (node_id, observed_at, cpu_utilization, memory_used_bytes, available_disk_bytes) +SELECT n.id, (n.health->'host'->>'observed_at')::timestamptz, + (n.health->'host'->>'cpu_utilization')::double precision, + CASE WHEN (n.health->'host'->>'total_memory_bytes')::bigint >= (n.health->'host'->>'available_memory_bytes')::bigint + THEN (n.health->'host'->>'total_memory_bytes')::bigint - (n.health->'host'->>'available_memory_bytes')::bigint END, + (n.health->'host'->>'available_disk_bytes')::bigint +FROM runtime_nodes n CROSS JOIN runtime_deployment d +WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id + AND n.connection_id IS NOT NULL AND n.connected_epoch=d.owner_epoch + AND n.last_seen_at > clock_timestamp()-interval '45 seconds' + AND (n.health->'host'->>'observed_at')::timestamptz > clock_timestamp()-interval '45 seconds' + AND (n.health->'host'->>'observed_at')::timestamptz <= clock_timestamp() +ON CONFLICT (node_id, observed_at) DO NOTHING +` + +func (q *Queries) SampleNodeHostHistory(ctx context.Context) (int64, error) { + result, err := q.db.Exec(ctx, sampleNodeHostHistory) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go index 15c9e1633..93873c344 100644 --- a/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/agents-api/internal/db/sqlc/runtime_nodes.sql.go @@ -464,7 +464,8 @@ SELECT n.id, n.installation_id, n.name, n.backend_fingerprint, n.credential_sha2 (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state='running' AND a.compute_phase IN('running','disabled'))::bigint AS running, (SELECT count(*) FROM runtime_allocations a WHERE a.node_id=n.id AND a.state<>'released' AND a.compute_state->'snapshot' IS NOT NULL AND a.compute_state->'snapshot'<>'null'::jsonb)::bigint AS snapshots FROM runtime_nodes n CROSS JOIN runtime_deployment d -WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id ORDER BY n.id +WHERE n.removed_at IS NULL AND n.installation_id=d.installation_id +AND ($1::uuid IS NULL OR n.id=$1::uuid) ORDER BY n.id ` type ListRuntimeNodesRow struct { @@ -494,8 +495,8 @@ type ListRuntimeNodesRow struct { Snapshots int64 `json:"snapshots"` } -func (q *Queries) ListRuntimeNodes(ctx context.Context) ([]ListRuntimeNodesRow, error) { - rows, err := q.db.Query(ctx, listRuntimeNodes) +func (q *Queries) ListRuntimeNodes(ctx context.Context, nodeID pgtype.UUID) ([]ListRuntimeNodesRow, error) { + rows, err := q.db.Query(ctx, listRuntimeNodes, nodeID) if err != nil { return nil, err } diff --git a/services/agents-api/internal/runtimehistory/postgresreader/reader.go b/services/agents-api/internal/runtimehistory/postgresreader/reader.go index 54ffbe3b1..88dbd26c4 100644 --- a/services/agents-api/internal/runtimehistory/postgresreader/reader.go +++ b/services/agents-api/internal/runtimehistory/postgresreader/reader.go @@ -115,7 +115,8 @@ func (r *Reader) Query(ctx context.Context, query runtimehistory.Query) (runtime } // Prune expires old telemetry even when no Runtime is being sampled. Each call -// deletes at most 4,096 rows in small lock-skipping batches under one deadline. +// deletes at most 8,192 Runtime and node-host rows in small lock-skipping batches +// under one deadline. func (r *Reader) Prune(ctx context.Context) (int64, error) { ctx, cancel := context.WithTimeout(ctx, r.queryTimeout) defer cancel() diff --git a/services/agents-api/internal/store/node_host_history.go b/services/agents-api/internal/store/node_host_history.go new file mode 100644 index 000000000..048c37aa4 --- /dev/null +++ b/services/agents-api/internal/store/node_host_history.go @@ -0,0 +1,132 @@ +package store + +import ( + "context" + "encoding/json" + "math" + "time" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/coremetrics" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/jackc/pgx/v5/pgtype" +) + +// Host observations are deployment telemetry, not sandbox capacity authority. +type RuntimeNodeHost struct { + EffectiveCPUCores *float64 `json:"effective_cpu_cores" extensions:"x-nullable"` + CPUUtilization *float64 `json:"cpu_utilization" extensions:"x-nullable"` + TotalMemoryBytes *int64 `json:"total_memory_bytes" extensions:"x-nullable"` + AvailableMemoryBytes *int64 `json:"available_memory_bytes" extensions:"x-nullable"` + AvailableDiskBytes *int64 `json:"available_disk_bytes" extensions:"x-nullable"` + ObservedAt *time.Time `json:"observed_at" extensions:"x-nullable"` +} +type runtimeNodeHealthRecord struct { + RuntimeNodeHealth + Host *RuntimeNodeHost `json:"host,omitempty"` +} +type NodeHostHistoryPoint struct { + Start time.Time `json:"start"` + CPUUtilizationMax *float64 `json:"cpu_utilization_max" extensions:"x-nullable"` + MemoryUsedBytesMax *int64 `json:"memory_used_bytes_max" extensions:"x-nullable"` + AvailableDiskBytesMin *int64 `json:"available_disk_bytes_min" extensions:"x-nullable"` +} +type NodeHostHistory struct { + ResolutionSeconds int64 `json:"resolution_seconds"` + Points []NodeHostHistoryPoint `json:"points"` +} +type RuntimeNodeDetail struct { + RuntimeNode + Host RuntimeNodeHost `json:"host"` + History NodeHostHistory `json:"history"` +} + +func validateRuntimeNodeHost(host *RuntimeNodeHost) error { + if host == nil { + return nil + } + if host.ObservedAt == nil || host.ObservedAt.IsZero() || host.ObservedAt.Year() < 1970 || host.ObservedAt.Year() > 9999 { + return ErrInvalidInput + } + for _, value := range []*float64{host.CPUUtilization, host.EffectiveCPUCores} { + if value != nil && (math.IsNaN(*value) || math.IsInf(*value, 0) || *value < 0) { + return ErrInvalidInput + } + } + if host.CPUUtilization != nil && *host.CPUUtilization > 1 || host.EffectiveCPUCores != nil && *host.EffectiveCPUCores == 0 { + return ErrInvalidInput + } + for _, value := range []*int64{host.TotalMemoryBytes, host.AvailableMemoryBytes, host.AvailableDiskBytes} { + if value != nil && (*value < 0 || *value > 1<<53-1) { + return ErrInvalidInput + } + } + if host.TotalMemoryBytes != nil && (*host.TotalMemoryBytes == 0 || host.AvailableMemoryBytes != nil && *host.AvailableMemoryBytes > *host.TotalMemoryBytes) { + return ErrInvalidInput + } + return nil +} + +// SampleNodeHostHistory runs with the existing Runtime sampler cadence. Only a +// fresh authenticated heartbeat is copied; neither reads nor offline nodes fill gaps. +func (s *Store) SampleNodeHostHistory(ctx context.Context) (int64, error) { + return s.queries.SampleNodeHostHistory(ctx) +} + +func (s *Store) GetRuntimeNodeDetail(ctx context.Context, id, name string) (RuntimeNodeDetail, error) { + nodeID, err := parseConnectionGeneration(id) + if err != nil { + return RuntimeNodeDetail{}, ErrInvalidInput + } + if name != "1h" && name != "6h" && name != "24h" { + return RuntimeNodeDetail{}, ErrInvalidInput + } + window, _ := coremetrics.Window(time.Now(), name) + rows, err := s.queries.ListRuntimeNodes(ctx, nodeID) + if err != nil { + return RuntimeNodeDetail{}, err + } + if len(rows) == 0 { + return RuntimeNodeDetail{}, ErrNotFound + } + nodes, err := runtimeNodeViews(rows) + if err != nil { + return RuntimeNodeDetail{}, err + } + var health runtimeNodeHealthRecord + if err := json.Unmarshal(rows[0].Health, &health); err != nil { + return RuntimeNodeDetail{}, err + } + value := RuntimeNodeDetail{RuntimeNode: nodes[0], History: NodeHostHistory{ResolutionSeconds: window.ResolutionSeconds, Points: make([]NodeHostHistoryPoint, 0)}} + if health.Host != nil { + value.Host = *health.Host + } + samples, err := s.queries.ListNodeHostHistory(ctx, sqlc.ListNodeHostHistoryParams{ + NodeID: nodeID, StartAt: pgtype.Timestamptz{Time: window.Start, Valid: true}, EndAt: pgtype.Timestamptz{Time: window.End, Valid: true}, + BucketWidth: pgtype.Interval{Microseconds: window.ResolutionSeconds * 1_000_000, Valid: true}, + }) + if err != nil { + return RuntimeNodeDetail{}, err + } + byStart := make(map[time.Time]NodeHostHistoryPoint, len(samples)) + for _, sample := range samples { + point := NodeHostHistoryPoint{Start: sample.Start.Time.UTC()} + if sample.CpuSamples > 0 { + point.CPUUtilizationMax = &sample.CpuUtilizationMax + } + if sample.MemorySamples > 0 { + point.MemoryUsedBytesMax = &sample.MemoryUsedBytesMax + } + if sample.DiskSamples > 0 { + point.AvailableDiskBytesMin = &sample.AvailableDiskBytesMin + } + byStart[point.Start] = point + } + for start := window.Start; start.Before(window.End); start = start.Add(time.Duration(window.ResolutionSeconds) * time.Second) { + point, ok := byStart[start] + if !ok { + point.Start = start + } + value.History.Points = append(value.History.Points, point) + } + return value, nil +} diff --git a/services/agents-api/internal/store/node_host_history_test.go b/services/agents-api/internal/store/node_host_history_test.go new file mode 100644 index 000000000..27ff4487f --- /dev/null +++ b/services/agents-api/internal/store/node_host_history_test.go @@ -0,0 +1,167 @@ +package store + +import ( + "encoding/json" + "errors" + "math" + "testing" + "time" + + "github.com/google/uuid" +) + +func hostPtr[T any](value T) *T { return &value } + +func TestNodeHostHistorySamplingAndDetail(t *testing.T) { + s, _, d := managerFixture(t, 2, 8) + connection := onlineManagerNode(t, s, d.LocalNodeID) + epoch := managerEpoch(t, s) + now := time.Now().UTC() + host := &RuntimeNodeHost{EffectiveCPUCores: hostPtr(2.0), CPUUtilization: hostPtr(0.35), TotalMemoryBytes: hostPtr(int64(4096)), AvailableMemoryBytes: hostPtr(int64(1024)), AvailableDiskBytes: hostPtr(int64(8192)), ObservedAt: &now} + health := RuntimeNodeHealth{ProviderReady: true, Host: host} + if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, connection, epoch, health); err != nil { + t.Fatal(err) + } + for i, want := range []int64{1, 0} { + if n, err := s.SampleNodeHostHistory(t.Context()); err != nil || n != want { + t.Fatal(i, n, err) + } + } + detail, err := s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, "1h") + if err != nil || detail.Host.TotalMemoryBytes == nil || *detail.Host.CPUUtilization != 0.35 || len(detail.History.Points) != 60 { + t.Fatal(detail, err) + } + // The current partial minute does not enter history yet. + for _, p := range detail.History.Points { + if p.CPUUtilizationMax != nil { + t.Fatal("partial bucket leaked", p) + } + } + list, err := s.ListRuntimeNodes(t.Context()) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(list[0]) + var fields map[string]json.RawMessage + _ = json.Unmarshal(raw, &fields) + for _, key := range []string{"host", "history", "cpu_utilization", "total_memory_bytes", "effective_cpu_cores"} { + if _, exists := fields[key]; exists { + t.Fatal("list contract expanded", key, string(raw)) + } + } + // Sample maxima/minima differ from the latest reading. + end := time.Now().UTC().Truncate(time.Minute) + start := end.Add(-time.Minute) + for _, sample := range []struct { + at time.Time + cpu any + mem any + disk any + }{ + {start.Add(time.Second), 0.1, int64(100), int64(800)}, + {start.Add(20 * time.Second), 0.8, int64(70), int64(900)}, + {start.Add(40 * time.Second), nil, int64(500), int64(600)}, + {start.Add(-time.Minute), nil, nil, nil}, + } { + if _, err := s.pool.Exec(t.Context(), "INSERT INTO node_host_history_samples(node_id, observed_at,cpu_utilization,memory_used_bytes,available_disk_bytes) VALUES($1,$2,$3,$4,$5)", d.LocalNodeID, sample.at, sample.cpu, sample.mem, sample.disk); err != nil { + t.Fatal(err) + } + } + detail, err = s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, "1h") + if err != nil { + t.Fatal(err) + } + last := detail.History.Points[len(detail.History.Points)-1] + if !last.Start.Equal(start) || last.CPUUtilizationMax == nil || *last.CPUUtilizationMax != 0.8 || *last.MemoryUsedBytesMax != 500 || *last.AvailableDiskBytesMin != 600 { + t.Fatal(last) + } + previous := detail.History.Points[len(detail.History.Points)-2] + if previous.CPUUtilizationMax != nil || previous.MemoryUsedBytesMax != nil || previous.AvailableDiskBytesMin != nil { + t.Fatal(previous) + } + for name, want := range map[string]int{"6h": 72, "24h": 96} { + detail, err := s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, name) + if err != nil || len(detail.History.Points) != want { + t.Fatal(name, detail, err) + } + } + if _, err := s.GetRuntimeNodeDetail(t.Context(), uuid.NewString(), "1h"); !errors.Is(err, ErrNotFound) { + t.Fatal(err) + } + for _, name := range []string{"", "7d", "other"} { + if _, err := s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, name); !errors.Is(err, ErrInvalidInput) { + t.Fatal(err) + } + } + // A disconnected node cannot create another history row, even with a fresh last observation. + next := now.Add(time.Millisecond) + host.ObservedAt = &next + if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, connection, epoch, health); err != nil { + t.Fatal(err) + } + if err := s.DisconnectRuntimeNode(t.Context(), d.LocalNodeID, connection, epoch); err != nil { + t.Fatal(err) + } + if n, err := s.SampleNodeHostHistory(t.Context()); err != nil || n != 0 { + t.Fatal(n, err) + } + // A restarted service can query existing durable history without re-sampling. + fresh := New(s.pool) + restored, err := fresh.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, "1h") + if err != nil || restored.Online || restored.History.Points[59].CPUUtilizationMax == nil { + t.Fatal(restored, err) + } + if !restored.Host.ObservedAt.Equal(next) { + t.Fatal(restored.Host) + } + // Retention uses the same existing cleanup operation and does not delete nodes. + old := now.Add(-8 * 24 * time.Hour) + if _, err := s.pool.Exec(t.Context(), "INSERT INTO node_host_history_samples(node_id, observed_at) VALUES($1,$2)", d.LocalNodeID, old); err != nil { + t.Fatal(err) + } + if n, err := s.PruneRuntimeHistorySamples(t.Context(), now.Add(-7*24*time.Hour).UnixNano()); err != nil || n != 1 { + t.Fatal(n, err) + } + if _, err := s.GetRuntimeNodeDetail(t.Context(), d.LocalNodeID, "1h"); err != nil { + t.Fatal(err) + } +} + +func TestNodeHostHistoryFencingAndUnknown(t *testing.T) { + s, _, d := managerFixture(t, 2, 8) + conn := onlineManagerNode(t, s, d.LocalNodeID) + epoch := managerEpoch(t, s) + for _, at := range []time.Time{time.Now().Add(-time.Minute), time.Now().Add(time.Hour)} { + if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, conn, epoch, RuntimeNodeHealth{Host: &RuntimeNodeHost{ObservedAt: &at}}); err != nil { + t.Fatal(err) + } + if n, err := s.SampleNodeHostHistory(t.Context()); err != nil || n != 0 { + t.Fatal(n, err) + } + } + now := time.Now().UTC() + health := RuntimeNodeHealth{Host: &RuntimeNodeHost{ObservedAt: &now}} + if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, uuid.NewString(), epoch, health); !errors.Is(err, ErrRuntimeNodeCredential) { + t.Fatal(err) + } + if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, conn, epoch, health); err != nil { + t.Fatal(err) + } + if n, err := s.SampleNodeHostHistory(t.Context()); err != nil || n != 1 { + t.Fatal(n, err) + } + var cpu *float64 + var memory, disk *int64 + if err := s.pool.QueryRow(t.Context(), "SELECT cpu_utilization,memory_used_bytes,available_disk_bytes FROM node_host_history_samples WHERE node_id=$1", d.LocalNodeID).Scan(&cpu, &memory, &disk); err != nil || cpu != nil || memory != nil || disk != nil { + t.Fatal(cpu, memory, disk, err) + } + for _, host := range []*RuntimeNodeHost{ + {ObservedAt: &now, CPUUtilization: hostPtr(1.1)}, {ObservedAt: &now, CPUUtilization: hostPtr(math.NaN())}, + {ObservedAt: &now, TotalMemoryBytes: hostPtr(int64(10)), AvailableMemoryBytes: hostPtr(int64(11))}, + {ObservedAt: &now, AvailableDiskBytes: hostPtr(int64(-1))}, {ObservedAt: &now, EffectiveCPUCores: hostPtr(0.0)}, {}, + } { + if err := s.HeartbeatRuntimeNode(t.Context(), d.LocalNodeID, conn, epoch, RuntimeNodeHealth{Host: host}); !errors.Is(err, ErrInvalidInput) { + t.Fatal(host, err) + } + } +} diff --git a/services/agents-api/internal/store/runtime_history.go b/services/agents-api/internal/store/runtime_history.go index 57d646b12..258688625 100644 --- a/services/agents-api/internal/store/runtime_history.go +++ b/services/agents-api/internal/store/runtime_history.go @@ -102,7 +102,12 @@ func (s *Store) ListRuntimeHistorySamples(ctx context.Context, tenantID, session } func (s *Store) PruneRuntimeHistorySamples(ctx context.Context, beforeNS int64) (int64, error) { - return s.queries.PruneRuntimeHistorySamples(ctx, beforeNS) + count, err := s.queries.PruneRuntimeHistorySamples(ctx, beforeNS) + if err != nil { + return count, err + } + nodes, err := s.queries.PruneNodeHostHistory(ctx, pgtype.Timestamptz{Time: time.Unix(0, beforeNS), Valid: true}) + return count + nodes, err } func historyFloat(value *float64) pgtype.Float8 { diff --git a/services/agents-api/internal/store/runtime_node_presence.go b/services/agents-api/internal/store/runtime_node_presence.go index cdf4d19df..ab811bd3f 100644 --- a/services/agents-api/internal/store/runtime_node_presence.go +++ b/services/agents-api/internal/store/runtime_node_presence.go @@ -48,7 +48,10 @@ func (s *Store) HeartbeatRuntimeNode(ctx context.Context, nodeID, connectionID s return ErrInvalidInput } } - raw, err := json.Marshal(health) + if err := validateRuntimeNodeHost(health.Host); err != nil { + return err + } + raw, err := json.Marshal(runtimeNodeHealthRecord{RuntimeNodeHealth: health, Host: health.Host}) if err != nil { return err } diff --git a/services/agents-api/internal/store/runtime_node_types.go b/services/agents-api/internal/store/runtime_node_types.go index 4a3651950..80db2ada1 100644 --- a/services/agents-api/internal/store/runtime_node_types.go +++ b/services/agents-api/internal/store/runtime_node_types.go @@ -38,11 +38,12 @@ type RuntimeNodeEnrollment struct { BackendFingerprint string `json:"backend_fingerprint"` } type RuntimeNodeHealth struct { - Diagnostic string `json:"diagnostic,omitempty"` - ProviderReady bool `json:"provider_ready"` - CPUCount *int64 `json:"cpu_count"` - AvailableMemoryBytes *int64 `json:"available_memory_bytes"` - AvailableDiskBytes *int64 `json:"available_disk_bytes"` + Host *RuntimeNodeHost `json:"-"` + Diagnostic string `json:"diagnostic,omitempty"` + ProviderReady bool `json:"provider_ready"` + CPUCount *int64 `json:"cpu_count"` + AvailableMemoryBytes *int64 `json:"available_memory_bytes"` + AvailableDiskBytes *int64 `json:"available_disk_bytes"` } type RuntimeNode struct { RuntimeNodeHealth diff --git a/services/agents-api/internal/store/runtime_nodes.go b/services/agents-api/internal/store/runtime_nodes.go index e8fee68cd..0a753b205 100644 --- a/services/agents-api/internal/store/runtime_nodes.go +++ b/services/agents-api/internal/store/runtime_nodes.go @@ -70,10 +70,14 @@ func runtimeUUID(id pgtype.UUID) string { return uuid.UUID(id.Bytes).String() } func (s *Store) ListRuntimeNodes(ctx context.Context) ([]RuntimeNode, error) { - rows, err := s.queries.ListRuntimeNodes(ctx) + rows, err := s.queries.ListRuntimeNodes(ctx, pgtype.UUID{}) if err != nil { return nil, err } + return runtimeNodeViews(rows) +} + +func runtimeNodeViews(rows []sqlc.ListRuntimeNodesRow) ([]RuntimeNode, error) { out := make([]RuntimeNode, 0, len(rows)) for _, n := range rows { var seen *time.Time @@ -221,7 +225,7 @@ func (s *Store) RemoveRuntimeNode(ctx context.Context, nodeID string) error { return err } return s.runtimeManagerTransaction(ctx, func(q *sqlc.Queries, d sqlc.RuntimeDeployment) error { - rows, err := q.ListRuntimeNodes(ctx) + rows, err := q.ListRuntimeNodes(ctx, pgtype.UUID{}) if err != nil { return err } diff --git a/services/agents-api/internal/store/runtime_observation.go b/services/agents-api/internal/store/runtime_observation.go index 1192d67b0..0df01ed63 100644 --- a/services/agents-api/internal/store/runtime_observation.go +++ b/services/agents-api/internal/store/runtime_observation.go @@ -35,7 +35,7 @@ func (s *Store) RuntimeNodeAvailable(ctx context.Context, node string) (bool, er return runtimeNodeAvailable(ctx, s.queries, id) } func runtimeNodeAvailable(ctx context.Context, q *sqlc.Queries, node pgtype.UUID) (bool, error) { - nodes, err := q.ListRuntimeNodes(ctx) + nodes, err := q.ListRuntimeNodes(ctx, pgtype.UUID{}) if err != nil { return false, err } diff --git a/services/agents-api/internal/store/runtime_placements.go b/services/agents-api/internal/store/runtime_placements.go index e5f3fcff9..2c6b92192 100644 --- a/services/agents-api/internal/store/runtime_placements.go +++ b/services/agents-api/internal/store/runtime_placements.go @@ -34,7 +34,7 @@ func reserveRuntimePlacement(ctx context.Context, q *sqlc.Queries, session pgtyp if d.Maintenance { return ErrRuntimeNodeUnavailable } - rows, err := q.ListRuntimeNodes(ctx) + rows, err := q.ListRuntimeNodes(ctx, pgtype.UUID{}) if err != nil { return err } @@ -97,7 +97,7 @@ func reserveRuntimeRestore(ctx context.Context, q *sqlc.Queries, node pgtype.UUI if _, err := q.LockRuntimeDeployment(ctx); err != nil { return err } - nodes, err := q.ListRuntimeNodes(ctx) + nodes, err := q.ListRuntimeNodes(ctx, pgtype.UUID{}) if err != nil { return err } diff --git a/services/agents-api/migrations/000071_node_host_history.sql b/services/agents-api/migrations/000071_node_host_history.sql new file mode 100644 index 000000000..cbdfe96ca --- /dev/null +++ b/services/agents-api/migrations/000071_node_host_history.sql @@ -0,0 +1,13 @@ +-- +goose Up +CREATE TABLE node_host_history_samples ( + node_id uuid NOT NULL REFERENCES runtime_nodes(id), + observed_at timestamptz NOT NULL, + cpu_utilization double precision CHECK (cpu_utilization >= 0 AND cpu_utilization <= 1), + memory_used_bytes bigint CHECK (memory_used_bytes >= 0), + available_disk_bytes bigint CHECK (available_disk_bytes >= 0), + PRIMARY KEY (node_id, observed_at) +); +CREATE INDEX node_host_history_retention ON node_host_history_samples(observed_at); + +-- +goose Down +DROP TABLE node_host_history_samples; diff --git a/services/core-console/sandbox_admin.go b/services/core-console/sandbox_admin.go index b83c7c7fc..d0ccff43a 100644 --- a/services/core-console/sandbox_admin.go +++ b/services/core-console/sandbox_admin.go @@ -32,7 +32,7 @@ func sandboxAdminRequest(r *http.Request) bool { } if len(parts) >= 2 && parts[0] == "nodes" && parts[1] != "" { if len(parts) == 2 { - return r.Method == http.MethodPatch || r.Method == http.MethodDelete + return r.Method == http.MethodGet || r.Method == http.MethodPatch || r.Method == http.MethodDelete } return len(parts) == 3 && parts[2] == "allocations" && r.Method == http.MethodGet } diff --git a/services/core-console/sandbox_admin_test.go b/services/core-console/sandbox_admin_test.go index 2f6c74a2d..4497ecc96 100644 --- a/services/core-console/sandbox_admin_test.go +++ b/services/core-console/sandbox_admin_test.go @@ -27,7 +27,7 @@ func TestSandboxAdminUsesAuthenticatedConsoleAndServerCredential(t *testing.T) { } w.WriteHeader(204) })) - for _, tc := range []struct{ method, path string }{{"GET", "/core/v1/sandbox/deployment"}, {"GET", "/core/v1/sandbox/nodes?limit=5"}, {"PATCH", "/core/v1/sandbox/nodes/node"}, {"DELETE", "/core/v1/sandbox/nodes/node"}, {"GET", "/core/v1/sandbox/nodes/node/allocations"}, {"POST", "/core/v1/sandbox/enrollment-tokens"}} { + for _, tc := range []struct{ method, path string }{{"GET", "/core/v1/sandbox/deployment"}, {"GET", "/core/v1/sandbox/nodes?limit=5"}, {"GET", "/core/v1/sandbox/nodes/node?range=1h"}, {"PATCH", "/core/v1/sandbox/nodes/node"}, {"DELETE", "/core/v1/sandbox/nodes/node"}, {"GET", "/core/v1/sandbox/nodes/node/allocations"}, {"POST", "/core/v1/sandbox/enrollment-tokens"}} { req := consoleRequest(t, server, tc.method, tc.path) req.Header.Set("X-Core-Console-Actor", "spoofed") response, _ := responseBody(t, server, req) @@ -41,7 +41,7 @@ func TestSandboxAdminUsesAuthenticatedConsoleAndServerCredential(t *testing.T) { t.Fatal("bearer bypassed console login") } } - if calls.Load() != 6 { + if calls.Load() != 7 { t.Fatal("unexpected sandbox proxy calls") } } From 7192e3f1a326a7c71086ff30c60b08ba8fbf131b Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Fri, 25 Sep 2026 17:16:45 +0800 Subject: [PATCH 4/4] Use process CPU fallback for an unbounded cgroup root --- contracts/agents-api/core-metrics.md | 2 +- docs/web/core-process-metrics-requirements.md | 3 +- .../internal/coremetrics/process_files.go | 22 +++++++-- .../coremetrics/process_files_test.go | 47 ++++++++++++++++++- 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/contracts/agents-api/core-metrics.md b/contracts/agents-api/core-metrics.md index e768b1d64..756635267 100644 --- a/contracts/agents-api/core-metrics.md +++ b/contracts/agents-api/core-metrics.md @@ -87,7 +87,7 @@ present. longer than 60 seconds reset the baseline; they never manufacture a zero. - `process.rss_bytes` is Linux `/proc/self/status` `VmRSS`, converted from KiB to bytes. `cpu_limit_cores` is the process's cgroup v2 `cpu.max` quota/period, - or `GOMAXPROCS` when its quota is `max`. `memory_limit_bytes` is that cgroup's + or `GOMAXPROCS` when its quota is `max` or the actual cgroup root has no quota interface. `memory_limit_bytes` is that cgroup's finite `memory.max`; `max` is null. Membership and mount information resolve the process's cgroup, including nested and subtree mounts. These are that cgroup's configured limits, not whole-host metrics or ancestor-limit discovery. diff --git a/docs/web/core-process-metrics-requirements.md b/docs/web/core-process-metrics-requirements.md index 35a6f6710..6cc86b46e 100644 --- a/docs/web/core-process-metrics-requirements.md +++ b/docs/web/core-process-metrics-requirements.md @@ -47,7 +47,8 @@ independent readings, so a missing CPU interval does not hide measured memory. Linux resolves cgroup v2 membership using `/proc/self/cgroup` and `/proc/self/mountinfo`; it reads the process's own cgroup rather than assuming -the mount root. Limits describe that cgroup's configuration; ancestor-limit +the mount root. The actual cgroup v2 root has no quota interface and uses +`GOMAXPROCS`; an unreadable interface remains unknown. Limits describe that cgroup's configuration; ancestor-limit discovery is outside this contract. Unreadable and malformed values stay null. All procfs and cgroup reads have byte bounds. Non-Linux builds return null for CPU usage, RSS and memory limit, with `GOMAXPROCS` as the CPU capacity fallback. diff --git a/services/agents-api/internal/coremetrics/process_files.go b/services/agents-api/internal/coremetrics/process_files.go index 389810f02..ef8ecdc9f 100644 --- a/services/agents-api/internal/coremetrics/process_files.go +++ b/services/agents-api/internal/coremetrics/process_files.go @@ -11,9 +11,19 @@ import ( func readProcessFiles(procSelf string, cpus int) processReading { reading := processReading{rssBytes: parseRSS(readBounded(filepath.Join(procSelf, "status"), 64<<10))} - group := cgroupDirectory(readBounded(filepath.Join(procSelf, "cgroup"), 64<<10), readBounded(filepath.Join(procSelf, "mountinfo"), 1<<20)) + group, root := cgroupDirectory(readBounded(filepath.Join(procSelf, "cgroup"), 64<<10), readBounded(filepath.Join(procSelf, "mountinfo"), 1<<20)) if group != "" { reading.cpuLimitCores = parseCPULimit(readBounded(filepath.Join(group, "cpu.max"), 4096), cpus) + // The actual v2 root has neither cpu.max nor cgroup.type. A namespace + // root backed by a non-root cgroup still has cgroup.type. + if root && reading.cpuLimitCores == nil && cpus > 0 { + _, quotaErr := os.Stat(filepath.Join(group, "cpu.max")) + _, typeErr := os.Stat(filepath.Join(group, "cgroup.type")) + _, controllerErr := os.Stat(filepath.Join(group, "cgroup.controllers")) + if os.IsNotExist(quotaErr) && os.IsNotExist(typeErr) && controllerErr == nil { + reading.cpuLimitCores = ptr(float64(cpus)) + } + } reading.memoryLimitBytes = parseMemoryLimit(readBounded(filepath.Join(group, "memory.max"), 4096)) } return reading @@ -83,21 +93,22 @@ func parseMemoryLimit(data string) *uint64 { // Resolve membership against the mounted cgroup v2 subtree. Do not assume that // the process belongs to the root or that cgroupfs is mounted at /sys/fs/cgroup. -func cgroupDirectory(membership, mountinfo string) string { +func cgroupDirectory(membership, mountinfo string) (string, bool) { group := "" for _, line := range strings.Split(membership, "\n") { if strings.HasPrefix(line, "0::") { if group != "" { - return "" + return "", false } group = strings.TrimPrefix(line, "0::") } } if !canonicalAbsolute(group) { - return "" + return "", false } decode := strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) selected, rootLength := "", -1 + actualRootCandidate := false for _, line := range strings.Split(mountinfo, "\n") { left, right, ok := strings.Cut(line, " - ") fields, fs := strings.Fields(left), strings.Fields(right) @@ -120,9 +131,10 @@ func cgroupDirectory(membership, mountinfo string) string { } if len(root) > rootLength { selected, rootLength = filepath.Join(point, relative), len(root) + actualRootCandidate = group == "/" && root == "/" } } - return selected + return selected, actualRootCandidate } func canonicalAbsolute(path string) bool { diff --git a/services/agents-api/internal/coremetrics/process_files_test.go b/services/agents-api/internal/coremetrics/process_files_test.go index f6c67ba0b..1eaa7c33f 100644 --- a/services/agents-api/internal/coremetrics/process_files_test.go +++ b/services/agents-api/internal/coremetrics/process_files_test.go @@ -55,7 +55,7 @@ func TestProcessCgroupDirectory(t *testing.T) { {"specific mount", "0::/work/core\n", "1 0 0:1 / /sys/fs/cgroup rw - cgroup2 cgroup rw\n2 0 0:1 /work /run/cgroup rw - cgroup2 cgroup rw\n", "/run/cgroup/core"}, } { t.Run(test.name, func(t *testing.T) { - if got := cgroupDirectory(test.membership, test.mountinfo); got != test.want { + if got, _ := cgroupDirectory(test.membership, test.mountinfo); got != test.want { t.Fatalf("got %q, want %q", got, test.want) } }) @@ -103,3 +103,48 @@ func TestProcessLinuxFiles(t *testing.T) { t.Fatal("missing or oversized reads must be unknown", got) } } + +func TestProcessRootCgroupWithoutQuota(t *testing.T) { + dir := t.TempDir() + group := filepath.Join(dir, "cgroup") + if err := os.Mkdir(group, 0700); err != nil { + t.Fatal(err) + } + write := func(path, data string) { + t.Helper() + if err := os.WriteFile(path, []byte(data), 0600); err != nil { + t.Fatal(err) + } + } + proc := filepath.Join(dir, "proc") + if err := os.Mkdir(proc, 0700); err != nil { + t.Fatal(err) + } + write(filepath.Join(proc, "cgroup"), "0::/\n") + write(filepath.Join(proc, "mountinfo"), "1 0 0:1 / "+group+" rw - cgroup2 cgroup rw\n") + write(filepath.Join(group, "cgroup.controllers"), "cpu memory\n") + got := readProcessFiles(proc, 3) + if got.cpuLimitCores == nil || *got.cpuLimitCores != 3 || got.memoryLimitBytes != nil { + t.Fatal("real root must use GOMAXPROCS", got) + } + write(filepath.Join(group, "cgroup.type"), "domain\n") + if got := readProcessFiles(proc, 3); got.cpuLimitCores != nil { + t.Fatal("namespace root missing quota must stay unknown", got) + } + if err := os.Remove(filepath.Join(group, "cgroup.type")); err != nil { + t.Fatal(err) + } + write(filepath.Join(group, "cpu.max"), "invalid\n") + if got := readProcessFiles(proc, 3); got.cpuLimitCores != nil { + t.Fatal("invalid quota became unlimited", got) + } + if err := os.Remove(filepath.Join(group, "cpu.max")); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(group, "cpu.max"), 0700); err != nil { + t.Fatal(err) + } + if got := readProcessFiles(proc, 3); got.cpuLimitCores != nil { + t.Fatal("failed read became unlimited", got) + } +}