Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .config/nextest.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ telemetry-contract = { max-threads = 1 }
# parallel load the file never appears (#5355). Serialize them; do not drop
# the tests.
exec-persistent-service = { max-threads = 1 }
# Fleet manager lifecycle tests start real `/bin/sh` workers under setsid and
# wait 10-15 s for the child's first line. Under full macOS CI load that line
# never ran for three overlapping tests (#6424, #6428 runs), while each
# passed alone from the same binary in under 1.2 s. Serialize the module; do
# not lengthen its deadlines or drop tests. The product side of the load
# (a durable heartbeat and a `ps` sample on every scheduler tick) is now
# bounded to once a second per worker.
fleet-manager-lifecycle = { max-threads = 1 }

# First matching test-group override wins. Keep these more-specific
# integration filters before binary(integration) so they are not stolen
Expand All @@ -56,6 +64,10 @@ test-group = 'telemetry-contract'
filter = 'binary(telemetry_kill_switch_dispatch)'
test-group = 'telemetry-contract'

[[profile.default.overrides]]
filter = 'package(codewhale-tui) & kind(lib) & test(/^fleet::manager::tests::/)'
test-group = 'fleet-manager-lifecycle'

[[profile.default.overrides]]
filter = 'binary(integration) & test(/^exec_persistent_service::/)'
test-group = 'exec-persistent-service'
Expand Down
13 changes: 12 additions & 1 deletion crates/tui/src/fleet/host.rs
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,14 @@ struct LocalWorkerProcess {
stopped: bool,
last_exit: Option<ExitStatus>,
last_memory_mb: Option<u64>,
/// When `ps` last sampled this worker. Status polls can run every few
/// milliseconds; memory is display data, so one sample a second is ample.
last_memory_sample: Option<std::time::Instant>,
}

/// Minimum spacing between `ps` memory samples for one worker.
const MEMORY_SAMPLE_INTERVAL: Duration = Duration::from_secs(1);

impl LocalProcessFleetHostAdapter {
pub fn new(workspace: impl AsRef<Path>) -> Self {
Self {
Expand Down Expand Up @@ -303,6 +309,7 @@ impl LocalProcessFleetHostAdapter {
stopped: false,
last_exit: None,
last_memory_mb: None,
last_memory_sample: None,
},
);
Ok(handle)
Expand Down Expand Up @@ -356,7 +363,11 @@ impl FleetHostAdapter for LocalProcessFleetHostAdapter {
match process.child.try_wait() {
Ok(None) => {
let pid = process.child.id();
let memory_mb = if process.host_kind == FleetHostKind::LocalProcess {
let due = process
.last_memory_sample
.is_none_or(|sampled| sampled.elapsed() >= MEMORY_SAMPLE_INTERVAL);
let memory_mb = if process.host_kind == FleetHostKind::LocalProcess && due {
process.last_memory_sample = Some(std::time::Instant::now());
sample_process_memory_mb(pid)
} else {
None
Expand Down
11 changes: 10 additions & 1 deletion crates/tui/src/fleet/manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -608,11 +608,20 @@ impl FleetManager {
.ok_or_else(|| anyhow!("Fleet run {} does not exist", run_id.0))?;
let worker_ids = worker_ids_for_run(&run, max_workers);

// Heartbeats are durable ledger appends (a full-drive flush on
// macOS). Timestamps have whole-second resolution and the stale window
// is minutes, so a worker already stamped this second needs no second
// record; a fast driver tick must not turn into a flush storm.
let now = timestamp();
for task in active_tasks_for_run(&state, run_id) {
if let Some(worker_id) = task.leased_to.as_deref()
&& worker_ids.iter().any(|id| id == worker_id)
&& state
.heartbeats
.get(worker_id)
.is_none_or(|heartbeat| heartbeat.timestamp != now)
{
self.ledger.heartbeat(worker_id, &timestamp(), None, None)?;
self.ledger.heartbeat(worker_id, &now, None, None)?;
report.heartbeats += 1;
}
}
Expand Down
Loading