supervisord.test.ts

import { expect } from 'vitest'
import { sandboxTest, wait } from './setup'

async function waitForHealth(sandbox: any, maxRetries = 10, intervalMs = 100) {
  for (let i = 0; i < maxRetries; i++) {
    try {
      const result = await sandbox.commands.run(
        'curl -s -o /dev/null -w "%{http_code}" http://0.0.0.0:49999/health'
      )
      if (result.stdout.trim() === '200') {
        return true
      }
    } catch {
      // Connection refused or other error, retry
    }
    await wait(intervalMs)
  }
  return false
}

sandboxTest('restart after jupyter kill', async ({ sandbox }) => {
  // Verify health is up initially
  const initialHealth = await waitForHealth(sandbox)
  expect(initialHealth).toBe(true)

  // Kill the jupyter process as root
  // The command handle may get killed too (since killing jupyter cascades to code-interpreter),
  // so we catch the error.
  try {
    await sandbox.commands.run("kill -9 $(pgrep -f 'jupyter server')", {
      user: 'root',
    })
  } catch {
    // Expected — the kill cascade may terminate the command handle
  }

  // Wait for supervisord to restart both services (jupyter startup + code-interpreter startup)
  const recovered = await waitForHealth(sandbox, 60, 500)
  expect(recovered).toBe(true)

  // Verify code execution works after recovery
  const result = await sandbox.runCode('x = 1; x')
  expect(result.text).toEqual('1')
})

sandboxTest('restart after jupyter kernel kill', async ({ sandbox }) => {
  // Verify code execution works initially
  const code = await sandbox.runCode('x = 42; x')
  expect(code.text).toEqual('42')

  // Kill all jupyter kernel processes as root
  try {
    await sandbox.commands.run("kill -9 $(pgrep -f 'ipykernel_launcher')", {
      user: 'root',
    })
  } catch {
    // Expected — the kill may terminate the command handle
  }

  // First call after kill returns context restarted error
  const code2 = await sandbox.runCode('x')
  expect(code2.error!.value).toEqual('Context was restarted')

  // Subsequent call works normally
  const code3 = await sandbox.runCode('y = 1; y')
  expect(code3.text).toEqual('1')
})

sandboxTest('restart after code-interpreter kill', async ({ sandbox }) => {
  // Verify health is up initially
  const initialHealth = await waitForHealth(sandbox)
  expect(initialHealth).toBe(true)

  // Kill the code-interpreter process as root
  try {
    await sandbox.commands.run('kill -9 $(cat /var/run/code-interpreter.pid)', {
      user: 'root',
    })
  } catch {
    // Expected — killing code-interpreter may terminate the command handle
  }

  // Wait for supervisord to restart it and health to come back
  const recovered = await waitForHealth(sandbox, 60, 500)
  expect(recovered).toBe(true)

  // Verify code execution works after recovery
  const result = await sandbox.runCode('x = 1; x')
  expect(result.text).toEqual('1')
})