diff --git a/core/http/react-ui/e2e/backends-installed-node-removal.spec.js b/core/http/react-ui/e2e/backends-installed-node-removal.spec.js new file mode 100644 index 000000000000..b6bc50a79220 --- /dev/null +++ b/core/http/react-ui/e2e/backends-installed-node-removal.spec.js @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: MIT +import { test, expect } from './coverage-fixtures.js' + +const BUILD = 'cpu-example-engine-development' +const SIBLING = 'cpu-example-engine' +const ids = ['worker-alpha', 'worker-beta'] +const remove = (page, id) => page.getByRole('button', { name: `Remove from ${id}…`, exact: true }) + +async function setup(page, options = {}) { + const state = { deletes: [], installs: [], inventoryCalls: {}, installedCalls: 0, failRefresh: false, ...options } + const nodes = ids.map(id => ({ id, name: id, node_type: 'backend', status: 'healthy', address: `${id}.example:50051` })) + const inventories = Object.fromEntries(ids.map(id => [id, [{ name: BUILD, is_system: false, is_meta: false }, { name: SIBLING, is_system: false, is_meta: false }]])) + state.nodes = nodes + state.inventories = inventories + await page.route('**/api/nodes', route => route.fulfill({ json: state.nodes })) + await page.route('**/api/backends?*', route => state.failRefresh + ? route.fulfill({ status: 500, json: { error: 'catalog unavailable' } }) + : route.fulfill({ json: { backends: [BUILD, SIBLING].map(name => ({ id: name, name, installed: true, isAlias: true, isDevelopment: name === BUILD, nodes: ids.map(id => ({ node_id: id, node_name: id })) })) } })) + await page.route('**/api/backends/upgrades', route => route.fulfill({ json: {} })) + await page.route('**/backends', async route => { + state.installedCalls++ + if (state.refreshWait && state.deletes.length) await state.refreshWait + return state.failRefresh ? route.fulfill({ status: 500, json: { error: 'installed unavailable' } }) : route.fulfill({ json: [BUILD, SIBLING].filter(Name => ids.some(id => inventories[id].some(b => b.name === Name))).map(Name => ({ Name, IsSystem: !!state.aggregateSystem, Nodes: ids.filter(id => inventories[id].some(b => b.name === Name)).map(id => ({ node_id: id, node_name: id, node_status: 'healthy' })) })) }) + }) + for (const id of ids) { + await page.route(`**/api/nodes/${id}/backends/install`, async route => { + state.installs.push({ id, method: route.request().method(), body: route.request().postDataJSON() }) + if (state.installWait) await state.installWait + return route.fulfill({ json: { message: 'queued' } }) + }) + await page.route(`**/api/nodes/${id}/backends`, async route => { + state.inventoryCalls[id] = (state.inventoryCalls[id] || 0) + 1 + if (state.inventoryWait) await state.inventoryWait + return state.inventoryError ? route.fulfill({ status: 500, json: { error: 'inventory unavailable' } }) : route.fulfill({ json: inventories[id] }) + }) + await page.route(`**/api/nodes/${id}/backends/delete`, async route => { + state.deletes.push({ id, method: route.request().method(), body: route.request().postDataJSON() }) + if (state.deleteWait) await state.deleteWait + if (state.deleteError) return route.fulfill({ status: 500, json: { error: 'removal rejected' } }) + inventories[id] = inventories[id].filter(b => b.name !== BUILD) + if (state.refreshError) state.failRefresh = true + return route.fulfill({ json: { message: 'removed' } }) + }) + } + await page.route('**/backends/delete/**', () => { throw new Error('Global deletion is forbidden') }) + return state +} +async function open(page, extra = '') { + await page.goto(`/app/backends?view=installed&show_all=1&development=1${extra}`) + await expect(page.locator(`[data-entity="${BUILD}"]`)).toBeVisible() + await page.locator(`[data-entity="${BUILD}"]`).click() +} +const confirm = page => page.getByRole('alertdialog').locator('button.btn-danger') + +test('Installed concrete build has independent lazy per-node actions and scoped deletion', async ({ page }) => { + const s = await setup(page) + await page.goto('/app/backends?view=installed&show_all=1&development=1') + await expect(page.locator(`[data-entity="${BUILD}"]`)).toBeVisible() + expect(s.inventoryCalls).toEqual({}) + await page.locator(`[data-entity="${BUILD}"]`).click() + await expect(remove(page, ids[0])).toBeEnabled() + await expect(remove(page, ids[1])).toBeEnabled() + await remove(page, ids[0]).click() + await expect(page.getByRole('alertdialog')).toContainText(BUILD) + await expect(page.getByRole('alertdialog')).toContainText(ids[0]) + expect(s.deletes).toEqual([]) + await confirm(page).click() + await expect(page.getByRole('alertdialog')).toBeHidden() + expect(s.deletes).toEqual([{ id: ids[0], method: 'POST', body: { backend: BUILD } }]) + await expect(remove(page, ids[1])).toBeEnabled() + await expect(page.locator(`[data-entity="${SIBLING}"]`)).toBeVisible() + expect(s.installedCalls).toBeGreaterThan(1) +}) + +for (const cancel of ['Cancel', 'Escape', 'backdrop']) test(`cancellation ${cancel} sends no request`, async ({ page }) => { + const s = await setup(page); await open(page) + await remove(page, ids[0]).click() + if (cancel === 'Escape') await page.keyboard.press('Escape') + else if (cancel === 'backdrop') await page.locator('.confirm-dialog-backdrop').click({ position: { x: 2, y: 2 } }) + else await page.getByRole('alertdialog').getByRole('button', { name: 'Cancel', exact: true }).click() + await expect(page.getByRole('alertdialog')).toBeHidden() + expect(s.deletes).toEqual([]) + await expect(remove(page, ids[0])).toBeEnabled() +}) + +for (const aggregateSystem of [true, false]) test(`actual mixed system inventory overrides aggregate ${aggregateSystem}`, async ({ page }) => { + const s = await setup(page, { aggregateSystem }); s.inventories[ids[0]][0].is_system = true + await open(page) + await expect(remove(page, ids[0])).toBeDisabled() + await expect(remove(page, ids[1])).toBeEnabled() +}) + +for (const mode of ['system', 'missing', 'offline', 'inventory error']) test(`confirmation revalidation blocks ${mode}`, async ({ page }) => { + const s = await setup(page); await open(page); await remove(page, ids[0]).click() + if (mode === 'system') s.inventories[ids[0]][0].is_system = true + if (mode === 'missing') s.inventories[ids[0]] = [] + if (mode === 'offline') s.nodes[0].status = 'offline' + if (mode === 'inventory error') s.inventoryError = true + await confirm(page).click() + await expect(page.getByRole('alertdialog')).toBeHidden() + await expect(page.getByRole('alert').filter({ hasText: ids[0] })).toBeVisible() + expect(s.deletes).toEqual([]) +}) + +for (const mode of ['unknown', 'meta', 'alias', 'offline', 'draining', 'missing target']) test(`fail closed for ${mode}`, async ({ page }) => { + const s = await setup(page) + if (mode === 'unknown') delete s.inventories[ids[0]][0].is_system + if (mode === 'meta') s.inventories[ids[0]][0].is_meta = true + if (mode === 'alias') s.inventories[ids[0]] = [] + if (mode === 'offline' || mode === 'draining') s.nodes[0].status = mode + if (mode === 'missing target') s.nodes.shift() + await open(page, `&target=${ids[0]}`) + if (mode === 'alias') await expect(remove(page, ids[0])).toHaveCount(0) + else await expect(remove(page, ids[0])).toBeDisabled() + await expect(remove(page, ids[1])).toHaveCount(0) + expect(s.deletes).toEqual([]) +}) + +test('inventory error retry and deferred loading fail closed', async ({ page }) => { + let release; const wait = new Promise(resolve => { release = resolve }) + const s = await setup(page, { inventoryWait: wait, inventoryError: true }); await open(page) + await expect(remove(page, ids[0])).toBeDisabled(); release() + await expect(page.getByRole('button', { name: `Retry inventory for ${ids[0]}` })).toBeVisible() + s.inventoryError = false + await page.getByRole('button', { name: `Retry inventory for ${ids[0]}` }).click() + await expect(remove(page, ids[0])).toBeEnabled() +}) + +test('immutable URL target and duplicate confirmation guard', async ({ page }) => { + let release; const wait = new Promise(resolve => { release = resolve }) + const s = await setup(page, { deleteWait: wait }); await open(page, `&target=${ids[0]}`) + await remove(page, ids[0]).click() + await page.evaluate(id => { const url = new URL(location.href); url.searchParams.set('target', id); history.pushState({}, '', url); dispatchEvent(new PopStateEvent('popstate')) }, ids[1]) + await confirm(page).evaluate(el => { el.click(); el.click() }) + await expect.poll(() => s.deletes.length).toBe(1) + await expect(confirm(page)).toBeDisabled() + release(); await expect(page.getByRole('alertdialog')).toBeHidden() + expect(s.deletes).toEqual([{ id: ids[0], method: 'POST', body: { backend: BUILD } }]) +}) + +test('API failure preserves rows and permits explicit confirmed retry', async ({ page }) => { + const s = await setup(page, { deleteError: true }); await open(page) + await remove(page, ids[0]).click(); await confirm(page).click() + await expect(page.getByRole('alertdialog')).toBeHidden() + await expect(page.getByRole('alert').filter({ hasText: ids[0] })).toBeVisible() + await expect(remove(page, ids[0])).toBeEnabled() + expect(s.deletes).toHaveLength(1) + s.deleteError = false + await remove(page, ids[0]).click(); expect(s.deletes).toHaveLength(1) + await confirm(page).click(); await expect(page.getByRole('alertdialog')).toBeHidden() + expect(s.deletes).toHaveLength(2) +}) + +test('successful deletion with refresh failure retains stale rows and offers refresh only', async ({ page }) => { + const s = await setup(page, { refreshError: true }); await open(page) + await remove(page, ids[0]).click(); await confirm(page).click() + await expect(page.getByRole('alertdialog')).toBeHidden() + await expect(page.getByRole('button', { name: 'Retry backend refresh' })).toBeVisible() + await expect(page.locator(`[data-entity="${BUILD}"]`)).toBeVisible() + await expect(page.locator(`[data-entity="${SIBLING}"]`)).toBeVisible() + await expect(page.getByText(/Deletion succeeded/i)).toBeVisible() + const before = s.installedCalls + await page.getByRole('button', { name: 'Retry backend refresh' }).click() + await expect.poll(() => s.installedCalls).toBeGreaterThan(before) + await expect(page.getByText(/Deletion succeeded/i)).toBeVisible() + s.failRefresh = false + await page.getByRole('button', { name: 'Retry backend refresh' }).click() + await expect(remove(page, ids[1])).toBeEnabled() + expect(s.deletes).toHaveLength(1) +}) + + +test('catalog target uses the same confirmed exact-node flow', async ({ page }) => { + const s = await setup(page) + await page.goto(`/app/backends?view=catalog&show_all=1&development=1&target=${ids[0]}&backend=${BUILD}`) + await expect(remove(page, ids[0])).toBeEnabled() + await remove(page, ids[0]).click() + expect(s.deletes).toHaveLength(0) + await expect(page.getByRole('alertdialog')).toContainText(BUILD) + await confirm(page).click() + await expect(page.getByRole('alertdialog')).toBeHidden() + expect(s.deletes).toEqual([{ id: ids[0], method: 'POST', body: { backend: BUILD } }]) +}) + +test('cold installed failure is not empty success, and retry recovers', async ({ page }) => { + const s = await setup(page, { failRefresh: true }) + await page.goto('/app/backends?view=installed&show_all=1&development=1') + await expect(page.getByRole('button', { name: 'Retry backend refresh' })).toBeVisible() + await expect(page.getByText('No backends installed', { exact: true })).toHaveCount(0) + s.failRefresh = false + await page.getByRole('button', { name: 'Retry backend refresh' }).click() + await expect(page.locator(`[data-entity="${BUILD}"]`)).toBeVisible() +}) + +test('background installed refresh retains rows on failure without claiming deletion success', async ({ page }) => { + const s = await setup(page); await open(page) + s.failRefresh = true + // An operations change triggers the existing background list refresh. + await page.route('**/api/operations', route => route.fulfill({ json: { operations: [{ name: SIBLING, jobID: 'synthetic-job', isQueued: true }] } })) + await expect(page.getByRole('button', { name: 'Retry backend refresh' })).toBeVisible({ timeout: 15000 }) + await expect(page.locator(`[data-entity="${BUILD}"]`)).toBeVisible() + expect(s.deletes).toHaveLength(0) +}) + +test('conflicting queued operation disables removal', async ({ page }) => { + const s = await setup(page) + await page.route('**/api/operations', route => route.fulfill({ json: { operations: [{ name: BUILD, jobID: 'synthetic-job', isQueued: true }] } })) + await open(page) + await expect(remove(page, ids[0])).toBeDisabled() + expect(s.deletes).toHaveLength(0) +}) + +test('standalone system protection has no scoped removal', async ({ page }) => { + const s = await setup(page, { aggregateSystem: true }) + await page.route('**/api/nodes', route => route.fulfill({ status: 503, json: { error: 'distributed disabled' } })) + await open(page) + await expect(remove(page, ids[0])).toHaveCount(0) + await expect(page.locator(`[data-entity="${BUILD}"]`)).toContainText('Protected') + expect(s.deletes).toHaveLength(0) +}) + +for (const initial of [true, false]) test(`unsuccessful ${initial ? 'initial' : 'background'} refresh retry never claims deletion`, async ({ page }) => { + const s = await setup(page, { failRefresh: initial }) + if (initial) await page.goto('/app/backends?view=installed&show_all=1&development=1') + else { + await open(page) + s.failRefresh = true + await page.route('**/api/operations', route => route.fulfill({ json: { operations: [{ name: SIBLING, jobID: 'synthetic-job', isQueued: true }] } })) + } + const retry = page.getByRole('button', { name: 'Retry backend refresh' }) + await expect(retry).toBeVisible({ timeout: 15000 }) + const before = s.installedCalls + await retry.click() + await expect.poll(() => s.installedCalls).toBeGreaterThan(before) + await expect(page.getByRole('alert').filter({ hasText: /refresh failed|could not|unable|failed to/i }).first()).toBeVisible() + await expect(page.getByText(/Deletion succeeded/i)).toHaveCount(0) + expect(s.deletes).toEqual([]) +}) + +const catalogTarget = page => page.goto(`/app/backends?view=catalog&show_all=1&development=1&target=${ids[0]}&backend=${BUILD}`) +const reinstall = page => page.getByRole('button', { name: 'Reinstall', exact: true }) + +test('deferred target reinstall synchronously blocks duplicate installs and removal', async ({ page }) => { + let release; const wait = new Promise(resolve => { release = resolve }) + const s = await setup(page, { installWait: wait }); await catalogTarget(page) + await expect(remove(page, ids[0])).toBeEnabled() + await reinstall(page).evaluate(el => { el.click(); el.click() }) + await expect.poll(() => s.installs.length).toBeGreaterThan(0) + await expect(reinstall(page)).toBeDisabled() + await expect(remove(page, ids[0])).toBeDisabled() + // Programmatic clicks still honor disabled state; no confirmation/mutation. + await remove(page, ids[0]).evaluate(el => el.click()) + await expect(page.getByRole('alertdialog')).toBeHidden() + expect(s.deletes).toEqual([]) + expect(s.installs).toEqual([{ id: ids[0], method: 'POST', body: { backend: BUILD } }]) + release() + await expect(reinstall(page)).toBeEnabled() +}) + +test('target reinstall stays blocked throughout deletion and deferred postdelete refresh', async ({ page }) => { + let releaseDelete, releaseRefresh + const s = await setup(page, { + deleteWait: new Promise(resolve => { releaseDelete = resolve }), + refreshWait: new Promise(resolve => { releaseRefresh = resolve }), + }) + await catalogTarget(page); await remove(page, ids[0]).click(); await confirm(page).click() + await expect.poll(() => s.deletes.length).toBe(1) + await expect(reinstall(page)).toBeDisabled() + releaseDelete() + await expect(page.getByRole('alertdialog')).toBeHidden() + await expect(reinstall(page)).toBeDisabled() + await expect(remove(page, ids[0])).toBeDisabled() + await reinstall(page).evaluate(el => el.click()) + expect(s.installs).toEqual([]) + expect(s.deletes).toHaveLength(1) + releaseRefresh() + await expect(reinstall(page)).toBeEnabled() +}) + +test('out-of-order inventory from previous row cannot overwrite current inventory', async ({ page }) => { + const s = await setup(page) + let release; const wait = new Promise(resolve => { release = resolve }) + let first = true + await page.route(`**/api/nodes/${ids[0]}/backends`, async route => { + if (first) { + first = false + await wait + return route.fulfill({ json: [{ name: SIBLING, is_system: false, is_meta: false }] }) + } + return route.fulfill({ json: [{ name: SIBLING, is_system: true, is_meta: false }] }) + }) + await open(page, `&target=${ids[0]}`) + await expect(remove(page, ids[0])).toBeDisabled() + await expect.poll(() => first).toBe(false) + await page.locator(`[data-entity="${SIBLING}"]`).click() + await expect(remove(page, ids[0])).toBeDisabled() + // Wait for the new protected response before releasing the obsolete managed one. + await expect(page.locator('.text-note').filter({ hasText: /protected|system/i })).toBeVisible() + const obsoleteResponse = page.waitForResponse(response => response.url().endsWith(`/api/nodes/${ids[0]}/backends`)) + release() + await obsoleteResponse + await expect(remove(page, ids[0])).toBeDisabled() + expect(s.deletes).toEqual([]) +}) + +test('last installation disappearance clears only selection and preserves URL refinements', async ({ page }) => { + const s = await setup(page) + s.inventories[ids[1]] = s.inventories[ids[1]].filter(b => b.name !== BUILD) + await open(page, `&target=${ids[0]}&q=example`) + await remove(page, ids[0]).click(); await confirm(page).click() + await expect(page.locator(`[data-entity="${BUILD}"]`)).toHaveCount(0) + await expect.poll(() => new URL(page.url()).searchParams.has('backend')).toBe(false) + const params = new URL(page.url()).searchParams + expect(Object.fromEntries(params)).toEqual({ view: 'installed', show_all: '1', development: '1', target: ids[0], q: 'example' }) + await expect(page.locator(`[data-entity="${SIBLING}"]`)).toBeVisible() + expect(s.deletes).toEqual([{ id: ids[0], method: 'POST', body: { backend: BUILD } }]) +}) + +const clearTarget = page => page.evaluate(() => { + const url = new URL(location.href) + url.searchParams.delete('target') + history.pushState({}, '', url) + dispatchEvent(new PopStateEvent('popstate')) +}) +const genericRemove = page => page.locator('.bk-remove') + +// A synthetic click must not bypass the disabled control. +const dispatchClick = locator => locator.evaluate(el => el.dispatchEvent(new MouseEvent('click', { bubbles: true }))) + +test('generic removal stays blocked during scoped postdelete refresh after target clears', async ({ page }) => { + let releaseRefresh + const s = await setup(page, { refreshWait: new Promise(resolve => { releaseRefresh = resolve }) }) + const globalDeletes = [] + await page.route('**/backends/delete/**', route => { + globalDeletes.push(route.request().url()) + return route.fulfill({ json: { message: 'removed' } }) + }) + await catalogTarget(page); await remove(page, ids[0]).click(); await confirm(page).click() + await expect(page.getByRole('alertdialog')).toBeHidden() + await expect.poll(() => s.installedCalls).toBeGreaterThan(1) + await clearTarget(page) + await expect(genericRemove(page)).toBeDisabled() + await dispatchClick(genericRemove(page)) + await expect(page.getByRole('alertdialog')).toBeHidden() + expect(globalDeletes).toEqual([]) + expect(s.deletes).toEqual([{ id: ids[0], method: 'POST', body: { backend: BUILD } }]) + releaseRefresh() + await expect(genericRemove(page)).toHaveCount(0) +}) + +test('generic removal rechecks scoped conflict after dependency lookup', async ({ page }) => { + let releaseCapabilities, releaseRefresh + const s = await setup(page, { refreshWait: new Promise(resolve => { releaseRefresh = resolve }) }) + const capabilitiesWait = new Promise(resolve => { releaseCapabilities = resolve }) + let capabilitiesStarted = false + await page.route('**/models/capabilities', async route => { + capabilitiesStarted = true + await capabilitiesWait + return route.fulfill({ json: { data: [] } }) + }) + await catalogTarget(page) + await clearTarget(page) + await genericRemove(page).click() + await expect.poll(() => capabilitiesStarted).toBe(true) + // The generic lookup is pending, so a scoped confirmation can start first. + await page.evaluate(id => { + const url = new URL(location.href); url.searchParams.set('target', id) + history.pushState({}, '', url); dispatchEvent(new PopStateEvent('popstate')) + }, ids[0]) + await expect(remove(page, ids[0])).toBeEnabled() + await remove(page, ids[0]).click(); await confirm(page).click() + await expect(page.getByRole('alertdialog')).toBeHidden() + const capabilitiesResponse = page.waitForResponse(response => response.url().endsWith('/models/capabilities')) + releaseCapabilities(); await capabilitiesResponse + await expect(page.getByRole('alertdialog')).toBeHidden() + expect(s.deletes).toHaveLength(1) + releaseRefresh() + await expect(reinstall(page)).toBeEnabled() +}) + +test('generic confirmation rechecks current row conflict', async ({ page }) => { + const s = await setup(page) + const globalDeletes = [] + await page.route('**/backends/delete/**', route => { + globalDeletes.push(route.request().url()) + return route.fulfill({ json: { message: 'removed' } }) + }) + await catalogTarget(page); await clearTarget(page); await genericRemove(page).click() + await expect(page.getByRole('alertdialog')).toBeVisible() + await page.route('**/api/operations', route => route.fulfill({ json: { operations: [{ name: BUILD, jobID: 'synthetic-job', isQueued: true }] } })) + await expect(page.locator(`[data-entity="${BUILD}"]`)).toContainText('Queued', { timeout: 15000 }) + await confirm(page).click() + await expect(page.getByRole('alertdialog')).toBeHidden() + expect(globalDeletes).toEqual([]) + expect(s.deletes).toEqual([]) +}) + +for (const beforeConfirm of [false, true]) test(`generic deletion blocks targeted install ${beforeConfirm ? 'at confirmation' : 'after target clears'}`, async ({ page }) => { + let releaseInstall + const s = await setup(page, { installWait: new Promise(resolve => { releaseInstall = resolve }) }) + const globalDeletes = [] + await page.route('**/backends/delete/**', route => { + globalDeletes.push(route.request().url()) + return route.fulfill({ json: { message: 'removed' } }) + }) + await catalogTarget(page) + if (beforeConfirm) { + await clearTarget(page); await genericRemove(page).click() + await expect(page.getByRole('alertdialog')).toBeVisible() + // Start the targeted request through its captured React handler while the + // existing generic confirmation remains open. + await page.evaluate(id => { + const url = new URL(location.href); url.searchParams.set('target', id) + history.pushState({}, '', url); dispatchEvent(new PopStateEvent('popstate')) + }, ids[0]) + } + await reinstall(page).evaluate(el => el.click()) + await expect.poll(() => s.installs.length).toBe(1) + if (beforeConfirm) { + await confirm(page).click() + await expect(page.getByRole('alertdialog')).toBeHidden() + } else { + await clearTarget(page) + await expect(genericRemove(page)).toBeDisabled() + await dispatchClick(genericRemove(page)) + await expect(page.getByRole('alertdialog')).toBeHidden() + } + expect(globalDeletes).toEqual([]) + expect(s.deletes).toEqual([]) + releaseInstall() +}) diff --git a/core/http/react-ui/public/locales/en/operate.json b/core/http/react-ui/public/locales/en/operate.json index fb6b2dfc8a7d..a6a04b6c889b 100644 --- a/core/http/react-ui/public/locales/en/operate.json +++ b/core/http/react-ui/public/locales/en/operate.json @@ -421,6 +421,22 @@ "license": "License", "tags": "Tags", "links": "Links" + }, + "nodeRemoval": { + "action": "Remove from {{node}}…", + "title": "Remove {{name}} from {{node}}?", + "message": "This affects only this node and can stop models using this backend.", + "inventoryUnknown": "Inventory not verified.", + "inventoryError": "Inventory unavailable. Retry to verify protection.", + "unreachable": "Node is unavailable or not a healthy backend node.", + "absent": "Exact backend is not installed on this node.", + "protected": "System, meta, or unknown protection: removal is unavailable.", + "managed": "Verified user-managed installation", + "retry": "Retry inventory for {{node}}", + "failed": "Could not remove {{name}} from {{node}}: {{message}}", + "refreshFailed": "Backend data could not be refreshed. Displayed data may be stale. Retry refresh.", + "stale": "Deletion succeeded, but backend data could not be refreshed. Displayed data may be stale. Retry refresh; do not repeat deletion.", + "refresh": "Retry backend refresh" } }, "logs": { diff --git a/core/http/react-ui/src/hooks/useDistributedMode.js b/core/http/react-ui/src/hooks/useDistributedMode.js index 49ad6eace065..5379f256d350 100644 --- a/core/http/react-ui/src/hooks/useDistributedMode.js +++ b/core/http/react-ui/src/hooks/useDistributedMode.js @@ -26,9 +26,11 @@ export function useDistributedMode() { const data = await nodesApi.list() setNodes(Array.isArray(data) ? data : []) setEnabled(true) + return true } catch { setEnabled(false) setNodes([]) + return false } finally { setLoading(false) } diff --git a/core/http/react-ui/src/pages/Backends.jsx b/core/http/react-ui/src/pages/Backends.jsx index 1ab1626f8ecd..b9874a11d71b 100644 --- a/core/http/react-ui/src/pages/Backends.jsx +++ b/core/http/react-ui/src/pages/Backends.jsx @@ -68,6 +68,25 @@ export default function Backends() { const [confirmDialog, setConfirmDialog] = useState(null) const [allBackends, setAllBackends] = useState([]) const [installed, setInstalled] = useState([]) + const [inventories, setInventories] = useState({}) + const [nodeErrors, setNodeErrors] = useState({}) + const [removalPending, setRemovalPending] = useState(false) + const [refreshError, setRefreshError] = useState(false) + const removalLock = useRef(null) + // Ref guards same-tick callbacks; state also rerenders the exact target controls. + const targetInstallLocks = useRef(new Set()) + const [targetInstalls, setTargetInstalls] = useState(() => new Set()) + const targetKey = (nodeID, name) => JSON.stringify([nodeID, name]) + const targetInstalling = (nodeID, name) => targetInstallLocks.current.has(targetKey(nodeID, name)) + const targetBusy = (nodeID, name) => rowConflict(name) + || targetInstalls.has(targetKey(nodeID, name)) + || removalLock.current === targetKey(nodeID, name) + const inventoryRequests = useRef({}) + const mounted = useRef(true) + useEffect(() => { + mounted.current = true + return () => { mounted.current = false; inventoryRequests.current = {} } + }, []) const [models, setModels] = useState([]) const [upgrades, setUpgrades] = useState({}) const [upgradingAll, setUpgradingAll] = useState(false) @@ -149,8 +168,10 @@ export default function Backends() { setPreferDev(true) setPreferDevLoaded(true) } + return true } catch (err) { addToast(t('backends.loadFailed', { message: err.message }), 'error') + return false } finally { loadedOnce.current = true setCatalogReady(true) @@ -166,15 +187,23 @@ export default function Backends() { setInstalledLoading(true) const data = await backendsApi.listInstalled() setInstalled(Array.isArray(data) ? data : []) + installedOnce.current = true setGlobalError('') + setSearchParams(prev => { + const selected = prev.get('backend') + if (prev.get('view') !== 'installed' || !selected || data.some(b => b.Name === selected)) return prev + const next = new URLSearchParams(prev) + next.delete('backend') + return next + }, { replace: true }) + return true } catch (err) { - setInstalled([]) setGlobalError(t('backends.installedLoadFailed', { message: err.message })) + return false } finally { - installedOnce.current = true setInstalledLoading(false) } - }, [t]) + }, [t, setSearchParams]) // Install and delete change what is installed, so the lists are read again // whenever the set of operations changes. @@ -232,27 +261,159 @@ export default function Backends() { const handleInstallOnTarget = async (id) => { if (!targetNode) return + const node = { id: targetNode.id, name: targetNode.name } + const key = targetKey(node.id, id) + if (targetBusy(node.id, id) || targetInstalling(node.id, id)) return + targetInstallLocks.current.add(key) + setTargetInstalls(new Set(targetInstallLocks.current)) setError(id, '') try { - await nodesApi.installBackend(targetNode.id, id) - addToast(t('backends.installingOn', { name: id, node: targetNode.name }), 'info') - setTimeout(() => { fetchBackends(); fetchInstalled(); refetchNodes() }, 1200) + await nodesApi.installBackend(node.id, id) + addToast(t('backends.installingOn', { name: id, node: node.name }), 'info') + await Promise.all([loadInventory(node.id), refreshBackendData()]) } catch (err) { - setError(id, t('backends.dispatchFailed', { node: targetNode.name, message: err.message })) + setError(id, t('backends.dispatchFailed', { node: node.name, message: err.message })) + } finally { + targetInstallLocks.current.delete(key) + if (mounted.current) setTargetInstalls(new Set(targetInstallLocks.current)) } } - const handleRemoveFromTarget = async (name) => { - setError(name, '') + // Inventory is page-local and fetched only for the expanded row's scoped nodes. + const loadInventory = useCallback(async (nodeID) => { + const request = Symbol(nodeID) + inventoryRequests.current[nodeID] = request + setInventories(prev => ({ ...prev, [nodeID]: { status: 'loading' } })) + try { + const data = await nodesApi.getBackends(nodeID) + if (!Array.isArray(data)) throw new Error(t('backends.nodeRemoval.inventoryUnknown')) + if (mounted.current && inventoryRequests.current[nodeID] === request) { + setInventories(prev => ({ ...prev, [nodeID]: { status: 'success', data } })) + } + return data + } catch (err) { + if (mounted.current && inventoryRequests.current[nodeID] === request) { + setInventories(prev => ({ ...prev, [nodeID]: { status: 'error', error: err.message } })) + } + return null + } + }, [t]) + + const visibleRefs = activeView === 'installed' + ? (installed.find(b => b.Name === selectedName)?.Nodes || installed.find(b => b.Name === selectedName)?.nodes || []) + : (allBackends.find(b => (b.name || b.id) === selectedName)?.nodes || []) + const visibleNodeIDs = distributedEnabled ? [...new Set(visibleRefs.map(n => n.node_id ?? n.NodeID))] + .filter(id => !targetNodeId || id === targetNodeId) : [] + const inventoryScope = JSON.stringify([selectedName, activeView, visibleNodeIDs, clusterNodes]) + useEffect(() => { + const ids = JSON.parse(inventoryScope)[2] + ids.forEach(id => { + const node = clusterNodes.find(n => n.id === id) + if (node?.node_type === 'backend' && node.status === 'healthy') loadInventory(id) + }) + return () => { inventoryRequests.current = {} } + // Responses from a previous expanded row or node status are ignored. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [inventoryScope, loadInventory]) + + const nodeReason = (node, data, name) => { + if (!node || node.node_type !== 'backend' || node.status !== 'healthy') return t('backends.nodeRemoval.unreachable') + const entry = data?.find(b => b.name === name) + if (!entry) return t('backends.nodeRemoval.absent') + if (entry.is_system !== false || entry.is_meta !== false) return t('backends.nodeRemoval.protected') + return '' + } + const rowConflict = (name) => { + const state = backendState({ installed: true, op: opForBackend(operations, name) }) + return pending.has(name) || upgradingAll || ['installing', 'queued', 'removing'].includes(state.kind) + } + // Async confirmation must use current operation state, not the dialog's closure. + const currentRowConflict = useRef(rowConflict) + currentRowConflict.current = rowConflict + + const refreshBackendData = async (afterDeletion = false) => { + const results = await Promise.all([fetchBackends(), fetchInstalled(), refetchNodes()]) + const failed = results.some(result => result === false) + setRefreshError(previous => failed ? (afterDeletion || previous === 'deletion' ? 'deletion' : 'refresh') : false) + return !failed + } + + const confirmNodeRemoval = async (snapshot) => { + if (removalLock.current || targetInstalling(snapshot.nodeID, snapshot.backendName)) return + const key = JSON.stringify([snapshot.nodeID, snapshot.backendName]) + removalLock.current = key + setRemovalPending(true) + setNodeErrors(prev => ({ ...prev, [key]: '' })) try { - await nodesApi.deleteBackend(targetNode.id, name) - addToast(t('backends.removedFrom', { name, node: targetNode.name }), 'success') - setTimeout(() => { fetchBackends(); fetchInstalled(); refetchNodes() }, 600) + // Refresh status too: a node can drain or disappear while the dialog is open. + const nodes = await nodesApi.list() + const node = nodes.find(n => n.id === snapshot.nodeID) + const data = await loadInventory(snapshot.nodeID) + const reason = nodeReason(node, data, snapshot.backendName) + if (!data || reason || currentRowConflict.current(snapshot.backendName) || targetInstalling(snapshot.nodeID, snapshot.backendName)) { + throw new Error(reason || t('backends.nodeRemoval.inventoryUnknown')) + } + if (!mounted.current) return + await nodesApi.deleteBackend(snapshot.nodeID, snapshot.backendName) + if (!mounted.current) return + setConfirmDialog(null) + addToast(t('backends.removedFrom', { name: snapshot.backendName, node: snapshot.nodeName }), 'success') + const inventory = await loadInventory(snapshot.nodeID) + const refreshed = await refreshBackendData(true) + if (!inventory || !refreshed) setRefreshError('deletion') } catch (err) { - setError(name, t('backends.removeFailed', { message: err.message })) + if (!mounted.current) return + setConfirmDialog(null) + setNodeErrors(prev => ({ ...prev, [key]: t('backends.nodeRemoval.failed', { name: snapshot.backendName, node: snapshot.nodeName, message: err.message }) })) + await loadInventory(snapshot.nodeID) + } finally { + removalLock.current = null + if (mounted.current) setRemovalPending(false) } } + const handleRemoveFromTarget = (snapshot) => { + if (removalLock.current || rowConflict(snapshot.backendName) || targetInstalling(snapshot.nodeID, snapshot.backendName)) return + setConfirmDialog({ + title: t('backends.nodeRemoval.title', { name: snapshot.backendName, node: snapshot.nodeName }), + message: <>{t('backends.nodeRemoval.message')}{usedBy(snapshot.backendName)}, + confirmLabel: t('backends.remove'), + onConfirm: () => confirmNodeRemoval(snapshot), + }) + } + + const nodeControls = (name, refs) => ( +
+ {refs.filter(ref => !targetNodeId || (ref.node_id ?? ref.NodeID) === targetNodeId).map(ref => { + const nodeID = ref.node_id ?? ref.NodeID + const node = clusterNodes.find(n => n.id === nodeID) + const nodeName = node?.name || ref.node_name || ref.NodeName || nodeID + const inventory = inventories[nodeID] + const key = JSON.stringify([nodeID, name]) + const reason = inventory?.status === 'success' + ? nodeReason(node, inventory.data, name) + : t(inventory?.status === 'error' ? 'backends.nodeRemoval.inventoryError' : 'backends.nodeRemoval.inventoryUnknown') + return ( +
+
+ {nodeName} + {reason || t('backends.nodeRemoval.managed')} + + {inventory?.status === 'error' && } +
+ {nodeErrors[key] &&
{nodeErrors[key]}
} +
+ ) + })} +
+ ) + const openPicker = (b, initialSelection = []) => { setPickerBackend(b) setPickerInitialSelection(initialSelection) @@ -334,7 +495,12 @@ export default function Backends() { // the backend they ask for and an installed meta backend names the concrete // one it points at, so both are read before asking, and the dialog says what // would stop working. + // Generic deletion can affect every node, so any targeted install of this + // build conflicts, even after navigation clears the selected node. + const deletionBlocked = (name) => removalLock.current || currentRowConflict.current(name) + || [...targetInstallLocks.current].some(key => JSON.parse(key)[1] === name) const handleDelete = async (name, how) => { + if (deletionBlocked(name)) return let dependents = { models: [], metas: [] } try { const data = await modelsApi.listCapabilities() @@ -344,6 +510,7 @@ export default function Backends() { } catch { dependents = backendDependents(name, { models, installed }) } + if (!mounted.current || deletionBlocked(name)) return setConfirmDialog({ title: t('backends.deleteTitle'), message: ( @@ -367,6 +534,7 @@ export default function Backends() { confirmLabel: t('backends.delete'), onConfirm: async () => { setConfirmDialog(null) + if (deletionBlocked(name)) return setError(name, '') try { if (how === 'installed') await backendsApi.deleteInstalled(name) @@ -484,7 +652,7 @@ export default function Backends() { const op = opForBackend(operations, name, b.id) const upgrade = upgrades[name] const state = backendState({ installed: b.installed, upgrade, op }) - const busy = pending.has(name) + const busy = pending.has(name) || removalPending const isNodeScoped = Boolean(targetNode) const onTarget = isNodeScoped && (b.nodes || []).some(n => (n.node_id ?? n.NodeID) === targetNode.id) @@ -496,11 +664,11 @@ export default function Backends() { } else if (state.kind === 'removing') { action = null } else if (state.kind === 'failed') { - action = + action = } else if (isNodeScoped) { action = onTarget ? null - : + : } else if (b.installed) { action = upgrade ? @@ -552,12 +720,10 @@ export default function Backends() {
{isNodeScoped && onTarget && ( <> - - + {nodeControls(name, b.nodes || [])} )} {!isNodeScoped && b.installed && state.kind !== 'installing' && state.kind !== 'removing' && ( @@ -570,7 +736,7 @@ export default function Backends() { - @@ -631,7 +797,7 @@ export default function Backends() { const upgrade = upgrades[name] const op = opForBackend(operations, name) const state = backendState({ installed: true, upgrade, op }) - const busy = pending.has(name) || state.kind === 'installing' || state.kind === 'queued' || state.kind === 'removing' + const busy = pending.has(name) || removalPending || state.kind === 'installing' || state.kind === 'queued' || state.kind === 'removing' const nodes = b.Nodes || b.nodes || [] let action = null @@ -656,7 +822,7 @@ export default function Backends() { items={[ { key: 'reinstall', icon: 'refresh', label: t('backends.reinstallBackend'), onClick: () => handleReinstall(name), disabled: busy }, { divider: true }, - { key: 'delete', icon: 'trash', label: t('backends.deleteBackend'), danger: true, onClick: () => handleDelete(name, 'installed') }, + { key: 'delete', icon: 'trash', label: t('backends.deleteBackend'), danger: true, disabled: busy, onClick: () => handleDelete(name, 'installed') }, ]} /> @@ -679,6 +845,7 @@ export default function Backends() {
{t('backends.installedOn')} + {nodeControls(name, nodes)}
)} {usedBy(name)} @@ -826,13 +993,12 @@ export default function Backends() {
)} - {globalError && ( + {(refreshError || globalError) && (
- - {globalError} + {refreshError ? t(refreshError === 'deletion' ? 'backends.nodeRemoval.stale' : 'backends.nodeRemoval.refreshFailed') : globalError} +
)} - {showManualInstall && (

{t('backends.manualTitle')}

@@ -870,7 +1036,7 @@ export default function Backends() { {showCatalogCold || showInstalledCold ? (
) : activeView === 'installed' ? ( - installed.length === 0 && !globalError ? ( + installed.length === 0 && globalError ? null : installed.length === 0 ? (

{t('backends.emptyTitle')}

@@ -943,8 +1109,9 @@ export default function Backends() { message={confirmDialog?.message} confirmLabel={confirmDialog?.confirmLabel} danger + pending={removalPending} onConfirm={confirmDialog?.onConfirm} - onCancel={() => setConfirmDialog(null)} + onCancel={() => { if (!removalLock.current) setConfirmDialog(null) }} /> cannot send API key headers). + {method: http.MethodGet, path: "/api/canvas/preview/:id"}: {}, + // Branding reads used by the login screen. Branding mutations are // intentionally absent and must receive 401. {method: http.MethodGet, path: "/api/branding"}: {}, diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index df466ba0d699..1fb02564898a 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -353,3 +353,35 @@ tokens. Set `max_tokens` on the model config, and keep `repeat_penalty` above ### Motion capture - [gem-x.cpp](https://github.com/localai-org/gem-x.cpp): live SOMA-77 and SMPL-24 human poses on CPU/Vulkan. See [Motion Capture](/features/motion/). + +### Remove an installed backend from one node + +In distributed mode, expand a row in **Backends → Installed** to see its node +installations. Use **Show variants** and **Development builds** when needed to +reveal concrete builds such as `cpu-example-engine-development`. Removal uses +that exact row name, not a generic alias or the recommended build. + +**Remove from worker-alpha…** asks for confirmation naming both the backend and +node. Only that installation is affected; other nodes and sibling builds remain. +Removal can stop models using the backend on that node. Cancel, Escape, or clicking +outside the dialog sends no removal request. A target-node URL restricts the +available actions to that node; it never falls back to a different worker. + +The page checks each expanded node's actual inventory. System and meta backends, +unknown protection metadata, absent builds, and unavailable or draining nodes +cannot be removed through these controls. A system installation on one node does +not prevent removal of a user-managed installation on another. Inventory and +node status are checked again when you confirm; server-side protection still +applies. Retry inventory explicitly if it could not be loaded. + +A removal error is scoped to the node and backend. Refresh and confirm again +before retrying; an API failure does not guarantee that nothing changed. If +removal succeeds but refresh fails, the page keeps the last successful installed +list and warns that it may be stale. **Retry** refreshes data without repeating +the deletion. Ordinary refresh failures do not report deletion success. Targeted +installation and removal controls block conflicting requests for the same node +and build, including while the post-removal refresh is pending. Generic catalog +removal also stays blocked during targeted installation, scoped removal, and +refresh, even if the URL +target is cleared. Its confirmation rechecks current row conflicts before +deleting. Standalone deletion and its system-backend protection are unchanged. diff --git a/scripts/run-coverage.sh b/scripts/run-coverage.sh index 430ca1f11fe9..24e125b47b6a 100755 --- a/scripts/run-coverage.sh +++ b/scripts/run-coverage.sh @@ -1,101 +1,150 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash +# shellcheck disable=SC2312 # -# run-coverage.sh OUTPUT_DIR MERGED_PROFILE FLAKE_ATTEMPTS UNIT_ROOT [UNIT_ROOT...] +# run-coverage.sh — run ginkgo per test root, merge coverprofiles, and +# enforce a coverage floor. Designed for POSIX sh (dash) and bash. # -# Runs the unit/suite tests under each UNIT_ROOT (recursively) plus the -# in-process integration suites in $COVERAGE_E2E_ROOTS, all instrumented to -# attribute coverage to $COVERAGE_COVERPKG, and merges everything into -# MERGED_PROFILE. +# Usage: +# run-coverage.sh OUTPUT_DIR MERGED_PROFILE FLAKE_ATTEMPTS UNIT_ROOT... # # Environment: -# GINKGO_TAGS build tags, e.g. "debug auth" -# COVERAGE_COVERPKG packages coverage is attributed to (comma-separated -# go patterns). Required so the in-process e2e suites -# credit the core/http handlers they drive over HTTP — -# with default per-package cover they'd only credit the -# e2e_test package itself. -# COVERAGE_E2E_ROOTS space-separated in-process integration suites, run -# NON-recursively (so tests/e2e/distributed, which needs -# containers, is excluded) with --label-filter. -# COVERAGE_E2E_LABELS ginkgo --label-filter for the e2e roots, e.g. -# "!real-models" (those specs need a downloaded model). -# COVERAGE_EXCLUDE_RE egrep pattern of profile lines to drop before merging, -# e.g. generated protobuf (grpc/proto/.*\.pb\.go). +# GINKGO_TAGS Space-separated build tags (e.g. "debug auth"). +# COVERAGE_COVERPKG Comma-separated --coverpkg pattern. +# COVERAGE_E2E_ROOTS Space-separated non-recursive e2e roots. +# COVERAGE_E2E_LABELS Ginkgo --label-filter for e2e roots. +# COVERAGE_EXCLUDE_RE Regex for coverprofile lines to drop on merge. # -# Why one ginkgo invocation per root: passing several recursive roots to a -# single ginkgo run only merges ONE root's coverprofile into --output-dir -# (verified ginkgo 2.29.0) — the rest are silently dropped. So each root runs -# separately. Because --coverpkg makes the same package appear in multiple -# root profiles, the merge SUMS per-block hit counts (a plain concat would -# duplicate blocks and miscount); summing is valid for covermode=atomic. -# -# POSIX sh (no bash): no arrays. Roots are space-free tokens iterated with -# word-splitting; the only flag that can contain a space (--tags="debug auth") -# is held in the positional parameters ("$@") so its quoting survives. +# Each ginkgo invocation's combined stdout+stderr is captured to a per-root +# log file (OUTPUT_DIR/.log). On failure the last 80 lines are +# printed to stderr so the error is visible under `make --output-sync=target` +# (which buffers all output and flushes at the end, hiding the actual error +# in interleaved output). On success the log is replayed to stdout. + set -u out_dir="${1:?usage: run-coverage.sh OUTPUT_DIR MERGED_PROFILE FLAKES UNIT_ROOT...}" merged="${2:?missing MERGED_PROFILE}" flakes="${3:?missing FLAKE_ATTEMPTS}" shift 3 -unit_roots="$*" # space-free tokens (./pkg ./core) +unit_roots="$*" -mkdir -p "$out_dir" -# Clear per-root profiles from a previous run: the merge collects them by glob, -# so a stale profile (e.g. from a root that failed to rebuild this run) must not -# leak into the merged result. -rm -f "$out_dir"/cover-*.out fail=0 +mkdir -p "$out_dir" -# Common optional flags go into "$@"; unquoted ${VAR:+...} would word-split a -# --tags value that contains a space. The unit roots were captured above, so -# overwriting the positional parameters here is safe. -set -- -[ -n "${GINKGO_TAGS:-}" ] && set -- "$@" "--tags=$GINKGO_TAGS" -[ -n "${COVERAGE_COVERPKG:-}" ] && set -- "$@" "--coverpkg=$COVERAGE_COVERPKG" +# Build flags shared by every ginkgo invocation. We store them as a +# newline-delimited list in $build_flags_list so we can iterate without +# word-splitting issues (e.g. --tags="debug auth" contains a space). +build_flags_list="" +if [ -n "${GINKGO_TAGS:-}" ]; then + build_flags_list="${build_flags_list}${build_flags_list:+ +}--tags=$GINKGO_TAGS" +fi +if [ -n "${COVERAGE_COVERPKG:-}" ]; then + build_flags_list="${build_flags_list}${build_flags_list:+ +}--coverpkg=$COVERAGE_COVERPKG" +fi # cover-.out, with path separators flattened (POSIX BRE, no \+). profile_name() { printf 'cover-%s.out' "$(printf '%s' "$1" | sed 's#[./][./]*#_#g; s#^_##; s#_$##')" } +# run_ginkgo ROOT RECURSIVE [LABEL_FILTER] +# Runs ginkgo for ROOT, captures combined output to a log file, and on +# failure prints the tail to stderr. +run_ginkgo() { + _root="$1" + _recursive="$2" + _label_filter="${3:-}" + _base="$(profile_name "$_root")" + _log="$out_dir/$_base.log" + + if [ "$_recursive" = "1" ]; then + _rflag="-r" + else + _rflag="" + fi + + # Build the ginkgo command. We use eval to properly handle --tags + # values that contain spaces. The build_flags_list is newline-delimited; + # we convert newlines to spaces for the eval string. Each flag is + # already in the form --key=value, and the value never contains shell + # metacharacters (tags are alphanum, coverpkg is a Go pattern). + _cmd="go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $flakes -v $_rflag" + # Append build flags (one per line -> one per arg) + if [ -n "$build_flags_list" ]; then + _bf=$(printf '%s' "$build_flags_list" | while IFS= read -r _line; do + # Quote the value part after = so spaces are preserved + printf '%s ' "${_line%%=*}=\"${_line#*=}\"" + done) + _cmd="$_cmd $_bf" + fi + if [ -n "$_label_filter" ]; then + _cmd="$_cmd --label-filter=$_label_filter" + fi + _cmd="$_cmd --cover --covermode=atomic --coverprofile=$_base --output-dir=$out_dir $_root" + + # Run ginkgo, capture all output to the log file. + eval "$_cmd" >"$_log" 2>&1 + _rc=$? + if [ "$_rc" -ne 0 ]; then + echo "run-coverage: ginkgo failed for $_root (exit $_rc), last 80 lines:" >&2 + tail -80 "$_log" >&2 + fail=1 + else + # Replay the log so test output is visible in CI. + cat "$_log" + fi + return "$_rc" +} + # Unit/suite roots: recursive. for root in $unit_roots; do - base="$(profile_name "$root")" - go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v -r "$@" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 + run_ginkgo "$root" 1 || true done # In-process integration roots: NON-recursive + optional label filter. for root in ${COVERAGE_E2E_ROOTS:-}; do - base="$(profile_name "$root")" - if [ -n "${COVERAGE_E2E_LABELS:-}" ]; then - go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \ - --label-filter="$COVERAGE_E2E_LABELS" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 + run_ginkgo "$root" 0 "${COVERAGE_E2E_LABELS:-}" || true +done + +######################################################### +## Merge coverprofiles +######################################################### + +# Collect all coverprofiles except the merged one. +profiles="" +for f in "$out_dir"/cover-*.out; do + [ -f "$f" ] || continue + [ "$f" = "$merged" ] && continue + if [ -z "$profiles" ]; then + profiles="$f" else - go run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts "$flakes" -v "$@" \ - --cover --covermode=atomic --coverprofile="$base" --output-dir="$out_dir" "$root" || fail=1 + profiles="$profiles $f" fi done -# Collect the per-root profiles by glob (space-safe, no list to track). -set -- "$out_dir"/cover-*.out -if [ ! -e "$1" ]; then - echo "run-coverage: no coverprofiles produced" >&2 - exit 2 +if [ -z "$profiles" ]; then + echo "run-coverage: no coverprofiles found in $out_dir" >&2 + exit 1 fi -# Merge: drop the per-file mode headers and any excluded (generated) lines, -# then sum hit counts per identical block, emitting a single mode header. -{ - echo "mode: atomic" - awk -v exre="${COVERAGE_EXCLUDE_RE:-}" ' - /^mode:/ { next } - exre != "" && $1 ~ exre { next } - { stmts[$1] = $2; cnt[$1] += $3 } - END { for (k in stmts) print k, stmts[k], cnt[k] } - ' "$@" -} > "$merged" +# Merge: drop the mode line from all but the first profile, optionally +# exclude generated proto files, then concatenate. +first=1 +for f in $profiles; do + if [ "$first" = "1" ]; then + cat "$f" + first=0 + else + tail -n +2 "$f" + fi +done | { if [ -n "${COVERAGE_EXCLUDE_RE:-}" ]; then + grep -vE "$COVERAGE_EXCLUDE_RE" +else + cat +fi; } >"$merged" + +echo "run-coverage: merged $(echo "$profiles" | wc -w) coverprofiles into $merged" exit "$fail"