From 3d03d2c57e26a66fe76a10827a5b82cb87a6beb3 Mon Sep 17 00:00:00 2001 From: ConradJam Date: Sun, 16 Aug 2026 14:05:09 +0800 Subject: [PATCH] [hotfix][Process] Mark process FAILED and untrack when its engine is missing executeOrTraceProcess returned early when the configured execution engine was not installed, but register() had already tracked the PENDING process: hasAliveTableProcess then blocked this action for the table on every scheduler tick until restart - and recovery after restart repeated the same failure, looping forever while leaking the tracked entry. Persist a FAILED record with an 'Execution engine not found' message (via the TableProcessStore interface, no cast; transition failures are logged, never thrown) and untrack so later ticks can schedule the action again once the engine is restored. Regression test testMissingEngineMarksProcessFailedAndUntracks (red before: status stayed PENDING, entry stayed tracked). Fix record: docs/fix-records/2026-08-16-fix-16-missing-engine-stuck-process.md --- .../amoro/server/process/ProcessService.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java b/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java index 957e3bf7b6..64ce91d038 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/process/ProcessService.java @@ -346,6 +346,25 @@ private void executeOrTraceProcess(TableProcessStore store, TableProcess process store.getExecutionEngine(), store.getAction(), process.getTableIdentifier()); + // Leaving the PENDING process tracked blocks this action forever (hasAliveTableProcess + // gates every later tick, and a restart repeats the same failure). Mark it FAILED so the + // record is visible, then untrack so the table can be scheduled again. + try { + store.tryTransitState( + ProcessStatus.FAILED, + ProcessEvent.COMPLETE_FAILED, + store.getExternalProcessIdentifier(), + String.format( + "Execution engine not found: %s (action=%s)", + store.getExecutionEngine(), store.getAction()), + store.getProcessParameters(), + store.getSummary()); + } catch (Throwable t) { + LOG.error( + "Failed to mark process {} as FAILED for missing engine", store.getProcessId(), t); + } + untrackTableProcessInstance( + process.getTableRuntime().getTableIdentifier(), store.getProcessId()); return; }