@@ -11,9 +11,16 @@ import { and, eq, exists, gt, inArray, isNull, lt, or, sql } from 'drizzle-orm'
1111import { alias } from 'drizzle-orm/pg-core'
1212import { type NextRequest , NextResponse } from 'next/server'
1313import { verifyCronAuth } from '@/lib/auth/internal'
14- import { JOB_PENDING_RETENTION_HOURS , JOB_RETENTION_HOURS , JOB_STATUS } from '@/lib/core/async-jobs'
14+ import {
15+ JOB_PENDING_RETENTION_HOURS ,
16+ JOB_RETENTION_HOURS ,
17+ JOB_STATUS ,
18+ MAX_JOB_DURATION_SECONDS ,
19+ MIN_JOB_DURATION_SECONDS ,
20+ } from '@/lib/core/async-jobs'
1521import {
1622 getExecutionReservationTtlMs ,
23+ getTimeoutErrorMessage ,
1724 RESERVATION_TTL_BUFFER_MS ,
1825} from '@/lib/core/execution-limits'
1926import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
@@ -24,6 +31,7 @@ const logger = createLogger('CleanupStaleExecutions')
2431const STALE_THRESHOLD_MS = getExecutionReservationTtlMs ( )
2532const STALE_THRESHOLD_MINUTES = Math . ceil ( STALE_THRESHOLD_MS / 60000 )
2633const GENERIC_STALE_PROCESSING_ERROR = `Job terminated: stuck in processing for more than ${ STALE_THRESHOLD_MINUTES } minutes`
34+ const EXECUTION_DEADLINE_ERROR = getTimeoutErrorMessage ( undefined )
2735const MAX_INT32 = 2_147_483_647
2836/** Terminal table-jobs older than this are pruned; only the latest job per table is ever read. */
2937const TABLE_JOB_RETENTION_HOURS = 24
@@ -142,7 +150,13 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
142150 executionData : sql `jsonb_set(
143151 COALESCE(execution_data, '{}'::jsonb),
144152 ARRAY['error'],
145- to_jsonb(${ `Execution terminated: worker timeout or crash after ${ staleDurationMinutes } minutes` } ::text)
153+ to_jsonb(
154+ CASE
155+ WHEN ${ workflowExecutionLogs . executionDeadlineAt } IS NOT NULL
156+ THEN ${ EXECUTION_DEADLINE_ERROR } ::text
157+ ELSE ${ `Execution terminated: worker timeout or crash after ${ staleDurationMinutes } minutes` } ::text
158+ END
159+ )
146160 )` ,
147161 } )
148162 . where (
@@ -187,18 +201,22 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
187201 let asyncJobsMarkedFailed = 0
188202
189203 try {
204+ const hasPositiveMaxDuration = sql < boolean > `CASE
205+ WHEN jsonb_typeof(${ asyncJobs . metadata } ->'maxDurationSeconds') = 'number'
206+ THEN (${ asyncJobs . metadata } ->>'maxDurationSeconds')::numeric >= ${ MIN_JOB_DURATION_SECONDS }
207+ AND (${ asyncJobs . metadata } ->>'maxDurationSeconds')::numeric <= ${ MAX_JOB_DURATION_SECONDS }
208+ AND trunc((${ asyncJobs . metadata } ->>'maxDurationSeconds')::numeric)
209+ = (${ asyncJobs . metadata } ->>'maxDurationSeconds')::numeric
210+ ELSE FALSE
211+ END`
212+ const staleProcessingDurationPredicate = sql < boolean > `CASE
213+ WHEN ${ hasPositiveMaxDuration }
214+ THEN ${ asyncJobs . startedAt } + ((${ asyncJobs . metadata } ->>'maxDurationSeconds')::double precision * interval '1 second') < ${ now }
215+ ELSE ${ asyncJobs . startedAt } < ${ staleThreshold }
216+ END`
190217 const staleProcessingPredicate = and (
191218 eq ( asyncJobs . status , JOB_STATUS . PROCESSING ) ,
192- or (
193- and (
194- sql `jsonb_typeof(${ asyncJobs . metadata } ->'maxDurationSeconds') = 'number'` ,
195- sql `${ asyncJobs . startedAt } + ((${ asyncJobs . metadata } ->>'maxDurationSeconds')::double precision * interval '1 second') < ${ now } `
196- ) ,
197- and (
198- sql `jsonb_typeof(${ asyncJobs . metadata } ->'maxDurationSeconds') IS DISTINCT FROM 'number'` ,
199- lt ( asyncJobs . startedAt , staleThreshold )
200- )
201- )
219+ staleProcessingDurationPredicate
202220 )
203221 const staleProcessingResult = await runBatchedMutation ( {
204222 batchSize : STATE_MUTATION_BATCH_SIZE ,
@@ -216,10 +234,10 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
216234 status : JOB_STATUS . FAILED ,
217235 completedAt : new Date ( ) ,
218236 error : sql < string > `CASE
219- WHEN jsonb_typeof( ${ asyncJobs . metadata } ->'maxDurationSeconds') = 'number'
220- THEN 'Job terminated: exceeded configured maximum duration of '
237+ WHEN ${ hasPositiveMaxDuration }
238+ THEN 'Job terminated: stuck in processing for more than '
221239 || (${ asyncJobs . metadata } ->>'maxDurationSeconds')
222- || ' seconds'
240+ || ' seconds (worker cleanup deadline) '
223241 ELSE ${ GENERIC_STALE_PROCESSING_ERROR }
224242 END` ,
225243 updatedAt : new Date ( ) ,
0 commit comments