From e94740c143d1e9f9a6e72969030d95b4c33e62fa Mon Sep 17 00:00:00 2001 From: Dmitry Grand Date: Tue, 28 Jul 2026 16:16:44 -0700 Subject: [PATCH 1/4] Store in firestore resutls of auto retry --- .../presubmit_subscription.dart | 8 ++ .../service/firestore/unified_check_run.dart | 55 ++++++++++++++ .../presubmit_luci_subscription_test.dart | 39 ++++++++++ .../firestore/unified_check_run_test.dart | 74 +++++++++++++++++++ 4 files changed, 176 insertions(+) diff --git a/app_dart/lib/src/request_handlers/presubmit_subscription.dart b/app_dart/lib/src/request_handlers/presubmit_subscription.dart index 968dbceff..a5af099ef 100644 --- a/app_dart/lib/src/request_handlers/presubmit_subscription.dart +++ b/app_dart/lib/src/request_handlers/presubmit_subscription.dart @@ -19,6 +19,7 @@ import '../model/common/presubmit_completed_check.dart'; import '../request_handling/exceptions.dart'; import '../request_handling/subscription_handler.dart'; import '../service/extensions/cache_service_test_suppression.dart'; +import '../service/firestore/unified_check_run.dart'; import '../service/luci_build_service/build_tags.dart'; import '../service/luci_build_service/user_data.dart'; import '../service/scheduler/ci_yaml_fetcher.dart'; @@ -164,6 +165,13 @@ base class PresubmitSubscription extends SubscriptionHandler { if (tagSet.currentAttempt < maxAttempt) { rescheduled = true; log.info('Rerunning failed task: $builderName'); + if (isUnifiedCheckRun) { + final check = PresubmitCompletedJob.fromBuild(build, userData); + await UnifiedCheckRun.reInitializeInProgressJob( + firestoreService: _firestore, + completedJob: check, + ); + } await _luciBuildService.reschedulePresubmitBuild( builderName: builderName, build: build, diff --git a/app_dart/lib/src/service/firestore/unified_check_run.dart b/app_dart/lib/src/service/firestore/unified_check_run.dart index dcb0d89d7..6ee16c8af 100644 --- a/app_dart/lib/src/service/firestore/unified_check_run.dart +++ b/app_dart/lib/src/service/firestore/unified_check_run.dart @@ -13,6 +13,7 @@ import 'package:googleapis/firestore/v1.dart' hide Status; import 'package:meta/meta.dart'; import '../../model/common/failed_presubmit_jobs.dart'; +import '../../model/common/presubmit_completed_check.dart'; import '../../model/common/presubmit_guard_conclusion.dart'; import '../../model/common/presubmit_job_state.dart'; import '../../model/firestore/base.dart'; @@ -259,6 +260,60 @@ final class UnifiedCheckRun { } } + /// Re-initializes an in-progress job that is being automatically rescheduled. + static Future reInitializeInProgressJob({ + required FirestoreService firestoreService, + required PresubmitCompletedJob completedJob, + @visibleForTesting DateTime Function() utcNow = DateTime.timestamp, + }) async { + final prNum = completedJob.prNum; + final logCrumb = + 'reInitializeInProgressJob(${completedJob.slug.fullName}, $prNum, ${completedJob.checkRunId}, ${completedJob.name})'; + + log.info('$logCrumb Re-initializing in-progress job.'); + + final currentJob = await PresubmitJob.fromFirestore( + firestoreService, + PresubmitJobId( + slug: completedJob.slug, + checkRunId: completedJob.checkRunId, + jobName: completedJob.name, + attemptNumber: completedJob.attempt, + ), + ); + + final creationTime = utcNow().millisecondsSinceEpoch; + final newJob = PresubmitJob.init( + slug: currentJob.slug, + jobName: currentJob.jobName, + checkRunId: currentJob.checkRunId, + creationTime: creationTime, + attemptNumber: currentJob.attemptNumber + 1, + ); + + currentJob.status = TaskStatus.failed; + if (completedJob.endTime != null) { + currentJob.endTime = completedJob.endTime!; + } + currentJob.summary = [ + if (completedJob.summary != null && completedJob.summary!.isNotEmpty) + completedJob.summary, + 'Automatically Rescheduled', + ].join('\n---\n'); + + try { + await firestoreService.writeViaTransaction([ + ...documentsToWrites([currentJob], exists: true), + ...documentsToWrites([newJob], exists: false), + ]); + log.info('$logCrumb: successfully re-initialized in-progress job.'); + return newJob.attemptNumber; + } catch (e) { + log.info('$logCrumb: failed to re-initialize in-progress job', e); + rethrow; + } + } + /// Returns _all_ jobs running against the specified github [checkRunId]. static Future> queryAllPresubmitJobsForGuard({ required FirestoreService firestoreService, diff --git a/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart b/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart index 0b10ce785..67bf33a67 100644 --- a/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart +++ b/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart @@ -875,6 +875,16 @@ void main() { ), ); + firestore.putDocument( + PresubmitJob.init( + slug: RepositorySlug('flutter', 'flutter'), + jobName: 'Linux presubmit_max_attempts=2', + checkRunId: 1, + creationTime: 12345, + attemptNumber: 1, + ), + ); + await tester.post(handler); verifyNever( @@ -889,6 +899,35 @@ void main() { ); verifyNever(mockScheduler.processCheckRunCompleted(any)); + + final updatedCurrentJob = await PresubmitJob.fromFirestore( + firestore, + PresubmitJobId( + slug: RepositorySlug('flutter', 'flutter'), + checkRunId: 1, + jobName: 'Linux presubmit_max_attempts=2', + attemptNumber: 1, + ), + ); + + expect(updatedCurrentJob.status, TaskStatus.failed); + expect( + updatedCurrentJob.summary, + 'test summary\n---\nAutomatically Rescheduled', + ); + + final newJob = await PresubmitJob.fromFirestore( + firestore, + PresubmitJobId( + slug: RepositorySlug('flutter', 'flutter'), + checkRunId: 1, + jobName: 'Linux presubmit_max_attempts=2', + attemptNumber: 2, + ), + ); + + expect(newJob.status, TaskStatus.waitingForBackfill); + expect(newJob.attemptNumber, 2); }); test( diff --git a/app_dart/test/service/firestore/unified_check_run_test.dart b/app_dart/test/service/firestore/unified_check_run_test.dart index dfb3d7985..87861e36e 100644 --- a/app_dart/test/service/firestore/unified_check_run_test.dart +++ b/app_dart/test/service/firestore/unified_check_run_test.dart @@ -5,6 +5,7 @@ import 'package:cocoon_common/task_status.dart'; import 'package:cocoon_integration_test/testing.dart'; import 'package:cocoon_server_test/test_logging.dart'; +import 'package:cocoon_service/src/model/common/presubmit_completed_check.dart'; import 'package:cocoon_service/src/model/common/presubmit_guard_conclusion.dart'; import 'package:cocoon_service/src/model/common/presubmit_job_state.dart'; import 'package:cocoon_service/src/model/firestore/base.dart'; @@ -697,5 +698,78 @@ void main() { expect(attempts[1].attemptNumber, 1); expect(attempts[1].summary, 'attempt 1'); }); + + test( + 'reInitializeInProgressJob creates newJob and updates currentJob', + () async { + final slug = RepositorySlug('flutter', 'flutter'); + final currentJob = PresubmitJob.init( + slug: slug, + jobName: 'linux', + checkRunId: 123, + creationTime: 1000, + attemptNumber: 1, + ); + + await firestoreService.writeViaTransaction( + documentsToWrites([currentJob], exists: false), + ); + + final completedJob = PresubmitCompletedJob( + name: 'linux', + sha: 'abc', + slug: slug, + status: TaskStatus.failed, + isMergeGroup: false, + checkRunId: 123, + checkSuiteId: 234, + headBranch: 'master', + isUnifiedCheckRun: true, + prNum: 567, + attempt: 1, + endTime: 2000, + summary: 'original summary', + ); + + final nextAttempt = await UnifiedCheckRun.reInitializeInProgressJob( + firestoreService: firestoreService, + completedJob: completedJob, + utcNow: () => DateTime.fromMillisecondsSinceEpoch(3000), + ); + + expect(nextAttempt, 2); + + final updatedCurrentJob = await PresubmitJob.fromFirestore( + firestoreService, + PresubmitJobId( + slug: slug, + checkRunId: 123, + jobName: 'linux', + attemptNumber: 1, + ), + ); + + expect(updatedCurrentJob.status, TaskStatus.failed); + expect(updatedCurrentJob.endTime, 2000); + expect( + updatedCurrentJob.summary, + 'original summary\n---\nAutomatically Rescheduled', + ); + + final newJob = await PresubmitJob.fromFirestore( + firestoreService, + PresubmitJobId( + slug: slug, + checkRunId: 123, + jobName: 'linux', + attemptNumber: 2, + ), + ); + + expect(newJob.creationTime, 3000); + expect(newJob.status, TaskStatus.waitingForBackfill); + expect(newJob.attemptNumber, 2); + }, + ); }); } From 5e907316c6e73d4d5a6ace2de58db937a589d844 Mon Sep 17 00:00:00 2001 From: Dmitry Grand Date: Tue, 28 Jul 2026 16:26:03 -0700 Subject: [PATCH 2/4] refactoring --- .../lib/src/request_handlers/presubmit_subscription.dart | 8 ++++++-- app_dart/lib/src/service/firestore/unified_check_run.dart | 6 +----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app_dart/lib/src/request_handlers/presubmit_subscription.dart b/app_dart/lib/src/request_handlers/presubmit_subscription.dart index a5af099ef..be629c233 100644 --- a/app_dart/lib/src/request_handlers/presubmit_subscription.dart +++ b/app_dart/lib/src/request_handlers/presubmit_subscription.dart @@ -166,10 +166,14 @@ base class PresubmitSubscription extends SubscriptionHandler { rescheduled = true; log.info('Rerunning failed task: $builderName'); if (isUnifiedCheckRun) { - final check = PresubmitCompletedJob.fromBuild(build, userData); await UnifiedCheckRun.reInitializeInProgressJob( firestoreService: _firestore, - completedJob: check, + completedJob: PresubmitCompletedJob.fromBuild( + build, + userData, + summaryPrepend: + '### ⚠️ Test failed but automatically rescheduled', + ), ); } await _luciBuildService.reschedulePresubmitBuild( diff --git a/app_dart/lib/src/service/firestore/unified_check_run.dart b/app_dart/lib/src/service/firestore/unified_check_run.dart index 6ee16c8af..bb5d7f690 100644 --- a/app_dart/lib/src/service/firestore/unified_check_run.dart +++ b/app_dart/lib/src/service/firestore/unified_check_run.dart @@ -295,11 +295,7 @@ final class UnifiedCheckRun { if (completedJob.endTime != null) { currentJob.endTime = completedJob.endTime!; } - currentJob.summary = [ - if (completedJob.summary != null && completedJob.summary!.isNotEmpty) - completedJob.summary, - 'Automatically Rescheduled', - ].join('\n---\n'); + currentJob.summary = completedJob.summary; try { await firestoreService.writeViaTransaction([ From d355b6de2be0f3dade34d4088a159362dc168a89 Mon Sep 17 00:00:00 2001 From: Dmitry Grand Date: Tue, 28 Jul 2026 16:31:49 -0700 Subject: [PATCH 3/4] fix unit tests --- .../request_handlers/presubmit_luci_subscription_test.dart | 2 +- app_dart/test/service/firestore/unified_check_run_test.dart | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart b/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart index 67bf33a67..058cc1766 100644 --- a/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart +++ b/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart @@ -913,7 +913,7 @@ void main() { expect(updatedCurrentJob.status, TaskStatus.failed); expect( updatedCurrentJob.summary, - 'test summary\n---\nAutomatically Rescheduled', + '### ⚠️ Test failed but automatically rescheduled\n---\ntest summary', ); final newJob = await PresubmitJob.fromFirestore( diff --git a/app_dart/test/service/firestore/unified_check_run_test.dart b/app_dart/test/service/firestore/unified_check_run_test.dart index 87861e36e..f9ea71007 100644 --- a/app_dart/test/service/firestore/unified_check_run_test.dart +++ b/app_dart/test/service/firestore/unified_check_run_test.dart @@ -728,7 +728,7 @@ void main() { prNum: 567, attempt: 1, endTime: 2000, - summary: 'original summary', + summary: 'summary', ); final nextAttempt = await UnifiedCheckRun.reInitializeInProgressJob( @@ -753,7 +753,7 @@ void main() { expect(updatedCurrentJob.endTime, 2000); expect( updatedCurrentJob.summary, - 'original summary\n---\nAutomatically Rescheduled', + 'summary', ); final newJob = await PresubmitJob.fromFirestore( From ecc476b935d6fa90719b07c948d1fcba97d49bc5 Mon Sep 17 00:00:00 2001 From: Dmitry Grand Date: Tue, 28 Jul 2026 16:36:12 -0700 Subject: [PATCH 4/4] fix ai code review --- .../service/firestore/unified_check_run.dart | 44 ++++++++++--------- .../firestore/unified_check_run_test.dart | 5 +-- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/app_dart/lib/src/service/firestore/unified_check_run.dart b/app_dart/lib/src/service/firestore/unified_check_run.dart index bb5d7f690..b763671ef 100644 --- a/app_dart/lib/src/service/firestore/unified_check_run.dart +++ b/app_dart/lib/src/service/firestore/unified_check_run.dart @@ -272,33 +272,36 @@ final class UnifiedCheckRun { log.info('$logCrumb Re-initializing in-progress job.'); - final currentJob = await PresubmitJob.fromFirestore( - firestoreService, - PresubmitJobId( + final transaction = await firestoreService.beginTransaction(); + try { + final checkDocName = PresubmitJob.documentNameFor( slug: completedJob.slug, checkRunId: completedJob.checkRunId, jobName: completedJob.name, attemptNumber: completedJob.attempt, - ), - ); + ); + final currentJobDocument = await firestoreService.getDocument( + checkDocName, + transaction: transaction, + ); + final currentJob = PresubmitJob.fromDocument(currentJobDocument); - final creationTime = utcNow().millisecondsSinceEpoch; - final newJob = PresubmitJob.init( - slug: currentJob.slug, - jobName: currentJob.jobName, - checkRunId: currentJob.checkRunId, - creationTime: creationTime, - attemptNumber: currentJob.attemptNumber + 1, - ); + final creationTime = utcNow().millisecondsSinceEpoch; + final newJob = PresubmitJob.init( + slug: currentJob.slug, + jobName: currentJob.jobName, + checkRunId: currentJob.checkRunId, + creationTime: creationTime, + attemptNumber: currentJob.attemptNumber + 1, + ); - currentJob.status = TaskStatus.failed; - if (completedJob.endTime != null) { - currentJob.endTime = completedJob.endTime!; - } - currentJob.summary = completedJob.summary; + currentJob.status = TaskStatus.failed; + if (completedJob.endTime != null) { + currentJob.endTime = completedJob.endTime!; + } + currentJob.summary = completedJob.summary; - try { - await firestoreService.writeViaTransaction([ + await firestoreService.commit(transaction, [ ...documentsToWrites([currentJob], exists: true), ...documentsToWrites([newJob], exists: false), ]); @@ -306,6 +309,7 @@ final class UnifiedCheckRun { return newJob.attemptNumber; } catch (e) { log.info('$logCrumb: failed to re-initialize in-progress job', e); + await firestoreService.rollback(transaction); rethrow; } } diff --git a/app_dart/test/service/firestore/unified_check_run_test.dart b/app_dart/test/service/firestore/unified_check_run_test.dart index f9ea71007..f6b486ace 100644 --- a/app_dart/test/service/firestore/unified_check_run_test.dart +++ b/app_dart/test/service/firestore/unified_check_run_test.dart @@ -751,10 +751,7 @@ void main() { expect(updatedCurrentJob.status, TaskStatus.failed); expect(updatedCurrentJob.endTime, 2000); - expect( - updatedCurrentJob.summary, - 'summary', - ); + expect(updatedCurrentJob.summary, 'summary'); final newJob = await PresubmitJob.fromFirestore( firestoreService,