diff --git a/app_dart/lib/src/request_handlers/presubmit_subscription.dart b/app_dart/lib/src/request_handlers/presubmit_subscription.dart index 968dbceff..be629c233 100644 --- a/app_dart/lib/src/request_handlers/presubmit_subscription.dart +++ b/app_dart/lib/src/request_handlers/presubmit_subscription.dart @@ -19,6 +19,7 @@ import '../model/common/presubmit_completed_check.dart'; import '../request_handling/exceptions.dart'; import '../request_handling/subscription_handler.dart'; import '../service/extensions/cache_service_test_suppression.dart'; +import '../service/firestore/unified_check_run.dart'; import '../service/luci_build_service/build_tags.dart'; import '../service/luci_build_service/user_data.dart'; import '../service/scheduler/ci_yaml_fetcher.dart'; @@ -164,6 +165,17 @@ base class PresubmitSubscription extends SubscriptionHandler { if (tagSet.currentAttempt < maxAttempt) { rescheduled = true; log.info('Rerunning failed task: $builderName'); + if (isUnifiedCheckRun) { + await UnifiedCheckRun.reInitializeInProgressJob( + firestoreService: _firestore, + completedJob: PresubmitCompletedJob.fromBuild( + build, + userData, + summaryPrepend: + '### ⚠️ Test failed but automatically rescheduled', + ), + ); + } await _luciBuildService.reschedulePresubmitBuild( builderName: builderName, build: build, diff --git a/app_dart/lib/src/service/firestore/unified_check_run.dart b/app_dart/lib/src/service/firestore/unified_check_run.dart index dcb0d89d7..b763671ef 100644 --- a/app_dart/lib/src/service/firestore/unified_check_run.dart +++ b/app_dart/lib/src/service/firestore/unified_check_run.dart @@ -13,6 +13,7 @@ import 'package:googleapis/firestore/v1.dart' hide Status; import 'package:meta/meta.dart'; import '../../model/common/failed_presubmit_jobs.dart'; +import '../../model/common/presubmit_completed_check.dart'; import '../../model/common/presubmit_guard_conclusion.dart'; import '../../model/common/presubmit_job_state.dart'; import '../../model/firestore/base.dart'; @@ -259,6 +260,60 @@ final class UnifiedCheckRun { } } + /// Re-initializes an in-progress job that is being automatically rescheduled. + static Future reInitializeInProgressJob({ + required FirestoreService firestoreService, + required PresubmitCompletedJob completedJob, + @visibleForTesting DateTime Function() utcNow = DateTime.timestamp, + }) async { + final prNum = completedJob.prNum; + final logCrumb = + 'reInitializeInProgressJob(${completedJob.slug.fullName}, $prNum, ${completedJob.checkRunId}, ${completedJob.name})'; + + log.info('$logCrumb Re-initializing in-progress job.'); + + final transaction = await firestoreService.beginTransaction(); + try { + final checkDocName = PresubmitJob.documentNameFor( + slug: completedJob.slug, + checkRunId: completedJob.checkRunId, + jobName: completedJob.name, + attemptNumber: completedJob.attempt, + ); + final currentJobDocument = await firestoreService.getDocument( + checkDocName, + transaction: transaction, + ); + final currentJob = PresubmitJob.fromDocument(currentJobDocument); + + final creationTime = utcNow().millisecondsSinceEpoch; + final newJob = PresubmitJob.init( + slug: currentJob.slug, + jobName: currentJob.jobName, + checkRunId: currentJob.checkRunId, + creationTime: creationTime, + attemptNumber: currentJob.attemptNumber + 1, + ); + + currentJob.status = TaskStatus.failed; + if (completedJob.endTime != null) { + currentJob.endTime = completedJob.endTime!; + } + currentJob.summary = completedJob.summary; + + await firestoreService.commit(transaction, [ + ...documentsToWrites([currentJob], exists: true), + ...documentsToWrites([newJob], exists: false), + ]); + log.info('$logCrumb: successfully re-initialized in-progress job.'); + return newJob.attemptNumber; + } catch (e) { + log.info('$logCrumb: failed to re-initialize in-progress job', e); + await firestoreService.rollback(transaction); + rethrow; + } + } + /// Returns _all_ jobs running against the specified github [checkRunId]. static Future> queryAllPresubmitJobsForGuard({ required FirestoreService firestoreService, diff --git a/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart b/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart index 0b10ce785..058cc1766 100644 --- a/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart +++ b/app_dart/test/request_handlers/presubmit_luci_subscription_test.dart @@ -875,6 +875,16 @@ void main() { ), ); + firestore.putDocument( + PresubmitJob.init( + slug: RepositorySlug('flutter', 'flutter'), + jobName: 'Linux presubmit_max_attempts=2', + checkRunId: 1, + creationTime: 12345, + attemptNumber: 1, + ), + ); + await tester.post(handler); verifyNever( @@ -889,6 +899,35 @@ void main() { ); verifyNever(mockScheduler.processCheckRunCompleted(any)); + + final updatedCurrentJob = await PresubmitJob.fromFirestore( + firestore, + PresubmitJobId( + slug: RepositorySlug('flutter', 'flutter'), + checkRunId: 1, + jobName: 'Linux presubmit_max_attempts=2', + attemptNumber: 1, + ), + ); + + expect(updatedCurrentJob.status, TaskStatus.failed); + expect( + updatedCurrentJob.summary, + '### ⚠️ Test failed but automatically rescheduled\n---\ntest summary', + ); + + final newJob = await PresubmitJob.fromFirestore( + firestore, + PresubmitJobId( + slug: RepositorySlug('flutter', 'flutter'), + checkRunId: 1, + jobName: 'Linux presubmit_max_attempts=2', + attemptNumber: 2, + ), + ); + + expect(newJob.status, TaskStatus.waitingForBackfill); + expect(newJob.attemptNumber, 2); }); test( diff --git a/app_dart/test/service/firestore/unified_check_run_test.dart b/app_dart/test/service/firestore/unified_check_run_test.dart index dfb3d7985..f6b486ace 100644 --- a/app_dart/test/service/firestore/unified_check_run_test.dart +++ b/app_dart/test/service/firestore/unified_check_run_test.dart @@ -5,6 +5,7 @@ import 'package:cocoon_common/task_status.dart'; import 'package:cocoon_integration_test/testing.dart'; import 'package:cocoon_server_test/test_logging.dart'; +import 'package:cocoon_service/src/model/common/presubmit_completed_check.dart'; import 'package:cocoon_service/src/model/common/presubmit_guard_conclusion.dart'; import 'package:cocoon_service/src/model/common/presubmit_job_state.dart'; import 'package:cocoon_service/src/model/firestore/base.dart'; @@ -697,5 +698,75 @@ void main() { expect(attempts[1].attemptNumber, 1); expect(attempts[1].summary, 'attempt 1'); }); + + test( + 'reInitializeInProgressJob creates newJob and updates currentJob', + () async { + final slug = RepositorySlug('flutter', 'flutter'); + final currentJob = PresubmitJob.init( + slug: slug, + jobName: 'linux', + checkRunId: 123, + creationTime: 1000, + attemptNumber: 1, + ); + + await firestoreService.writeViaTransaction( + documentsToWrites([currentJob], exists: false), + ); + + final completedJob = PresubmitCompletedJob( + name: 'linux', + sha: 'abc', + slug: slug, + status: TaskStatus.failed, + isMergeGroup: false, + checkRunId: 123, + checkSuiteId: 234, + headBranch: 'master', + isUnifiedCheckRun: true, + prNum: 567, + attempt: 1, + endTime: 2000, + summary: 'summary', + ); + + final nextAttempt = await UnifiedCheckRun.reInitializeInProgressJob( + firestoreService: firestoreService, + completedJob: completedJob, + utcNow: () => DateTime.fromMillisecondsSinceEpoch(3000), + ); + + expect(nextAttempt, 2); + + final updatedCurrentJob = await PresubmitJob.fromFirestore( + firestoreService, + PresubmitJobId( + slug: slug, + checkRunId: 123, + jobName: 'linux', + attemptNumber: 1, + ), + ); + + expect(updatedCurrentJob.status, TaskStatus.failed); + expect(updatedCurrentJob.endTime, 2000); + expect(updatedCurrentJob.summary, 'summary'); + + final newJob = await PresubmitJob.fromFirestore( + firestoreService, + PresubmitJobId( + slug: slug, + checkRunId: 123, + jobName: 'linux', + attemptNumber: 2, + ), + ); + + expect(newJob.creationTime, 3000); + expect(newJob.status, TaskStatus.waitingForBackfill); + expect(newJob.attemptNumber, 2); + }, + ); }); }