diff --git a/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java b/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java index 8f0e022a8..d1e1c9ed0 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java +++ b/backend/src/main/java/com/bablsoft/accessflow/audit/api/AuditAction.java @@ -287,6 +287,8 @@ public enum AuditAction { /** #985: undecided deployment reviewers were re-nudged on the plan's nudge_interval_hours. */ DEPLOYMENT_REVIEW_NUDGED, DEPLOYMENT_CANCELLED, + /** #1026: a post-deployment verification breached (system row, {@code trigger=verification}). */ + DEPLOYMENT_VERIFICATION_FAILED, /** #695: a deployment break-glass retro-review acknowledged (was generic BREAK_GLASS_REVIEWED). */ DEPLOYMENT_BREAK_GLASS_REVIEWED, diff --git a/backend/src/main/java/com/bablsoft/accessflow/deploygov/internal/DeploymentRequestStateService.java b/backend/src/main/java/com/bablsoft/accessflow/deploygov/internal/DeploymentRequestStateService.java index b96a07d94..4c93e2604 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/deploygov/internal/DeploymentRequestStateService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/deploygov/internal/DeploymentRequestStateService.java @@ -22,6 +22,7 @@ import java.time.Instant; import java.util.EnumMap; import java.util.EnumSet; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -179,7 +180,7 @@ public boolean markNudged(UUID id, Instant at) { * verification — a lost race, not an error. */ @Transactional - public boolean markVerificationFailed(UUID id, String detail) { + public boolean markVerificationFailed(UUID id, String check, String detail) { var entity = require(id); if (!isRunningVerification(entity)) { return false; @@ -188,7 +189,18 @@ public boolean markVerificationFailed(UUID id, String detail) { entity.setVerificationFinishedAt(clock.instant()); entity.setVerificationDetail(detail); apply(entity, QueryStatus.FAILED); - // #1026: the DEPLOYMENT_VERIFICATION_FAILED audit row is recorded here. + // #1026: system row — null actor; check is null when the config snapshot was unreadable. + var metadata = new LinkedHashMap(); + metadata.put("trigger", "verification"); + metadata.put("pipeline_id", entity.getPipelineId().toString()); + metadata.put("version", entity.getVersion()); + if (check != null) { + metadata.put("check", check); + } + metadata.put("detail", detail); + auditWriter.record(AuditAction.DEPLOYMENT_VERIFICATION_FAILED, + AuditResourceType.DEPLOYMENT_REQUEST, entity.getId(), entity.getOrganizationId(), + null, metadata, null); eventPublisher.publishEvent(new DeploymentVerificationFailedEvent( entity.getOrganizationId(), entity.getId(), entity.getPipelineId(), detail)); return true; diff --git a/backend/src/main/java/com/bablsoft/accessflow/deploygov/internal/verification/DefaultDeploymentVerificationService.java b/backend/src/main/java/com/bablsoft/accessflow/deploygov/internal/verification/DefaultDeploymentVerificationService.java index ff3458ed2..a84cfa1ff 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/deploygov/internal/verification/DefaultDeploymentVerificationService.java +++ b/backend/src/main/java/com/bablsoft/accessflow/deploygov/internal/verification/DefaultDeploymentVerificationService.java @@ -76,7 +76,7 @@ public VerificationTickResult verifyOne(UUID deploymentRequestId) { // Fail closed: a running verification whose snapshot cannot be read must not pass. log.warn("Deployment request {} has an unreadable verification snapshot", deploymentRequestId); - return stateService.markVerificationFailed(deploymentRequestId, UNREADABLE_CONFIG) + return stateService.markVerificationFailed(deploymentRequestId, null, UNREADABLE_CONFIG) ? VerificationTickResult.BREACHED : VerificationTickResult.SKIPPED; } var now = clock.instant(); @@ -110,7 +110,7 @@ private VerificationTickResult record(UUID id, DeploymentVerificationConfig conf if (failed >= config.failureThreshold()) { var detail = check.name() + ": " + failed + "/" + config.failureThreshold() + " failed samples"; - return stateService.markVerificationFailed(id, detail) + return stateService.markVerificationFailed(id, check.name(), detail) ? VerificationTickResult.BREACHED : VerificationTickResult.SKIPPED; } } diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/api/NotificationEventType.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/api/NotificationEventType.java index 61deb5542..6e1ac7e44 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/notifications/api/NotificationEventType.java +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/api/NotificationEventType.java @@ -68,6 +68,12 @@ public enum NotificationEventType { * every active org admin including PagerDuty, mirroring {@code BREAK_GLASS_EXECUTED}. */ DEPLOYMENT_BREAK_GLASS_EXECUTED, + /** + * A post-deployment verification breached (#1026) — notifies the approvers and the submitter. + * A recommendation only: the release is still running and AccessFlow did not roll it back. + * Never pages, never tickets. + */ + DEPLOYMENT_VERIFICATION_FAILED, /** * A schema change set promotion entered review (#882) — its request group reached diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/DeploymentNotificationListener.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/DeploymentNotificationListener.java index 795acc41a..af650d5a9 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/DeploymentNotificationListener.java +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/DeploymentNotificationListener.java @@ -8,6 +8,7 @@ import com.bablsoft.accessflow.deploygov.events.DeploymentReviewEscalatedEvent; import com.bablsoft.accessflow.deploygov.events.DeploymentReviewNudgedEvent; import com.bablsoft.accessflow.deploygov.events.DeploymentStatusChangedEvent; +import com.bablsoft.accessflow.deploygov.events.DeploymentVerificationFailedEvent; import com.bablsoft.accessflow.notifications.api.NotificationEventType; import lombok.RequiredArgsConstructor; import org.springframework.modulith.events.ApplicationModuleListener; @@ -20,7 +21,8 @@ * deployment that a routing policy auto-decides never pings them. A decided deployment alerts the * submitter ({@code TIMED_OUT} folds into {@code DEPLOYMENT_REJECTED} with its * {@code review_timeout} reason); a {@code FAILED} or {@code ROLLED_BACK} outcome alerts the - * reviewers who approved it; a break-glass release alerts every org admin; an idle review is + * reviewers who approved it; a breached post-deployment verification (#1026) alerts those approvers + * plus the submitter; a break-glass release alerts every org admin; an idle review is * escalated / nudged via the shared {@code REVIEW_ESCALATED} / {@code REVIEW_NUDGE} types (#985). * Break-glass publishes no {@code DeploymentDecidedEvent}, so the submitter never self-notifies on * that path. Delivery is best-effort and never blocks the deployment flow. @@ -56,6 +58,13 @@ void onOutcomeReported(DeploymentOutcomeReportedEvent event) { } } + // #1026: the breached check rides in the decision-reason slot. + @ApplicationModuleListener + void onVerificationFailed(DeploymentVerificationFailedEvent event) { + dispatcher.dispatchDeployment(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, + event.deploymentRequestId(), null, event.detail()); + } + @ApplicationModuleListener void onBreakGlassExecuted(DeploymentBreakGlassExecutedEvent event) { dispatcher.dispatchDeployment(NotificationEventType.DEPLOYMENT_BREAK_GLASS_EXECUTED, diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationContext.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationContext.java index c30e1f2df..ffe568f09 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationContext.java +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationContext.java @@ -50,7 +50,8 @@ * for the submission (or break-glass) justification. {@code deploymentOutcome} is only set for * {@code DEPLOYMENT_OUTCOME_FAILED}; {@code deploymentDecisionReason} carries the decision * provenance ({@code "routing:<policyId>"}, {@code "freeze:<windowId>"}, - * {@code "review_timeout"}, or null for a reviewer verdict). + * {@code "review_timeout"}, or null for a reviewer verdict) — or, for + * {@code DEPLOYMENT_VERIFICATION_FAILED} (#1026), the verification breach detail. * *

The {@code schemaChange*} fields are only populated for the {@code SCHEMA_CHANGE_PROMOTION_*} * events and {@code driftNewFindingCount} only for {@code SCHEMA_DRIFT_DETECTED} (#882). Both reuse diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationContextBuilder.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationContextBuilder.java index 953f8fbbf..76179c33b 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationContextBuilder.java +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationContextBuilder.java @@ -195,6 +195,7 @@ private List resolveRecipients(NotificationEventType eventType, ERASURE_APPROVED, DEPLOYMENT_SUBMITTED, DEPLOYMENT_APPROVED, DEPLOYMENT_REJECTED, DEPLOYMENT_OUTCOME_FAILED, DEPLOYMENT_BREAK_GLASS_EXECUTED, + DEPLOYMENT_VERIFICATION_FAILED, SCHEMA_CHANGE_PROMOTION_SUBMITTED, SCHEMA_CHANGE_PROMOTION_APPLIED, SCHEMA_CHANGE_PROMOTION_FAILED, SCHEMA_DRIFT_DETECTED, DATA_BUDGET_THRESHOLD_REACHED, DATA_BUDGET_EXHAUSTED -> List.of(); @@ -573,6 +574,14 @@ private List deploymentRecipients(NotificationEventType eventType // The reviewers who granted the deployment learn it failed or was rolled back. case DEPLOYMENT_OUTCOME_FAILED -> toActiveRecipients( deploymentNotificationLookupService.findApproverUserIds(view.id())); + // #1026: a breached verification also reaches the submitter, the person most able to + // roll back. Auto-approved and break-glass requests have no approvers: submitter only. + case DEPLOYMENT_VERIFICATION_FAILED -> { + var ids = new LinkedHashSet<>( + deploymentNotificationLookupService.findApproverUserIds(view.id())); + ids.add(view.submittedByUserId()); + yield toActiveRecipients(List.copyOf(ids)); + } // #985: a nudge reminds the eligible reviewers who have not decided yet; an escalation // adds every active org admin, because the point is that those reviewers did not act. // Notify-only — neither widens who may decide, and the submitter is never included. diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationDispatcher.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationDispatcher.java index c4e1ffbca..0af6ab2cc 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationDispatcher.java +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/NotificationDispatcher.java @@ -325,6 +325,11 @@ private String buildPayload(NotificationContext ctx) { if (ctx.deploymentOutcome() != null) { payload.put("outcome", ctx.deploymentOutcome().name()); } + // #1026: the bell is the one surface every recipient gets, so it names the failed check. + if (ctx.eventType() == NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED + && ctx.deploymentDecisionReason() != null) { + payload.put("verification_detail", ctx.deploymentDecisionReason()); + } if (ctx.schemaChangePromotionId() != null) { payload.put("schema_change_promotion_id", ctx.schemaChangePromotionId().toString()); } @@ -410,6 +415,7 @@ private List resolveChannels(NotificationEventType ev || eventType == NotificationEventType.DEPLOYMENT_REJECTED || eventType == NotificationEventType.DEPLOYMENT_OUTCOME_FAILED || eventType == NotificationEventType.DEPLOYMENT_BREAK_GLASS_EXECUTED + || eventType == NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED // #985: the shared REVIEW_ESCALATED / REVIEW_NUDGE on a deployment carry the // pipeline in datasourceId, so they fan out org-wide like every deployment event. || ctx.deploymentRequestId() != null diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/DeploymentVerificationText.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/DeploymentVerificationText.java new file mode 100644 index 000000000..03c115524 --- /dev/null +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/DeploymentVerificationText.java @@ -0,0 +1,28 @@ +package com.bablsoft.accessflow.notifications.internal.strategy; + +import com.bablsoft.accessflow.notifications.api.NotificationEventType; +import com.bablsoft.accessflow.notifications.internal.NotificationContext; + +/** + * Shared chat rendering for {@code DEPLOYMENT_VERIFICATION_FAILED} (#1026). The wording is a + * recommendation: AccessFlow never rolls back, so no channel may imply that it did. + */ +final class DeploymentVerificationText { + + static final String TITLE = "🚨 Deployment Verification Failed"; + static final String RECOMMENDATION = "The release is still running — AccessFlow did not roll it " + + "back. Decide whether to roll back, then report ROLLED_BACK."; + + private DeploymentVerificationText() { + } + + static boolean applies(NotificationContext ctx) { + return ctx.eventType() == NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED; + } + + /** The breached check and its sample count, bounded like every other chat field. */ + static String detail(NotificationContext ctx) { + var detail = ctx.deploymentDecisionReason(); + return detail == null || detail.isBlank() ? "—" : SchemaChangeText.truncate(detail); + } +} diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/DiscordPayloadFactory.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/DiscordPayloadFactory.java index 91b46f957..6ba75b9a4 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/DiscordPayloadFactory.java +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/DiscordPayloadFactory.java @@ -146,6 +146,10 @@ private static Map buildEventEmbed(NotificationContext ctx) { if (ctx.deploymentOutcome() != null) { addField(fields, "Outcome", ctx.deploymentOutcome().name()); } + if (DeploymentVerificationText.applies(ctx)) { + addField(fields, "Verification", DeploymentVerificationText.detail(ctx)); + addField(fields, "Next step", DeploymentVerificationText.RECOMMENDATION); + } embed.put("fields", fields); return embed; } @@ -236,6 +240,7 @@ private static String headerLabel(NotificationContext ctx) { case DEPLOYMENT_REJECTED -> "❌ Deployment Rejected"; case DEPLOYMENT_OUTCOME_FAILED -> "🚨 Deployment Failed or Rolled Back"; case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "🚨 Break-glass Deployment Executed"; + case DEPLOYMENT_VERIFICATION_FAILED -> DeploymentVerificationText.TITLE; case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "🧱 Schema Change Awaiting Review"; case SCHEMA_CHANGE_PROMOTION_APPLIED -> "✅ Schema Change Applied"; case SCHEMA_CHANGE_PROMOTION_FAILED -> "🚨 Schema Change Failed"; diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/EmailNotificationStrategy.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/EmailNotificationStrategy.java index 33f997754..e3973f693 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/EmailNotificationStrategy.java +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/EmailNotificationStrategy.java @@ -316,6 +316,7 @@ static String templateName(NotificationEventType eventType) { case DEPLOYMENT_REJECTED -> "email/deployment-rejected"; case DEPLOYMENT_OUTCOME_FAILED -> "email/deployment-outcome-failed"; case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "email/deployment-break-glass-executed"; + case DEPLOYMENT_VERIFICATION_FAILED -> "email/deployment-verification-failed"; // #882: schema-change promotions and drift. case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "email/schema-change-promotion-submitted"; case SCHEMA_CHANGE_PROMOTION_APPLIED -> "email/schema-change-promotion-applied"; @@ -352,7 +353,8 @@ private String subject(NotificationContext ctx) { // Spelled out — this switch has a silent default and would otherwise pass only the // pipeline, leaving "{1}" unfilled in every deployment subject. case DEPLOYMENT_SUBMITTED, DEPLOYMENT_APPROVED, DEPLOYMENT_REJECTED, - DEPLOYMENT_OUTCOME_FAILED, DEPLOYMENT_BREAK_GLASS_EXECUTED -> + DEPLOYMENT_OUTCOME_FAILED, DEPLOYMENT_BREAK_GLASS_EXECUTED, + DEPLOYMENT_VERIFICATION_FAILED -> new Object[]{ctx.datasourceName(), ctx.deploymentVersion()}; // #882: "{0} → {1}" is the change set and the environment it was promoted to. Spelled // out — the silent default would pass the pipeline where the change set belongs. @@ -406,6 +408,8 @@ private static String subjectKey(NotificationEventType eventType) { "notification.email.subject.deployment_outcome_failed"; case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "notification.email.subject.deployment_break_glass_executed"; + case DEPLOYMENT_VERIFICATION_FAILED -> + "notification.email.subject.deployment_verification_failed"; case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "notification.email.subject.schema_change_promotion_submitted"; case SCHEMA_CHANGE_PROMOTION_APPLIED -> diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/MsTeamsPayloadFactory.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/MsTeamsPayloadFactory.java index 83b894718..d5f5aaa74 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/MsTeamsPayloadFactory.java +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/MsTeamsPayloadFactory.java @@ -154,6 +154,10 @@ private static Map buildEventCard(NotificationContext ctx) { if (ctx.deploymentOutcome() != null) { facts.add(fact("Outcome", ctx.deploymentOutcome().name())); } + if (DeploymentVerificationText.applies(ctx)) { + facts.add(fact("Verification", DeploymentVerificationText.detail(ctx))); + facts.add(fact("Next step", DeploymentVerificationText.RECOMMENDATION)); + } body.add(factSet(facts)); card.put("body", body); return card; @@ -294,6 +298,7 @@ private static String headerLabel(NotificationContext ctx) { case DEPLOYMENT_REJECTED -> "❌ Deployment Rejected"; case DEPLOYMENT_OUTCOME_FAILED -> "🚨 Deployment Failed or Rolled Back"; case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "🚨 Break-glass Deployment Executed"; + case DEPLOYMENT_VERIFICATION_FAILED -> DeploymentVerificationText.TITLE; case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "🧱 Schema Change Awaiting Review"; case SCHEMA_CHANGE_PROMOTION_APPLIED -> "✅ Schema Change Applied"; case SCHEMA_CHANGE_PROMOTION_FAILED -> "🚨 Schema Change Failed"; diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/SlackBlockKitFactory.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/SlackBlockKitFactory.java index 446828d5d..e27ea2c62 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/SlackBlockKitFactory.java +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/SlackBlockKitFactory.java @@ -154,6 +154,10 @@ private static SectionBlock deploymentSection(NotificationContext ctx) { if (ctx.deploymentOutcome() != null) { fields.add(mrkdwn("*Outcome:*\n" + ctx.deploymentOutcome().name())); } + if (DeploymentVerificationText.applies(ctx)) { + fields.add(mrkdwn("*Verification:*\n" + DeploymentVerificationText.detail(ctx))); + fields.add(mrkdwn("*Next step:*\n" + DeploymentVerificationText.RECOMMENDATION)); + } return SectionBlock.builder().fields(fields).build(); } @@ -303,6 +307,7 @@ private static String headerLabel(NotificationContext ctx) { case DEPLOYMENT_REJECTED -> "❌ Deployment Rejected"; case DEPLOYMENT_OUTCOME_FAILED -> "🚨 Deployment Failed or Rolled Back"; case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "🚨 Break-glass Deployment Executed"; + case DEPLOYMENT_VERIFICATION_FAILED -> DeploymentVerificationText.TITLE; case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "🧱 Schema Change Awaiting Review"; case SCHEMA_CHANGE_PROMOTION_APPLIED -> "✅ Schema Change Applied"; case SCHEMA_CHANGE_PROMOTION_FAILED -> "🚨 Schema Change Failed"; diff --git a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/TelegramMessageFactory.java b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/TelegramMessageFactory.java index c1063c39f..a4b3ce15f 100644 --- a/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/TelegramMessageFactory.java +++ b/backend/src/main/java/com/bablsoft/accessflow/notifications/internal/strategy/TelegramMessageFactory.java @@ -124,6 +124,10 @@ private static String buildEventText(NotificationContext ctx) { if (ctx.deploymentOutcome() != null) { appendField(sb, "Outcome", ctx.deploymentOutcome().name()); } + if (DeploymentVerificationText.applies(ctx)) { + appendField(sb, "Verification", DeploymentVerificationText.detail(ctx)); + appendField(sb, "Next step", DeploymentVerificationText.RECOMMENDATION); + } return sb.toString(); } appendField(sb, "Datasource", nullToDash(ctx.datasourceName())); @@ -204,6 +208,7 @@ private static String headerLabel(NotificationContext ctx) { case DEPLOYMENT_REJECTED -> "❌ Deployment Rejected"; case DEPLOYMENT_OUTCOME_FAILED -> "🚨 Deployment Failed or Rolled Back"; case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "🚨 Break-glass Deployment Executed"; + case DEPLOYMENT_VERIFICATION_FAILED -> DeploymentVerificationText.TITLE; case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "🧱 Schema Change Awaiting Review"; case SCHEMA_CHANGE_PROMOTION_APPLIED -> "✅ Schema Change Applied"; case SCHEMA_CHANGE_PROMOTION_FAILED -> "🚨 Schema Change Failed"; diff --git a/backend/src/main/resources/i18n/messages.properties b/backend/src/main/resources/i18n/messages.properties index 97db6618a..7d923422a 100644 --- a/backend/src/main/resources/i18n/messages.properties +++ b/backend/src/main/resources/i18n/messages.properties @@ -1294,6 +1294,7 @@ notification.email.subject.deployment_submitted=[AccessFlow] Deployment awaiting notification.email.subject.deployment_approved=[AccessFlow] Deployment approved: {0} {1} notification.email.subject.deployment_rejected=[AccessFlow] Deployment rejected: {0} {1} notification.email.subject.deployment_outcome_failed=[AccessFlow] Deployment failed or rolled back: {0} {1} +notification.email.subject.deployment_verification_failed=[AccessFlow] Deployment verification failed: {0} {1} notification.email.subject.deployment_break_glass_executed=[AccessFlow] Break-glass deployment {1} executed on {0} notification.email.deployment.pipeline_label=Pipeline: notification.email.deployment.environment_label=Environment: @@ -1316,6 +1317,11 @@ notification.email.deployment_outcome_failed.title=Deployment failed notification.email.deployment_outcome_failed.heading=A deployment you approved went wrong notification.email.deployment_outcome_failed.body_failed=The pipeline reported the deployment as failed. notification.email.deployment_outcome_failed.body_rolled_back=The pipeline reported the deployment as rolled back. +notification.email.deployment_verification_failed.title=Deployment verification failed +notification.email.deployment_verification_failed.heading=A released deployment failed its post-deployment verification +notification.email.deployment_verification_failed.body=The post-deployment verification failed after the pipeline reported the deployment as succeeded. +notification.email.deployment_verification_failed.detail_label=Detail: +notification.email.deployment_verification_failed.recommendation=The release is still running — AccessFlow did not roll it back. Decide whether to roll back, then report ROLLED_BACK from the pipeline. notification.email.deployment_break_glass_executed.title=Break-glass deployment executed notification.email.deployment_break_glass_executed.heading=A break-glass deployment was released notification.email.deployment_break_glass_executed.body=An emergency deployment bypassed review and was force-approved for release. It requires a mandatory retrospective review. diff --git a/backend/src/main/resources/i18n/messages_de.properties b/backend/src/main/resources/i18n/messages_de.properties index 10d0b7128..c67a5e4b3 100644 --- a/backend/src/main/resources/i18n/messages_de.properties +++ b/backend/src/main/resources/i18n/messages_de.properties @@ -1273,6 +1273,7 @@ notification.email.subject.deployment_submitted=[AccessFlow] Deployment wartet a notification.email.subject.deployment_approved=[AccessFlow] Deployment genehmigt: {0} {1} notification.email.subject.deployment_rejected=[AccessFlow] Deployment abgelehnt: {0} {1} notification.email.subject.deployment_outcome_failed=[AccessFlow] Deployment fehlgeschlagen oder zurückgerollt: {0} {1} +notification.email.subject.deployment_verification_failed=[AccessFlow] Verifizierung des Deployments fehlgeschlagen: {0} {1} notification.email.subject.deployment_break_glass_executed=[AccessFlow] Break-Glass-Deployment {1} ausgeführt auf {0} notification.email.deployment.pipeline_label=Pipeline: notification.email.deployment.environment_label=Umgebung: @@ -1295,6 +1296,11 @@ notification.email.deployment_outcome_failed.title=Deployment fehlgeschlagen notification.email.deployment_outcome_failed.heading=Ein von Ihnen genehmigtes Deployment ist schiefgelaufen notification.email.deployment_outcome_failed.body_failed=Die Pipeline hat das Deployment als fehlgeschlagen gemeldet. notification.email.deployment_outcome_failed.body_rolled_back=Die Pipeline hat für das Deployment einen Rollback gemeldet. +notification.email.deployment_verification_failed.title=Deployment-Verifizierung fehlgeschlagen +notification.email.deployment_verification_failed.heading=Ein ausgeliefertes Deployment hat die Verifizierung nach dem Deployment nicht bestanden +notification.email.deployment_verification_failed.body=Die Verifizierung nach dem Deployment ist fehlgeschlagen, nachdem die Pipeline das Deployment als erfolgreich gemeldet hatte. +notification.email.deployment_verification_failed.detail_label=Details: +notification.email.deployment_verification_failed.recommendation=Das Release läuft weiterhin — AccessFlow hat keinen Rollback durchgeführt. Entscheiden Sie, ob Sie zurückrollen, und melden Sie danach ROLLED_BACK aus der Pipeline. notification.email.deployment_break_glass_executed.title=Break-Glass-Deployment ausgeführt notification.email.deployment_break_glass_executed.heading=Ein Break-Glass-Deployment wurde freigegeben notification.email.deployment_break_glass_executed.body=Ein Notfall-Deployment hat die Prüfung umgangen und wurde zwangsweise zur Freigabe genehmigt. Es erfordert eine obligatorische nachträgliche Prüfung. diff --git a/backend/src/main/resources/i18n/messages_es.properties b/backend/src/main/resources/i18n/messages_es.properties index cb356b4c9..1acdd30d5 100644 --- a/backend/src/main/resources/i18n/messages_es.properties +++ b/backend/src/main/resources/i18n/messages_es.properties @@ -1273,6 +1273,7 @@ notification.email.subject.deployment_submitted=[AccessFlow] Despliegue pendient notification.email.subject.deployment_approved=[AccessFlow] Despliegue aprobado: {0} {1} notification.email.subject.deployment_rejected=[AccessFlow] Despliegue rechazado: {0} {1} notification.email.subject.deployment_outcome_failed=[AccessFlow] Despliegue fallido o revertido: {0} {1} +notification.email.subject.deployment_verification_failed=[AccessFlow] Falló la verificación del despliegue: {0} {1} notification.email.subject.deployment_break_glass_executed=[AccessFlow] Despliegue de break-glass {1} ejecutado en {0} notification.email.deployment.pipeline_label=Pipeline: notification.email.deployment.environment_label=Entorno: @@ -1295,6 +1296,11 @@ notification.email.deployment_outcome_failed.title=Despliegue fallido notification.email.deployment_outcome_failed.heading=Un despliegue que usted aprobó salió mal notification.email.deployment_outcome_failed.body_failed=El pipeline informó el despliegue como fallido. notification.email.deployment_outcome_failed.body_rolled_back=El pipeline informó el despliegue como revertido. +notification.email.deployment_verification_failed.title=Falló la verificación del despliegue +notification.email.deployment_verification_failed.heading=Un despliegue publicado no superó la verificación posterior al despliegue +notification.email.deployment_verification_failed.body=La verificación posterior al despliegue falló después de que el pipeline informara el despliegue como correcto. +notification.email.deployment_verification_failed.detail_label=Detalle: +notification.email.deployment_verification_failed.recommendation=La versión sigue en ejecución: AccessFlow no la revirtió. Decida si revertirla y, después, informe ROLLED_BACK desde el pipeline. notification.email.deployment_break_glass_executed.title=Despliegue de break-glass ejecutado notification.email.deployment_break_glass_executed.heading=Se liberó un despliegue de break-glass notification.email.deployment_break_glass_executed.body=Un despliegue de emergencia omitió la revisión y fue aprobado de forma forzosa para su liberación. Requiere una revisión retrospectiva obligatoria. diff --git a/backend/src/main/resources/i18n/messages_fr.properties b/backend/src/main/resources/i18n/messages_fr.properties index 910a46d06..ba28c3aa7 100644 --- a/backend/src/main/resources/i18n/messages_fr.properties +++ b/backend/src/main/resources/i18n/messages_fr.properties @@ -1279,6 +1279,7 @@ notification.email.subject.deployment_submitted=[AccessFlow] Déploiement en att notification.email.subject.deployment_approved=[AccessFlow] Déploiement approuvé : {0} {1} notification.email.subject.deployment_rejected=[AccessFlow] Déploiement rejeté : {0} {1} notification.email.subject.deployment_outcome_failed=[AccessFlow] Échec ou rollback du déploiement : {0} {1} +notification.email.subject.deployment_verification_failed=[AccessFlow] Échec de la vérification du déploiement : {0} {1} notification.email.subject.deployment_break_glass_executed=[AccessFlow] Déploiement break-glass {1} exécuté sur {0} notification.email.deployment.pipeline_label=Pipeline : notification.email.deployment.environment_label=Environnement : @@ -1301,6 +1302,11 @@ notification.email.deployment_outcome_failed.title=Déploiement échoué notification.email.deployment_outcome_failed.heading=Un déploiement que vous avez approuvé a mal tourné notification.email.deployment_outcome_failed.body_failed=Le pipeline a signalé le déploiement comme échoué. notification.email.deployment_outcome_failed.body_rolled_back=Le pipeline a signalé un rollback du déploiement. +notification.email.deployment_verification_failed.title=Échec de la vérification du déploiement +notification.email.deployment_verification_failed.heading=Un déploiement livré a échoué à sa vérification post-déploiement +notification.email.deployment_verification_failed.body=La vérification post-déploiement a échoué après que le pipeline a signalé le déploiement comme réussi. +notification.email.deployment_verification_failed.detail_label=Détail : +notification.email.deployment_verification_failed.recommendation=La version est toujours en service — AccessFlow ne l'a pas annulée. Décidez s'il faut faire un rollback, puis signalez ROLLED_BACK depuis le pipeline. notification.email.deployment_break_glass_executed.title=Déploiement break-glass exécuté notification.email.deployment_break_glass_executed.heading=Un déploiement break-glass a été libéré notification.email.deployment_break_glass_executed.body=Un déploiement d'urgence a contourné la relecture et a été approuvé de force pour être libéré. Il nécessite une revue rétrospective obligatoire. diff --git a/backend/src/main/resources/i18n/messages_hy.properties b/backend/src/main/resources/i18n/messages_hy.properties index 490793030..b95e7fbfd 100644 --- a/backend/src/main/resources/i18n/messages_hy.properties +++ b/backend/src/main/resources/i18n/messages_hy.properties @@ -1273,6 +1273,7 @@ notification.email.subject.deployment_submitted=[AccessFlow] Տեղակայու notification.email.subject.deployment_approved=[AccessFlow] Տեղակայումը հաստատվել է՝ {0} {1} notification.email.subject.deployment_rejected=[AccessFlow] Տեղակայումը մերժվել է՝ {0} {1} notification.email.subject.deployment_outcome_failed=[AccessFlow] Տեղակայումը ձախողվել կամ հետ է գլորվել՝ {0} {1} +notification.email.subject.deployment_verification_failed=[AccessFlow] Տեղակայման ստուգումը ձախողվեց՝ {0} {1} notification.email.subject.deployment_break_glass_executed=[AccessFlow] Break-glass տեղակայում {1}-ը կատարվեց {0}-ում notification.email.deployment.pipeline_label=Փիփլայն՝ notification.email.deployment.environment_label=Միջավայր՝ @@ -1295,6 +1296,11 @@ notification.email.deployment_outcome_failed.title=Տեղակայումը ձախ notification.email.deployment_outcome_failed.heading=Ձեր հաստատած տեղակայումը խնդիր ունեցավ notification.email.deployment_outcome_failed.body_failed=Փիփլայնը զեկուցեց, որ տեղակայումը ձախողվել է։ notification.email.deployment_outcome_failed.body_rolled_back=Փիփլայնը զեկուցեց, որ տեղակայումը հետ է գլորվել։ +notification.email.deployment_verification_failed.title=Տեղակայման ստուգումը ձախողվեց +notification.email.deployment_verification_failed.heading=Թողարկված տեղակայումը չանցավ հետտեղակայման ստուգումը +notification.email.deployment_verification_failed.body=Հետտեղակայման ստուգումը ձախողվեց այն բանից հետո, երբ փիփլայնը տեղակայումը զեկուցեց որպես հաջողված։ +notification.email.deployment_verification_failed.detail_label=Մանրամասներ՝ +notification.email.deployment_verification_failed.recommendation=Թողարկումը դեռ աշխատում է — AccessFlow-ը այն հետ չի գլորել։ Որոշեք՝ արդյոք հետ գլորել, ապա փիփլայնից զեկուցեք ROLLED_BACK։ notification.email.deployment_break_glass_executed.title=Break-glass տեղակայումը կատարվեց notification.email.deployment_break_glass_executed.heading=Թողարկվեց break-glass տեղակայում notification.email.deployment_break_glass_executed.body=Արտակարգ տեղակայումը շրջանցեց վերանայումը և հարկադրաբար հաստատվեց թողարկման համար։ Այն պահանջում է պարտադիր հետադարձ վերանայում։ diff --git a/backend/src/main/resources/i18n/messages_ru.properties b/backend/src/main/resources/i18n/messages_ru.properties index f1b351162..7f3afc840 100644 --- a/backend/src/main/resources/i18n/messages_ru.properties +++ b/backend/src/main/resources/i18n/messages_ru.properties @@ -1273,6 +1273,7 @@ notification.email.subject.deployment_submitted=[AccessFlow] Развертыв notification.email.subject.deployment_approved=[AccessFlow] Развертывание одобрено: {0} {1} notification.email.subject.deployment_rejected=[AccessFlow] Развертывание отклонено: {0} {1} notification.email.subject.deployment_outcome_failed=[AccessFlow] Сбой или откат развертывания: {0} {1} +notification.email.subject.deployment_verification_failed=[AccessFlow] Проверка развертывания не пройдена: {0} {1} notification.email.subject.deployment_break_glass_executed=[AccessFlow] Аварийное развертывание (break-glass) {1} выполнено в {0} notification.email.deployment.pipeline_label=Пайплайн: notification.email.deployment.environment_label=Окружение: @@ -1295,6 +1296,11 @@ notification.email.deployment_outcome_failed.title=Сбой развертыва notification.email.deployment_outcome_failed.heading=Одобренное вами развертывание завершилось неудачно notification.email.deployment_outcome_failed.body_failed=Пайплайн сообщил, что развертывание завершилось сбоем. notification.email.deployment_outcome_failed.body_rolled_back=Пайплайн сообщил об откате развертывания. +notification.email.deployment_verification_failed.title=Проверка развертывания не пройдена +notification.email.deployment_verification_failed.heading=Выпущенное развертывание не прошло проверку после развертывания +notification.email.deployment_verification_failed.body=Проверка после развертывания не пройдена после того, как пайплайн сообщил об успешном развертывании. +notification.email.deployment_verification_failed.detail_label=Подробности: +notification.email.deployment_verification_failed.recommendation=Релиз по-прежнему работает — AccessFlow не выполнял откат. Решите, нужен ли откат, затем сообщите ROLLED_BACK из пайплайна. notification.email.deployment_break_glass_executed.title=Аварийное развертывание break-glass выполнено notification.email.deployment_break_glass_executed.heading=Выпущено аварийное развертывание break-glass notification.email.deployment_break_glass_executed.body=Аварийное развертывание обошло проверку и было принудительно одобрено к выпуску. Оно требует обязательной ретроспективной проверки. diff --git a/backend/src/main/resources/i18n/messages_zh_CN.properties b/backend/src/main/resources/i18n/messages_zh_CN.properties index 6b417a0d5..a2824c4b4 100644 --- a/backend/src/main/resources/i18n/messages_zh_CN.properties +++ b/backend/src/main/resources/i18n/messages_zh_CN.properties @@ -1273,6 +1273,7 @@ notification.email.subject.deployment_submitted=[AccessFlow] 部署等待审核 notification.email.subject.deployment_approved=[AccessFlow] 部署已批准:{0} {1} notification.email.subject.deployment_rejected=[AccessFlow] 部署已拒绝:{0} {1} notification.email.subject.deployment_outcome_failed=[AccessFlow] 部署失败或已回滚:{0} {1} +notification.email.subject.deployment_verification_failed=[AccessFlow] 部署验证失败:{0} {1} notification.email.subject.deployment_break_glass_executed=[AccessFlow] 已在 {0} 上执行 break-glass 部署 {1} notification.email.deployment.pipeline_label=流水线: notification.email.deployment.environment_label=环境: @@ -1295,6 +1296,11 @@ notification.email.deployment_outcome_failed.title=部署失败 notification.email.deployment_outcome_failed.heading=您批准的部署出了问题 notification.email.deployment_outcome_failed.body_failed=流水线报告该部署失败。 notification.email.deployment_outcome_failed.body_rolled_back=流水线报告该部署已回滚。 +notification.email.deployment_verification_failed.title=部署验证失败 +notification.email.deployment_verification_failed.heading=已发布的部署未通过部署后验证 +notification.email.deployment_verification_failed.body=在流水线报告部署成功之后,部署后验证失败。 +notification.email.deployment_verification_failed.detail_label=详情: +notification.email.deployment_verification_failed.recommendation=该版本仍在运行——AccessFlow 未执行回滚。请决定是否回滚,然后从流水线报告 ROLLED_BACK。 notification.email.deployment_break_glass_executed.title=Break-glass 部署已执行 notification.email.deployment_break_glass_executed.heading=已发布一个 break-glass 部署 notification.email.deployment_break_glass_executed.body=一个应急部署绕过了审核并被强制批准发布。它需要进行强制性的事后复核。 diff --git a/backend/src/main/resources/templates/email/deployment-verification-failed.html b/backend/src/main/resources/templates/email/deployment-verification-failed.html new file mode 100644 index 000000000..7925c558c --- /dev/null +++ b/backend/src/main/resources/templates/email/deployment-verification-failed.html @@ -0,0 +1,40 @@ + + + + + Deployment verification failed + + + +

A released deployment failed its post-deployment verification

+

+ The post-deployment verification failed after the pipeline reported the deployment as succeeded. +

+

+ Detail: +

+
health: 3/3 failed samples
+

+ Pipeline: + —
+ Environment: + —
+ Version: + —
+ Submitted by: + — +

+

+ The release is still running — AccessFlow did not roll it back. Decide whether to roll back, then report ROLLED_BACK from the pipeline. +

+

+ Open in AccessFlow +

+ + diff --git a/backend/src/test/java/com/bablsoft/accessflow/deploygov/internal/DeploymentRequestStateServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/deploygov/internal/DeploymentRequestStateServiceTest.java index f0e879199..7ee9fbb8b 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/deploygov/internal/DeploymentRequestStateServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/deploygov/internal/DeploymentRequestStateServiceTest.java @@ -400,7 +400,8 @@ void markVerificationFailedFlipsToFailedStampsAndPublishesWithoutTouchingTheOutc var entity = verifying(QueryStatus.EXECUTED, VerificationStatus.RUNNING); when(repository.findById(entity.getId())).thenReturn(Optional.of(entity)); - assertThat(service.markVerificationFailed(entity.getId(), "health: 3/3 failed samples")) + assertThat(service.markVerificationFailed(entity.getId(), "health", + "health: 3/3 failed samples")) .isTrue(); assertThat(entity.getStatus()).isEqualTo(QueryStatus.FAILED); @@ -417,6 +418,41 @@ void markVerificationFailedFlipsToFailedStampsAndPublishesWithoutTouchingTheOutc verifyNoInteractions(versionTracker); } + @Test + @SuppressWarnings("unchecked") + void markVerificationFailedWritesASystemAuditRow() { + var entity = verifying(QueryStatus.EXECUTED, VerificationStatus.RUNNING); + when(repository.findById(entity.getId())).thenReturn(Optional.of(entity)); + + service.markVerificationFailed(entity.getId(), "health", "health: 3/3 failed samples"); + + ArgumentCaptor> metadata = ArgumentCaptor.forClass(Map.class); + verify(auditWriter).record(eq(AuditAction.DEPLOYMENT_VERIFICATION_FAILED), + eq(AuditResourceType.DEPLOYMENT_REQUEST), eq(entity.getId()), + eq(entity.getOrganizationId()), isNull(), metadata.capture(), isNull()); + assertThat(metadata.getValue()) + .containsEntry("trigger", "verification") + .containsEntry("pipeline_id", entity.getPipelineId().toString()) + .containsEntry("version", entity.getVersion()) + .containsEntry("check", "health") + .containsEntry("detail", "health: 3/3 failed samples"); + } + + @Test + @SuppressWarnings("unchecked") + void markVerificationFailedOmitsTheCheckKeyWhenNoCheckBreached() { + var entity = verifying(QueryStatus.EXECUTED, VerificationStatus.RUNNING); + when(repository.findById(entity.getId())).thenReturn(Optional.of(entity)); + + service.markVerificationFailed(entity.getId(), null, "unreadable"); + + ArgumentCaptor> metadata = ArgumentCaptor.forClass(Map.class); + verify(auditWriter).record(eq(AuditAction.DEPLOYMENT_VERIFICATION_FAILED), any(), any(), + any(), isNull(), metadata.capture(), isNull()); + assertThat(metadata.getValue()).doesNotContainKey("check") + .containsEntry("detail", "unreadable"); + } + @ParameterizedTest @CsvSource({"EXECUTED,PASSED", "EXECUTED,SKIPPED", "EXECUTED,FAILED", "FAILED,RUNNING", "APPROVED,RUNNING"}) @@ -425,12 +461,12 @@ void markVerificationFailedIsANoOpUnlessARunningVerificationOfAnExecutedRequest( var entity = verifying(status, verification); when(repository.findById(entity.getId())).thenReturn(Optional.of(entity)); - assertThat(service.markVerificationFailed(entity.getId(), "x")).isFalse(); + assertThat(service.markVerificationFailed(entity.getId(), "c", "x")).isFalse(); assertThat(entity.getStatus()).isEqualTo(status); assertThat(entity.getVerificationStatus()).isEqualTo(verification); verify(repository, never()).save(any()); - verifyNoInteractions(eventPublisher, versionTracker); + verifyNoInteractions(eventPublisher, versionTracker, auditWriter); } @Test diff --git a/backend/src/test/java/com/bablsoft/accessflow/deploygov/internal/verification/DefaultDeploymentVerificationServiceTest.java b/backend/src/test/java/com/bablsoft/accessflow/deploygov/internal/verification/DefaultDeploymentVerificationServiceTest.java index 61f091bff..4441129ec 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/deploygov/internal/verification/DefaultDeploymentVerificationServiceTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/deploygov/internal/verification/DefaultDeploymentVerificationServiceTest.java @@ -28,6 +28,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -78,7 +80,7 @@ void setUp() { when(codec.fromJson(JSON)).thenReturn(config(3, HEALTH, READY)); when(evaluators.evaluate(HEALTH)).thenReturn(passed(HEALTH)); when(evaluators.evaluate(READY)).thenReturn(passed(READY)); - when(stateService.markVerificationFailed(any(), anyString())).thenReturn(true); + when(stateService.markVerificationFailed(any(), any(), anyString())).thenReturn(true); when(stateService.markVerificationPassed(any())).thenReturn(true); } @@ -99,7 +101,7 @@ void persistsOneSamplePerCheckAndStampsLastVerifiedAt() { assertThat(first.getObservedValue()).isEqualTo("200"); assertThat(request.getLastVerifiedAt()).isEqualTo(NOW); verify(requestRepository).save(request); - verify(stateService, never()).markVerificationFailed(any(), any()); + verify(stateService, never()).markVerificationFailed(any(), any(), any()); verify(stateService, never()).markVerificationPassed(any()); } @@ -146,7 +148,7 @@ void breachesOnTheCheckThatReachesItsThresholdOnly() { assertThat(service.verifyOne(request.getId())).isEqualTo(VerificationTickResult.BREACHED); - verify(stateService).markVerificationFailed(request.getId(), "ready: 3/3 failed samples"); + verify(stateService).markVerificationFailed(request.getId(), "ready", "ready: 3/3 failed samples"); verify(stateService, never()).markVerificationPassed(any()); } @@ -158,14 +160,14 @@ void belowTheThresholdKeepsSampling() { assertThat(service.verifyOne(request.getId())).isEqualTo(VerificationTickResult.SAMPLED); - verify(stateService, never()).markVerificationFailed(any(), any()); + verify(stateService, never()).markVerificationFailed(any(), any(), any()); } @Test void aBreachLostToARaceIsReportedAsSkipped() { when(verificationRepository.countByDeploymentRequestIdAndCheckNameAndPassedFalse( request.getId(), "health")).thenReturn(3L); - when(stateService.markVerificationFailed(any(), anyString())).thenReturn(false); + when(stateService.markVerificationFailed(any(), any(), anyString())).thenReturn(false); assertThat(service.verifyOne(request.getId())).isEqualTo(VerificationTickResult.SKIPPED); } @@ -219,20 +221,20 @@ void anUnreadableSnapshotFailsClosed() { assertThat(service.verifyOne(request.getId())).isEqualTo(VerificationTickResult.BREACHED); - verify(stateService).markVerificationFailed(request.getId(), - DefaultDeploymentVerificationService.UNREADABLE_CONFIG); + verify(stateService).markVerificationFailed(eq(request.getId()), isNull(), + eq(DefaultDeploymentVerificationService.UNREADABLE_CONFIG)); verifyNoInteractions(evaluators, verificationRepository); } @Test void aMissingSnapshotFailsClosedAndALostRaceIsSkipped() { when(codec.fromJson(JSON)).thenReturn(null); - when(stateService.markVerificationFailed(any(), anyString())).thenReturn(false); + when(stateService.markVerificationFailed(any(), any(), anyString())).thenReturn(false); assertThat(service.verifyOne(request.getId())).isEqualTo(VerificationTickResult.SKIPPED); - verify(stateService).markVerificationFailed(request.getId(), - DefaultDeploymentVerificationService.UNREADABLE_CONFIG); + verify(stateService).markVerificationFailed(eq(request.getId()), isNull(), + eq(DefaultDeploymentVerificationService.UNREADABLE_CONFIG)); } @Test @@ -262,7 +264,7 @@ void anOverlappingTickThatAlreadySampledTheRowDropsThisOne() { assertThat(service.verifyOne(request.getId())).isEqualTo(VerificationTickResult.SKIPPED); verify(verificationRepository, never()).save(any()); - verify(stateService, never()).markVerificationFailed(any(), any()); + verify(stateService, never()).markVerificationFailed(any(), any(), any()); } @Test diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/DeploymentNotificationListenerTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/DeploymentNotificationListenerTest.java index 9102e74b0..d5d99070d 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/DeploymentNotificationListenerTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/DeploymentNotificationListenerTest.java @@ -8,6 +8,7 @@ import com.bablsoft.accessflow.deploygov.events.DeploymentReviewEscalatedEvent; import com.bablsoft.accessflow.deploygov.events.DeploymentReviewNudgedEvent; import com.bablsoft.accessflow.deploygov.events.DeploymentStatusChangedEvent; +import com.bablsoft.accessflow.deploygov.events.DeploymentVerificationFailedEvent; import com.bablsoft.accessflow.notifications.api.NotificationEventType; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -106,6 +107,15 @@ void rolledBackOutcomeNotifiesTheGrantingApprovers() { requestId, DeploymentOutcome.ROLLED_BACK, null); } + @Test + void verificationFailureNotifiesWithTheBreachedCheckAsTheReason() { + listener.onVerificationFailed(new DeploymentVerificationFailedEvent(orgId, requestId, + pipelineId, "health: 3/3 failed samples")); + verify(dispatcher).dispatchDeployment( + NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, requestId, null, + "health: 3/3 failed samples"); + } + @Test void succeededOutcomeIsNotNotified() { listener.onOutcomeReported(new DeploymentOutcomeReportedEvent(orgId, requestId, diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/NotificationContextBuilderTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/NotificationContextBuilderTest.java index efda206a0..8277c5b2a 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/NotificationContextBuilderTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/NotificationContextBuilderTest.java @@ -677,6 +677,69 @@ void deploymentOutcomeFailureNotifiesTheGrantingApprovers() { .isEqualTo(com.bablsoft.accessflow.deploygov.api.DeploymentOutcome.FAILED); } + @Test + void deploymentVerificationFailureNotifiesApproversAndTheSubmitterOnce() { + var deploymentRequestId = UUID.randomUUID(); + var approver = user(UUID.randomUUID(), "approver@example.com", UserRoleType.REVIEWER); + var submitter = user(submitterId, "dev@example.com", UserRoleType.ANALYST); + when(deploymentLookup.find(deploymentRequestId)).thenReturn(Optional.of(deploymentView( + deploymentRequestId))); + when(userQuery.findById(submitterId)).thenReturn(Optional.of(submitter)); + // The submitter also approved (a no-op in practice, but must not duplicate the recipient). + when(deploymentLookup.findApproverUserIds(deploymentRequestId)) + .thenReturn(List.of(approver.id(), submitterId)); + when(userQuery.findByIds(List.of(approver.id(), submitterId))) + .thenReturn(List.of(approver, submitter)); + + var ctx = builder.buildDeployment(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, + deploymentRequestId, null, "health: 3/3 failed samples").orElseThrow(); + + assertThat(ctx.recipients()).extracting(RecipientView::userId) + .containsExactly(approver.id(), submitterId); + assertThat(ctx.deploymentDecisionReason()).isEqualTo("health: 3/3 failed samples"); + assertThat(ctx.deploymentOutcome()).isNull(); + } + + @Test + void deploymentVerificationFailureWithoutApproversReachesOnlyTheActiveSubmitter() { + var deploymentRequestId = UUID.randomUUID(); + var submitter = user(submitterId, "dev@example.com", UserRoleType.ANALYST); + when(deploymentLookup.find(deploymentRequestId)).thenReturn(Optional.of(deploymentView( + deploymentRequestId))); + when(userQuery.findById(submitterId)).thenReturn(Optional.of(submitter)); + when(deploymentLookup.findApproverUserIds(deploymentRequestId)).thenReturn(List.of()); + when(userQuery.findByIds(List.of(submitterId))).thenReturn(List.of(submitter)); + + var ctx = builder.buildDeployment(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, + deploymentRequestId, null, "x").orElseThrow(); + + assertThat(ctx.recipients()).extracting(RecipientView::userId) + .containsExactly(submitterId); + } + + @Test + void deploymentVerificationFailureFiltersInactiveRecipients() { + var deploymentRequestId = UUID.randomUUID(); + var approver = user(UUID.randomUUID(), "approver@example.com", UserRoleType.REVIEWER); + var submitter = user(submitterId, "dev@example.com", UserRoleType.ANALYST); + var inactiveSubmitter = new UserView(submitterId, "dev@example.com", "Bob", + UserRoleType.ANALYST, orgId, false, AuthProviderType.LOCAL, "hash", null, null, + false, Instant.now()); + when(deploymentLookup.find(deploymentRequestId)).thenReturn(Optional.of(deploymentView( + deploymentRequestId))); + when(userQuery.findById(submitterId)).thenReturn(Optional.of(submitter)); + when(deploymentLookup.findApproverUserIds(deploymentRequestId)) + .thenReturn(List.of(approver.id())); + when(userQuery.findByIds(List.of(approver.id(), submitterId))) + .thenReturn(List.of(approver, inactiveSubmitter)); + + var ctx = builder.buildDeployment(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, + deploymentRequestId, null, "x").orElseThrow(); + + assertThat(ctx.recipients()).extracting(RecipientView::userId) + .containsExactly(approver.id()); + } + @Test void deploymentBreakGlassFansOutToActiveOrgAdmins() { var deploymentRequestId = UUID.randomUUID(); diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/NotificationDispatcherTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/NotificationDispatcherTest.java index 6e4694379..3072e29c0 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/NotificationDispatcherTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/NotificationDispatcherTest.java @@ -281,6 +281,32 @@ void persistsDeploymentNotificationWithDeploymentRequestIdAndPayload() { .contains("\"datasource\":\"payments-pipeline\""); } + @Test + void verificationFailedPayloadCarriesTheBreachDetailForTheBell() { + var reviewer = UUID.randomUUID(); + var deploymentRequestId = UUID.randomUUID(); + var base = sampleDeploymentContext(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, + deploymentRequestId, null, List.of(new RecipientView(reviewer, "a@x", "A"))); + var ctx = org.mockito.Mockito.spy(base); + org.mockito.Mockito.doReturn("health: 3/3 failed samples").when(ctx).deploymentDecisionReason(); + when(contextBuilder.buildDeployment( + eq(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED), eq(deploymentRequestId), + isNull(), eq("health: 3/3 failed samples"))) + .thenReturn(Optional.of(ctx)); + when(channelRepository.findAllByOrganizationIdAndActiveTrue(orgId)).thenReturn(List.of()); + + dispatcher.dispatchDeployment(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, + deploymentRequestId, null, "health: 3/3 failed samples"); + + var payloadCaptor = ArgumentCaptor.forClass(String.class); + verify(userNotificationService).recordForUsers( + eq(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED), eq(Set.of(reviewer)), + eq(orgId), isNull(), isNull(), eq(deploymentRequestId), isNull(), + payloadCaptor.capture()); + assertThat(payloadCaptor.getValue()) + .contains("\"verification_detail\":\"health: 3/3 failed samples\""); + } + /** * #695: deployment pipelines carry no review-plan channel binding, so every deployment event * must fan out org-wide. Falling through to the plan-channel lookup would silently deliver to @@ -297,6 +323,7 @@ void deploymentEventsUseAllActiveChannelsForOrg() { NotificationEventType.DEPLOYMENT_REJECTED, NotificationEventType.DEPLOYMENT_OUTCOME_FAILED, NotificationEventType.DEPLOYMENT_BREAK_GLASS_EXECUTED, + NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, // #985: the shared reminder types, on a deployment context. NotificationEventType.REVIEW_ESCALATED, NotificationEventType.REVIEW_NUDGE)) { @@ -307,7 +334,7 @@ void deploymentEventsUseAllActiveChannelsForOrg() { dispatcher.dispatchDeployment(eventType, deploymentRequestId, null, null); } - verify(emailStrategy, org.mockito.Mockito.times(7)).deliver(any(), eq(emailCh)); + verify(emailStrategy, org.mockito.Mockito.times(8)).deliver(any(), eq(emailCh)); verify(contextBuilder, never()).lookupPlanChannelIds(any()); } diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/codec/PagerDutyTriggerTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/codec/PagerDutyTriggerTest.java index 9d433b5ad..242bea0ac 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/codec/PagerDutyTriggerTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/codec/PagerDutyTriggerTest.java @@ -58,6 +58,9 @@ void forEventReturnsEmptyForUnmappedEvents() { assertThat(PagerDutyTrigger.forEvent(NotificationEventType.DEPLOYMENT_REJECTED)).isEmpty(); assertThat(PagerDutyTrigger.forEvent(NotificationEventType.DEPLOYMENT_OUTCOME_FAILED)) .isEmpty(); + // #1026: a breached verification recommends a rollback; it never pages. + assertThat(PagerDutyTrigger.forEvent( + NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED)).isEmpty(); // #895: schema drift on an API connector is a review item, not an incident. assertThat(PagerDutyTrigger.forEvent(NotificationEventType.API_CONNECTOR_DRIFT)).isEmpty(); } diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/codec/TicketingTriggerTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/codec/TicketingTriggerTest.java index b0ebfd602..8c63eea12 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/codec/TicketingTriggerTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/codec/TicketingTriggerTest.java @@ -29,6 +29,8 @@ void bothDeploymentOpeningEventsResolveToTheDeploymentTrigger() { assertThat(TicketingTrigger.forEvent(NotificationEventType.DEPLOYMENT_APPROVED)) .contains(TicketingTrigger.DEPLOYMENT_SUBMITTED); assertThat(TicketingTrigger.forEvent(NotificationEventType.DEPLOYMENT_REJECTED)).isEmpty(); + assertThat(TicketingTrigger.forEvent( + NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED)).isEmpty(); assertThat(TicketingTrigger.forEvent( NotificationEventType.DEPLOYMENT_BREAK_GLASS_EXECUTED)).isEmpty(); } diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DeploymentNotificationTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DeploymentNotificationTest.java index c0ec185af..f4bf30bfb 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DeploymentNotificationTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DeploymentNotificationTest.java @@ -27,7 +27,7 @@ import static org.mockito.Mockito.when; /** - * Covers the #695 deployment-event fan-out sites the compiler cannot: the five Thymeleaf + * Covers the #695 deployment-event fan-out sites the compiler cannot: the six Thymeleaf * templates (in every shipped locale), and the two trigger registries — only * {@code DEPLOYMENT_BREAK_GLASS_EXECUTED} may page, and no deployment event may open a ticket. * Chat-factory bodies are pinned per-factory in the {@code *FactoryTest} classes. @@ -39,14 +39,16 @@ class DeploymentNotificationTest { NotificationEventType.DEPLOYMENT_APPROVED, NotificationEventType.DEPLOYMENT_REJECTED, NotificationEventType.DEPLOYMENT_OUTCOME_FAILED, - NotificationEventType.DEPLOYMENT_BREAK_GLASS_EXECUTED); + NotificationEventType.DEPLOYMENT_BREAK_GLASS_EXECUTED, + NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED); private static final List TEMPLATES = List.of( "email/deployment-pending-review", "email/deployment-approved", "email/deployment-rejected", "email/deployment-outcome-failed", - "email/deployment-break-glass-executed"); + "email/deployment-break-glass-executed", + "email/deployment-verification-failed"); private static SpringTemplateEngine buildEngine() { var resolver = new ClassLoaderTemplateResolver(); @@ -127,6 +129,25 @@ void outcomeFailedTemplateBranchesOnTheOutcome() { .doesNotContain("rolled back"); } + /** #1026: the copy recommends a rollback and never claims one happened. */ + @Test + void verificationFailedTemplateCarriesTheCheckAndTheRecommendation() { + var ctx = templateContext(Locale.ENGLISH); + ctx.setVariable("deploymentOutcome", null); + ctx.setVariable("deploymentDecisionReason", "health: 3/3 failed samples"); + var html = buildEngine().process("email/deployment-verification-failed", ctx); + + assertThat(html) + .contains("health: 3/3 failed samples") + .contains("payments-pipeline") + .contains("production") + .contains("2.4.1") + .contains("still running") + .contains("did not roll it back") + .contains("ROLLED_BACK") + .doesNotContain("href=\"null\""); + } + @Test void breakGlassTemplateCarriesTheJustificationAndNoDeadCta() { var html = buildEngine().process("email/deployment-break-glass-executed", @@ -177,14 +198,15 @@ void openingEventsMapToTheChangeTicketTrigger(NotificationEventType event) { /** Rejection, outcome and break-glass never open a change ticket. */ @ParameterizedTest @EnumSource(names = {"DEPLOYMENT_REJECTED", "DEPLOYMENT_OUTCOME_FAILED", - "DEPLOYMENT_BREAK_GLASS_EXECUTED"}) + "DEPLOYMENT_BREAK_GLASS_EXECUTED", "DEPLOYMENT_VERIFICATION_FAILED"}) void neverOpensATicket(NotificationEventType event) { assertThat(TicketingTrigger.forEvent(event)).isEmpty(); } @ParameterizedTest @EnumSource(names = {"DEPLOYMENT_SUBMITTED", "DEPLOYMENT_APPROVED", "DEPLOYMENT_REJECTED", - "DEPLOYMENT_OUTCOME_FAILED", "DEPLOYMENT_BREAK_GLASS_EXECUTED"}) + "DEPLOYMENT_OUTCOME_FAILED", "DEPLOYMENT_BREAK_GLASS_EXECUTED", + "DEPLOYMENT_VERIFICATION_FAILED"}) void everyDeploymentEventRoutesToItsOwnEmailTemplate(NotificationEventType event) { assertThat(EmailNotificationStrategy.hasTemplateFor(event)).isTrue(); } diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DeploymentVerificationTextTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DeploymentVerificationTextTest.java new file mode 100644 index 000000000..2792332ba --- /dev/null +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DeploymentVerificationTextTest.java @@ -0,0 +1,47 @@ +package com.bablsoft.accessflow.notifications.internal.strategy; + +import com.bablsoft.accessflow.notifications.api.NotificationEventType; +import com.bablsoft.accessflow.notifications.internal.NotificationContext; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class DeploymentVerificationTextTest { + + @Test + void appliesOnlyToTheVerificationFailedEvent() { + assertThat(DeploymentVerificationText.applies( + ctx(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, null))).isTrue(); + assertThat(DeploymentVerificationText.applies( + ctx(NotificationEventType.DEPLOYMENT_OUTCOME_FAILED, null))).isFalse(); + } + + @Test + void detailFallsBackToADashAndIsBounded() { + var type = NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED; + assertThat(DeploymentVerificationText.detail(ctx(type, null))).isEqualTo("—"); + assertThat(DeploymentVerificationText.detail(ctx(type, " "))).isEqualTo("—"); + assertThat(DeploymentVerificationText.detail(ctx(type, "health: 3/3 failed samples"))) + .isEqualTo("health: 3/3 failed samples"); + assertThat(DeploymentVerificationText.detail(ctx(type, + "x".repeat(SchemaChangeText.ERROR_MAX_LENGTH + 5)))) + .hasSize(SchemaChangeText.ERROR_MAX_LENGTH); + } + + @Test + void recommendationNeverClaimsARollbackHappened() { + assertThat(DeploymentVerificationText.RECOMMENDATION) + .contains("still running") + .contains("did not roll it back") + .contains("ROLLED_BACK"); + } + + private static NotificationContext ctx(NotificationEventType type, String reason) { + var ctx = mock(NotificationContext.class); + when(ctx.eventType()).thenReturn(type); + when(ctx.deploymentDecisionReason()).thenReturn(reason); + return ctx; + } +} diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DiscordPayloadFactoryTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DiscordPayloadFactoryTest.java index 04a2170ec..dee5bde91 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DiscordPayloadFactoryTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/DiscordPayloadFactoryTest.java @@ -136,6 +136,7 @@ void weeklyDigestRendersDigestFields() { DEPLOYMENT_REJECTED | ❌ Deployment Rejected DEPLOYMENT_OUTCOME_FAILED | 🚨 Deployment Failed or Rolled Back DEPLOYMENT_BREAK_GLASS_EXECUTED | 🚨 Break-glass Deployment Executed + DEPLOYMENT_VERIFICATION_FAILED | 🚨 Deployment Verification Failed """) void deploymentHeadersAreUsed(NotificationEventType eventType, String expectedHeader) { var body = factory.buildEventBody(deploymentCtx(eventType, null), @@ -169,12 +170,29 @@ void deploymentOutcomeFailedEmbedCarriesOutcomeName() { .contains("ROLLED_BACK"); } + @Test + void deploymentVerificationFailedEmbedCarriesTheCheckAndTheRecommendation() { + var body = factory.buildEventBody( + deploymentCtx(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, null, + "health: 3/3 failed samples"), + new DiscordChannelConfig(URI.create("https://discord.com/api/webhooks/x"), null, null)); + assertThat(body).contains("Deployment Verification Failed") + .contains("health: 3/3 failed samples") + .contains("did not roll it back"); + } + /** * A DEPLOYMENT_* context (#695): pipeline name rides in datasourceName, submitter in * submitterEmail/DisplayName; fullSqlText/queryType/reviewUrl are null. */ private static NotificationContext deploymentCtx(NotificationEventType eventType, DeploymentOutcome outcome) { + return deploymentCtx(eventType, outcome, null); + } + + private static NotificationContext deploymentCtx(NotificationEventType eventType, + DeploymentOutcome outcome, + String decisionReason) { return new NotificationContext( eventType, UUID.randomUUID(), @@ -211,7 +229,7 @@ private static NotificationContext deploymentCtx(NotificationEventType eventType "production", "v2.14.0", outcome, - null); + decisionReason); } private static NotificationContext digestCtx() { diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/EmailNotificationStrategyTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/EmailNotificationStrategyTest.java index 30c2793e9..e0ed5e819 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/EmailNotificationStrategyTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/EmailNotificationStrategyTest.java @@ -205,7 +205,9 @@ void deliverRoutesEachDeploymentEventToItsOwnTemplate() { NotificationEventType.DEPLOYMENT_REJECTED, "email/deployment-rejected", NotificationEventType.DEPLOYMENT_OUTCOME_FAILED, "email/deployment-outcome-failed", NotificationEventType.DEPLOYMENT_BREAK_GLASS_EXECUTED, - "email/deployment-break-glass-executed"); + "email/deployment-break-glass-executed", + NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, + "email/deployment-verification-failed"); expected.forEach((eventType, template) -> { strategy.deliver(deploymentCtx(eventType), channel()); verify(templateEngine).process(eq(template), any()); @@ -228,6 +230,19 @@ void deploymentSubjectArgsCarryPipelineAndVersion() { assertThat(argsCaptor.getValue()).containsExactly("payments-pipeline", "2.4.1"); } + @Test + void verificationFailedSubjectUsesItsOwnKeyWithPipelineAndVersion() { + strategy.deliver(deploymentCtx(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED), + channel()); + + var argsCaptor = ArgumentCaptor.forClass(Object[].class); + verify(messageSource).getMessage( + eq("notification.email.subject.deployment_verification_failed"), + argsCaptor.capture(), + any(Locale.class)); + assertThat(argsCaptor.getValue()).containsExactly("payments-pipeline", "2.4.1"); + } + /** * #882: the subject-args switch has a silent default that would pass the pipeline where the * change set belongs; each schema-change event must pass its own arguments. diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/MsTeamsPayloadFactoryTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/MsTeamsPayloadFactoryTest.java index 3dfb8c0ba..fba190a99 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/MsTeamsPayloadFactoryTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/MsTeamsPayloadFactoryTest.java @@ -250,12 +250,29 @@ private static NotificationContext ctx(NotificationEventType eventType, Integer approvalTimeoutHours); } + @Test + void deploymentVerificationFailedCardCarriesTheCheckAndTheRecommendation() { + var body = factory.buildEventBody(deploymentCtx( + NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, null, + "health: 3/3 failed samples")); + assertThat(body).contains("🚨 Deployment Verification Failed") + .contains("health: 3/3 failed samples") + .contains("did not roll it back") + .contains("ROLLED_BACK"); + } + /** * A DEPLOYMENT_* context (#695) — the pipeline name rides in {@code datasourceName}, the * deployment fields trail the record; fullSqlText/queryType/reviewUrl are null. */ private static NotificationContext deploymentCtx(NotificationEventType eventType, DeploymentOutcome outcome) { + return deploymentCtx(eventType, outcome, null); + } + + private static NotificationContext deploymentCtx(NotificationEventType eventType, + DeploymentOutcome outcome, + String decisionReason) { return new NotificationContext( eventType, UUID.randomUUID(), @@ -288,7 +305,7 @@ private static NotificationContext deploymentCtx(NotificationEventType eventType "production", "v2.4.1", outcome, - null); + decisionReason); } private static NotificationContext ctxWithoutReviewUrl(NotificationEventType eventType) { diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/SlackBlockKitFactoryTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/SlackBlockKitFactoryTest.java index d0dd07906..c1cfaa255 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/SlackBlockKitFactoryTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/SlackBlockKitFactoryTest.java @@ -352,12 +352,39 @@ void deploymentOutcomeFailedPayloadCarriesTheOutcomeName() { .anyMatch(t -> t.contains("Outcome:") && t.contains("ROLLED_BACK")); } + @Test + void deploymentVerificationFailedCarriesTheCheckAndTheRecommendation() { + var payload = factory.buildEventPayload( + deploymentCtx(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, null, + "health: 3/3 failed samples"), null); + assertThat(payload.getText()).isEqualTo("🚨 Deployment Verification Failed"); + var summary = (SectionBlock) payload.getBlocks().get(1); + assertThat(summary.getFields()).extracting(t -> ((MarkdownTextObject) t).getText()) + .anyMatch(t -> t.contains("Verification:") && t.contains("health: 3/3")) + .anyMatch(t -> t.contains("did not roll it back") && t.contains("ROLLED_BACK")); + } + + @Test + void otherDeploymentEventsCarryNoVerificationField() { + var payload = factory.buildEventPayload( + deploymentCtx(NotificationEventType.DEPLOYMENT_APPROVED, null, "x"), null); + var summary = (SectionBlock) payload.getBlocks().get(1); + assertThat(summary.getFields()).extracting(t -> ((MarkdownTextObject) t).getText()) + .noneMatch(t -> t.contains("Verification:")); + } + /** * A DEPLOYMENT_* context (#695): the pipeline rides in {@code datasourceName}, no SQL, no * queryType, no reviewUrl; the deployment fields are the trailing canonical components. */ private static NotificationContext deploymentCtx(NotificationEventType eventType, DeploymentOutcome outcome) { + return deploymentCtx(eventType, outcome, null); + } + + private static NotificationContext deploymentCtx(NotificationEventType eventType, + DeploymentOutcome outcome, + String decisionReason) { return new NotificationContext( eventType, UUID.randomUUID(), @@ -394,7 +421,7 @@ private static NotificationContext deploymentCtx(NotificationEventType eventType "production", "v2.4.1", outcome, - null); + decisionReason); } @Test diff --git a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/TelegramMessageFactoryTest.java b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/TelegramMessageFactoryTest.java index 4778bd08b..6adc9d5a3 100644 --- a/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/TelegramMessageFactoryTest.java +++ b/backend/src/test/java/com/bablsoft/accessflow/notifications/internal/strategy/TelegramMessageFactoryTest.java @@ -164,9 +164,26 @@ void deploymentOutcomeFailedCarriesOutcomeName() { assertThat(body).contains("Outcome").contains("ROLLED\\\\_BACK"); } + @Test + void deploymentVerificationFailedCarriesTheCheckAndTheRecommendation() { + var body = factory.buildEventBody( + deploymentCtx(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED, null, + "health 3 failed"), "-100"); + assertThat(body).contains("Deployment Verification Failed") + .contains("health 3 failed") + .contains("did not roll it back") + .contains("ROLLED"); + } + /** A DEPLOYMENT_* context (#695) — pipeline in datasourceName, no SQL, no review URL. */ - private static NotificationContext deploymentCtx( - NotificationEventType eventType, DeploymentOutcome outcome) { + private static NotificationContext deploymentCtx(NotificationEventType eventType, + DeploymentOutcome outcome) { + return deploymentCtx(eventType, outcome, null); + } + + private static NotificationContext deploymentCtx(NotificationEventType eventType, + DeploymentOutcome outcome, + String decisionReason) { return new NotificationContext( eventType, UUID.randomUUID(), @@ -203,7 +220,7 @@ private static NotificationContext deploymentCtx( "production", "2.4.0", outcome, - null); + decisionReason); } private static NotificationContext digestCtx() { diff --git a/docs/03-data-model.md b/docs/03-data-model.md index 28d664420..5035a4710 100644 --- a/docs/03-data-model.md +++ b/docs/03-data-model.md @@ -1955,6 +1955,7 @@ The hash chain (added in V26) is per organization. Inserts are serialized by a P | `DEPLOYMENT_APPROVED` / `DEPLOYMENT_REJECTED` | A deployment decision lands (#695). Reviewer verdicts are written at the controller with the caller's IP + user-agent, one row per decision (before quorum, mirroring apigov). System decisions carry a **null actor** and a `trigger` naming the mechanism: `routing` (+ `policy_id`), `freeze` (+ `freeze_window_id`), `ticket` (+ `ticket_system`, `external_key`, `channel_id`, `actor` — a synced ServiceNow / Jira change ticket decided it, #992), or `environment_policy` (approved because the environment needs no review). Resource: `deployment_request`. | | `DEPLOYMENT_TIMED_OUT` | The review timeout auto-rejected a deployment past its window (#695, via `DeploymentTimeoutJob`). Resource: `deployment_request`. Null actor; metadata `trigger: "timeout"`, `pipeline_id`, `version`. | | `DEPLOYMENT_REVIEW_ESCALATED` / `DEPLOYMENT_REVIEW_NUDGED` | `DeploymentReviewEscalationJob` escalated a deployment review idle past its plan's `escalation_after_hours`, or re-nudged its undecided reviewers on `nudge_interval_hours` (#985). Resource: `deployment_request`. Null actor; metadata `trigger: "escalation"` / `"nudge"`, `pipeline_id`. Notify-only — written once per stamp of `escalated_at` / `last_nudged_at`. | +| `DEPLOYMENT_VERIFICATION_FAILED` | A post-deployment verification breached and the request flipped `EXECUTED → FAILED` (#1026, via `DeploymentVerificationJob` → `DeploymentRequestStateService.markVerificationFailed`). Resource: `deployment_request`. Null actor; metadata `trigger: "verification"`, `pipeline_id`, `version`, `check` (absent when the verification config snapshot was unreadable), `detail`. Nothing is rolled back. | | `DEPLOYMENT_CANCELLED` | The submitter cancelled a pending or scheduled-approved deployment (#695). Resource: `deployment_request`. Written at the controller with IP + user-agent, only after the service accepted the cancel. | | `HELP_AGENT_CONFIG_UPDATED` | An admin saved the in-app help agent's settings (#901). Resource: `help_agent_config`. Metadata carries only what changed: `enabled`, `ai_config_bound` (plus `ai_config_id` when it is bound — audit metadata rejects null values, so an unbind reads as the flag going false), `retrieval_enabled`, `retention_days`, `send_user_context`. | | `SCHEMA_CHANGE_PROMOTION_SUBMITTED` | A schema change set was promoted to an environment (#880). Resource: `schema_change_promotion`. Actor = the promoter, with the request's IP + user-agent. Metadata: `change_set_id`, `environment_id`, `environment_name`, `datasource_id`, `request_group_id`, `statement_count`, `statements_checksum`. Written only after every gate check passed and the row was persisted. | diff --git a/docs/04-api-spec.md b/docs/04-api-spec.md index 69ec2c334..3046979b6 100644 --- a/docs/04-api-spec.md +++ b/docs/04-api-spec.md @@ -8623,7 +8623,7 @@ events (`API_REQUEST_*`, `API_CONNECTOR_OAUTH2_TOKEN_FAILED` — AF-500), the co events (`API_CONNECTOR_DRIFT` \| `API_CONNECTOR_UNHEALTHY` — #895), and the deployment events (`DEPLOYMENT_SUBMITTED` \| `DEPLOYMENT_APPROVED` \| `DEPLOYMENT_REJECTED` \| `DEPLOYMENT_OUTCOME_FAILED` \| -`DEPLOYMENT_BREAK_GLASS_EXECUTED` — #695), and the schema-change events +`DEPLOYMENT_BREAK_GLASS_EXECUTED` — #695 \| `DEPLOYMENT_VERIFICATION_FAILED` — #1026), and the schema-change events (`SCHEMA_CHANGE_PROMOTION_SUBMITTED` \| `SCHEMA_CHANGE_PROMOTION_APPLIED` \| `SCHEMA_CHANGE_PROMOTION_FAILED` \| `SCHEMA_DRIFT_DETECTED` — #882), and the data-budget events (`DATA_BUDGET_THRESHOLD_REACHED` \| `DATA_BUDGET_EXHAUSTED` — #942). At most one of diff --git a/docs/05-backend.md b/docs/05-backend.md index c8d201400..91b0f766b 100644 --- a/docs/05-backend.md +++ b/docs/05-backend.md @@ -5558,7 +5558,9 @@ with a **null actor** and a `trigger` naming the mechanism — `routing` (+ `pol `DeploymentReviewStateMachine`, `freeze` (+ `freeze_window_id`) on the submission auto-reject, `environment_policy` when an environment needs no review, `ticket` (+ `ticket_system`, `external_key`, `channel_id`, `actor`) from `DefaultDeploymentExternalDecisionService` (#992). `DeploymentRequestStateService -.markTimedOut` writes `DEPLOYMENT_TIMED_OUT` (`trigger=timeout`), and the cancel endpoint writes +.markTimedOut` writes `DEPLOYMENT_TIMED_OUT` (`trigger=timeout`), `markVerificationFailed` writes +`DEPLOYMENT_VERIFICATION_FAILED` (`trigger=verification`, + `check` when one breached, `detail` — #1026), +and the cancel endpoint writes `DEPLOYMENT_CANCELLED` (controller layer, IP + user-agent, only after the service accepts). These join the pre-#695 rows: `DEPLOYMENT_BREAK_GLASS_EXECUTED` (#692, an AF-385 compensating control), and the #693 machine contract — `DEPLOYMENT_EXECUTED` (`trigger=pipeline`) on diff --git a/docs/06-frontend.md b/docs/06-frontend.md index 6badd89b0..776e14c29 100644 --- a/docs/06-frontend.md +++ b/docs/06-frontend.md @@ -947,9 +947,11 @@ Live refresh: the backend pushes `deployment.status_changed` to the **submitter* transition; `websocketManager.invokeDefault` invalidates the deployment detail (incl. its gate child key), list, and review queue. Reviewer queues refresh off `notification.created` — a `DEPLOYMENT_SUBMITTED` notification additionally invalidates the review queue and a -`DEPLOYMENT_OUTCOME_FAILED` one the rollback-review list. `NotificationBell` routes deployment -notifications into the pages (submission → the hub's Deployments tab, outcome-failed → its -Rollbacks tab, the rest → the deployment detail). Schema-change promotions (#882) are reviewed and +`DEPLOYMENT_OUTCOME_FAILED` one the rollback-review list, and a `DEPLOYMENT_VERIFICATION_FAILED` +one (#1026) the deployment detail and list, since its approvers get no status push. +`NotificationBell` routes deployment notifications into the pages (submission → the hub's +Deployments tab, a `ROLLED_BACK` outcome → its Rollbacks tab, the rest — verification failures +included, because nothing was rolled back — → the deployment detail). Schema-change promotions (#882) are reviewed and run as request groups, so their notifications route into those pages instead — a `SCHEMA_CHANGE_PROMOTION_SUBMITTED` notification to `/request-groups/reviews` (and it invalidates `['request-groups', 'reviews']`), `_APPLIED` / `_FAILED` to the promoter's `/request-groups`, and diff --git a/docs/08-notifications.md b/docs/08-notifications.md index f32e6d91b..7e24f8749 100644 --- a/docs/08-notifications.md +++ b/docs/08-notifications.md @@ -34,6 +34,7 @@ The dispatcher runs on virtual-thread executors and consumes events using Spring | `DEPLOYMENT_APPROVED` | A deployment is approved — reviewer quorum, routing-policy `AUTO_APPROVE`, or an environment that needs no review (#695) | The submitter. Org-wide channel fanout; never pages. Opens the change ticket when no `DEPLOYMENT_SUBMITTED` opened one — an auto-approved deployment never reaches review (#991). | implemented | | `DEPLOYMENT_REJECTED` | A deployment is rejected — reviewer verdict, routing-policy `AUTO_REJECT`, an active `REJECT` freeze window at submission, or the review timeout (`TIMED_OUT` folds into this event with its `review_timeout` reason) (#695) | The submitter. Org-wide channel fanout; never pages, never opens a ticket. | implemented | | `DEPLOYMENT_OUTCOME_FAILED` | The pipeline reports a `FAILED` or `ROLLED_BACK` outcome for an executed deployment (#695) | The reviewers who **approved** the deployment — the people who granted it learn it went wrong. Org-wide channel fanout; never pages, never opens a ticket. | implemented | +| `DEPLOYMENT_VERIFICATION_FAILED` | A post-deployment verification breaches and the request flips `EXECUTED → FAILED` (#1026, epic #1022). The breach detail rides in the decision-reason slot (`deployment.decision_reason` in the webhook body): `: / failed samples` — the count of failed samples against the threshold, not against the samples taken — or the literal `verification config unreadable` on the fail-closed path for an unreadable config snapshot; the in-app payload repeats it as `verification_detail` so the bell names the check | The reviewers who **approved** the deployment ∪ the submitter (de-duplicated, inactive users dropped; auto-approved and break-glass requests have no approvers, so the submitter alone). The copy is a **recommendation** — the release is still running and AccessFlow did not roll it back; the recipient decides whether to roll back and, if so, the pipeline reports `ROLLED_BACK`. Org-wide channel fanout; never pages, never opens a ticket. | implemented | | `DEPLOYMENT_BREAK_GLASS_EXECUTED` | A break-glass deployment force-approves and releases past review (#695, AF-385 mirror) | All active ADMIN users in the org. Fanned out to **all** active org channels including PagerDuty via the existing `BREAK_GLASS` trigger (one operator knob covers break-glass queries and deployments). | implemented | | `SCHEMA_CHANGE_PROMOTION_SUBMITTED` | A schema change set promotion reaches `IN_REVIEW` — its request group entered `PENDING_REVIEW` (#882, epic #870). Deliberately **not** on submission (`PENDING`): a promotion whose environment needs no review never pings anyone | The target datasource's eligible reviewers — the set the promotion's request group is actually decided by: every approver rule of the datasource's review plan (a group has one approval stage) unioned with the per-datasource reviewer assignments, else the `REVIEW_OVERRIDE` holders, the only users who can act on a group nobody is named for — excluding the promoter. Fanned out to **all** active org channels. Never pages, never opens a ticket. | implemented | | `SCHEMA_CHANGE_PROMOTION_APPLIED` | Every statement of a promotion executed on its target environment (`APPLIED`, #882) | The promoter. Org-wide channel fanout; never pages, never opens a ticket. | implemented | @@ -99,6 +100,7 @@ Email bodies are rendered using **Thymeleaf** HTML templates located in `resourc - `email/deployment-approved.html` — `DEPLOYMENT_APPROVED` (#695; pipeline, environment, version, green accent) - `email/deployment-rejected.html` — `DEPLOYMENT_REJECTED` (#695; adds an explicit timed-out note when the decision reason is `review_timeout`) - `email/deployment-outcome-failed.html` — `DEPLOYMENT_OUTCOME_FAILED` (#695; copy branches on `FAILED` vs `ROLLED_BACK`, red banner) +- `email/deployment-verification-failed.html` — `DEPLOYMENT_VERIFICATION_FAILED` (#1026; red banner, the breach detail, pipeline/environment/version/submitter, and the roll-back-or-not recommendation; the CTA opens the deployment) - `email/deployment-break-glass-executed.html` — `DEPLOYMENT_BREAK_GLASS_EXECUTED` (#695; the AF-385-style red emergency banner, executed-by, and the break-glass justification) - `email/schema-change-promotion-submitted.html` — `SCHEMA_CHANGE_PROMOTION_SUBMITTED` (#882; change set, pipeline, environment, promoter; the CTA opens the request-group review queue, and the applied/failed emails open the promoter's request groups) - `email/schema-change-promotion-applied.html` — `SCHEMA_CHANGE_PROMOTION_APPLIED` (#882; green accent) @@ -578,6 +580,7 @@ in `NotificationContextBuilder`: | `DEPLOYMENT_SUBMITTED` | The deployment's eligible reviewers (stage-1 plan approver rules when present, else REVIEWER ∪ ADMIN role holders), excluding the submitter | | `DEPLOYMENT_APPROVED` / `DEPLOYMENT_REJECTED` | The submitter | | `DEPLOYMENT_OUTCOME_FAILED` | The reviewers who approved the deployment | +| `DEPLOYMENT_VERIFICATION_FAILED` | The reviewers who approved the deployment ∪ the submitter | | `DEPLOYMENT_BREAK_GLASS_EXECUTED` | All active org admins | | `SCHEMA_CHANGE_PROMOTION_SUBMITTED` | The target datasource's eligible reviewers (every plan approver rule ∪ datasource reviewer assignments, else `REVIEW_OVERRIDE` holders), excluding the promoter | | `SCHEMA_CHANGE_PROMOTION_APPLIED` / `SCHEMA_CHANGE_PROMOTION_FAILED` | The promoter | diff --git a/docs/18-deployment-governance.md b/docs/18-deployment-governance.md index 049389c66..584434c84 100644 --- a/docs/18-deployment-governance.md +++ b/docs/18-deployment-governance.md @@ -177,7 +177,9 @@ hold a snapshot of the config and the verdict, and the `deployment_verifications running, so it is still shown as current until the pipeline reports `ROLLED_BACK`, which is accepted after a failed verification as the one exception to "the first outcome is final". A clean window ends `PASSED`. The gate and the request carry `verification_status`, and - `GET /deployment-requests/{id}/verification-samples` lists every sample. + `GET /deployment-requests/{id}/verification-samples` lists every sample. A breach is audited as + `DEPLOYMENT_VERIFICATION_FAILED` and notifies the approvers plus the submitter (see + [Audit & notifications](#audit--notifications)). - **Save-time rules.** A config is rejected with `422 DEPLOYMENT_VERIFICATION_INVALID` when its window is outside 1 minute–24 hours, its interval outside 30 seconds–window, its failure threshold one the window cannot reach, two checks share a name, or a URL is refused or does not @@ -828,8 +830,9 @@ is the [AF-859 access explainer](05-backend.md#access-explainer-af-859). Every transition lands in the INSERT-only `audit_log`: `DEPLOYMENT_SUBMITTED`, `DEPLOYMENT_APPROVED`, `DEPLOYMENT_REJECTED`, `DEPLOYMENT_TIMED_OUT`, `DEPLOYMENT_CANCELLED`, `DEPLOYMENT_REVIEW_ESCALATED`, `DEPLOYMENT_REVIEW_NUDGED`, -`DEPLOYMENT_EXECUTED`, `DEPLOYMENT_OUTCOME_REPORTED`, `DEPLOYMENT_BREAK_GLASS_EXECUTED`, -`DEPLOYMENT_BREAK_GLASS_REVIEWED` and `DEPLOYMENT_ROLLBACK_REVIEWED`, over the resource types +`DEPLOYMENT_EXECUTED`, `DEPLOYMENT_OUTCOME_REPORTED`, `DEPLOYMENT_VERIFICATION_FAILED` (#1026), +`DEPLOYMENT_BREAK_GLASS_EXECUTED`, `DEPLOYMENT_BREAK_GLASS_REVIEWED` and +`DEPLOYMENT_ROLLBACK_REVIEWED`, over the resource types `deployment_request` and `deployment_rollback_review`. Change tickets add `TICKET_CREATED` and `TICKET_STATUS_SYNCED` rows over the `deployment_ticket` resource type (#991, #992), and on a `change_ticket_required` environment `DEPLOYMENT_EXECUTED` also carries `change_ticket_required` @@ -837,20 +840,24 @@ plus `change_ticket_key` — or `change_ticket_bypassed = break_glass` (#993). R verdict, before quorum**, so the record shows who decided what and when, not merely the outcome. System-attributed rows carry a **null actor** plus a `trigger` key naming the mechanism — `routing` (with `policy_id`), `environment_policy`, `freeze` (with `freeze_window_id`), `timeout`, -`escalation` / `nudge` (#985), +`escalation` / `nudge` (#985), `verification` (a breached post-deployment verification, with +`pipeline_id`, `version`, `check` — absent when the config snapshot was unreadable — and `detail`; +#1026), `ticket` (a synced change-ticket resolution, with `ticket_system`, `external_key`, `channel_id`, `actor` — #992), or `pipeline`. A failed audit write is logged and swallowed rather than breaking the governed operation. -Five notification event types fan out across every configured channel (see +Six notification event types fan out across every configured channel (see [08-notifications.md](08-notifications.md)): **`DEPLOYMENT_SUBMITTED`** to the eligible reviewers — fired on the `PENDING_REVIEW` *transition*, not on submission, so an auto-decided deployment never pings anyone; **`DEPLOYMENT_APPROVED`** and **`DEPLOYMENT_REJECTED`** to the submitter (a timeout folds into the rejection with reason `review_timeout`); **`DEPLOYMENT_OUTCOME_FAILED`** to the -approvers who granted the release when the pipeline reports `FAILED` or `ROLLED_BACK`; and +approvers who granted the release when the pipeline reports `FAILED` or `ROLLED_BACK`; **`DEPLOYMENT_BREAK_GLASS_EXECUTED`** to every org admin, routed to PagerDuty through the existing -`BREAK_GLASS` trigger. Notifications carry the pipeline, environment, version and outcome, and the -bell links straight to the request — `user_notifications.deployment_request_id` (V155) is the third +`BREAK_GLASS` trigger; and **`DEPLOYMENT_VERIFICATION_FAILED`** (#1026) to those approvers plus the +submitter when a post-deployment verification breaches. That one is a recommendation: it says the +release is still running and was not rolled back, and it never pages or opens a ticket. +Notifications carry the pipeline, environment, version and outcome, and the bell links straight to the request — `user_notifications.deployment_request_id` (V155) is the third mutually-exclusive target column alongside queries and API requests. The frontend additionally receives a `deployment.status_changed` WebSocket event on every transition. diff --git a/frontend/src/components/common/NotificationBell.tsx b/frontend/src/components/common/NotificationBell.tsx index 594229a13..61a2e7757 100644 --- a/frontend/src/components/common/NotificationBell.tsx +++ b/frontend/src/components/common/NotificationBell.tsx @@ -277,6 +277,14 @@ function renderMessage( // Translated via enums.deployment_outcome — never the raw backend enum name. outcome: payload.outcome ? t(`enums.deployment_outcome.${payload.outcome}`) : '—', }); + // #1026 — a recommendation: the release is still running, nothing was rolled back. + case 'DEPLOYMENT_VERIFICATION_FAILED': + return t('notifications.events.DEPLOYMENT_VERIFICATION_FAILED', { + pipeline: datasource, + environment: payload.environment ?? '—', + version: payload.version ?? '—', + detail: payload.verification_detail ?? '—', + }); case 'DEPLOYMENT_BREAK_GLASS_EXECUTED': return payload.submitter ? t('notifications.events.DEPLOYMENT_BREAK_GLASS_EXECUTED', { @@ -404,7 +412,9 @@ export function routeForNotification(item: UserNotification): string | null { if ( item.event_type === 'DEPLOYMENT_APPROVED' || item.event_type === 'DEPLOYMENT_REJECTED' || - item.event_type === 'DEPLOYMENT_BREAK_GLASS_EXECUTED' + item.event_type === 'DEPLOYMENT_BREAK_GLASS_EXECUTED' || + // #1026: always the deployment itself — nothing was rolled back, so no rollback review. + item.event_type === 'DEPLOYMENT_VERIFICATION_FAILED' ) { return item.deployment_request_id ? `/deployments/${item.deployment_request_id}` : '/deployments'; } diff --git a/frontend/src/components/common/__tests__/NotificationBell.test.tsx b/frontend/src/components/common/__tests__/NotificationBell.test.tsx index ec88a4e03..8ddbebea0 100644 --- a/frontend/src/components/common/__tests__/NotificationBell.test.tsx +++ b/frontend/src/components/common/__tests__/NotificationBell.test.tsx @@ -558,6 +558,17 @@ describe('NotificationBell', () => { /deployment you approved on payments-pipeline reported rolled back/, '/reviews?tab=rollbacks', ], + [ + 'DEPLOYMENT_VERIFICATION_FAILED' as const, + { + datasource: 'payments-pipeline', + environment: 'production', + version: '2.4.1', + verification_detail: 'health: 3/3 failed samples', + }, + /Verification failed for 2\.4\.1 on payments-pipeline \/ production \(health: 3\/3 failed samples\)/, + '/deployments/dr-1', + ], [ 'DEPLOYMENT_BREAK_GLASS_EXECUTED' as const, { datasource: 'payments-pipeline', submitter: 'dev@acme.com' }, diff --git a/frontend/src/locales/de.json b/frontend/src/locales/de.json index 5ffcb01d2..9fe6f90f0 100644 --- a/frontend/src/locales/de.json +++ b/frontend/src/locales/de.json @@ -3036,6 +3036,7 @@ "DEPLOYMENT_OUTCOME_FAILED": "Ein von dir genehmigtes Deployment auf {{pipeline}} hat {{outcome}} gemeldet", "DEPLOYMENT_BREAK_GLASS_EXECUTED": "{{submitter}} hat ein Break-Glass-Deployment auf {{pipeline}} freigegeben", "DEPLOYMENT_BREAK_GLASS_EXECUTED_no_submitter": "Ein Break-Glass-Deployment wurde auf {{pipeline}} freigegeben", + "DEPLOYMENT_VERIFICATION_FAILED": "Verifizierung für {{version}} auf {{pipeline}} / {{environment}} fehlgeschlagen ({{detail}}) — läuft weiter, entscheide über einen Rollback", "SCHEMA_CHANGE_PROMOTION_SUBMITTED": "Eine Schemaänderung {{changeSet}} → {{environment}} wartet auf Ihre Prüfung", "SCHEMA_CHANGE_PROMOTION_APPLIED": "Ihre Schemaänderung {{changeSet}} wurde auf {{environment}} angewendet", "SCHEMA_CHANGE_PROMOTION_FAILED": "Ihre Schemaänderung {{changeSet}} ist auf {{environment}} fehlgeschlagen", diff --git a/frontend/src/locales/en.json b/frontend/src/locales/en.json index 96c3dfa14..89c19e783 100644 --- a/frontend/src/locales/en.json +++ b/frontend/src/locales/en.json @@ -3036,6 +3036,7 @@ "DEPLOYMENT_OUTCOME_FAILED": "A deployment you approved on {{pipeline}} reported {{outcome}}", "DEPLOYMENT_BREAK_GLASS_EXECUTED": "{{submitter}} released a break-glass deployment on {{pipeline}}", "DEPLOYMENT_BREAK_GLASS_EXECUTED_no_submitter": "A break-glass deployment was released on {{pipeline}}", + "DEPLOYMENT_VERIFICATION_FAILED": "Verification failed for {{version}} on {{pipeline}} / {{environment}} ({{detail}}) — still running, decide whether to roll back", "SCHEMA_CHANGE_PROMOTION_SUBMITTED": "A schema change {{changeSet}} → {{environment}} is awaiting your review", "SCHEMA_CHANGE_PROMOTION_APPLIED": "Your schema change {{changeSet}} was applied to {{environment}}", "SCHEMA_CHANGE_PROMOTION_FAILED": "Your schema change {{changeSet}} failed on {{environment}}", diff --git a/frontend/src/locales/es.json b/frontend/src/locales/es.json index 00c430ecf..574db694f 100644 --- a/frontend/src/locales/es.json +++ b/frontend/src/locales/es.json @@ -3036,6 +3036,7 @@ "DEPLOYMENT_OUTCOME_FAILED": "Un despliegue que aprobaste en {{pipeline}} informó {{outcome}}", "DEPLOYMENT_BREAK_GLASS_EXECUTED": "{{submitter}} liberó un despliegue de break-glass en {{pipeline}}", "DEPLOYMENT_BREAK_GLASS_EXECUTED_no_submitter": "Se liberó un despliegue de break-glass en {{pipeline}}", + "DEPLOYMENT_VERIFICATION_FAILED": "Falló la verificación de {{version}} en {{pipeline}} / {{environment}} ({{detail}}): sigue en ejecución, decide si revertir", "SCHEMA_CHANGE_PROMOTION_SUBMITTED": "Un cambio de esquema {{changeSet}} → {{environment}} espera su revisión", "SCHEMA_CHANGE_PROMOTION_APPLIED": "Su cambio de esquema {{changeSet}} se aplicó en {{environment}}", "SCHEMA_CHANGE_PROMOTION_FAILED": "Su cambio de esquema {{changeSet}} falló en {{environment}}", diff --git a/frontend/src/locales/fr.json b/frontend/src/locales/fr.json index 51f880b24..dbfdedf08 100644 --- a/frontend/src/locales/fr.json +++ b/frontend/src/locales/fr.json @@ -3036,6 +3036,7 @@ "DEPLOYMENT_OUTCOME_FAILED": "Un déploiement que tu as approuvé sur {{pipeline}} a signalé {{outcome}}", "DEPLOYMENT_BREAK_GLASS_EXECUTED": "{{submitter}} a libéré un déploiement break-glass sur {{pipeline}}", "DEPLOYMENT_BREAK_GLASS_EXECUTED_no_submitter": "Un déploiement break-glass a été libéré sur {{pipeline}}", + "DEPLOYMENT_VERIFICATION_FAILED": "Échec de la vérification de {{version}} sur {{pipeline}} / {{environment}} ({{detail}}) — toujours en service, décide d'un éventuel rollback", "SCHEMA_CHANGE_PROMOTION_SUBMITTED": "Une modification de schéma {{changeSet}} → {{environment}} attend votre relecture", "SCHEMA_CHANGE_PROMOTION_APPLIED": "Votre modification de schéma {{changeSet}} a été appliquée sur {{environment}}", "SCHEMA_CHANGE_PROMOTION_FAILED": "Votre modification de schéma {{changeSet}} a échoué sur {{environment}}", diff --git a/frontend/src/locales/hy.json b/frontend/src/locales/hy.json index 97b3962d7..9234ca6d3 100644 --- a/frontend/src/locales/hy.json +++ b/frontend/src/locales/hy.json @@ -3036,6 +3036,7 @@ "DEPLOYMENT_OUTCOME_FAILED": "Ձեր հաստատած տեղակայումը {{pipeline}}-ում զեկուցել է {{outcome}}", "DEPLOYMENT_BREAK_GLASS_EXECUTED": "{{submitter}} թողարկեց break-glass տեղակայում {{pipeline}}-ում", "DEPLOYMENT_BREAK_GLASS_EXECUTED_no_submitter": "Break-glass տեղակայում թողարկվեց {{pipeline}}-ում", + "DEPLOYMENT_VERIFICATION_FAILED": "{{version}}-ի ստուգումը ձախողվեց {{pipeline}} / {{environment}}-ում ({{detail}}) — դեռ աշխատում է, որոշեք՝ արդյոք հետ գլորել", "SCHEMA_CHANGE_PROMOTION_SUBMITTED": "Սխեմայի փոփոխությունը {{changeSet}} → {{environment}} սպասում է ձեր վերանայմանը", "SCHEMA_CHANGE_PROMOTION_APPLIED": "Ձեր սխեմայի փոփոխությունը {{changeSet}} կիրառվեց {{environment}}-ում", "SCHEMA_CHANGE_PROMOTION_FAILED": "Ձեր սխեմայի փոփոխությունը {{changeSet}} ձախողվեց {{environment}}-ում", diff --git a/frontend/src/locales/ru.json b/frontend/src/locales/ru.json index 7409441cd..90d376c7f 100644 --- a/frontend/src/locales/ru.json +++ b/frontend/src/locales/ru.json @@ -3036,6 +3036,7 @@ "DEPLOYMENT_OUTCOME_FAILED": "Одобренное вами развертывание на {{pipeline}} сообщило результат {{outcome}}", "DEPLOYMENT_BREAK_GLASS_EXECUTED": "{{submitter}} выпустил аварийное развертывание (break-glass) на {{pipeline}}", "DEPLOYMENT_BREAK_GLASS_EXECUTED_no_submitter": "Аварийное развертывание (break-glass) было выпущено на {{pipeline}}", + "DEPLOYMENT_VERIFICATION_FAILED": "Проверка {{version}} на {{pipeline}} / {{environment}} не пройдена ({{detail}}) — релиз работает, решите, нужен ли откат", "SCHEMA_CHANGE_PROMOTION_SUBMITTED": "Изменение схемы {{changeSet}} → {{environment}} ожидает вашей проверки", "SCHEMA_CHANGE_PROMOTION_APPLIED": "Ваше изменение схемы {{changeSet}} применено в {{environment}}", "SCHEMA_CHANGE_PROMOTION_FAILED": "Ваше изменение схемы {{changeSet}} не выполнено в {{environment}}", diff --git a/frontend/src/locales/zh-CN.json b/frontend/src/locales/zh-CN.json index e13457c41..b827ae7a5 100644 --- a/frontend/src/locales/zh-CN.json +++ b/frontend/src/locales/zh-CN.json @@ -3036,6 +3036,7 @@ "DEPLOYMENT_OUTCOME_FAILED": "您在 {{pipeline}} 上批准的部署报告了 {{outcome}}", "DEPLOYMENT_BREAK_GLASS_EXECUTED": "{{submitter}} 在 {{pipeline}} 上发布了一个 break-glass 部署", "DEPLOYMENT_BREAK_GLASS_EXECUTED_no_submitter": "{{pipeline}} 上发布了一个 break-glass 部署", + "DEPLOYMENT_VERIFICATION_FAILED": "{{version}} 在 {{pipeline}} / {{environment}} 上的验证失败({{detail}})——仍在运行,请决定是否回滚", "SCHEMA_CHANGE_PROMOTION_SUBMITTED": "架构变更 {{changeSet}} → {{environment}} 等待您审核", "SCHEMA_CHANGE_PROMOTION_APPLIED": "您的架构变更 {{changeSet}} 已应用到 {{environment}}", "SCHEMA_CHANGE_PROMOTION_FAILED": "您的架构变更 {{changeSet}} 在 {{environment}} 上失败", diff --git a/frontend/src/pages/admin/AuditLogPage.tsx b/frontend/src/pages/admin/AuditLogPage.tsx index 5574b30f2..8ee829a2e 100644 --- a/frontend/src/pages/admin/AuditLogPage.tsx +++ b/frontend/src/pages/admin/AuditLogPage.tsx @@ -103,6 +103,7 @@ const ACTIONS = [ 'DEPLOYMENT_CANCELLED', 'DEPLOYMENT_EXECUTED', 'DEPLOYMENT_OUTCOME_REPORTED', + 'DEPLOYMENT_VERIFICATION_FAILED', 'DEPLOYMENT_BREAK_GLASS_EXECUTED', 'DEPLOYMENT_BREAK_GLASS_REVIEWED', 'DEPLOYMENT_ROLLBACK_REVIEWED', diff --git a/frontend/src/realtime/__tests__/websocketManager.test.ts b/frontend/src/realtime/__tests__/websocketManager.test.ts index 79d9534ab..ead9804ab 100644 --- a/frontend/src/realtime/__tests__/websocketManager.test.ts +++ b/frontend/src/realtime/__tests__/websocketManager.test.ts @@ -228,6 +228,22 @@ describe('websocketManager', () => { }); expect(spy).toHaveBeenCalledWith({ queryKey: ['deployment-rollback-reviews'] }); expect(spy).not.toHaveBeenCalledWith({ queryKey: ['deployment-reviews', 'list'] }); + + // #1026: a breached verification refreshes the deployment views, never the rollback worklist. + spy.mockReset(); + sock(0).triggerMessage({ + event: 'notification.created', + timestamp: 'now', + data: { + notification_id: 'n4', + event_type: 'DEPLOYMENT_VERIFICATION_FAILED', + query_id: null, + created_at: '2026-11-10T10:00:00Z', + }, + }); + expect(spy).toHaveBeenCalledWith({ queryKey: ['deployments', 'detail'] }); + expect(spy).toHaveBeenCalledWith({ queryKey: ['deployments', 'list'] }); + expect(spy).not.toHaveBeenCalledWith({ queryKey: ['deployment-rollback-reviews'] }); }); it('a SCHEMA_CHANGE_PROMOTION_SUBMITTED notification refreshes the group review queue (#882)', () => { diff --git a/frontend/src/realtime/websocketManager.ts b/frontend/src/realtime/websocketManager.ts index b57cf6f51..7061bcda7 100644 --- a/frontend/src/realtime/websocketManager.ts +++ b/frontend/src/realtime/websocketManager.ts @@ -213,6 +213,11 @@ class WebSocketManager { this.queryClient.invalidateQueries({ queryKey: ['deployment-reviews', 'list'] }); } else if (eventType === 'DEPLOYMENT_OUTCOME_FAILED') { this.queryClient.invalidateQueries({ queryKey: ['deployment-rollback-reviews'] }); + } else if (eventType === 'DEPLOYMENT_VERIFICATION_FAILED') { + // #1026: approvers get no deployment.status_changed push, so an open detail page or + // list refreshes off the notification (EXECUTED → FAILED). + this.queryClient.invalidateQueries({ queryKey: ['deployments', 'detail'] }); + this.queryClient.invalidateQueries({ queryKey: ['deployments', 'list'] }); } else if (eventType === 'API_REQUEST_SUBMITTED') { // Same shape for API requests (#772): there is no reviewer-side WS event, so the // unified queue's API tab and its badge count refresh off the notification. diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index de5d7fd15..f933413de 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -2829,6 +2829,7 @@ export type UserNotificationEventType = | 'DEPLOYMENT_REJECTED' | 'DEPLOYMENT_OUTCOME_FAILED' | 'DEPLOYMENT_BREAK_GLASS_EXECUTED' + | 'DEPLOYMENT_VERIFICATION_FAILED' | 'SCHEMA_CHANGE_PROMOTION_SUBMITTED' | 'SCHEMA_CHANGE_PROMOTION_APPLIED' | 'SCHEMA_CHANGE_PROMOTION_FAILED' @@ -2862,6 +2863,8 @@ export interface UserNotificationPayload { environment?: string; version?: string; outcome?: 'FAILED' | 'ROLLED_BACK'; + // #1026: DEPLOYMENT_VERIFICATION_FAILED only — the breached check and its failed-sample count. + verification_detail?: string; schema_change_promotion_id?: string; change_set?: string; promotion_status?: 'APPLIED' | 'FAILED' | 'PARTIALLY_APPLIED' | 'IN_REVIEW'; diff --git a/help-corpus/manifest.json b/help-corpus/manifest.json index 2ddff93ab..8eca2ac6f 100644 --- a/help-corpus/manifest.json +++ b/help-corpus/manifest.json @@ -1,8 +1,8 @@ { "schemaVersion": 1, "corpusVersion": "debe778a1394", - "generatedAt": "2026-11-10T13:12:40.411Z", - "sourceCommit": "3b4ef08e97069a2ebcc770c70964e4932d17a28d", + "generatedAt": "2026-11-10T15:03:00.861Z", + "sourceCommit": "9a82bd18a235668d39ebecd1627ac4ae4b088727", "chunkCount": 615, "sha256": "debe778a139480445a9c4945403fec4d253471c837b75d6236b7484cddc70d4d", "quickReferenceSha256": "24edf02e4c124fee2ce843e9333a470905f9a55868ddc3cdef869315ca473072", @@ -429,7 +429,7 @@ "url": "https://accessflow.io/docs/", "section": "Navigation", "chunks": 9, - "sha256": "ac7468e28b155a590b11d18a5e610edcb4d3763303e56c3d6da538b5bcbe48f2" + "sha256": "198c5a970d39cd6fee394942c8870147e1e3f405892d7ca377b3a1fb77c77177" } ] }