Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,8 @@ public enum AuditAction {
/** #985: undecided deployment reviewers were re-nudged on the plan's nudge_interval_hours. */
DEPLOYMENT_REVIEW_NUDGED,
DEPLOYMENT_CANCELLED,
/** #1026: a post-deployment verification breached (system row, {@code trigger=verification}). */
DEPLOYMENT_VERIFICATION_FAILED,
/** #695: a deployment break-glass retro-review acknowledged (was generic BREAK_GLASS_REVIEWED). */
DEPLOYMENT_BREAK_GLASS_REVIEWED,

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.time.Instant;
import java.util.EnumMap;
import java.util.EnumSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
Expand Down Expand Up @@ -179,7 +180,7 @@ public boolean markNudged(UUID id, Instant at) {
* verification — a lost race, not an error.
*/
@Transactional
public boolean markVerificationFailed(UUID id, String detail) {
public boolean markVerificationFailed(UUID id, String check, String detail) {
var entity = require(id);
if (!isRunningVerification(entity)) {
return false;
Expand All @@ -188,7 +189,18 @@ public boolean markVerificationFailed(UUID id, String detail) {
entity.setVerificationFinishedAt(clock.instant());
entity.setVerificationDetail(detail);
apply(entity, QueryStatus.FAILED);
// #1026: the DEPLOYMENT_VERIFICATION_FAILED audit row is recorded here.
// #1026: system row — null actor; check is null when the config snapshot was unreadable.
var metadata = new LinkedHashMap<String, Object>();
metadata.put("trigger", "verification");
metadata.put("pipeline_id", entity.getPipelineId().toString());
metadata.put("version", entity.getVersion());
if (check != null) {
metadata.put("check", check);
}
metadata.put("detail", detail);
auditWriter.record(AuditAction.DEPLOYMENT_VERIFICATION_FAILED,
AuditResourceType.DEPLOYMENT_REQUEST, entity.getId(), entity.getOrganizationId(),
null, metadata, null);
eventPublisher.publishEvent(new DeploymentVerificationFailedEvent(
entity.getOrganizationId(), entity.getId(), entity.getPipelineId(), detail));
return true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public VerificationTickResult verifyOne(UUID deploymentRequestId) {
// Fail closed: a running verification whose snapshot cannot be read must not pass.
log.warn("Deployment request {} has an unreadable verification snapshot",
deploymentRequestId);
return stateService.markVerificationFailed(deploymentRequestId, UNREADABLE_CONFIG)
return stateService.markVerificationFailed(deploymentRequestId, null, UNREADABLE_CONFIG)
? VerificationTickResult.BREACHED : VerificationTickResult.SKIPPED;
}
var now = clock.instant();
Expand Down Expand Up @@ -110,7 +110,7 @@ private VerificationTickResult record(UUID id, DeploymentVerificationConfig conf
if (failed >= config.failureThreshold()) {
var detail = check.name() + ": " + failed + "/" + config.failureThreshold()
+ " failed samples";
return stateService.markVerificationFailed(id, detail)
return stateService.markVerificationFailed(id, check.name(), detail)
? VerificationTickResult.BREACHED : VerificationTickResult.SKIPPED;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,12 @@ public enum NotificationEventType {
* every active org admin including PagerDuty, mirroring {@code BREAK_GLASS_EXECUTED}.
*/
DEPLOYMENT_BREAK_GLASS_EXECUTED,
/**
* A post-deployment verification breached (#1026) — notifies the approvers and the submitter.
* A recommendation only: the release is still running and AccessFlow did not roll it back.
* Never pages, never tickets.
*/
DEPLOYMENT_VERIFICATION_FAILED,

/**
* A schema change set promotion entered review (#882) — its request group reached
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import com.bablsoft.accessflow.deploygov.events.DeploymentReviewEscalatedEvent;
import com.bablsoft.accessflow.deploygov.events.DeploymentReviewNudgedEvent;
import com.bablsoft.accessflow.deploygov.events.DeploymentStatusChangedEvent;
import com.bablsoft.accessflow.deploygov.events.DeploymentVerificationFailedEvent;
import com.bablsoft.accessflow.notifications.api.NotificationEventType;
import lombok.RequiredArgsConstructor;
import org.springframework.modulith.events.ApplicationModuleListener;
Expand All @@ -20,7 +21,8 @@
* deployment that a routing policy auto-decides never pings them. A decided deployment alerts the
* submitter ({@code TIMED_OUT} folds into {@code DEPLOYMENT_REJECTED} with its
* {@code review_timeout} reason); a {@code FAILED} or {@code ROLLED_BACK} outcome alerts the
* reviewers who approved it; a break-glass release alerts every org admin; an idle review is
* reviewers who approved it; a breached post-deployment verification (#1026) alerts those approvers
* plus the submitter; a break-glass release alerts every org admin; an idle review is
* escalated / nudged via the shared {@code REVIEW_ESCALATED} / {@code REVIEW_NUDGE} types (#985).
* Break-glass publishes no {@code DeploymentDecidedEvent}, so the submitter never self-notifies on
* that path. Delivery is best-effort and never blocks the deployment flow.
Expand Down Expand Up @@ -56,6 +58,13 @@ void onOutcomeReported(DeploymentOutcomeReportedEvent event) {
}
}

// #1026: the breached check rides in the decision-reason slot.
@ApplicationModuleListener
void onVerificationFailed(DeploymentVerificationFailedEvent event) {
dispatcher.dispatchDeployment(NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED,
event.deploymentRequestId(), null, event.detail());
}

@ApplicationModuleListener
void onBreakGlassExecuted(DeploymentBreakGlassExecutedEvent event) {
dispatcher.dispatchDeployment(NotificationEventType.DEPLOYMENT_BREAK_GLASS_EXECUTED,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@
* for the submission (or break-glass) justification. {@code deploymentOutcome} is only set for
* {@code DEPLOYMENT_OUTCOME_FAILED}; {@code deploymentDecisionReason} carries the decision
* provenance ({@code "routing:&lt;policyId&gt;"}, {@code "freeze:&lt;windowId&gt;"},
* {@code "review_timeout"}, or null for a reviewer verdict).
* {@code "review_timeout"}, or null for a reviewer verdict) — or, for
* {@code DEPLOYMENT_VERIFICATION_FAILED} (#1026), the verification breach detail.
*
* <p>The {@code schemaChange*} fields are only populated for the {@code SCHEMA_CHANGE_PROMOTION_*}
* events and {@code driftNewFindingCount} only for {@code SCHEMA_DRIFT_DETECTED} (#882). Both reuse
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ private List<RecipientView> resolveRecipients(NotificationEventType eventType,
ERASURE_APPROVED,
DEPLOYMENT_SUBMITTED, DEPLOYMENT_APPROVED, DEPLOYMENT_REJECTED,
DEPLOYMENT_OUTCOME_FAILED, DEPLOYMENT_BREAK_GLASS_EXECUTED,
DEPLOYMENT_VERIFICATION_FAILED,
SCHEMA_CHANGE_PROMOTION_SUBMITTED, SCHEMA_CHANGE_PROMOTION_APPLIED,
SCHEMA_CHANGE_PROMOTION_FAILED, SCHEMA_DRIFT_DETECTED,
DATA_BUDGET_THRESHOLD_REACHED, DATA_BUDGET_EXHAUSTED -> List.of();
Expand Down Expand Up @@ -573,6 +574,14 @@ private List<RecipientView> deploymentRecipients(NotificationEventType eventType
// The reviewers who granted the deployment learn it failed or was rolled back.
case DEPLOYMENT_OUTCOME_FAILED -> toActiveRecipients(
deploymentNotificationLookupService.findApproverUserIds(view.id()));
// #1026: a breached verification also reaches the submitter, the person most able to
// roll back. Auto-approved and break-glass requests have no approvers: submitter only.
case DEPLOYMENT_VERIFICATION_FAILED -> {
var ids = new LinkedHashSet<>(
deploymentNotificationLookupService.findApproverUserIds(view.id()));
ids.add(view.submittedByUserId());
yield toActiveRecipients(List.copyOf(ids));
}
// #985: a nudge reminds the eligible reviewers who have not decided yet; an escalation
// adds every active org admin, because the point is that those reviewers did not act.
// Notify-only — neither widens who may decide, and the submitter is never included.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,11 @@ private String buildPayload(NotificationContext ctx) {
if (ctx.deploymentOutcome() != null) {
payload.put("outcome", ctx.deploymentOutcome().name());
}
// #1026: the bell is the one surface every recipient gets, so it names the failed check.
if (ctx.eventType() == NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED
&& ctx.deploymentDecisionReason() != null) {
payload.put("verification_detail", ctx.deploymentDecisionReason());
}
if (ctx.schemaChangePromotionId() != null) {
payload.put("schema_change_promotion_id", ctx.schemaChangePromotionId().toString());
}
Expand Down Expand Up @@ -410,6 +415,7 @@ private List<NotificationChannelEntity> resolveChannels(NotificationEventType ev
|| eventType == NotificationEventType.DEPLOYMENT_REJECTED
|| eventType == NotificationEventType.DEPLOYMENT_OUTCOME_FAILED
|| eventType == NotificationEventType.DEPLOYMENT_BREAK_GLASS_EXECUTED
|| eventType == NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED
// #985: the shared REVIEW_ESCALATED / REVIEW_NUDGE on a deployment carry the
// pipeline in datasourceId, so they fan out org-wide like every deployment event.
|| ctx.deploymentRequestId() != null
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package com.bablsoft.accessflow.notifications.internal.strategy;

import com.bablsoft.accessflow.notifications.api.NotificationEventType;
import com.bablsoft.accessflow.notifications.internal.NotificationContext;

/**
* Shared chat rendering for {@code DEPLOYMENT_VERIFICATION_FAILED} (#1026). The wording is a
* recommendation: AccessFlow never rolls back, so no channel may imply that it did.
*/
final class DeploymentVerificationText {

static final String TITLE = "🚨 Deployment Verification Failed";
static final String RECOMMENDATION = "The release is still running — AccessFlow did not roll it "
+ "back. Decide whether to roll back, then report ROLLED_BACK.";

private DeploymentVerificationText() {
}

static boolean applies(NotificationContext ctx) {
return ctx.eventType() == NotificationEventType.DEPLOYMENT_VERIFICATION_FAILED;
}

/** The breached check and its sample count, bounded like every other chat field. */
static String detail(NotificationContext ctx) {
var detail = ctx.deploymentDecisionReason();
return detail == null || detail.isBlank() ? "—" : SchemaChangeText.truncate(detail);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,10 @@ private static Map<String, Object> buildEventEmbed(NotificationContext ctx) {
if (ctx.deploymentOutcome() != null) {
addField(fields, "Outcome", ctx.deploymentOutcome().name());
}
if (DeploymentVerificationText.applies(ctx)) {
addField(fields, "Verification", DeploymentVerificationText.detail(ctx));
addField(fields, "Next step", DeploymentVerificationText.RECOMMENDATION);
}
embed.put("fields", fields);
return embed;
}
Expand Down Expand Up @@ -236,6 +240,7 @@ private static String headerLabel(NotificationContext ctx) {
case DEPLOYMENT_REJECTED -> "❌ Deployment Rejected";
case DEPLOYMENT_OUTCOME_FAILED -> "🚨 Deployment Failed or Rolled Back";
case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "🚨 Break-glass Deployment Executed";
case DEPLOYMENT_VERIFICATION_FAILED -> DeploymentVerificationText.TITLE;
case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "🧱 Schema Change Awaiting Review";
case SCHEMA_CHANGE_PROMOTION_APPLIED -> "✅ Schema Change Applied";
case SCHEMA_CHANGE_PROMOTION_FAILED -> "🚨 Schema Change Failed";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,7 @@ static String templateName(NotificationEventType eventType) {
case DEPLOYMENT_REJECTED -> "email/deployment-rejected";
case DEPLOYMENT_OUTCOME_FAILED -> "email/deployment-outcome-failed";
case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "email/deployment-break-glass-executed";
case DEPLOYMENT_VERIFICATION_FAILED -> "email/deployment-verification-failed";
// #882: schema-change promotions and drift.
case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "email/schema-change-promotion-submitted";
case SCHEMA_CHANGE_PROMOTION_APPLIED -> "email/schema-change-promotion-applied";
Expand Down Expand Up @@ -352,7 +353,8 @@ private String subject(NotificationContext ctx) {
// Spelled out — this switch has a silent default and would otherwise pass only the
// pipeline, leaving "{1}" unfilled in every deployment subject.
case DEPLOYMENT_SUBMITTED, DEPLOYMENT_APPROVED, DEPLOYMENT_REJECTED,
DEPLOYMENT_OUTCOME_FAILED, DEPLOYMENT_BREAK_GLASS_EXECUTED ->
DEPLOYMENT_OUTCOME_FAILED, DEPLOYMENT_BREAK_GLASS_EXECUTED,
DEPLOYMENT_VERIFICATION_FAILED ->
new Object[]{ctx.datasourceName(), ctx.deploymentVersion()};
// #882: "{0} → {1}" is the change set and the environment it was promoted to. Spelled
// out — the silent default would pass the pipeline where the change set belongs.
Expand Down Expand Up @@ -406,6 +408,8 @@ private static String subjectKey(NotificationEventType eventType) {
"notification.email.subject.deployment_outcome_failed";
case DEPLOYMENT_BREAK_GLASS_EXECUTED ->
"notification.email.subject.deployment_break_glass_executed";
case DEPLOYMENT_VERIFICATION_FAILED ->
"notification.email.subject.deployment_verification_failed";
case SCHEMA_CHANGE_PROMOTION_SUBMITTED ->
"notification.email.subject.schema_change_promotion_submitted";
case SCHEMA_CHANGE_PROMOTION_APPLIED ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,10 @@ private static Map<String, Object> buildEventCard(NotificationContext ctx) {
if (ctx.deploymentOutcome() != null) {
facts.add(fact("Outcome", ctx.deploymentOutcome().name()));
}
if (DeploymentVerificationText.applies(ctx)) {
facts.add(fact("Verification", DeploymentVerificationText.detail(ctx)));
facts.add(fact("Next step", DeploymentVerificationText.RECOMMENDATION));
}
body.add(factSet(facts));
card.put("body", body);
return card;
Expand Down Expand Up @@ -294,6 +298,7 @@ private static String headerLabel(NotificationContext ctx) {
case DEPLOYMENT_REJECTED -> "❌ Deployment Rejected";
case DEPLOYMENT_OUTCOME_FAILED -> "🚨 Deployment Failed or Rolled Back";
case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "🚨 Break-glass Deployment Executed";
case DEPLOYMENT_VERIFICATION_FAILED -> DeploymentVerificationText.TITLE;
case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "🧱 Schema Change Awaiting Review";
case SCHEMA_CHANGE_PROMOTION_APPLIED -> "✅ Schema Change Applied";
case SCHEMA_CHANGE_PROMOTION_FAILED -> "🚨 Schema Change Failed";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,10 @@ private static SectionBlock deploymentSection(NotificationContext ctx) {
if (ctx.deploymentOutcome() != null) {
fields.add(mrkdwn("*Outcome:*\n" + ctx.deploymentOutcome().name()));
}
if (DeploymentVerificationText.applies(ctx)) {
fields.add(mrkdwn("*Verification:*\n" + DeploymentVerificationText.detail(ctx)));
fields.add(mrkdwn("*Next step:*\n" + DeploymentVerificationText.RECOMMENDATION));
}
return SectionBlock.builder().fields(fields).build();
}

Expand Down Expand Up @@ -303,6 +307,7 @@ private static String headerLabel(NotificationContext ctx) {
case DEPLOYMENT_REJECTED -> "❌ Deployment Rejected";
case DEPLOYMENT_OUTCOME_FAILED -> "🚨 Deployment Failed or Rolled Back";
case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "🚨 Break-glass Deployment Executed";
case DEPLOYMENT_VERIFICATION_FAILED -> DeploymentVerificationText.TITLE;
case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "🧱 Schema Change Awaiting Review";
case SCHEMA_CHANGE_PROMOTION_APPLIED -> "✅ Schema Change Applied";
case SCHEMA_CHANGE_PROMOTION_FAILED -> "🚨 Schema Change Failed";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,10 @@ private static String buildEventText(NotificationContext ctx) {
if (ctx.deploymentOutcome() != null) {
appendField(sb, "Outcome", ctx.deploymentOutcome().name());
}
if (DeploymentVerificationText.applies(ctx)) {
appendField(sb, "Verification", DeploymentVerificationText.detail(ctx));
appendField(sb, "Next step", DeploymentVerificationText.RECOMMENDATION);
}
return sb.toString();
}
appendField(sb, "Datasource", nullToDash(ctx.datasourceName()));
Expand Down Expand Up @@ -204,6 +208,7 @@ private static String headerLabel(NotificationContext ctx) {
case DEPLOYMENT_REJECTED -> "❌ Deployment Rejected";
case DEPLOYMENT_OUTCOME_FAILED -> "🚨 Deployment Failed or Rolled Back";
case DEPLOYMENT_BREAK_GLASS_EXECUTED -> "🚨 Break-glass Deployment Executed";
case DEPLOYMENT_VERIFICATION_FAILED -> DeploymentVerificationText.TITLE;
case SCHEMA_CHANGE_PROMOTION_SUBMITTED -> "🧱 Schema Change Awaiting Review";
case SCHEMA_CHANGE_PROMOTION_APPLIED -> "✅ Schema Change Applied";
case SCHEMA_CHANGE_PROMOTION_FAILED -> "🚨 Schema Change Failed";
Expand Down
Loading
Loading