[fm] Add saga diagnosis engine

dev-tools/omdb/tests/successes.out

@@ -724,6 +724,8 @@ task: "fm_analysis"
 
       no in-service control plane disks
 
+      no non-terminal sagas observed
+
     analysis report:
       sitrep ID: ..........<REDACTED_UUID>...........
       no analysis log
@@ -784,6 +786,8 @@ task: "fm_sitrep_gc"
       batches:                                                                   1
     orphaned fm_fact_physical_disk rows deleted:                                 0
       batches:                                                                   1
+    orphaned fm_fact_saga rows deleted:                                          0
+      batches:                                                                   1
     orphaned fm_sitrep_analysis_report rows deleted:                             <ORPHANED_FMSITREPANALYSISREPORT_ROWS_DELETED_REDACTED>
       batches:                                                                   1
     orphaned fm_support_bundle_request rows deleted:                             0
@@ -1446,6 +1450,8 @@ task: "fm_analysis"
 
       no in-service control plane disks
 
+      no non-terminal sagas observed
+
     analysis report:
       sitrep ID: ..........<REDACTED_UUID>...........
       no analysis log
@@ -1506,6 +1512,8 @@ task: "fm_sitrep_gc"
       batches:                                                                   1
     orphaned fm_fact_physical_disk rows deleted:                                 0
       batches:                                                                   1
+    orphaned fm_fact_saga rows deleted:                                          0
+      batches:                                                                   1
     orphaned fm_sitrep_analysis_report rows deleted:                             <ORPHANED_FMSITREPANALYSISREPORT_ROWS_DELETED_REDACTED>
       batches:                                                                   1
     orphaned fm_support_bundle_request rows deleted:                             0

nexus/db-model/src/db_metadata.rs

@@ -44,7 +44,7 @@ impl DbMetadata {
 impl_enum_type!(
     DbMetadataNexusStateEnum:
 
-    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Serialize, Deserialize)]
+    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Serialize, Deserialize, strum::VariantArray)]
     pub enum DbMetadataNexusState;
 
     // Enum values
@@ -53,6 +53,10 @@ impl_enum_type!(
     Quiesced => b"quiesced"
 );
 
+impl DbMetadataNexusState {
+    pub const ALL: &'static [Self] = <Self as strum::VariantArray>::VARIANTS;
+}
+
 impl fmt::Display for DbMetadataNexusState {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(match self {

nexus/db-model/src/fm.rs

@@ -27,10 +27,12 @@ mod case;
 pub use case::*;
 mod diagnosis_engine;
 pub use diagnosis_engine::*;
-mod rendezvous_created;
-pub use rendezvous_created::*;
 mod fact_physical_disk;
 pub use fact_physical_disk::*;
+mod fact_saga;
+pub use fact_saga::*;
+mod rendezvous_created;
+pub use rendezvous_created::*;
 mod support_bundle_request;
 pub use support_bundle_request::*;
 mod sitrep_analysis_report;

nexus/db-model/src/fm/diagnosis_engine.rs

@@ -25,6 +25,7 @@ impl_enum_type!(
 
     PowerShelf => b"power_shelf"
     PhysicalDisk => b"physical_disk"
+    Saga => b"saga"
 
 );
 
@@ -35,6 +36,7 @@ impl From<DiagnosisEngine> for fm::DiagnosisEngineKind {
             DiagnosisEngine::PhysicalDisk => {
                 fm::DiagnosisEngineKind::PhysicalDisk
             }
+            DiagnosisEngine::Saga => fm::DiagnosisEngineKind::Saga,
         }
     }
 }
@@ -46,6 +48,7 @@ impl From<fm::DiagnosisEngineKind> for DiagnosisEngine {
             fm::DiagnosisEngineKind::PhysicalDisk => {
                 DiagnosisEngine::PhysicalDisk
             }
+            fm::DiagnosisEngineKind::Saga => DiagnosisEngine::Saga,
         }
     }
 }

nexus/db-model/src/fm/fact_physical_disk.rs

@@ -156,9 +156,11 @@ fn db_kind(disk_fact: &DiskFact) -> FmFactPhysicalDiskKind {
 }
 
 fn missing_column(kind: FmFactPhysicalDiskKind, column: &str) -> Error {
-    Error::internal_error(&format!(
-        "fm_fact_physical_disk row of kind {kind:?} has a NULL {column}, \
-         violating the CHECK constraint requiring it to be non-NULL for \
-         this kind"
-    ))
+    Error::InternalError {
+        internal_message: format!(
+            "fm_fact_physical_disk row of kind {kind:?} has a NULL {column}, \
+             violating the CHECK constraint requiring it to be non-NULL for \
+             this kind"
+        ),
+    }
 }

nexus/db-model/src/fm/fact_saga.rs

@@ -0,0 +1,228 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Database representation of the saga diagnosis engine's facts.
+//!
+//! Each saga fact is stored as typed columns in the `fm_fact_saga` table. The
+//! `kind` discriminant selects which payload columns are populated; per-kind
+//! CHECK constraints (e.g. `not_progressing_columns_present`) enforce that
+//! the right columns are non-NULL for each kind. See
+//! [`nexus_types::fm::SagaFact`] for semantics.
+
+use crate::DbTypedUuid;
+use crate::SagaState;
+use crate::impl_enum_type;
+use chrono::{DateTime, Utc};
+use nexus_db_schema::schema::fm_fact_saga;
+use nexus_types::fm;
+use nexus_types::fm::case::FactMetadata;
+use nexus_types::fm::{
+    FactPayload, SagaAbandonedFactPayload, SagaFact,
+    SagaNotProgressingFactPayload, SagaOwnerNotCurrentFactPayload,
+};
+use nexus_types::observed_saga::{OrphanedReason, SagaProgressState};
+use omicron_common::api::external::Error;
+use omicron_uuid_kinds::{CaseKind, FactKind, OmicronZoneKind, SitrepKind};
+use uuid::Uuid;
+
+impl_enum_type!(
+    FmFactSagaKindEnum:
+
+    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq)]
+    pub enum FmFactSagaKind;
+
+    NotProgressing => b"not_progressing"
+    OwnerNotCurrentGeneration => b"owner_not_current_generation"
+    Abandoned => b"abandoned"
+);
+
+impl_enum_type!(
+    FmFactSagaOrphanReasonEnum:
+
+    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq)]
+    pub enum FmFactSagaOrphanReason;
+
+    Quiesced => b"quiesced"
+    Expunged => b"expunged"
+);
+
+impl From<OrphanedReason> for FmFactSagaOrphanReason {
+    fn from(reason: OrphanedReason) -> Self {
+        match reason {
+            OrphanedReason::Quiesced => FmFactSagaOrphanReason::Quiesced,
+            OrphanedReason::Expunged => FmFactSagaOrphanReason::Expunged,
+        }
+    }
+}
+
+impl From<FmFactSagaOrphanReason> for OrphanedReason {
+    fn from(reason: FmFactSagaOrphanReason) -> Self {
+        match reason {
+            FmFactSagaOrphanReason::Quiesced => OrphanedReason::Quiesced,
+            FmFactSagaOrphanReason::Expunged => OrphanedReason::Expunged,
+        }
+    }
+}
+
+impl From<SagaProgressState> for SagaState {
+    fn from(state: SagaProgressState) -> Self {
+        match state {
+            SagaProgressState::Running => SagaState::Running,
+            SagaProgressState::Unwinding => SagaState::Unwinding,
+        }
+    }
+}
+
+/// Convert a DB `saga_state` back into the [`SagaProgressState`] recorded on
+/// a `NotProgressing` fact. Only live (running or unwinding) sagas carry that
+/// fact: a done saga has no facts at all, and an abandoned saga carries an
+/// `Abandoned` fact, whose payload has no saga state. A stored `Done` or
+/// `Abandoned` here therefore indicates a corrupt row.
+fn saga_progress_state(state: SagaState) -> Result<SagaProgressState, Error> {
+    match state {
+        SagaState::Running => Ok(SagaProgressState::Running),
+        SagaState::Unwinding => Ok(SagaProgressState::Unwinding),
+        SagaState::Done | SagaState::Abandoned => Err(Error::InternalError {
+            internal_message: format!(
+                "fm_fact_saga row has saga_state {state:?}, which is \
+                     never recorded on a NotProgressing saga fact"
+            ),
+        }),
+    }
+}
+
+/// Diesel row for the `fm_fact_saga` table.
+///
+/// The payload columns are populated according to `kind`: a column is `Some`
+/// if it belongs to that `kind`'s payload, and `None` otherwise.
+#[derive(Queryable, Insertable, Clone, Debug, Selectable)]
+#[diesel(table_name = fm_fact_saga)]
+pub struct FmFactSaga {
+    pub id: DbTypedUuid<FactKind>,
+    /// The sitrep to which this fact belongs.
+    ///
+    /// This will change as the fact is carried forward from one sitrep to the
+    /// next.
+    pub sitrep_id: DbTypedUuid<SitrepKind>,
+    pub case_id: DbTypedUuid<CaseKind>,
+    /// Sitrep in which this fact was first added.
+    ///
+    /// Preserved unchanged when the fact is carried forward; debug-only.
+    pub created_sitrep_id: DbTypedUuid<SitrepKind>,
+    pub comment: String,
+
+    /// The saga this fact is about.
+    pub saga_id: Uuid,
+    pub kind: FmFactSagaKind,
+
+    // Columns for the `NotProgressing` kind.
+    pub saga_state: Option<SagaState>,
+    pub last_event_time: Option<DateTime<Utc>>,
+
+    // Columns for the `OwnerNotCurrentGeneration` kind.
+    pub current_sec: Option<DbTypedUuid<OmicronZoneKind>>,
+    pub orphan_reason: Option<FmFactSagaOrphanReason>,
+}
+
+impl FmFactSaga {
+    /// Build a row from a fact's shared metadata (`metadata`) and its
+    /// already-dispatched saga payload (`saga_fact`).
+    pub fn from_sitrep(
+        sitrep_id: impl Into<DbTypedUuid<SitrepKind>>,
+        case_id: impl Into<DbTypedUuid<CaseKind>>,
+        metadata: &FactMetadata,
+        saga_fact: &SagaFact,
+    ) -> Self {
+        let FactMetadata { id, created_sitrep_id, comment } = metadata;
+        let base = Self {
+            id: (*id).into(),
+            sitrep_id: sitrep_id.into(),
+            case_id: case_id.into(),
+            created_sitrep_id: (*created_sitrep_id).into(),
+            comment: comment.clone(),
+            saga_id: saga_fact.saga_id().0,
+            kind: FmFactSagaKind::NotProgressing,
+            saga_state: None,
+            last_event_time: None,
+            current_sec: None,
+            orphan_reason: None,
+        };
+        match saga_fact {
+            SagaFact::NotProgressing(p) => Self {
+                kind: FmFactSagaKind::NotProgressing,
+                saga_state: Some(p.saga_state.into()),
+                last_event_time: Some(p.last_event_time),
+                ..base
+            },
+            SagaFact::OwnerNotCurrentGeneration(p) => Self {
+                kind: FmFactSagaKind::OwnerNotCurrentGeneration,
+                current_sec: Some(p.current_sec.into()),
+                orphan_reason: Some(p.orphan_reason.into()),
+                ..base
+            },
+            SagaFact::Abandoned(_) => {
+                Self { kind: FmFactSagaKind::Abandoned, ..base }
+            }
+        }
+    }
+
+    /// Reconstruct an in-memory fact from a row.
+    pub fn into_fact(self) -> Result<fm::case::Fact, Error> {
+        let kind = self.kind;
+        let saga_id = steno::SagaId(self.saga_id);
+        let payload = match kind {
+            FmFactSagaKind::NotProgressing => FactPayload::Saga(
+                SagaFact::NotProgressing(SagaNotProgressingFactPayload {
+                    saga_id,
+                    saga_state: saga_progress_state(
+                        self.saga_state.ok_or_else(|| {
+                            missing_column(kind, "saga_state")
+                        })?,
+                    )?,
+                    last_event_time: self.last_event_time.ok_or_else(|| {
+                        missing_column(kind, "last_event_time")
+                    })?,
+                }),
+            ),
+            FmFactSagaKind::OwnerNotCurrentGeneration => {
+                FactPayload::Saga(SagaFact::OwnerNotCurrentGeneration(
+                    SagaOwnerNotCurrentFactPayload {
+                        saga_id,
+                        current_sec: self
+                            .current_sec
+                            .ok_or_else(|| missing_column(kind, "current_sec"))?
+                            .into(),
+                        orphan_reason: self
+                            .orphan_reason
+                            .ok_or_else(|| {
+                                missing_column(kind, "orphan_reason")
+                            })?
+                            .into(),
+                    },
+                ))
+            }
+            FmFactSagaKind::Abandoned => FactPayload::Saga(
+                SagaFact::Abandoned(SagaAbandonedFactPayload { saga_id }),
+            ),
+        };
+        Ok(fm::case::Fact {
+            metadata: fm::case::FactMetadata {
+                id: self.id.into(),
+                created_sitrep_id: self.created_sitrep_id.into(),
+                comment: self.comment,
+            },
+            payload,
+        })
+    }
+}
+
+fn missing_column(kind: FmFactSagaKind, column: &str) -> Error {
+    Error::InternalError {
+        internal_message: format!(
+            "fm_fact_saga row of kind {kind:?} has a NULL {column}, \
+             violating the CHECK constraint requiring it to be non-NULL for \
+             this kind"
+        ),
+    }
+}