diff --git a/internal/ingest/delta/delta.go b/internal/ingest/delta/delta.go new file mode 100644 index 0000000..22eab54 --- /dev/null +++ b/internal/ingest/delta/delta.go @@ -0,0 +1,1302 @@ +// Package delta is Lane A step A.14: steady-state delta ingestion. +// +// =========================================================================== +// WHAT THIS PACKAGE IS FOR +// =========================================================================== +// +// A.8 fills the cache once, from a 570 MB bulk archive. A.7 asks "did anything +// change" for the price of a conditional GET. This package is what happens +// BETWEEN those two: when a poll says something changed, it works out the +// cheapest set of bytes that describes the change, decodes them, and upserts +// exactly the rows that moved. +// +// research/06 Recommendation §3, "Steady state per feed, by value density", is +// the whole specification, and its central finding is a cost model: +// +// "The hourly delta is CUMULATIVE SINCE THE MIDNIGHT BASELINE, not per-hour +// incremental." Measured: 22 B at 0000Z, 25,882 B at 0100Z, 17,312,299 B at +// 2300Z. "Polling hourly and downloading each delta re-transfers everything +// already held. Rough integral over a day: ~200 MB/day if polled hourly, vs +// ~17 MB/day if the end-of-day delta is taken once." +// +// So the cheap path is not the delta archive at all. It is `deltaLog.json` — +// "a rolling 30 days worth of CVE record modification history" — polled every +// 15 minutes, which NAMES what changed without carrying it, followed by a fetch +// of only the named records. A.14's packet makes re-downloading the cumulative +// zip on every poll a forbidden action, and PreferDeltaLog below is where that +// preference is made structural rather than advisory. +// +// =========================================================================== +// EVERY CLOCK IN THIS PACKAGE COMES FROM THE FEED TABLE +// =========================================================================== +// +// There is no cadence written in Go here and there must never be one. +// research/06 Recommendation §4: "Every cadence above lives in config, never in +// code", and internal/ingest/config carries three of them per row for exactly +// this step and A.15 — `interval_seconds`, `reconcile_interval_seconds`, +// `baseline_interval_seconds`. A.1's feeds_test.go asserts mechanically that no +// cadence literal appears in its own source; delta_test.go carries the same +// assertion against this file, because the same defect on the CONSUMING side is +// just as fatal and is easier to commit. +// +// Due() is the whole scheduler and it is a PURE FUNCTION of (feed row, last +// success, now). A daemon calls it; nothing in here sleeps, and nothing in here +// knows what a week is. +// +// =========================================================================== +// THE CURSOR IS A QUERY, NOT A COLUMN +// =========================================================================== +// +// feed_state has exactly one cursor column and A.8 already owns it: it writes a +// bootstrap Progress token into `watermark` and hands over through +// bootstrap.Handoff. A second delta cursor squeezed into the same column would +// be two writers on one value. +// +// So the delta cursor is derived from the rows themselves: CachedModified reads +// what the cache holds for one (source, source_id), and a record whose delta +// log entry is not NEWER than that is never fetched. That has a property a +// stored cursor does not — it cannot disagree with the data. A cursor that +// advanced past a failed write would drop the record permanently and silently. +// +// =========================================================================== +// WHAT CROSSES FROM FEED CONTENT INTO A REQUEST, AND WHAT DOES NOT +// =========================================================================== +// +// The deltaLog route reads a document written by strangers and then FETCHES +// THINGS IT NAMES. That is the highest-risk pattern in Lane A: a feed document +// that could choose a URL turns a scheduled background job into a request to +// wherever it likes, with the feed's own credential attached. +// +// Exactly one thing crosses that boundary, and it is a CVE identifier that +// passed IsCVEID — `CVE-` then digits, a dash, then digits, and nothing else. +// It is an ALLOWLIST of structure, not a denylist of dangerous characters: +// this project has lost three guards to a symbol, a verb or a wording nobody +// listed, and `CVE-2024-0001/../../etc/passwd` is precisely the string a +// denylist misses. The links a real deltaLog carries (`githubLink`, +// `cveOrgLink`) are PARSED AND DISCARDED — see DeltaLogEntry. +// +// Turning an identifier into a URL is the Source hook's job, for the same +// reason poller.Watermarker is an injected interface: a package that knew where +// one feed's records live would be a hard-coded feed table wearing a different +// hat. +// +// =========================================================================== +// THREE ROUTES ARE PLANNED HERE AND DELIBERATELY NOT RUN HERE +// =========================================================================== +// +// RouteReconcile, RouteBaseline and RouteGitFetch are recognised, scheduled and +// reported by Due(), and refused with a named sentinel unless a delegate is +// wired. Each refusal has a reason that is about correctness, not effort: +// +// - RouteReconcile wants the ~17 MB end-of-day delta asset. A.8's importer +// resolves the LARGEST archive asset of a release, which is the 570 MB +// midnight baseline. Wiring reconcile to it would cost 570 MB/day and +// would be the very re-download A.14's packet forbids, so this package +// refuses rather than defaults. Choosing WHICH asset of a release is the +// reconciliation artifact is a per-feed fact and belongs in A.1's table. +// - RouteBaseline is A.15's weekly self-heal. Due() computes its clock +// because the clock is in the feed row and one planner should own all +// three; running it here would be two implementations of one pass. +// - RouteGitFetch needs A.8's clone directory and would have to rewrite +// A.8's watermark token afterwards — two writers on one column, which is +// the thing the cursor design above avoids. +// +// A refusal is loud, typed and counted. It is not a silent no-op, and it is +// not a t.Skip. +package delta + +import ( + "archive/zip" + "bytes" + "compress/gzip" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + "strings" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/config" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/license" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/poller" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/sanitize" +) + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +var ( + // ErrDelta is satisfied by every error this package originates, so a + // caller can tell "the delta pipeline declined" from "the database + // failed" without listing every sentinel. + ErrDelta = errors.New("delta") + + // ErrSyncRefused is satisfied by every refusal: a decision Anvil made, + // as opposed to something that went wrong. + ErrSyncRefused = fmt.Errorf("%w: refused", ErrDelta) + + // ErrNoCache is a Syncer built without the A.2 ingestion cache. + ErrNoCache = fmt.Errorf("%w: no ingestion cache", ErrSyncRefused) + + // ErrNoPoller is a Syncer built without A.7. There is no fallback path: + // a delta sync that fetched without the poller would be a second, + // unreviewed implementation of the conditional-GET, scope and credential + // rules that package exists to hold. + ErrNoPoller = fmt.Errorf("%w: no poller", ErrSyncRefused) + + // ErrNoSource is the deltaLog route with no Source hook wired. It is not + // fatal: SyncDelta falls back to decoding the polled body, which is the + // correct behaviour for every feed whose body IS the delta. + ErrNoSource = fmt.Errorf("%w: no delta source", ErrSyncRefused) + + // ErrNoDeltaLog is what a Source returns to say "this feed has no delta + // log". It is a normal answer, not a fault. + ErrNoDeltaLog = fmt.Errorf("%w: feed has no delta log", ErrSyncRefused) + + // ErrNoReconciler is RouteReconcile with nothing wired to run it. See the + // package comment: defaulting it to A.8's bulk importer would cost 570 MB + // a day. + ErrNoReconciler = fmt.Errorf("%w: no reconciler", ErrSyncRefused) + + // ErrDelegated is a route this package plans and does not run. + ErrDelegated = fmt.Errorf("%w: route is delegated", ErrSyncRefused) + + // ErrRecordName is the CVE-identifier allowlist refusing a name a delta + // log offered. It is the boundary between feed content and a request. + ErrRecordName = fmt.Errorf("%w: record name", ErrSyncRefused) + + // ErrStatementNotAllowed is the SQL allowlist refusing a statement. It is + // what stands between a delta batch and a full FTS rebuild. + ErrStatementNotAllowed = fmt.Errorf("%w: statement not on the allowlist", ErrSyncRefused) + + // ErrUnsanitized is a record reaching the write path with a string A.3 + // would have changed. + ErrUnsanitized = fmt.Errorf("%w: unsanitized field", ErrSyncRefused) + + // ErrBadRecord is a decoded record that cannot be written: no primary + // key, no raw document. + ErrBadRecord = fmt.Errorf("%w: unwritable record", ErrSyncRefused) + + // ErrUnrecognisedShape is a fetched document in no shape this package + // decodes. SyncDelta turns it into a routing decision rather than a + // dropped change. + ErrUnrecognisedShape = fmt.Errorf("%w: unrecognised document shape", ErrSyncRefused) + + // ErrDocumentTooLarge, ErrArchiveTooLarge and ErrBatchTooLarge are the + // three size bounds. Each is a refusal about what ARRIVED, never about + // what a header claimed. + ErrDocumentTooLarge = fmt.Errorf("%w: document too large", ErrSyncRefused) + ErrArchiveTooLarge = fmt.Errorf("%w: archive too large", ErrSyncRefused) + ErrBatchTooLarge = fmt.Errorf("%w: batch too large", ErrSyncRefused) +) + +// refuse builds a refusal: a decision Anvil made. +func refuse(sentinel error, format string, args ...any) error { + return fmt.Errorf("%w: %s", sentinel, fmt.Sprintf(format, args...)) +} + +// --------------------------------------------------------------------------- +// Routes +// --------------------------------------------------------------------------- + +// Route names the transport one sync used, or would use. +// +// It is Lane-A-local vocabulary with no counterpart among the record contract's +// six frozen enums, so declaring it here does not violate the single-owner rule +// in plan/IMPLEMENTATION-PLAN.md §6. It exists so a caller switches on a +// constant rather than re-deriving the decision from a sync mechanism and a +// body it would have to sniff again. +type Route string + +const ( + // RouteNone is nothing to do: the row is disabled, not due, or carries no + // steady-state poll at all. + RouteNone Route = "none" + + // RouteDerived is a feed whose content arrives inside another feed's + // payload. CISA Vulnrichment is the worked example: it is delivered in + // the CVE record's ADP container, so a separate sync would be a second + // copy of the same bytes. + RouteDerived Route = "derived" + + // RoutePoll is the plan-time answer for any polled feed: poll it, then + // let the bytes decide between RouteDeltaLog and RouteFeedBody. Due() + // never returns the other two, because which one applies is a fact about + // the response and not about the row. + RoutePoll Route = "poll" + + // RouteDeltaLog is the cheap path: a delta LOG names what changed, and + // only the named records are fetched. This is the route research/06 §3 + // prescribes for cvelistV5 at 15 minutes. + RouteDeltaLog Route = "delta_log" + + // RouteFeedBody is the polled body itself carrying the changed records — + // KEV's catalogue, an OSV ecosystem archive. There is nothing cheaper for + // these feeds: their publishers offer no delta mechanism. + RouteFeedBody Route = "feed_body" + + // RouteReconcile is the periodic wider-window pass on + // reconcile_interval_seconds. See the package comment for why it is + // planned here and refused unless delegated. + RouteReconcile Route = "reconcile" + + // RouteBaseline is A.15's full-baseline self-heal on + // baseline_interval_seconds. + RouteBaseline Route = "baseline" + + // RouteGitFetch is GHSA's incremental `git fetch` against A.8's blobless + // clone. + RouteGitFetch Route = "git_fetch" +) + +// RouteValues returns every legal Route, in declaration order. +func RouteValues() []Route { + return []Route{ + RouteNone, RouteDerived, RoutePoll, RouteDeltaLog, + RouteFeedBody, RouteReconcile, RouteBaseline, RouteGitFetch, + } +} + +// Valid reports whether r is one of the declared routes. +func (r Route) Valid() bool { + for _, v := range RouteValues() { + if r == v { + return true + } + } + return false +} + +// --------------------------------------------------------------------------- +// Due — the whole scheduler, as a pure function +// --------------------------------------------------------------------------- + +// Plan is what Due concluded about one feed at one instant. +// +// It is returned whole rather than as a bare boolean because "not due" and +// "not scheduled at all" and "disabled" are three different operational facts, +// and a feed that has gone quiet is diagnosed by which one it is. +type Plan struct { + // FeedID echoes the row. + FeedID string + + // Route is the transport a steady sync would use, and is one of + // RouteNone, RouteDerived, RoutePoll or RouteGitFetch. The two + // content-decided routes are never planned; see RoutePoll. + Route Route + + // Due is whether the steady sync should run now. + Due bool + + // Because is the sentence an operator reads when Due is false. + Because string + + // Interval is the steady cadence from the feed row, and NextDueAt is when + // the steady sync may next run. NextDueAt is the zero time when the feed + // has never succeeded, which is also when Due is true regardless of the + // clock. + Interval time.Duration + NextDueAt time.Time + + // LastOK is feed_state.last_ok_at as read before the poll. + LastOK time.Time + + // ReconcileDue and BaselineDue are the two wider passes. They are + // reported rather than run; see the package comment. + // + // Both are WINDOW-BOUNDARY tests, not elapsed-time tests: reconcile is due + // when `now` and `LastOK` fall in different reconcile windows. For the + // 86,400-second reconcile interval the feed table ships, that is exactly + // "the UTC day changed since the last successful sync", which is what + // research/06's "end-of-day delta once daily" means and what an + // elapsed-time test does not give — a feed polled at 23:50 and again at + // 00:10 has elapsed 20 minutes and has crossed the day. + ReconcileDue bool + BaselineDue bool + + // ReconcileInterval and BaselineInterval are the two cadences as read + // from the feed row. Zero means the row schedules no such pass. + ReconcileInterval time.Duration + BaselineInterval time.Duration +} + +// Due decides what is scheduled for one feed at one instant. +// +// EVERY DURATION IN THE ANSWER COMES FROM feed. There is no default cadence, no +// minimum, no clamp and no Go constant: an operator who sets +// `interval_seconds: 86400` on a constrained host gets a daily poll, which is +// research/06 Recommendation §4's stated purpose for putting cadences in +// config at all. +// +// lastOK is feed_state.last_ok_at. The zero time means "never succeeded", and +// everything is due. +func Due(feed config.FeedConfig, lastOK, now time.Time) Plan { + p := Plan{ + FeedID: feed.ID, + Route: RouteNone, + Interval: feed.Interval(), + LastOK: lastOK, + ReconcileInterval: feed.ReconcileInterval(), + BaselineInterval: feed.BaselineInterval(), + } + + // The wider passes are computed for every row, including one that is not + // polled at all: a bulk-only feed (sync_mechanism: none) still has a + // baseline cadence, and that is the only thing that refreshes it. + p.ReconcileDue = windowCrossed(lastOK, now, p.ReconcileInterval) + p.BaselineDue = windowCrossed(lastOK, now, p.BaselineInterval) + + if !feed.Enabled { + p.Because = "the feed row is disabled; nothing about it is scheduled" + p.ReconcileDue, p.BaselineDue = false, false + return p + } + + switch feed.SyncMechanism { + case config.SyncDerived: + p.Route = RouteDerived + p.Because = fmt.Sprintf( + "the feed is derived from %q and arrives inside that feed's payload; syncing it separately "+ + "would fetch the same bytes twice", feed.DerivedFrom) + return p + + case config.SyncNone: + p.Because = "the feed carries no steady-state poll; only its baseline pass refreshes it" + return p + + case config.SyncGitBloblessFetch: + p.Route = RouteGitFetch + + default: + p.Route = RoutePoll + } + + if p.Interval <= 0 { + p.Route = RouteNone + p.Because = "the feed row declares a zero poll interval, so it has no steady-state cadence" + return p + } + + if lastOK.IsZero() { + p.Due = true + p.Because = "the feed has never recorded a successful sync" + return p + } + + p.NextDueAt = lastOK.Add(p.Interval) + if now.Before(p.NextDueAt) { + p.Because = fmt.Sprintf("the last success was at %s and the row's cadence is %s, so the next sync is at %s", + lastOK.UTC().Format(time.RFC3339), p.Interval, p.NextDueAt.UTC().Format(time.RFC3339)) + return p + } + p.Due = true + p.Because = fmt.Sprintf("the last success was at %s, which is at least the row's %s cadence ago", + lastOK.UTC().Format(time.RFC3339), p.Interval) + return p +} + +// windowCrossed reports whether now and last fall in different windows of the +// given width. +// +// A ZERO WIDTH IS "no such pass" AND RETURNS FALSE. A zero last is "never +// succeeded" and returns true. +// +// time.Time.Truncate rounds toward the zero time, which is midnight UTC on +// 1 January year 1, so a 24-hour width gives UTC day boundaries and a +// 168-hour width gives a fixed weekly boundary. That is the property this +// wants: the pass happens once per calendar window rather than drifting later +// by however long the previous run took. +func windowCrossed(last, now time.Time, width time.Duration) bool { + if width <= 0 { + return false + } + if last.IsZero() { + return true + } + return !now.UTC().Truncate(width).Equal(last.UTC().Truncate(width)) +} + +// --------------------------------------------------------------------------- +// The delta log +// --------------------------------------------------------------------------- + +// DeltaLogEntry is one entry of a delta log: a moment, and the records that +// changed at it. +// +// THE LINK FIELDS ARE ABSENT ON PURPOSE. A real cvelistV5 deltaLog entry +// carries `githubLink` and `cveOrgLink` per record, and binding them would be +// the obvious way to fetch: the document tells you where the record lives. +// It is also an SSRF with the feed's credential attached, decided by a +// document written by strangers. encoding/json drops unknown fields, so those +// links are parsed and discarded by omission — which is stronger than dropping +// them in code, because there is no field for a later change to start using. +// +// What survives is the identifier and the claimed update time. The identifier +// is checked against IsCVEID before it reaches a Source; the time is compared +// with what the cache already holds. +type DeltaLogEntry struct { + // FetchTime is when the publisher recorded this batch of changes. + FetchTime string `json:"fetchTime"` + + // NumberOfChanges is the publisher's own count. It is carried for + // diagnostics and never trusted as a length. + NumberOfChanges int `json:"numberOfChanges"` + + New []DeltaLogRecord `json:"new"` + Updated []DeltaLogRecord `json:"updated"` + Error []DeltaLogRecord `json:"error"` +} + +// DeltaLogRecord names one changed record. +type DeltaLogRecord struct { + // CVEID is the only field that may influence a request, and only after + // IsCVEID accepts it. + CVEID string `json:"cveId"` + + // DateUpdated is the publisher's claimed modification time. It is + // compared against the cache's `modified` to decide whether the record is + // worth fetching at all, and a value in an unrecognised shape means + // "fetch it" — see isNewer, which fails toward fetching. + DateUpdated string `json:"dateUpdated"` +} + +// MaxDeltaLogBytes bounds a delta log document. research/06 records the +// cvelistV5 log as "a rolling 30 days worth of CVE record modification +// history" and its release-notes sibling at 65 KB; 64 MiB is three orders of +// magnitude of headroom and still refuses a memory-exhaustion payload. +const MaxDeltaLogBytes = 64 << 20 + +// ParseDeltaLog reads a delta log document into entries. +// +// It accepts both shapes a log is published in — a bare array of entries, and +// an object with an `entries` array — because the second is what a mirror +// wrapping the first tends to produce, and refusing it would make a legitimate +// mirror unusable for no security gain. It accepts nothing else. +func ParseDeltaLog(raw []byte) ([]DeltaLogEntry, error) { + if len(raw) > MaxDeltaLogBytes { + return nil, refuse(ErrDocumentTooLarge, + "a %d-byte delta log exceeds the %d-byte cap", len(raw), MaxDeltaLogBytes) + } + trimmed := bytes.TrimLeft(raw, " \t\r\n\ufeff") + if len(trimmed) == 0 { + return nil, refuse(ErrUnrecognisedShape, "the delta log document is empty") + } + + if trimmed[0] == '[' { + var entries []DeltaLogEntry + if err := json.Unmarshal(trimmed, &entries); err != nil { + return nil, refuse(ErrUnrecognisedShape, "the delta log is not an array of entries: %v", err) + } + return entries, nil + } + if trimmed[0] == '{' { + var wrapper struct { + Entries []DeltaLogEntry `json:"entries"` + } + if err := json.Unmarshal(trimmed, &wrapper); err != nil { + return nil, refuse(ErrUnrecognisedShape, "the delta log is not an object carrying `entries`: %v", err) + } + if wrapper.Entries == nil { + return nil, refuse(ErrUnrecognisedShape, + "the delta log is a JSON object with no `entries` array; a delta log names what changed") + } + return wrapper.Entries, nil + } + return nil, refuse(ErrUnrecognisedShape, "the delta log is neither a JSON array nor a JSON object") +} + +// checkRecordName is THE BOUNDARY between a document written by strangers and a +// request Anvil makes. +// +// It is an allowlist and it is deliberately narrow: `CVE-` then at least four +// digits, a dash, then at least one digit, and nothing else at all. Every +// traversal sequence, every scheme, every host, every wildcard and every +// encoding trick fails it, not because any of them is listed but because none +// of them is a CVE identifier. +// +// A rejected name is COUNTED, not silently dropped: a delta log whose names +// stopped parsing is a feed that changed shape, and the number is how an +// operator finds out before the cache quietly stops moving. +// It also requires the name to be ALREADY CANONICAL — equal to itself with +// surrounding whitespace removed. IsCVEID trims before it judges, because it +// also classifies aliases inside advisory documents where leading space is +// meaningless noise; that leniency is wrong HERE, and delta_test.go caught it: +// a guard that normalises and then accepts is not checking the string the +// caller goes on to use unless the caller applies the identical normalisation. +// Requiring the input to already be canonical removes the gap instead of +// duplicating the normalisation on both sides and hoping they stay equal. +func checkRecordName(feedID, name string) error { + if name == strings.TrimSpace(name) && IsCVEID(name) { + return nil + } + return refuse(ErrRecordName, + "feed %q: the delta log named %q, which is not a CVE identifier. Only a name matching "+ + "CVE-- may be turned into a fetch; the log's own link fields are not read "+ + "at all, because a document written by strangers must not choose where Anvil sends a "+ + "credentialed request.", + feedID, clip(name, 120)) +} + +// clip bounds a value quoted back into an error, so a hostile document cannot +// write a megabyte into a log line. +func clip(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} + +// --------------------------------------------------------------------------- +// Injected dependencies +// --------------------------------------------------------------------------- + +// FeedPoller is A.7. *poller.Poller satisfies it. +// +// It is an interface so that delta_test.go can count polls and so that the +// daemon supplies one configured Poller rather than this package constructing +// an HTTP client of its own — which would be a second implementation of the +// authentication, redirect-scope and body-cap rules A.7 exists to hold. +type FeedPoller interface { + Poll(ctx context.Context, feed config.FeedConfig) (poller.PollResult, error) +} + +// Source is the FEED-SPECIFIC knowledge the deltaLog route needs and this +// package deliberately does not hold. +// +// The rationale is poller.Watermarker's, verbatim in spirit: where a feed's +// delta log lives, and where one named record lives, are facts about a FEED. +// A package that knew them would be a hard-coded feed table wearing a different +// hat, and A.1's whole design is that Lane A knows nothing about a feed that is +// not in the table. +// +// A nil Source is legal and common. It makes RouteDeltaLog unreachable and +// SyncDelta decodes the polled body instead, which is correct for every feed +// whose publisher offers no delta log — which is most of them. +type Source interface { + // DeltaLog returns the feed's delta log document for this poll. + // + // It returns an error satisfying ErrNoDeltaLog when the feed has none; + // that is a normal answer and SyncDelta falls through to the body route. + // poll is passed so an implementation can read the release manifest, the + // ETag or the watermark the poll just produced without making a second + // request for them. + DeltaLog(ctx context.Context, feed config.FeedConfig, poll poller.PollResult) ([]byte, error) + + // Record returns one advisory record verbatim. + // + // id has ALREADY passed checkRecordName; an implementation may rely on + // that and must not relax it. An implementation must also apply the same + // scope discipline A.7 applies — same host as the feed row, no cross-host + // redirect, credentials from the row's credential_env and nowhere else. + Record(ctx context.Context, feed config.FeedConfig, id string) ([]byte, error) +} + +// Reconciler runs RouteReconcile. See the package comment for why this package +// refuses rather than defaulting the route to A.8's bulk importer. +type Reconciler interface { + Reconcile(ctx context.Context, feed config.FeedConfig) (BatchStats, error) +} + +// --------------------------------------------------------------------------- +// Syncer +// --------------------------------------------------------------------------- + +// Options configures a Syncer. DB and Poller are required. +type Options struct { + // DB is the A.2 ingestion cache, already migrated. It is NOT + // internal/store: that is the audit store of record and nothing here may + // touch it. + DB *sql.DB + + // Poller is A.7. Nothing in this package makes an HTTP request except + // through it and through Source. + Poller FeedPoller + + // THERE IS NO Mirror FIELD, AND ITS ABSENCE IS THE POINT. A.4's licence + // gate is resolved by A.7 BEFORE the request goes out, and the decision + // arrives on PollResult bound to the bytes it admitted. A Mirror here + // would let this package resolve the gate a second time, which means two + // answers to one question and a way to write rows under a decision the + // fetch was not made under. + + // Source is the deltaLog route's feed-specific hook. Nil disables that + // route. + Source Source + + // Reconcile runs RouteReconcile. Nil makes SyncReconcile a typed refusal. + Reconcile Reconciler + + // Now is the clock. Nil means time.Now. It is injected so that a cadence + // test asserts a boundary rather than approximately asserts one. + Now func() time.Time +} + +// Syncer performs delta syncs. It holds no per-sync mutable state and is safe +// for concurrent use across feeds; two concurrent syncs of the SAME feed are +// not useful but are not unsafe, because every write is an upsert keyed on +// (source, source_id). +type Syncer struct { + db *sql.DB + poll FeedPoller + source Source + reconciler Reconciler + now func() time.Time +} + +// SyncStats is what one SyncDelta did. +// +// It is returned even on an error, because "what did we do before it went +// wrong" is the first question an operator asks: whether the feed was polled, +// whether the licence gate refused, how many records were fetched, and whether +// anything reached the cache. +type SyncStats struct { + // FeedID echoes the row. + FeedID string + + // Plan is what Due concluded BEFORE the poll. Its LastOK is the value + // that decided the schedule, which the poll then moves. + Plan Plan + + // Route is the transport actually used. It is RouteDeltaLog or + // RouteFeedBody for a sync that ran, and one of the others for a sync + // that did not. + Route Route + + // Skipped is true when nothing ran because nothing was due. It is not an + // error and the returned error is nil. + Skipped bool + + // Polled is whether A.7 was called, and PollStatus its typed outcome. + Polled bool + PollStatus poller.Status + + // Decision is A.4's licence decision, and Refused says the gate declined. + // A.7 resolves the gate BEFORE the request, so a refusal here means no + // bytes were fetched at all. + Decision license.Decision + Refused bool + RefusedBecause string + + // Delegated marks a route this package plans and does not run. + Delegated bool + + // DeltaLogEntries, NamesSeen, NamesRejected and NamesUpToDate describe the + // cheap path's arithmetic. + // + // NamesUpToDate IS THE COST MODEL. It counts records the delta log named + // that the cache already holds at or past the log's own dateUpdated, and + // therefore records NOT fetched. research/06's ~200 MB/day figure is what + // happens when this number is zero because nobody looked. + DeltaLogEntries int + NamesSeen int + NamesRejected int + NamesUpToDate int + + // RecordFetches is how many individual record documents were fetched, and + // RecordBytes how many bytes those cost. BodyBytes is what the poll + // itself transferred. + RecordFetches int + RecordBytes int64 + BodyBytes int64 + + // Documents is how many documents were decoded (a body may be an archive + // of many), and Records how many advisories came out of them. + Documents int + Records int + + // Batch is what reached the cache. + Batch BatchStats + + // Sanitize is the merged A.3 report over everything decoded. A non-zero + // count is not an error; it is the ordinary state of text written by + // strangers. + Sanitize sanitize.SanitizeStats + + // AsOf is the timestamp stamped on every row this sync wrote, and + // StalenessSeconds spine S6's age of the DATA at write time — measured + // from the response's Last-Modified where the feed sent one, never from + // the age of the write. + AsOf time.Time + StalenessSeconds int + + // NextSyncAfter is the shortest delay before this feed may be synced + // again. It is A.7's answer where A.7 ran, because a server that asked + // for longer than the feed table's cadence has to be honoured. + NextSyncAfter time.Duration + + // Note is a sentence for an operator when the outcome needs one: a route + // refused, a body in a shape this path does not decode, a delta log the + // Source declined to provide. + Note string +} + +// New builds a Syncer. DB and Poller are the two hard requirements: without the +// cache nothing can be written, and without A.7 nothing may be fetched. +func New(opts Options) (*Syncer, error) { + if opts.DB == nil { + return nil, refuse(ErrNoCache, "a delta sync writes rows and needs the A.2 ingestion cache") + } + if opts.Poller == nil { + return nil, refuse(ErrNoPoller, + "a delta sync fetches only through A.7; a client built here would be a second implementation "+ + "of its authentication, redirect-scope and body-cap rules") + } + s := &Syncer{ + db: opts.DB, + source: opts.Source, + reconciler: opts.Reconcile, + poll: opts.Poller, + now: opts.Now, + } + if s.now == nil { + s.now = time.Now + } + return s, nil +} + +// --------------------------------------------------------------------------- +// SyncDelta +// --------------------------------------------------------------------------- + +// SyncDelta performs one steady-state delta sync of one feed. +// +// It is A.14's `SyncDelta(ctx, feed FeedConfig) (SyncStats, error)`; the +// dependencies that signature has no room for — the cache, the poller, the +// clock, the record source — live on the receiver so that no call site can +// supply a different one per call and no default can be reached by accident. +// +// THE ORDER OF WHAT FOLLOWS IS THE CONTRACT: +// +// 1. feed_state is read (no network) +// 2. Due decides what is scheduled (pure) +// 3. A.7 polls — which resolves A.4's licence gate BEFORE the request, +// sends the conditional headers, and refuses an off-host redirect +// 4. a 304 ends the sync having written nothing +// 5. the delta log is preferred; only records it names, and only records +// the cache does not already hold, are fetched +// 6. every document is decoded, sanitized field by field +// 7. one row-scoped upsert per changed record +// +// Step 5 is the one A.14's packet is about. Step 3 is the one A.7's ordering +// rule is about and it is not optional: the licence gate runs before the +// request, so a feed with no acquired licence body costs no bytes at all. +// +// A non-nil error is returned WITH a populated SyncStats, never instead of one. +func (s *Syncer) SyncDelta(ctx context.Context, feed config.FeedConfig) (SyncStats, error) { + now := s.now().UTC() + stats := SyncStats{ + FeedID: feed.ID, + Route: RouteNone, + AsOf: now, + Decision: license.Decision{Tier: config.LicenseTier(license.NoTier)}, + } + + lastOK, err := s.lastSuccess(ctx, feed.ID) + if err != nil { + return stats, err + } + plan := Due(feed, lastOK, now) + stats.Plan = plan + stats.Route = plan.Route + stats.NextSyncAfter = plan.Interval + + if !plan.Due { + stats.Skipped = true + stats.Note = plan.Because + return stats, nil + } + + switch plan.Route { + case RouteGitFetch: + stats.Delegated = true + stats.Note = "the row's sync_mechanism is git_blobless_fetch, which fetches into A.8's clone and " + + "would then have to rewrite A.8's watermark token; that is two writers on one column, so this " + + "package plans the route and does not run it" + return stats, refuse(ErrDelegated, "feed %q: %s", feed.ID, stats.Note) + case RoutePoll: + // fall through + default: + stats.Skipped = true + stats.Note = plan.Because + return stats, nil + } + + // --- 3. A.7. The licence gate runs inside it, before the request. --- + res, err := s.poll.Poll(ctx, feed) + stats.Polled = true + stats.PollStatus = res.Status + stats.Decision = res.Decision + stats.BodyBytes = res.BodyBytes + stats.Sanitize.Merge(res.Sanitize) + if res.NextPollAfter > 0 { + stats.NextSyncAfter = res.NextPollAfter + } + if err != nil { + if errors.Is(err, license.ErrLicenseRefused) || res.Decision.Refused() { + stats.Refused = true + stats.RefusedBecause = err.Error() + stats.Note = "the licence gate declined this feed, so no bytes were fetched. " + + "That is the ordinary state of a fresh clone: no publisher licence body has been " + + "acquired into mirror/ yet. See " + license.AcquireCommand + } + return stats, err + } + if res.Decision.Refused() { + stats.Refused = true + stats.Note = "the poll returned no error but produced a refusing licence decision" + return stats, refuse(license.ErrLicenseRefused, + "feed %q: tier %d, dir %q", feed.ID, res.Decision.Tier.Int(), res.Decision.Dir) + } + + // --- 4. A 304 writes nothing. Exit criterion 3: advisory, affected and + // advisory_fts must be byte-identical after one. --- + if res.Status != poller.StatusUpdated { + stats.Note = fmt.Sprintf("the poll returned %q, so nothing changed and no row was written", res.Status) + return stats, nil + } + + stats.StalenessSeconds = stalenessSeconds(now, res.LastModified) + + // --- 5. Prefer the delta log. --- + recs, route, note, err := s.collect(ctx, feed, res, &stats) + stats.Route = route + if note != "" { + stats.Note = note + } + if err != nil { + return stats, err + } + stats.Records = len(recs) + if len(recs) == 0 { + return stats, nil + } + + // --- 7. One upsert per changed record. --- + batch, err := Apply(ctx, s.db, feed, res.Decision, recs, now, stats.StalenessSeconds) + stats.Batch = batch + if err != nil { + return stats, err + } + return stats, nil +} + +// collect resolves the poll into decoded records, preferring the delta log. +// +// PREFERENCE IS STRUCTURAL, NOT ADVISORY. The delta log is asked for first, and +// the polled body is decoded ONLY when there is no delta log to be had — no +// Source wired, or the Source saying this feed has none. There is no size +// threshold, no "if the body is small enough", and no flag: a threshold is +// exactly how the cumulative-zip re-download A.14's packet forbids gets +// reintroduced as an optimisation. +func (s *Syncer) collect( + ctx context.Context, + feed config.FeedConfig, + res poller.PollResult, + stats *SyncStats, +) ([]Record, Route, string, error) { + if s.source != nil { + raw, err := s.source.DeltaLog(ctx, feed, res) + switch { + case err == nil: + recs, err := s.fromDeltaLog(ctx, feed, raw, stats) + return recs, RouteDeltaLog, "", err + case errors.Is(err, ErrNoDeltaLog): + // Normal. Fall through to the body. + default: + return nil, RouteDeltaLog, "", err + } + } + + recs, note, err := s.fromBody(feed, res, stats) + return recs, RouteFeedBody, note, err +} + +// fromDeltaLog is the cheap path. +// +// It reads a document that NAMES changes, checks every name against the CVE-ID +// allowlist, drops the names the cache already holds at or past the log's own +// dateUpdated, and fetches only what is left. The two counters it fills — +// NamesUpToDate and RecordFetches — are the cost model made observable. +func (s *Syncer) fromDeltaLog( + ctx context.Context, + feed config.FeedConfig, + raw []byte, + stats *SyncStats, +) ([]Record, error) { + entries, err := ParseDeltaLog(raw) + if err != nil { + return nil, fmt.Errorf("feed %q: %w", feed.ID, err) + } + stats.DeltaLogEntries = len(entries) + stats.RecordBytes += int64(len(raw)) + + // Names are de-duplicated across entries and sorted, so that a log naming + // the same CVE in three consecutive entries costs one fetch, and so that + // the request order is deterministic — a test that asserts a fetch count + // against a fixture should not depend on map iteration order. + claimed := map[string]string{} + var order []string + for _, e := range entries { + for _, group := range [][]DeltaLogRecord{e.New, e.Updated, e.Error} { + for _, r := range group { + stats.NamesSeen++ + if err := checkRecordName(feed.ID, r.CVEID); err != nil { + stats.NamesRejected++ + continue + } + // The name is used EXACTLY as checkRecordName accepted it. + // No trim, no case fold, no normalisation of any kind: the + // string that was checked and the string that becomes a fetch + // have to be the same bytes, or the check was of something + // else. + name := r.CVEID + if prev, seen := claimed[name]; !seen || isNewer(r.DateUpdated, prev) { + claimed[name] = strings.TrimSpace(r.DateUpdated) + if !seen { + order = append(order, name) + } + } + } + } + } + sort.Strings(order) + + var out []Record + for _, name := range order { + cached, present, err := CachedModified(ctx, s.db, feed.ID, name) + if err != nil { + return out, err + } + if present && !isNewer(claimed[name], cached) { + // THE WHOLE POINT. The cache already holds this record at or past + // the log's own claimed update time, so there is nothing to + // transfer. research/06's ~200 MB/day figure is what happens when + // this branch does not exist. + stats.NamesUpToDate++ + continue + } + + doc, err := s.source.Record(ctx, feed, name) + if err != nil { + return out, fmt.Errorf("feed %q: fetching record %s: %w", feed.ID, name, err) + } + stats.RecordFetches++ + stats.RecordBytes += int64(len(doc)) + stats.Documents++ + + recs, sstats, err := Decode(feed.ID, doc) + stats.Sanitize.Merge(sstats) + if err != nil { + return out, fmt.Errorf("feed %q: record %s: %w", feed.ID, name, err) + } + out = append(out, recs...) + } + return out, nil +} + +// fromBody decodes the polled body itself. +// +// This is the correct route for every feed whose publisher offers no delta +// mechanism — KEV's catalogue, an OSV ecosystem archive — and there is nothing +// cheaper for them: research/06 records "full-file download is the only option +// OVAL documents anyway". +// +// A body in a shape this package does not decode is NOT an error that fails the +// sync. It is a ROUTING FACT, returned as a note with zero records, because the +// answer for a CSAF directory listing or an EPSS CSV is A.8's bulk path and not +// a second decoder here. Failing the sync would make a correctly-configured +// feed look broken; dropping it silently would lose the change. +func (s *Syncer) fromBody(feed config.FeedConfig, res poller.PollResult, stats *SyncStats) ([]Record, string, error) { + body, err := res.Payload.Bytes() + if err != nil { + return nil, "", err + } + + docs, err := unwrap(feed.ID, body) + if err != nil { + if errors.Is(err, ErrUnrecognisedShape) { + return nil, unroutableNote(feed.ID, err), nil + } + return nil, "", err + } + stats.Documents = len(docs) + if len(docs) > MaxBatchRecords { + return nil, "", refuse(ErrBatchTooLarge, + "feed %q: the polled body unpacks to %d documents, which is a bulk artifact and belongs on "+ + "A.8's resumable path rather than in a delta batch", feed.ID, len(docs)) + } + + var out []Record + for _, d := range docs { + recs, sstats, err := Decode(feed.ID, d) + stats.Sanitize.Merge(sstats) + if err != nil { + if errors.Is(err, ErrUnrecognisedShape) { + return nil, unroutableNote(feed.ID, err), nil + } + return nil, "", err + } + out = append(out, recs...) + } + return out, "", nil +} + +func unroutableNote(feedID string, err error) string { + return fmt.Sprintf( + "feed %q polled successfully but its body is not a shape the delta decoder reads, so no row was "+ + "written and nothing was dropped silently: %v. Feeds whose steady state is a full-file "+ + "refresh (CSAF directory listings, per-branch distro secdb, the EPSS CSV) reach the cache "+ + "through A.8's bulk path.", feedID, err) +} + +// --------------------------------------------------------------------------- +// SyncReconcile +// --------------------------------------------------------------------------- + +// SyncReconcile runs the periodic wider-window pass on +// reconcile_interval_seconds. +// +// IT REFUSES UNLESS A Reconciler IS WIRED, and the refusal is the design. The +// artifact this pass wants is cvelistV5's ~17 MB end-of-day delta; A.8's +// importer resolves the LARGEST archive asset of a release, which is the 570 MB +// midnight baseline. Wiring this route to A.8 by default would cost 570 MB a +// day and would be precisely the re-download A.14's packet forbids — and it +// would do it silently, which is worse than doing it loudly. +// +// Choosing WHICH asset of a release is the reconciliation artifact is a +// per-feed fact, so it belongs in A.1's table beside the cadence that +// schedules it. Until it is there, this package plans the pass and declines to +// guess. +func (s *Syncer) SyncReconcile(ctx context.Context, feed config.FeedConfig) (SyncStats, error) { + now := s.now().UTC() + stats := SyncStats{ + FeedID: feed.ID, + Route: RouteReconcile, + AsOf: now, + Decision: license.Decision{Tier: config.LicenseTier(license.NoTier)}, + } + lastOK, err := s.lastSuccess(ctx, feed.ID) + if err != nil { + return stats, err + } + stats.Plan = Due(feed, lastOK, now) + + if !stats.Plan.ReconcileDue { + stats.Skipped = true + if stats.Plan.ReconcileInterval <= 0 { + stats.Note = "the feed row schedules no reconciliation pass" + } else { + stats.Note = fmt.Sprintf( + "the last success at %s falls in the same %s reconciliation window as now", + lastOK.UTC().Format(time.RFC3339), stats.Plan.ReconcileInterval) + } + return stats, nil + } + + if s.reconciler == nil { + stats.Delegated = true + stats.Note = "the reconciliation pass is due and no Reconciler is wired. This package will not " + + "default it to A.8's bulk importer: that importer resolves the largest asset of a release " + + "(the 570 MB midnight baseline) rather than the ~17 MB end-of-day delta, so the default " + + "would be a 570 MB/day re-download of data already held" + return stats, refuse(ErrNoReconciler, "feed %q: %s", feed.ID, stats.Note) + } + + batch, err := s.reconciler.Reconcile(ctx, feed) + stats.Batch = batch + return stats, err +} + +// --------------------------------------------------------------------------- +// feed_state +// --------------------------------------------------------------------------- + +// lastSuccess reads feed_state.last_ok_at, which is the only durable input the +// scheduler has. +// +// A feed with no row has never been polled and everything about it is due. A +// row whose last_ok_at does not parse is treated the same way, deliberately: a +// clock we cannot read must not be allowed to postpone a sync indefinitely, and +// re-syncing early costs one conditional GET. +func (s *Syncer) lastSuccess(ctx context.Context, feedID string) (time.Time, error) { + row, err := queryRowDB(ctx, s.db, cache.SelectFeedStateSQL, feedID) + if err != nil { + return time.Time{}, err + } + var ( + etag, lastModified, watermark, lastOK sql.NullString + failures, tier int + ) + switch err := row.Scan(&etag, &lastModified, &watermark, &lastOK, &failures, &tier); { + case err == sql.ErrNoRows: + return time.Time{}, nil + case err != nil: + return time.Time{}, fmt.Errorf("delta: reading feed_state for %q: %w", feedID, err) + } + if !lastOK.Valid { + return time.Time{}, nil + } + // A.7 and A.8 write this column in slightly different renderings of the + // same instant, so both are accepted rather than one being declared + // canonical from here. A value in neither shape is treated as "never + // succeeded": a clock we cannot read must not be able to postpone a sync + // indefinitely, and re-syncing early costs one conditional GET. + v := strings.TrimSpace(lastOK.String) + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05.000000000Z"} { + if t, err := time.Parse(layout, v); err == nil { + return t.UTC(), nil + } + } + return time.Time{}, nil +} + +// stalenessSeconds is the age of the DATA at write time, floored at zero. +// +// spine S6 requires as_of and staleness_seconds on every record, and +// research/06 Risk #5 is why: "never fail the scan — serve stale data with an +// as_of timestamp and a staleness_seconds field. A scan run on 3-day-old KEV +// data must say so." A publisher clock ahead of ours must not produce a +// negative age, which the cache's staleness_nonneg CHECK would refuse anyway. +func stalenessSeconds(now time.Time, lastModified string) int { + v := strings.TrimSpace(lastModified) + if v == "" { + return 0 + } + t, err := time.Parse(time.RFC1123, v) + if err != nil { + if t, err = time.Parse(time.RFC1123Z, v); err != nil { + return 0 + } + } + if d := int(now.Sub(t).Seconds()); d > 0 { + return d + } + return 0 +} + +// --------------------------------------------------------------------------- +// Unwrapping a polled body +// --------------------------------------------------------------------------- + +// MaxUnpackedBytes bounds the total uncompressed size of one polled body. +// +// It is the decompression-bomb bound and it is measured on what ARRIVED, never +// on what a zip header claimed: a member that lies about its uncompressed size +// is stopped by the running total, not by its own metadata. +const MaxUnpackedBytes = 512 << 20 + +// unwrap turns a polled body into the documents inside it. +// +// A bare JSON body is one document. A zip is its members. A gzip is what it +// decompresses to. Anything else is ErrUnrecognisedShape, which SyncDelta turns +// into a routing note rather than a failure. +// +// FORMAT IS DECIDED BY THE BYTES. There is no feed-id-to-format table here for +// the same reason there is no feed-id-to-parser table in Decode: a mapping +// compiled into Go breaks the moment an operator points a row at a mirror, and +// what a body IS is a property of the body. +func unwrap(feedID string, body []byte) ([][]byte, error) { + trimmed := bytes.TrimLeft(body, " \t\r\n\ufeff") + if len(trimmed) == 0 { + return nil, refuse(ErrUnrecognisedShape, "feed %q: the polled body is empty", feedID) + } + + switch { + case trimmed[0] == '{' || trimmed[0] == '[': + return [][]byte{trimmed}, nil + + case bytes.HasPrefix(trimmed, []byte{0x50, 0x4b, 0x03, 0x04}), // "PK\x03\x04" + bytes.HasPrefix(trimmed, []byte{0x50, 0x4b, 0x05, 0x06}): + return unwrapZip(feedID, trimmed) + + case bytes.HasPrefix(trimmed, []byte{0x1f, 0x8b}): + return unwrapGzip(feedID, trimmed) + } + return nil, refuse(ErrUnrecognisedShape, + "feed %q: the polled body is neither JSON, nor a zip, nor gzip", feedID) +} + +func unwrapZip(feedID string, body []byte) ([][]byte, error) { + zr, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + return nil, refuse(ErrUnrecognisedShape, "feed %q: the body has a zip signature but does not open as one: %v", feedID, err) + } + var ( + out [][]byte + total int64 + ) + for _, f := range zr.File { + if f.FileInfo().IsDir() { + continue + } + rc, err := f.Open() + if err != nil { + return nil, fmt.Errorf("delta: feed %q: opening zip member %q: %w", feedID, clip(f.Name, 200), err) + } + data, err := readCapped(rc, MaxDocumentBytes) + _ = rc.Close() + if err != nil { + return nil, fmt.Errorf("delta: feed %q: reading zip member %q: %w", feedID, clip(f.Name, 200), err) + } + total += int64(len(data)) + if total > MaxUnpackedBytes { + return nil, refuse(ErrArchiveTooLarge, + "feed %q: the polled body unpacks past the %d-byte cap", feedID, MaxUnpackedBytes) + } + trimmed := bytes.TrimLeft(data, " \t\r\n\ufeff") + if len(trimmed) == 0 { + continue + } + // A MEMBER THAT IS NOT JSON IS SKIPPED, NOT FATAL. This is the one + // place this package skips anything, and it follows A.8's reasoning + // exactly: an ecosystem archive is thousands of files written by + // strangers, and a README or a checksums file must not cost the + // advisories beside it. It is bounded to "the bytes are not a JSON + // document at all" \u2014 a member that IS JSON and is in no shape this + // decoder recognises still fails loudly, because that one is a + // dropped advisory rather than a text file. + if trimmed[0] != '{' && trimmed[0] != '[' { + continue + } + out = append(out, trimmed) + } + if len(out) == 0 { + return nil, refuse(ErrUnrecognisedShape, "feed %q: the zip holds no JSON member", feedID) + } + return out, nil +} + +func unwrapGzip(feedID string, body []byte) ([][]byte, error) { + gr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, refuse(ErrUnrecognisedShape, "feed %q: the body has a gzip signature but does not open as one: %v", feedID, err) + } + defer func() { _ = gr.Close() }() + data, err := readCapped(gr, MaxDocumentBytes) + if err != nil { + return nil, fmt.Errorf("delta: feed %q: decompressing the body: %w", feedID, err) + } + trimmed := bytes.TrimLeft(data, " \t\r\n\ufeff") + if len(trimmed) == 0 { + return nil, refuse(ErrUnrecognisedShape, "feed %q: the gzip decompresses to nothing", feedID) + } + return [][]byte{trimmed}, nil +} + +// readCapped reads at most limit bytes and REFUSES at limit+1, so that a member +// lying about its size is stopped by what actually arrived. +func readCapped(r io.Reader, limit int64) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > limit { + return nil, refuse(ErrDocumentTooLarge, "a document exceeded %d bytes while being read", limit) + } + return data, nil +} diff --git a/internal/ingest/delta/delta_test.go b/internal/ingest/delta/delta_test.go new file mode 100644 index 0000000..4204748 --- /dev/null +++ b/internal/ingest/delta/delta_test.go @@ -0,0 +1,1987 @@ +// delta_test.go is A.14's evidence. +// +// The two claims A.14's packet asks to be measured are measured, not asserted: +// +// 1. "A 200-row delta batch produces exactly 200 upserts and zero full-table +// statements." Counted from a SQL TRACE taken at the driver layer, so it +// also covers statements database/sql synthesises on the caller's behalf, +// and so that the number is not read back out of a struct field the code +// under test filled in itself. +// 2. "The deltaLog.json path is used in preference to re-downloading the +// cumulative hourly zip on every poll." Measured as bytes: the same feed is +// synced twice with a Source and twice without one, and the two transfer +// totals are compared against research/06's own cost model. +// +// Everything else here follows the rules this project has already paid for: +// +// - EVERY GUARD IS VERIFIED RED. checkStatement, checkRecordName and the +// no-cadence-literal scanner are each run against a corpus that must fail +// them, and a green run over an empty corpus is treated as a broken test +// rather than as a pass. +// - NO CORPUS COMES FROM THE IMPLEMENTATION. The conformance test compares +// this package's decoder against A.8's over the same bytes; the fixture +// documents are written here and consumed by both. +// - NO NETWORK. httptest only, and the licence gate's fixture mirror is an +// fstest.MapFS. No test reads the process environment, so a machine with a +// real ANVIL_GITHUB_TOKEN behaves identically to one without. +package delta + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "database/sql" + "database/sql/driver" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "net/http" + "net/http/httptest" + "os" + "path" + "path/filepath" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "testing" + "testing/fstest" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/bootstrap" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/config" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/license" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/poller" +) + +// --------------------------------------------------------------------------- +// Fixture constants +// --------------------------------------------------------------------------- + +// fixtureToken is NOT a credential. It is a string chosen so a test can search +// an error or a rendered result for it; nothing anywhere accepts it. The real +// PAT is operator-provisioned, lives in the environment variable the feed row +// names, and is never read by this suite. +const fixtureToken = "not-a-real-token-0000-test-only" + +// cc0Verbatim is the publisher licence body the synthetic mirror pins. A.4 +// classifies BODIES, so a fixture that wants an admission has to supply one. +const cc0Verbatim = `Creative Commons Legal Code + +CC0 1.0 Universal + +The person who associated a work with this deed has dedicated the work to the +public domain by waiving all rights to the work worldwide under copyright law.` + +const cc0Notes = `SPDX-License-Identifier: CC0-1.0 + +Anvil's record: this source is public domain and carries no obligation.` + +var fixtureClock = time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC) + +// --------------------------------------------------------------------------- +// A tracing driver, so "no full-table statement" is an observation +// --------------------------------------------------------------------------- + +const traceDriverName = "sqlite-anvil-delta-trace" + +func init() { + // sql.Open resolves the driver immediately and connects lazily, so this + // neither creates a file nor opens a connection. + probe, err := sql.Open("sqlite", "file:anvil-delta-driver-probe?mode=memory") + if err != nil { + panic("delta_test: cannot resolve the sqlite driver: " + err.Error()) + } + base := probe.Driver() + _ = probe.Close() + sql.Register(traceDriverName, traceDriver{base: base}) +} + +// --------------------------------------------------------------------------- +// Cache and mirror fixtures +// --------------------------------------------------------------------------- + +// openTracedCache opens a migrated cache through the tracing driver. +// +// It does not call cache.Open, which resolves its own driver name: the trace +// has to sit under this package's statements, and the DSN is taken from +// cache.DSN so the connection pragmas (WAL, foreign_keys, busy_timeout) are the +// ones A.2 requires rather than a set assembled here. +func openTracedCache(t *testing.T) (*sql.DB, *sqlTrace) { + t.Helper() + dsn, err := cache.DSN(filepath.Join(t.TempDir(), "anvil-cache.sqlite")) + if err != nil { + t.Fatalf("cache.DSN: %v", err) + } + db, err := sql.Open(traceDriverName, dsn) + if err != nil { + t.Fatalf("opening traced cache: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + tr := globalTrace + tr.reset() + t.Cleanup(tr.reset) + if err := db.PingContext(t.Context()); err != nil { + t.Fatalf("pinging traced cache: %v", err) + } + if err := cache.CheckWAL(t.Context(), db); err != nil { + t.Fatalf("the traced cache is not in WAL mode: %v", err) + } + if err := cache.CheckFTS5(t.Context(), db); err != nil { + t.Fatalf("the traced cache has no FTS5: %v", err) + } + if _, err := cache.Migrate(t.Context(), db); err != nil { + t.Fatalf("migrating the traced cache: %v", err) + } + tr.reset() + return db, tr +} + +// openPlainCache opens a migrated cache through the ordinary driver, for the +// tests that do not need a trace. +func openPlainCache(t *testing.T) *sql.DB { + t.Helper() + db, err := cache.Open(t.Context(), filepath.Join(t.TempDir(), "anvil-cache.sqlite")) + if err != nil { + t.Fatalf("cache.Open: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if _, err := cache.Migrate(t.Context(), db); err != nil { + t.Fatalf("cache.Migrate: %v", err) + } + return db +} + +func digestOf(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +// admittingMirror renders the mirror tree A.4 reads: a pinned manifest, the +// publisher's acquired text at the digest the pin names, and Anvil's own +// record. +// +// IT IS BUILT THE WAY internal/ingest/license's OWN FIXTURES ARE BUILT. The +// gate's admission path is exacting and a mirror assembled by guesswork simply +// refuses — which would make every test below pass for the wrong reason, since +// a refused feed writes nothing and a suite that only ever exercised refusal +// would look green. +func admittingMirror(t *testing.T, feeds ...config.FeedConfig) fs.FS { + t.Helper() + fsys := fstest.MapFS{} + var man strings.Builder + man.WriteString("# synthetic manifest, delta_test\n") + man.WriteString("schema_version = 1\n") + man.WriteString("generated_utc = \"2026-08-09\"\n") + man.WriteString("generated_by = \"delta_test\"\n") + + notes := map[config.LicenseTier]*strings.Builder{} + for _, f := range feeds { + dir := f.MirrorDir + if dir == "" { + dir = f.ID + } + fmt.Fprintf(&man, "\n[[body]]\nfeed_id = %q\ntier = %d\ndir = %q\n"+ + "spdx_id = %q\ntext_url = \"https://example.invalid/LICENSE\"\n"+ + "sha256 = %q\nclaim_source = \"delta_test fixture\"\n", + f.ID, f.LicenseTier.Int(), dir, f.LicenseSPDX, digestOf(cc0Verbatim)) + fsys[path.Join(license.TierDir(f.LicenseTier), dir, license.VerbatimFileName)] = + &fstest.MapFile{Data: []byte(cc0Verbatim)} + + b, ok := notes[f.LicenseTier] + if !ok { + b = &strings.Builder{} + b.WriteString("# fixture notes\n") + notes[f.LicenseTier] = b + } + fmt.Fprintf(b, "\n%s\n%s\n%s\n", + license.BodyBeginMarker(f.ID), cc0Notes, license.BodyEndMarker(f.ID)) + } + for tier, b := range notes { + fsys[path.Join(license.TierDir(tier), license.NotesFileName)] = &fstest.MapFile{Data: []byte(b.String())} + } + fsys[license.ManifestFileName] = &fstest.MapFile{Data: []byte(man.String())} + return fsys +} + +// fixtureFeed builds one admitted, polled feed row. +// +// The cadences are PARAMETERS, never defaults with a value hidden in here: a +// helper that supplied a cadence would be the hard-coded cadence this package +// forbids, wearing a test's clothes. +func fixtureFeed(id, rawURL string, intervalSeconds, reconcileSeconds, baselineSeconds int) config.FeedConfig { + return config.FeedConfig{ + ID: id, + URL: rawURL, + Enabled: true, + AuthMode: config.AuthNone, + SyncMechanism: config.SyncConditionalGetETag, + IntervalSeconds: intervalSeconds, + ReconcileIntervalSeconds: reconcileSeconds, + BaselineIntervalSeconds: baselineSeconds, + FreshnessSLOSeconds: intervalSeconds * 8, + OnFailure: config.OnFailureServeStale, + LicenseTier: config.LicenseTier0, + LicenseSPDX: "CC0-1.0", + MirrorDir: id, + BootstrapMechanism: config.BootstrapBulkArchive, + } +} + +// newTestSyncer wires a Syncer to a REAL A.7 poller pointed at an httptest +// server. Using the real poller is the point: it is what makes the licence +// gate run before the request, and a fake would let this suite pass with the +// gate bypassed. +func newTestSyncer(t *testing.T, db *sql.DB, feed config.FeedConfig, mirror fs.FS, tr http.RoundTripper, src Source, now func() time.Time) *Syncer { + t.Helper() + p, err := poller.New(poller.Options{ + DB: db, + Mirror: mirror, + Transport: tr, + Credentials: fixtureCredentials{}, + Now: now, + }) + if err != nil { + t.Fatalf("poller.New: %v", err) + } + s, err := New(Options{DB: db, Poller: p, Source: src, Now: now}) + if err != nil { + t.Fatalf("delta.New: %v", err) + } + return s +} + +// fixtureCredentials answers nothing. No test in this file reads the process +// environment, so a machine that happens to carry a real feed credential +// behaves exactly like one that does not. +type fixtureCredentials struct{} + +func (fixtureCredentials) Credential(string) (string, bool) { return "", false } + +// --------------------------------------------------------------------------- +// Document fixtures — written HERE, consumed by both decoders +// --------------------------------------------------------------------------- + +// cve5Record renders one CVE 5.x record. It is the shape a deltaLog names. +func cve5Record(id string, updated string) []byte { + doc := map[string]any{ + "dataType": "CVE_RECORD", + "dataVersion": "5.1", + "cveMetadata": map[string]any{ + "cveId": id, + "state": "PUBLISHED", + "datePublished": "2026-01-01T00:00:00Z", + "dateUpdated": updated, + }, + "containers": map[string]any{ + "cna": map[string]any{ + "descriptions": []any{map[string]any{ + "lang": "en", + "value": "A synthetic advisory about " + id + " affecting tokenalpha" + strings.TrimPrefix(id, "CVE-"), + }}, + "references": []any{map[string]any{"url": "https://example.invalid/adv/" + id}}, + "metrics": []any{map[string]any{"cvssV3_1": map[string]any{ + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL", + }}}, + "affected": []any{map[string]any{ + "vendor": "example", + "product": "widget", + "packageName": "widget", + "versions": []any{map[string]any{ + "version": "1.0.0", + "lessThan": "1.2.3", + "status": "affected", + "versionType": "semver", + "lessThanOrEqual": "", + "changesUnexpected": false, + }}, + }}, + }, + }, + } + b, _ := json.Marshal(doc) + return b +} + +// osvRecord renders one OSV advisory, which is also GHSA's format. +func osvRecord(i int) []byte { + doc := map[string]any{ + "schema_version": "1.6.0", + "id": fmt.Sprintf("GHSA-test-%06d", i), + "aliases": []string{fmt.Sprintf("CVE-2026-%06d", i)}, + "published": "2026-01-01T00:00:00Z", + "modified": "2026-02-01T00:00:00Z", + "summary": fmt.Sprintf("Synthetic advisory %d", i), + "details": "Details of a synthetic advisory about tokengamma" + strconv.Itoa(i) + ".", + "severity": []any{map[string]any{ + "type": "CVSS_V3", "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + }}, + "references": []any{map[string]any{"type": "ADVISORY", "url": fmt.Sprintf("https://example.invalid/a/%d", i)}}, + "affected": []any{map[string]any{ + "package": map[string]any{"ecosystem": "PyPI", "name": fmt.Sprintf("pkg-%d", i%7)}, + "ranges": []any{map[string]any{ + "type": "ECOSYSTEM", + "events": []any{ + map[string]any{"introduced": "0"}, + map[string]any{"fixed": fmt.Sprintf("1.%d.0", i%5)}, + }, + }}, + }}, + } + b, _ := json.Marshal(doc) + return b +} + +// ubuntuOSVRecord is an OSV record from a DISTRO, which must come out with +// distro_backport set. research/12 §3 is the reason that column exists. +func ubuntuOSVRecord(i int) []byte { + doc := map[string]any{ + "schema_version": "1.6.0", + "id": fmt.Sprintf("USN-%06d-1", i), + "aliases": []string{fmt.Sprintf("CVE-2025-%06d", i)}, + "published": "2026-01-01T00:00:00Z", + "modified": "2026-02-01T00:00:00Z", + "summary": "A synthetic distro advisory", + "details": "The distro backported the fix without moving the upstream version.", + "affected": []any{map[string]any{ + "package": map[string]any{"ecosystem": "Ubuntu:22.04:LTS", "name": "openssl"}, + "ranges": []any{map[string]any{ + "type": "ECOSYSTEM", + "events": []any{map[string]any{"introduced": "0"}, map[string]any{"fixed": "3.0.2-0ubuntu1.10"}}, + }}, + }}, + } + b, _ := json.Marshal(doc) + return b +} + +// kevCatalogue renders a KEV-shaped document. +func kevCatalogue(ids ...string) []byte { + vulns := make([]any, 0, len(ids)) + for _, id := range ids { + vulns = append(vulns, map[string]any{ + "cveID": id, + "vendorProject": "Example", + "product": "Widget", + "vulnerabilityName": "Example Widget Remote Code Execution", + "dateAdded": "2026-03-01", + "shortDescription": "A synthetic known-exploited entry about tokendelta.", + "requiredAction": "Apply mitigations per vendor instructions.", + "dueDate": "2026-03-22", + "notes": "https://example.invalid/kev/" + id, + }) + } + b, _ := json.Marshal(map[string]any{ + "title": "Synthetic KEV Catalog", + "catalogVersion": "2026.03.01", + "count": len(ids), + "vulnerabilities": vulns, + }) + return b +} + +// deltaLogDocument renders a delta log naming records. +// +// IT CARRIES THE LINK FIELDS A REAL ONE CARRIES. That is deliberate: the +// security claim is that Anvil never reads them, and a fixture without them +// could not distinguish "never read" from "never present". +func deltaLogDocument(t *testing.T, fetchTime string, updated map[string]string, hostileLinkHost string) []byte { + t.Helper() + recs := make([]any, 0, len(updated)) + names := make([]string, 0, len(updated)) + for id := range updated { + names = append(names, id) + } + sort.Strings(names) + for _, id := range names { + recs = append(recs, map[string]any{ + "cveId": id, + "cveOrgLink": "https://" + hostileLinkHost + "/cve/" + id, + "githubLink": "https://" + hostileLinkHost + "/raw/" + id + ".json", + "dateUpdated": updated[id], + }) + } + b, err := json.Marshal([]any{map[string]any{ + "fetchTime": fetchTime, + "numberOfChanges": len(recs), + "new": []any{}, + "updated": recs, + "error": []any{}, + }}) + if err != nil { + t.Fatalf("rendering delta log: %v", err) + } + return b +} + +// --------------------------------------------------------------------------- +// A Source fixture that RECORDS every request it was asked to make +// --------------------------------------------------------------------------- + +// fixtureSource is the injected feed-specific hook. It records every id it was +// handed, so a test can assert not only what was fetched but that nothing else +// was — which is how the "a feed document never chooses a URL" claim is +// checked rather than believed. +type fixtureSource struct { + deltaLog []byte + // docs maps a CVE id to the record document served for it. + docs map[string][]byte + // asked is every id Record was called with, in order. + asked []string + // logCalls counts DeltaLog calls. + logCalls int + // noLog makes DeltaLog answer ErrNoDeltaLog, the ordinary answer for a + // feed whose publisher offers none. + noLog bool +} + +func (f *fixtureSource) DeltaLog(context.Context, config.FeedConfig, poller.PollResult) ([]byte, error) { + f.logCalls++ + if f.noLog { + return nil, fmt.Errorf("%w: fixture feed publishes no delta log", ErrNoDeltaLog) + } + return f.deltaLog, nil +} + +func (f *fixtureSource) Record(_ context.Context, _ config.FeedConfig, id string) ([]byte, error) { + f.asked = append(f.asked, id) + doc, ok := f.docs[id] + if !ok { + return nil, fmt.Errorf("fixture source has no document for %q", id) + } + return doc, nil +} + +// TestEveryRouteIsListed keeps the enum and its value list from drifting. A +// Route added without an entry in RouteValues would report Valid() == false +// about itself, which is the kind of quiet inconsistency §6's single-owner rule +// exists to prevent for the record contract's own enums. +func TestEveryRouteIsListed(t *testing.T) { + seen := map[Route]bool{} + for _, r := range RouteValues() { + if !r.Valid() { + t.Errorf("RouteValues lists %q but Valid rejects it", r) + } + if seen[r] { + t.Errorf("RouteValues lists %q twice", r) + } + seen[r] = true + } + for _, r := range []Route{"", "rebuild", "Poll", "delta-log"} { + if r.Valid() { + t.Errorf("Valid accepted %q", r) + } + } + // Every route a Plan or a SyncStats can carry must be in the list, or a + // caller switching on it has an arm that never fires. + for _, r := range []Route{ + RouteNone, RouteDerived, RoutePoll, RouteDeltaLog, + RouteFeedBody, RouteReconcile, RouteBaseline, RouteGitFetch, + } { + if !seen[r] { + t.Errorf("route %q is reachable but not listed by RouteValues", r) + } + } +} + +// --------------------------------------------------------------------------- +// (a) The scheduler — every clock comes from the feed row +// --------------------------------------------------------------------------- + +// TestDueTakesEveryCadenceFromTheFeedRow varies only the row's numbers and +// asserts the schedule follows them. +// +// It is the consuming half of A.1's rule. feeds_test.go proves the feed table's +// own source carries no cadence literal; this proves the consumer does not +// quietly substitute one when the row says something unusual, which is the +// defect that would make an operator's `interval_seconds: 86400` on a +// constrained host silently not happen. +func TestDueTakesEveryCadenceFromTheFeedRow(t *testing.T) { + base := time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC) + + cases := []struct { + name string + interval int + elapsed time.Duration + wantDue bool + }{ + {"one second short of a fifteen-minute row", 900, 899 * time.Second, false}, + {"exactly a fifteen-minute row", 900, 900 * time.Second, true}, + {"a one-second row polled after two seconds", 1, 2 * time.Second, true}, + {"a ninety-day row polled after a day", 7776000, 24 * time.Hour, false}, + {"a ninety-day row polled after a hundred days", 7776000, 2400 * time.Hour, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + feed := fixtureFeed("f", "https://example.invalid/f", tc.interval, 0, 0) + p := Due(feed, base, base.Add(tc.elapsed)) + if p.Due != tc.wantDue { + t.Fatalf("Due=%v want %v (%s)", p.Due, tc.wantDue, p.Because) + } + if p.Interval != time.Duration(tc.interval)*time.Second { + t.Fatalf("Interval=%s want %ds", p.Interval, tc.interval) + } + }) + } + + t.Run("a feed that has never succeeded is always due", func(t *testing.T) { + feed := fixtureFeed("f", "https://example.invalid/f", 7776000, 0, 0) + if p := Due(feed, time.Time{}, base); !p.Due { + t.Fatalf("a feed with no recorded success is not due: %s", p.Because) + } + }) + + t.Run("a disabled row schedules nothing at all", func(t *testing.T) { + feed := fixtureFeed("f", "https://example.invalid/f", 900, 86400, 604800) + feed.Enabled = false + p := Due(feed, time.Time{}, base) + if p.Due || p.ReconcileDue || p.BaselineDue { + t.Fatalf("a disabled row scheduled something: %+v", p) + } + }) + + t.Run("a derived row is never synced on its own account", func(t *testing.T) { + feed := fixtureFeed("cisa-vulnrichment", "", 0, 0, 0) + feed.SyncMechanism = config.SyncDerived + feed.DerivedFrom = "cvelistv5" + p := Due(feed, time.Time{}, base) + if p.Route != RouteDerived || p.Due { + t.Fatalf("a derived row planned %q due=%v; it arrives inside its carrier's payload", p.Route, p.Due) + } + }) +} + +// TestReconcileAndBaselineAreWindowBoundariesNotElapsedTime is the difference +// between "once a day" and "every 24 hours", and research/06 asks for the +// first: "cvelistV5 end-of-day delta once daily", "full baseline weekly". +// +// An elapsed-time test gets this wrong in the operationally common direction: a +// feed polled at 23:50 and again at 00:10 has elapsed twenty minutes and has +// crossed the day, so the end-of-day pass would never fire on a busy feed. +func TestReconcileAndBaselineAreWindowBoundariesNotElapsedTime(t *testing.T) { + feed := fixtureFeed("cvelistv5", "https://example.invalid/c", 900, 86400, 604800) + + lateYesterday := time.Date(2026, 8, 9, 23, 50, 0, 0, time.UTC) + earlyToday := time.Date(2026, 8, 10, 0, 10, 0, 0, time.UTC) + if p := Due(feed, lateYesterday, earlyToday); !p.ReconcileDue { + t.Fatalf("20 minutes that cross midnight did not make the daily reconcile due: %+v", p) + } + + midday := time.Date(2026, 8, 10, 12, 0, 0, 0, time.UTC) + lateSameDay := time.Date(2026, 8, 10, 23, 0, 0, 0, time.UTC) + if p := Due(feed, midday, lateSameDay); p.ReconcileDue { + t.Fatalf("11 hours inside one day made the daily reconcile due: %+v", p) + } + + // The weekly baseline uses the same mechanism against a wider window, so + // eleven hours must not trigger it and eight days must. + if p := Due(feed, midday, lateSameDay); p.BaselineDue { + t.Fatalf("11 hours made the weekly baseline due: %+v", p) + } + if p := Due(feed, midday, midday.Add(8*24*time.Hour)); !p.BaselineDue { + t.Fatalf("8 days did not make the weekly baseline due: %+v", p) + } + + // A row that schedules no such pass never has one due, whatever the clock + // says. This is the case that would otherwise make every feed in the table + // run a daily bulk pull. + plain := fixtureFeed("cisa-kev", "https://example.invalid/k", 900, 0, 0) + if p := Due(plain, midday, midday.Add(365*24*time.Hour)); p.ReconcileDue || p.BaselineDue { + t.Fatalf("a row with no reconcile or baseline cadence scheduled one after a year: %+v", p) + } +} + +// TestNoCadenceLiteralIsWrittenInThisPackage is A.1's rule applied to the +// consumer, and it is the assertion A.14's brief names as the defect A.1 has an +// AST test against. +// +// THE SCANNER IS VERIFIED RED against synthetic source in the same run. A +// scanner that found nothing because it was looking for the wrong thing would +// be indistinguishable from a clean tree, and this repository has already +// certified a defect that way once. +func TestNoCadenceLiteralIsWrittenInThisPackage(t *testing.T) { + // The negative control first: if this does not flag, nothing below means + // anything. + const probe = `package probe + +import "time" + +const pollEvery = 900 * time.Second + +func f() time.Duration { return 86400 * time.Second } +` + if got := scanForCadenceLiterals(t, "probe.go", probe); len(got) < 2 { + t.Fatalf("the cadence scanner found %d findings in source that contains two; "+ + "it is not measuring what it claims: %v", len(got), got) + } + + for _, name := range []string{"delta.go", "upsert.go"} { + src, err := readSource(name) + if err != nil { + t.Fatalf("reading %s: %v", name, err) + } + if found := scanForCadenceLiterals(t, name, src); len(found) > 0 { + t.Errorf("%s writes a cadence in Go:\n\t%s\n"+ + "Every cadence lives in the feed table (research/06 Recommendation item 4); a duration "+ + "compiled in here cannot be dialled down by an operator on a constrained host.", + name, strings.Join(found, "\n\t")) + } + } +} + +// cadenceSeconds are the values the shipped feed table uses. A literal equal to +// one of them in this package's source is a cadence by any other name. +var cadenceSeconds = map[string]bool{ + "900": true, "3600": true, "7200": true, "21600": true, "86400": true, + "259200": true, "604800": true, "7776000": true, "15552000": true, +} + +// scanForCadenceLiterals reports every place src writes a duration. +func scanForCadenceLiterals(t *testing.T, name, src string) []string { + t.Helper() + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, name, src, 0) + if err != nil { + t.Fatalf("parsing %s: %v", name, err) + } + var found []string + timeUnits := map[string]bool{ + "Nanosecond": true, "Microsecond": true, "Millisecond": true, + "Second": true, "Minute": true, "Hour": true, + } + ast.Inspect(f, func(n ast.Node) bool { + switch e := n.(type) { + case *ast.SelectorExpr: + pkg, ok := e.X.(*ast.Ident) + if ok && pkg.Name == "time" && timeUnits[e.Sel.Name] { + found = append(found, fmt.Sprintf("%s: time.%s", fset.Position(e.Pos()), e.Sel.Name)) + } + case *ast.BasicLit: + if e.Kind == token.INT && cadenceSeconds[e.Value] { + found = append(found, fmt.Sprintf("%s: the literal %s is a cadence in the feed table", + fset.Position(e.Pos()), e.Value)) + } + } + return true + }) + return found +} + +// readSource reads one of this package's own source files. Tests run with the +// package directory as the working directory, so the name is enough. +func readSource(name string) (string, error) { + b, err := os.ReadFile(name) + return string(b), err +} + +// --------------------------------------------------------------------------- +// (b) The two guards, each verified RED +// --------------------------------------------------------------------------- + +// TestStatementAllowlistRefusesEveryFullTableStatement is the guard behind +// A.14's forbidden action, "do not rebuild advisory_fts wholesale on any delta +// batch, regardless of batch size". +// +// It is checked in BOTH directions. A guard that refused everything would pass +// a refusal-only test and would also stop the package working, so the +// allowlisted statements are asserted to be accepted in the same run. +func TestStatementAllowlistRefusesEveryFullTableStatement(t *testing.T) { + forbidden := []string{ + `DROP TABLE advisory_fts`, + `DROP TABLE IF EXISTS advisory_fts`, + `CREATE VIRTUAL TABLE advisory_fts USING fts5(description, references_text)`, + `INSERT INTO advisory_fts(advisory_fts) VALUES('rebuild')`, + `INSERT INTO advisory_fts(advisory_fts) VALUES('optimize')`, + `DELETE FROM advisory_fts`, + `DELETE FROM advisory`, + `DELETE FROM affected`, + `UPDATE advisory SET raw_json = ?`, + `REPLACE INTO advisory (source, source_id) VALUES (?, ?)`, + `ALTER TABLE advisory RENAME TO advisory_old`, + `VACUUM`, + // Two shapes a denylist of verbs would miss, listed to make the point + // that this is not a denylist: neither carries DROP, REBUILD or + // CREATE as a leading verb. + ` insert into advisory_fts(advisory_fts) values('rebuild') `, + `INSERT OR REPLACE INTO advisory_fts (rowid, description, references_text) VALUES (?, ?, ?), (?, ?, ?)`, + } + for _, q := range forbidden { + if err := checkStatement(q); err == nil { + t.Errorf("the allowlist accepted %q; a delta batch costs one upsert per changed row and "+ + "nothing else", condense(q)) + } else if !strings.Contains(err.Error(), "allowlist") { + t.Errorf("the refusal of %q does not mention the allowlist: %v", condense(q), err) + } + } + + if len(allowedStatements) == 0 { + t.Fatal("the allowlist is empty, so accepting nothing proves nothing") + } + for q, reason := range allowedStatements { + if err := checkStatement(q); err != nil { + t.Errorf("the allowlist refuses its own member (%s): %v", reason, err) + } + if strings.TrimSpace(reason) == "" { + t.Errorf("an allowlist entry carries no reason:\n\t%s", condense(q)) + } + } +} + +// TestOnlyACVEIdentifierCrossesFromFeedContentIntoAFetch is the other guard. +// +// The corpus is written to defeat a DENYLIST: traversal in three encodings, a +// scheme, an authority, a null byte, a newline-smuggled header, a homoglyph +// digit, and a name that is a valid CVE id with one extra character. None of +// them is refused by being listed; all of them are refused by not being a CVE +// identifier. +func TestOnlyACVEIdentifierCrossesFromFeedContentIntoAFetch(t *testing.T) { + hostile := []string{ + "", + " ", + "CVE-2024-0001/../../etc/passwd", + "CVE-2024-0001%2f..%2f..%2fetc%2fpasswd", + "CVE-2024-0001\\..\\..\\windows\\win.ini", + "../CVE-2024-0001", + "https://evil.invalid/CVE-2024-0001.json", + "//evil.invalid/CVE-2024-0001", + "file:///etc/passwd", + "CVE-2024-0001\x00.json", + "CVE-2024-0001\r\nX-Injected: 1", + "CVE-2024-0001 ", + "CVE-2024-0001.json", + "CVE-2024-٠٠٠١", + "cve-2024-0001", + "CVE-24-0001", + "CVE-2024-", + "CVE-2024", + "CVE-2024-0001-2", + "CVE--2024-0001", + strings.Repeat("CVE-2024-0001", 500), + } + for _, name := range hostile { + if err := checkRecordName("fixture", name); err == nil { + t.Errorf("a delta log name %q was accepted as a CVE identifier; only that identifier may "+ + "become a fetch", name) + } + } + + // The positive control. A guard that refused everything would pass the + // loop above and would also stop the deltaLog route working. + for _, name := range []string{"CVE-2024-0001", "CVE-1999-0001", "CVE-2026-1234567"} { + if err := checkRecordName("fixture", name); err != nil { + t.Errorf("a well-formed identifier %q was refused: %v", name, err) + } + } +} + +// TestADeltaLogHasNowhereToPutAURL is a STRUCTURAL guard. +// +// checkRecordName stops a hostile IDENTIFIER. This stops the other half: a +// later change deciding that following the log's own `githubLink` would be +// convenient. There is no field to put it in, so the change cannot be made by +// accident — it has to add a field, and this test is what fails when it does. +func TestADeltaLogHasNowhereToPutAURL(t *testing.T) { + want := map[string]bool{"CVEID": true, "DateUpdated": true} + rt := reflect.TypeOf(DeltaLogRecord{}) + for i := 0; i < rt.NumField(); i++ { + if !want[rt.Field(i).Name] { + t.Errorf("DeltaLogRecord carries a field %q. A delta log is written by strangers; the only "+ + "thing that may cross from it into a request is a CVE identifier that passed IsCVEID.", + rt.Field(i).Name) + } + } + if rt.NumField() != len(want) { + t.Errorf("DeltaLogRecord has %d fields, want %d", rt.NumField(), len(want)) + } +} + +// --------------------------------------------------------------------------- +// (c) A.14's headline validation +// --------------------------------------------------------------------------- + +// fullTablePatterns are the statement shapes that must NEVER appear in a delta +// batch's trace. Each is a rebuild of an index whose whole design premise is +// that it never needs one. +var fullTablePatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?is)\bDROP\b`), + regexp.MustCompile(`(?is)\bCREATE\s+VIRTUAL\s+TABLE\b`), + regexp.MustCompile(`(?is)\bCREATE\s+TABLE\b`), + regexp.MustCompile(`(?is)\bALTER\s+TABLE\b`), + regexp.MustCompile(`(?is)'rebuild'`), + regexp.MustCompile(`(?is)'optimize'`), + regexp.MustCompile(`(?is)\bVACUUM\b`), +} + +// unscopedFTSDelete matches a delete from the index; the caller then requires +// it to be scoped to one rowid. It is two steps rather than one regex because +// Go's RE2 has no lookahead, and a pattern that pretended to express "delete +// without a rowid clause" would either not compile or quietly match nothing — +// which is the shape of guard this project has already been burned by. +var unscopedFTSDelete = regexp.MustCompile(`(?is)\bDELETE\s+FROM\s+advisory_fts\b`) + +var rowidScoped = regexp.MustCompile(`(?is)\bWHERE\s+rowid\s*=\s*\?`) + +// TestTwoHundredRowDeltaBatchCostsExactlyTwoHundredUpserts is the number A.14's +// packet asks for, taken from a driver-layer trace. +func TestTwoHundredRowDeltaBatchCostsExactlyTwoHundredUpserts(t *testing.T) { + const rows = 200 + const feedID = "cvelistv5" + + srv := newFixtureServer(t, []byte(`{"assets":[]}`), "application/json") + feed := fixtureFeed(feedID, srv.URL+"/releases/latest", 900, 86400, 604800) + + updated := map[string]string{} + docs := map[string][]byte{} + for i := range rows { + id := fmt.Sprintf("CVE-2026-%04d", i) + updated[id] = "2026-08-09T00:00:00Z" + docs[id] = cve5Record(id, "2026-08-09T00:00:00Z") + } + src := &fixtureSource{ + deltaLog: deltaLogDocument(t, "2026-08-09T00:07:00Z", updated, "evil.invalid"), + docs: docs, + } + + db, trace := openTracedCache(t) + s := newTestSyncer(t, db, feed, admittingMirror(t, feed), srv.Client().Transport, src, + func() time.Time { return fixtureClock }) + + trace.reset() + stats, err := s.SyncDelta(t.Context(), feed) + if err != nil { + t.Fatalf("SyncDelta: %v (note: %s)", err, stats.Note) + } + batchTrace := trace.snapshot() + + if stats.Route != RouteDeltaLog { + t.Fatalf("route %q, want %q; the delta log is the cheap path and must be preferred", stats.Route, RouteDeltaLog) + } + if stats.Batch.Upserts != rows { + t.Errorf("%d advisory upserts, want %d", stats.Batch.Upserts, rows) + } + if stats.Batch.FTSUpserts != rows { + t.Errorf("%d advisory_fts writes, want %d", stats.Batch.FTSUpserts, rows) + } + + // The counts above come from the code under test. These come from the + // driver. + upserts := countStatement(batchTrace, cache.UpsertAdvisorySQL) + ftsWrites := countStatement(batchTrace, cache.UpsertAdvisoryFTSSQL) + if upserts != rows { + t.Errorf("the driver saw %d UpsertAdvisorySQL statements, want %d", upserts, rows) + } + if ftsWrites != rows { + t.Errorf("the driver saw %d UpsertAdvisoryFTSSQL statements, want %d", ftsWrites, rows) + } + assertNoFullTableStatement(t, batchTrace) + + // And the rows are really there. + if got := countRows(t, db, `SELECT count(*) FROM advisory WHERE source = ?`, feedID); got != rows { + t.Errorf("%d advisory rows, want %d", got, rows) + } + if got := countRows(t, db, `SELECT count(*) FROM advisory_fts`); got != rows { + t.Errorf("%d advisory_fts rows, want %d", got, rows) + } +} + +// TestFTSStaysQueryConsistentWithAdvisoryAfterABatch is A.14's stop condition. +// +// It is a ROUND TRIP: the text is written through the delta path and then read +// back through a MATCH query joined to `advisory`. It also re-runs the batch +// with changed text, because the failure this catches is not "the index is +// empty" but "the index still matches the OLD text", which is what a +// contentless FTS5 table does without contentless_delete=1 and which no row +// count would reveal. +func TestFTSStaysQueryConsistentWithAdvisoryAfterABatch(t *testing.T) { + const feedID = "cvelistv5" + srv := newFixtureServer(t, []byte(`{"assets":[]}`), "application/json") + feed := fixtureFeed(feedID, srv.URL+"/releases/latest", 900, 0, 0) + + id := "CVE-2026-4242" + first := map[string]string{id: "2026-08-09T00:00:00Z"} + src := &fixtureSource{ + deltaLog: deltaLogDocument(t, "2026-08-09T00:07:00Z", first, "evil.invalid"), + docs: map[string][]byte{id: cve5Record(id, "2026-08-09T00:00:00Z")}, + } + + db := openPlainCache(t) + clock := fixtureClock + s := newTestSyncer(t, db, feed, admittingMirror(t, feed), srv.Client().Transport, src, + func() time.Time { return clock }) + + if _, err := s.SyncDelta(t.Context(), feed); err != nil { + t.Fatalf("first sync: %v", err) + } + + // The round trip: find the row through the index, and prove the index's + // rowid resolves to the right advisory. + var gotID string + err := db.QueryRowContext(t.Context(), ` + SELECT a.source_id FROM advisory_fts f + JOIN advisory a ON a.rowid = f.rowid + WHERE advisory_fts MATCH ?`, "tokenalpha2026*").Scan(&gotID) + if err != nil { + t.Fatalf("the batch's text is not queryable through advisory_fts: %v", err) + } + if gotID != id { + t.Fatalf("MATCH resolved to %q, want %q", gotID, id) + } + + // Now change the text and re-sync. The old term must stop matching. + clock = clock.Add(2 * time.Hour) + changed := cve5Record(id, "2026-08-10T00:00:00Z") + changed = []byte(strings.ReplaceAll(string(changed), "tokenalpha2026-4242", "tokenomega2026")) + src.docs[id] = changed + src.deltaLog = deltaLogDocument(t, "2026-08-10T00:07:00Z", + map[string]string{id: "2026-08-10T00:00:00Z"}, "evil.invalid") + + if _, err := s.SyncDelta(t.Context(), feed); err != nil { + t.Fatalf("second sync: %v", err) + } + if n := ftsHits(t, db, "tokenalpha2026*"); n != 0 { + t.Errorf("the OLD term still matches %d rows after the record changed; the index was not "+ + "row-scoped-replaced", n) + } + if n := ftsHits(t, db, "tokenomega2026"); n != 1 { + t.Errorf("the NEW term matches %d rows, want 1", n) + } + if n := countRows(t, db, `SELECT count(*) FROM advisory WHERE source = ?`, feedID); n != 1 { + t.Errorf("%d advisory rows after two syncs of one record, want 1; the upsert is not keyed on "+ + "(source, source_id)", n) + } +} + +// --------------------------------------------------------------------------- +// (d) The cost model +// --------------------------------------------------------------------------- + +// TestTheDeltaLogIsPreferredOverRedownloadingTheCumulativeArchive is A.14's +// second named validation, and it is measured in bytes. +// +// research/06's finding is the whole reason this package exists: the hourly +// delta is CUMULATIVE since the midnight baseline, so re-downloading it every +// poll re-transfers everything already held — "~200 MB/day if polled hourly, vs +// ~17 MB/day if the end-of-day delta is taken once". The fixture reproduces the +// shape at a testable scale: a body that grows with every poll, and a delta log +// that names only what actually changed. +func TestTheDeltaLogIsPreferredOverRedownloadingTheCumulativeArchive(t *testing.T) { + const feedID = "cvelistv5" + + // A cumulative body that grows on every poll, exactly as the real hourly + // delta does. Each poll serves a fresh ETag so nothing 304s. + var cumulativeBytes int64 + polls := 0 + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + polls++ + // Every poll carries every record seen so far — the cumulative shape. + var docs []json.RawMessage + for i := range polls * 4 { + docs = append(docs, json.RawMessage(osvRecord(i))) + } + body, _ := json.Marshal(docs) + cumulativeBytes += int64(len(body)) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("ETag", fmt.Sprintf(`"poll-%d"`, polls)) + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + + feed := fixtureFeed(feedID, srv.URL+"/delta", 900, 0, 0) + mirror := admittingMirror(t, feed) + + // --- The route WITHOUT a delta log: the body is all there is. --- + bodyDB := openPlainCache(t) + bodyClock := fixtureClock + bodySyncer := newTestSyncer(t, bodyDB, feed, mirror, srv.Client().Transport, nil, + func() time.Time { return bodyClock }) + + var bodyTransfer int64 + for range 4 { + st, err := bodySyncer.SyncDelta(t.Context(), feed) + if err != nil { + t.Fatalf("body-route sync: %v (%s)", err, st.Note) + } + if st.Route != RouteFeedBody { + t.Fatalf("without a Source the route is %q, want %q", st.Route, RouteFeedBody) + } + bodyTransfer += st.BodyBytes + bodyClock = bodyClock.Add(time.Duration(feed.IntervalSeconds) * time.Second) + } + + // --- The route WITH a delta log: only the changed records move. --- + changed := map[string]string{ + "CVE-2026-0001": "2026-08-09T00:00:00Z", + "CVE-2026-0002": "2026-08-09T00:00:00Z", + "CVE-2026-0003": "2026-08-09T00:00:00Z", + } + docs := map[string][]byte{} + for id, when := range changed { + docs[id] = cve5Record(id, when) + } + src := &fixtureSource{ + deltaLog: deltaLogDocument(t, "2026-08-09T00:07:00Z", changed, "evil.invalid"), + docs: docs, + } + + logDB := openPlainCache(t) + logClock := fixtureClock + logSyncer := newTestSyncer(t, logDB, feed, mirror, srv.Client().Transport, src, + func() time.Time { return logClock }) + + var ( + logTransfer int64 + fetchesFirst int + fetchesLater int + ) + for i := range 4 { + st, err := logSyncer.SyncDelta(t.Context(), feed) + if err != nil { + t.Fatalf("delta-log sync %d: %v (%s)", i, err, st.Note) + } + if st.Route != RouteDeltaLog { + t.Fatalf("sync %d took route %q, want %q", i, st.Route, RouteDeltaLog) + } + logTransfer += st.RecordBytes + if i == 0 { + fetchesFirst = st.RecordFetches + } else { + fetchesLater += st.RecordFetches + if st.NamesUpToDate != len(changed) { + t.Errorf("sync %d re-fetched records the cache already held: NamesUpToDate=%d want %d", + i, st.NamesUpToDate, len(changed)) + } + } + logClock = logClock.Add(time.Duration(feed.IntervalSeconds) * time.Second) + } + + if fetchesFirst != len(changed) { + t.Errorf("the first delta-log sync fetched %d records, want %d", fetchesFirst, len(changed)) + } + if fetchesLater != 0 { + t.Errorf("later syncs fetched %d records although the delta log named nothing new; "+ + "the per-record freshness probe is not being consulted", fetchesLater) + } + + // THE COST MODEL. The delta-log route must transfer strictly and + // substantially less than re-reading a cumulative body every poll. + if logTransfer >= bodyTransfer { + t.Fatalf("the delta-log route transferred %d bytes and the cumulative-body route %d; "+ + "the cheap path is not cheaper", logTransfer, bodyTransfer) + } + if logTransfer*4 >= bodyTransfer { + t.Errorf("the delta-log route transferred %d bytes against the body route's %d — less, but not "+ + "by the margin research/06's cost model describes (a cumulative artifact re-transfers "+ + "everything already held on every poll)", logTransfer, bodyTransfer) + } + + // And the delta log route never touched the cumulative body at all: the + // only thing it read from the poll was the fact that something changed. + if src.logCalls != 4 { + t.Errorf("the Source was asked for a delta log %d times across 4 syncs", src.logCalls) + } +} + +// TestTheDeltaLogsOwnLinksAreNeverFetched is the security half of the cheap +// path. +// +// The fixture's delta log carries `githubLink` and `cveOrgLink` pointing at a +// host Anvil must never contact, alongside identifiers that are not +// identifiers. Nothing but a well-formed CVE id may reach the Source. +func TestTheDeltaLogsOwnLinksAreNeverFetched(t *testing.T) { + const feedID = "cvelistv5" + srv := newFixtureServer(t, []byte(`{"assets":[]}`), "application/json") + feed := fixtureFeed(feedID, srv.URL+"/releases/latest", 900, 0, 0) + + // A log mixing three good names with five that must never become a fetch. + log := []any{map[string]any{ + "fetchTime": "2026-08-09T00:07:00Z", + "numberOfChanges": 8, + "updated": []any{ + map[string]any{"cveId": "CVE-2026-0001", "githubLink": "https://evil.invalid/a", "dateUpdated": "2026-08-09T00:00:00Z"}, + map[string]any{"cveId": "https://evil.invalid/x.json", "dateUpdated": "2026-08-09T00:00:00Z"}, + map[string]any{"cveId": "CVE-2026-0002/../../../etc/passwd", "dateUpdated": "2026-08-09T00:00:00Z"}, + map[string]any{"cveId": "CVE-2026-0002", "githubLink": "https://evil.invalid/b", "dateUpdated": "2026-08-09T00:00:00Z"}, + map[string]any{"cveId": "//evil.invalid/c", "dateUpdated": "2026-08-09T00:00:00Z"}, + map[string]any{"cveId": "", "dateUpdated": "2026-08-09T00:00:00Z"}, + map[string]any{"cveId": "CVE-2026-0003", "cveOrgLink": "https://evil.invalid/d", "dateUpdated": "2026-08-09T00:00:00Z"}, + map[string]any{"cveId": "CVE-2026-0003.json", "dateUpdated": "2026-08-09T00:00:00Z"}, + }, + }} + raw, err := json.Marshal(log) + if err != nil { + t.Fatalf("rendering the hostile delta log: %v", err) + } + src := &fixtureSource{ + deltaLog: raw, + docs: map[string][]byte{ + "CVE-2026-0001": cve5Record("CVE-2026-0001", "2026-08-09T00:00:00Z"), + "CVE-2026-0002": cve5Record("CVE-2026-0002", "2026-08-09T00:00:00Z"), + "CVE-2026-0003": cve5Record("CVE-2026-0003", "2026-08-09T00:00:00Z"), + }, + } + + db := openPlainCache(t) + s := newTestSyncer(t, db, feed, admittingMirror(t, feed), srv.Client().Transport, src, + func() time.Time { return fixtureClock }) + + stats, err := s.SyncDelta(t.Context(), feed) + if err != nil { + t.Fatalf("SyncDelta: %v (%s)", err, stats.Note) + } + + want := []string{"CVE-2026-0001", "CVE-2026-0002", "CVE-2026-0003"} + if !reflect.DeepEqual(src.asked, want) { + t.Fatalf("the Source was asked for %v, want %v", src.asked, want) + } + for _, asked := range src.asked { + if strings.Contains(asked, "evil.invalid") || strings.ContainsAny(asked, "/\\:") { + t.Fatalf("a request name %q carries something a CVE identifier cannot", asked) + } + } + if stats.NamesRejected != 5 { + t.Errorf("NamesRejected=%d, want 5; a name the guard drops must be counted, not silently "+ + "discarded — a delta log whose names stopped parsing is how a cache quietly stops moving", + stats.NamesRejected) + } + if stats.NamesSeen != 8 { + t.Errorf("NamesSeen=%d, want 8", stats.NamesSeen) + } +} + +// --------------------------------------------------------------------------- +// (e) Cross-producer conformance with A.8 +// --------------------------------------------------------------------------- + +// TestDeltaAndBootstrapDecodeTheSameBytesIntoTheSameRows is the guard on the +// duplication this package was forced into. +// +// internal/ingest/bootstrap's decoders are unexported, so A.14 re-derives them, +// and two producers writing one table from one document is precisely the drift +// spine S6 names for the fingerprint. If they diverge, A.15's weekly self-heal +// "restores" the same rows forever and nothing surfaces why. +// +// The fixture documents are written in THIS file and handed to both importers. +// Neither decoder's output is the other's expectation, and neither is compared +// against a table derived from itself. +func TestDeltaAndBootstrapDecodeTheSameBytesIntoTheSameRows(t *testing.T) { + const feedID = "conformance" + + members := []zipMember{ + {"cves/CVE-2026-0001.json", cve5Record("CVE-2026-0001", "2026-08-09T00:00:00Z")}, + {"osv/GHSA-test-000001.json", osvRecord(1)}, + {"osv/USN-000002-1.json", ubuntuOSVRecord(2)}, + {"kev/known_exploited_vulnerabilities.json", kevCatalogue("CVE-2026-9001", "CVE-2026-9002")}, + } + archive := buildZip(t, members) + + srv := newFixtureServer(t, archive, "application/zip") + feed := fixtureFeed(feedID, srv.URL+"/all.zip", 900, 0, 0) + feed.BootstrapURL = srv.URL + "/all.zip" + mirror := admittingMirror(t, feed) + + // --- A.8's importer. --- + bootDB := openPlainCache(t) + b := &bootstrap.Bootstrapper{ + DB: bootDB, + Mirror: mirror, + WorkDir: t.TempDir(), + HTTP: srv.Client(), + Clock: func() time.Time { return fixtureClock }, + Lookup: func(string) (string, bool) { return fixtureToken, true }, + } + res, err := b.Bootstrap(t.Context(), feed) + if err != nil { + t.Fatalf("bootstrap: %v (refused: %s)", err, res.RefusedBecause) + } + if res.RecordsUpserted == 0 { + t.Fatalf("A.8 imported nothing, so there is nothing to compare against") + } + + // --- This package's write path over the same bytes. --- + decision, err := license.Resolve(license.FromFeed(feed, "", mirror)) + if err != nil { + t.Fatalf("resolving the fixture licence: %v", err) + } + deltaDB := openPlainCache(t) + var batch []Record + for _, m := range members { + recs, _, err := Decode(feedID, m.body) + if err != nil { + t.Fatalf("delta decoding %s: %v", m.name, err) + } + batch = append(batch, recs...) + } + if _, err := Apply(t.Context(), deltaDB, feed, decision, batch, fixtureClock, 0); err != nil { + t.Fatalf("delta Apply: %v", err) + } + + assertSameAdvisoryRows(t, bootDB, deltaDB) +} + +// assertSameAdvisoryRows compares the two caches column by column. +// +// as_of and staleness_seconds are EXCLUDED and the exclusion is stated rather +// than hidden: they record when the import ran and how old the artifact was, +// which are properties of the run and not of the document. Every column that is +// a function of the bytes is compared, including raw_json byte for byte. +func assertSameAdvisoryRows(t *testing.T, a, b *sql.DB) { + t.Helper() + const q = ` +SELECT source, source_id, ifnull(cve_id,''), ifnull(published,''), ifnull(modified,''), + state, ifnull(tombstoned_at,''), ifnull(severity,''), ifnull(cvss_vector,''), + ifnull(cvss_score,-1), ifnull(epss_score,-1), ifnull(epss_as_of,''), kev, + ifnull(license_spdx,''), ifnull(license_manual_note,''), license_tier, anvil_trust, + parse_degraded, ifnull(data_version,''), hex(raw_json) +FROM advisory ORDER BY source, source_id` + left := dumpRows(t, a, q) + right := dumpRows(t, b, q) + if len(left) == 0 { + t.Fatal("the reference cache is empty; the comparison would pass vacuously") + } + if len(left) != len(right) { + t.Fatalf("A.8 wrote %d advisory rows and A.14 wrote %d from the same documents:\nA.8: %v\nA.14: %v", + len(left), len(right), rowKeys(left), rowKeys(right)) + } + for i := range left { + if left[i] != right[i] { + t.Errorf("advisory row %d differs between the two importers.\nA.8: %s\nA.14: %s\n"+ + "Two producers writing one table from one document must agree; a divergence here makes "+ + "A.15's weekly self-heal restore the same rows forever with nothing surfacing why.", + i, left[i], right[i]) + } + } + + const affectedQ = ` +SELECT source, source_id, ecosystem, package, ifnull(purl,''), ifnull(introduced,''), + ifnull(fixed,''), distro_backport +FROM affected ORDER BY source, source_id, ecosystem, package, ifnull(introduced,''), ifnull(fixed,'')` + la, ra := dumpRows(t, a, affectedQ), dumpRows(t, b, affectedQ) + if !reflect.DeepEqual(la, ra) { + t.Errorf("the two importers disagree about `affected`:\nA.8: %v\nA.14: %v", la, ra) + } + + const aliasQ = `SELECT cve_id, source, source_id FROM cve_alias ORDER BY cve_id, source, source_id` + lc, rc := dumpRows(t, a, aliasQ), dumpRows(t, b, aliasQ) + if !reflect.DeepEqual(lc, rc) { + t.Errorf("the two importers disagree about `cve_alias`:\nA.8: %v\nA.14: %v", lc, rc) + } +} + +// --------------------------------------------------------------------------- +// (f) The refusal paths, which are the ORDINARY paths today +// --------------------------------------------------------------------------- + +// TestALicenceRefusalCostsNoRequestAndWritesNoRow is A.7's ordering seen from +// A.14: the gate runs BEFORE the request, so a feed with no acquired licence +// body costs no bytes at all. +// +// This is the state of a fresh clone — internal/ingest/license currently admits +// nothing, because no publisher body has been acquired into mirror/ — so this +// is the path the daemon takes on a real machine today and it must be a clean, +// counted refusal rather than a crash or a partial write. +func TestALicenceRefusalCostsNoRequestAndWritesNoRow(t *testing.T) { + const feedID = "cvelistv5" + hits := 0 + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + + feed := fixtureFeed(feedID, srv.URL+"/x", 900, 0, 0) + db := openPlainCache(t) + // An EMPTY mirror: nothing acquired, nothing pinned, nothing admitted. + s := newTestSyncer(t, db, feed, fstest.MapFS{}, srv.Client().Transport, nil, + func() time.Time { return fixtureClock }) + + stats, err := s.SyncDelta(t.Context(), feed) + if err == nil { + t.Fatal("an unlicensed feed synced without error") + } + if !stats.Refused { + t.Errorf("the refusal was not reported as one: %+v", stats) + } + if hits != 0 { + t.Errorf("the server was contacted %d times for a feed the licence gate refuses; the gate runs "+ + "before the request precisely so that no rate-limit budget is spent on bytes that must "+ + "then be discarded", hits) + } + if n := countRows(t, db, `SELECT count(*) FROM advisory`); n != 0 { + t.Errorf("%d advisory rows were written under a refused licence", n) + } + if stats.Note == "" { + t.Error("a refusal carried no operator-readable note") + } +} + +// TestANotModifiedResponseWritesNothing is the A.2 cache's exit criterion 3 +// seen from the delta path. +func TestANotModifiedResponseWritesNothing(t *testing.T) { + const feedID = "cisa-kev" + body := kevCatalogue("CVE-2026-9001") + serve := http.StatusOK + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("ETag", `"kev-1"`) + if serve == http.StatusNotModified { + w.WriteHeader(http.StatusNotModified) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + + feed := fixtureFeed(feedID, srv.URL+"/kev.json", 900, 0, 0) + db, trace := openTracedCache(t) + clock := fixtureClock + s := newTestSyncer(t, db, feed, admittingMirror(t, feed), srv.Client().Transport, nil, + func() time.Time { return clock }) + + if st, err := s.SyncDelta(t.Context(), feed); err != nil { + t.Fatalf("first sync: %v (%s)", err, st.Note) + } else if st.Batch.Upserts != 1 { + t.Fatalf("the first sync wrote %d rows, want 1", st.Batch.Upserts) + } + + before := snapshotAdvisories(t, db) + serve = http.StatusNotModified + clock = clock.Add(time.Duration(feed.IntervalSeconds) * time.Second) + trace.reset() + + st, err := s.SyncDelta(t.Context(), feed) + if err != nil { + t.Fatalf("the 304 sync failed: %v (%s)", err, st.Note) + } + if st.PollStatus != poller.StatusNotModified { + t.Fatalf("poll status %q, want %q", st.PollStatus, poller.StatusNotModified) + } + if st.Batch.Upserts != 0 || st.Batch.FTSUpserts != 0 { + t.Errorf("a 304 wrote %d advisory and %d FTS rows", st.Batch.Upserts, st.Batch.FTSUpserts) + } + if after := snapshotAdvisories(t, db); !reflect.DeepEqual(before, after) { + t.Errorf("a 304 changed `advisory`:\nbefore: %v\nafter: %v", before, after) + } + if n := countStatement(trace.snapshot(), cache.UpsertAdvisorySQL); n != 0 { + t.Errorf("the driver saw %d advisory upserts on a 304", n) + } +} + +// TestDelegatedRoutesRefuseLoudly checks the three routes this package plans +// and does not run. +// +// A route that quietly did nothing would look exactly like a route that +// succeeded, and the feed would stop moving with nothing to see. Each refusal +// must carry a named sentinel and a sentence. +func TestDelegatedRoutesRefuseLoudly(t *testing.T) { + db := openPlainCache(t) + srv := newFixtureServer(t, []byte(`{}`), "application/json") + + t.Run("git_blobless_fetch", func(t *testing.T) { + feed := fixtureFeed("ghsa", "https://github.invalid/github/advisory-database", 3600, 0, 0) + feed.SyncMechanism = config.SyncGitBloblessFetch + feed.BootstrapMechanism = config.BootstrapBloblessClone + s := newTestSyncer(t, db, feed, admittingMirror(t, feed), srv.Client().Transport, nil, + func() time.Time { return fixtureClock }) + st, err := s.SyncDelta(t.Context(), feed) + if err == nil { + t.Fatal("the git-fetch route ran") + } + if !isDeltaError(err, ErrDelegated) { + t.Errorf("the refusal does not satisfy ErrDelegated: %v", err) + } + if !st.Delegated || st.Note == "" { + t.Errorf("the delegation was not reported: %+v", st) + } + }) + + t.Run("reconcile without a Reconciler", func(t *testing.T) { + feed := fixtureFeed("cvelistv5", srv.URL+"/r", 900, 86400, 604800) + s := newTestSyncer(t, db, feed, admittingMirror(t, feed), srv.Client().Transport, nil, + func() time.Time { return fixtureClock }) + st, err := s.SyncReconcile(t.Context(), feed) + if err == nil { + t.Fatal("a due reconciliation pass ran with nothing wired to run it") + } + if !isDeltaError(err, ErrNoReconciler) { + t.Errorf("the refusal does not satisfy ErrNoReconciler: %v", err) + } + if !strings.Contains(st.Note, "570") { + t.Errorf("the note does not say why the route is not defaulted to A.8's importer: %q", st.Note) + } + }) + + t.Run("reconcile that is not due", func(t *testing.T) { + feed := fixtureFeed("cisa-kev", srv.URL+"/k", 900, 0, 0) + s := newTestSyncer(t, db, feed, admittingMirror(t, feed), srv.Client().Transport, nil, + func() time.Time { return fixtureClock }) + st, err := s.SyncReconcile(t.Context(), feed) + if err != nil { + t.Fatalf("a row that schedules no reconciliation pass returned an error: %v", err) + } + if !st.Skipped { + t.Errorf("a row with no reconcile cadence was not skipped: %+v", st) + } + }) +} + +// TestABodyInAnUnreadShapeIsARoutingNoteNotADroppedChange. +// +// CSAF directory listings, per-branch distro secdb files and the EPSS CSV all +// reach the cache through A.8's bulk path. When one of them arrives here the +// correct outcome is a stated routing fact — not a failed sync (which makes a +// correctly-configured feed look broken) and not a silent success (which loses +// the change). +func TestABodyInAnUnreadShapeIsARoutingNoteNotADroppedChange(t *testing.T) { + const feedID = "epss" + srv := newFixtureServer(t, []byte("#model_version:v2025.03.14\ncve,epss,percentile\nCVE-2026-0001,0.5,0.9\n"), "text/csv") + feed := fixtureFeed(feedID, srv.URL+"/epss.csv", 86400, 0, 0) + + db := openPlainCache(t) + s := newTestSyncer(t, db, feed, admittingMirror(t, feed), srv.Client().Transport, nil, + func() time.Time { return fixtureClock }) + + st, err := s.SyncDelta(t.Context(), feed) + if err != nil { + t.Fatalf("an unreadable body failed the sync rather than routing it: %v", err) + } + if st.Batch.Upserts != 0 { + t.Errorf("%d rows were written from a body this path does not decode", st.Batch.Upserts) + } + if !strings.Contains(st.Note, "A.8") { + t.Errorf("the note does not name the path that does handle it: %q", st.Note) + } +} + +// --------------------------------------------------------------------------- +// (g) The write path's own invariants +// --------------------------------------------------------------------------- + +// TestApplyRefusesARecordThatSkippedTheSanitizer is the precondition check that +// makes A.3's obligation true for a caller this package cannot see — A.15 +// builds Records from its own baseline read and reaches the same statements. +// +// It is verified in both directions in one run: the same record fails with an +// invisible character in its description and succeeds once Decode has been +// through it. +func TestApplyRefusesARecordThatSkippedTheSanitizer(t *testing.T) { + const feedID = "cvelistv5" + srv := newFixtureServer(t, []byte(`{}`), "application/json") + feed := fixtureFeed(feedID, srv.URL+"/x", 900, 0, 0) + mirror := admittingMirror(t, feed) + decision, err := license.Resolve(license.FromFeed(feed, "", mirror)) + if err != nil { + t.Fatalf("resolving the fixture licence: %v", err) + } + db := openPlainCache(t) + + // U+200B ZERO WIDTH SPACE, hand-built and never sanitized. + dirty := Record{ + Source: feedID, + SourceID: "CVE-2026-0001", + CVEID: "CVE-2026-0001", + State: cache.AdvisoryPublished, + Description: "a description with a zero\u200bwidth space in it", + Raw: []byte(`{"id":"CVE-2026-0001"}`), + } + if _, err := Apply(t.Context(), db, feed, decision, []Record{dirty}, fixtureClock, 0); err == nil { + t.Fatal("a record carrying an unsanitized string reached the write path") + } else if !isDeltaError(err, ErrUnsanitized) { + t.Errorf("the refusal does not satisfy ErrUnsanitized: %v", err) + } + if n := countRows(t, db, `SELECT count(*) FROM advisory`); n != 0 { + t.Errorf("%d rows survived a refused batch; Apply is one transaction", n) + } + + // The positive control: the same document through Decode is accepted. + doc := cve5Record("CVE-2026-0001", "2026-08-09T00:00:00Z") + recs, _, err := Decode(feedID, doc) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if _, err := Apply(t.Context(), db, feed, decision, recs, fixtureClock, 0); err != nil { + t.Fatalf("a sanitized record was refused: %v", err) + } + if n := countRows(t, db, `SELECT count(*) FROM advisory`); n != 1 { + t.Errorf("%d rows after a clean batch, want 1", n) + } +} + +// TestApplyRefusesAnUnadmittedLicenceDecision. Decision's zero value carries +// Tier 0 — the most permissive tier this system has — so a write path that +// checked nothing would treat "nobody filled this in" as "fully permissive". +func TestApplyRefusesAnUnadmittedLicenceDecision(t *testing.T) { + db := openPlainCache(t) + feed := fixtureFeed("cvelistv5", "https://example.invalid/x", 900, 0, 0) + recs, _, err := Decode(feed.ID, cve5Record("CVE-2026-0001", "2026-08-09T00:00:00Z")) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if _, err := Apply(t.Context(), db, feed, license.Decision{}, recs, fixtureClock, 0); err == nil { + t.Fatal("the zero licence Decision was accepted; it carries Tier 0, the most permissive tier there is") + } + if n := countRows(t, db, `SELECT count(*) FROM advisory`); n != 0 { + t.Errorf("%d rows were written under a zero Decision", n) + } +} + +// TestATombstonedAdvisoryLeavesTheIndexAndKeepsItsRow is A.2 exit criterion 22 +// on the delta path: a REJECTED CVE record is tombstoned, never deleted, so a +// finding that depended on it becomes invalidated rather than vanishing — and +// its text stops matching, so nothing retrieves it as live advice. +func TestATombstonedAdvisoryLeavesTheIndexAndKeepsItsRow(t *testing.T) { + const feedID = "cvelistv5" + srv := newFixtureServer(t, []byte(`{}`), "application/json") + feed := fixtureFeed(feedID, srv.URL+"/x", 900, 0, 0) + mirror := admittingMirror(t, feed) + decision, err := license.Resolve(license.FromFeed(feed, "", mirror)) + if err != nil { + t.Fatalf("resolving the fixture licence: %v", err) + } + db := openPlainCache(t) + + live := cve5Record("CVE-2026-0001", "2026-08-09T00:00:00Z") + recs, _, err := Decode(feedID, live) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if _, err := Apply(t.Context(), db, feed, decision, recs, fixtureClock, 0); err != nil { + t.Fatalf("Apply (live): %v", err) + } + if n := ftsHits(t, db, "tokenalpha2026*"); n != 1 { + t.Fatalf("the live advisory matches %d rows, want 1", n) + } + + rejected := []byte(strings.Replace(string(live), `"state":"PUBLISHED"`, `"state":"REJECTED"`, 1)) + if string(rejected) == string(live) { + t.Fatal("the fixture's state field did not change; the test would prove nothing") + } + recs, _, err = Decode(feedID, rejected) + if err != nil { + t.Fatalf("Decode (rejected): %v", err) + } + st, err := Apply(t.Context(), db, feed, decision, recs, fixtureClock, 0) + if err != nil { + t.Fatalf("Apply (rejected): %v", err) + } + if st.Tombstoned != 1 || st.FTSDeletes != 1 { + t.Errorf("tombstoned=%d ftsDeletes=%d, want 1 and 1", st.Tombstoned, st.FTSDeletes) + } + if n := countRows(t, db, `SELECT count(*) FROM advisory WHERE state = 'rejected' AND tombstoned_at IS NOT NULL`); n != 1 { + t.Errorf("%d tombstoned rows, want 1; a rejected advisory is tombstoned and never deleted", n) + } + if n := ftsHits(t, db, "tokenalpha2026*"); n != 0 { + t.Errorf("a tombstoned advisory still matches %d rows in the index", n) + } +} + +// TestDecodeRefusesADeltaLogHandedToItAsAnAdvisory. A delta log NAMES changes; +// it does not carry them. Decoding one as an advisory would write a row whose +// content is a list of identifiers, which is exactly the kind of quiet garbage +// a shape-sniffing decoder produces if it has no opinion about what it is +// looking at. +func TestDecodeRefusesADeltaLogHandedToItAsAnAdvisory(t *testing.T) { + log := deltaLogDocument(t, "2026-08-09T00:07:00Z", + map[string]string{"CVE-2026-0001": "2026-08-09T00:00:00Z"}, "example.invalid") + // The array form decodes element-wise, so hand it the element. + var elems []json.RawMessage + if err := json.Unmarshal(log, &elems); err != nil { + t.Fatalf("unmarshalling the fixture log: %v", err) + } + if _, _, err := Decode("cvelistv5", elems[0]); err == nil { + t.Fatal("a delta log entry was decoded as an advisory") + } else if !isDeltaError(err, ErrUnrecognisedShape) { + t.Errorf("the refusal does not satisfy ErrUnrecognisedShape: %v", err) + } +} + +// TestAZipBodyIsUnpackedIntoItsMembers covers the OSV ecosystem archives, whose +// steady state is a full-file refresh because their publishers document no +// delta mechanism. +func TestAZipBodyIsUnpackedIntoItsMembers(t *testing.T) { + const feedID = "osv-pypi" + members := []zipMember{ + {"GHSA-test-000001.json", osvRecord(1)}, + {"GHSA-test-000002.json", osvRecord(2)}, + {"README.txt", []byte("not an advisory")}, + } + archive := buildZip(t, members) + srv := newFixtureServer(t, archive, "application/zip") + feed := fixtureFeed(feedID, srv.URL+"/all.zip", 86400, 0, 0) + + db := openPlainCache(t) + s := newTestSyncer(t, db, feed, admittingMirror(t, feed), srv.Client().Transport, nil, + func() time.Time { return fixtureClock }) + + st, err := s.SyncDelta(t.Context(), feed) + if err != nil { + t.Fatalf("SyncDelta over a zip body: %v (%s)", err, st.Note) + } + if st.Route != RouteFeedBody { + t.Fatalf("route %q, want %q", st.Route, RouteFeedBody) + } + // The README is skipped and the two advisories are not. A zip that lost + // its advisories to a text file beside them would be the silent-drop + // failure this package exists to avoid. + if st.Batch.Upserts != 2 { + t.Fatalf("a two-advisory zip produced %d upserts (note: %s)", st.Batch.Upserts, st.Note) + } + if n := countRows(t, db, `SELECT count(*) FROM advisory WHERE source = ?`, feedID); n != 2 { + t.Errorf("%d advisory rows, want 2", n) + } + // The distro-backport column is what defeats the CVE-2023-32681 / + // RHSA-2023:4520 false-positive class, so a PyPI range must NOT carry it. + if n := countRows(t, db, `SELECT count(*) FROM affected WHERE distro_backport = 1`); n != 0 { + t.Errorf("%d PyPI ranges were marked distro_backport", n) + } +} + +// TestADistroOSVRecordCarriesTheBackportFlag is the other half: research/12 §3 +// records that a distro backports a fix without moving the upstream version, so +// an upstream range calls a patched package vulnerable. The column only helps +// A.17 if it is actually set. +func TestADistroOSVRecordCarriesTheBackportFlag(t *testing.T) { + // The row is a tier 0 fixture on purpose. The share-alike QUARANTINE is + // A.4's own tested territory and needs a share-alike licence BODY to + // exercise; what is under test here is that a distro's OSV export produces + // a backported range whatever tier it is admitted at, because the flag is + // a property of the ecosystem and not of the licence. + const feedID = "ubuntu-osv-mirror" + srv := newFixtureServer(t, []byte(`{}`), "application/json") + feed := fixtureFeed(feedID, srv.URL+"/x", 86400, 0, 0) + mirror := admittingMirror(t, feed) + decision, err := license.Resolve(license.FromFeed(feed, "", mirror)) + if err != nil { + t.Fatalf("resolving the fixture licence: %v", err) + } + + db := openPlainCache(t) + recs, _, err := Decode(feedID, ubuntuOSVRecord(2)) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if _, err := Apply(t.Context(), db, feed, decision, recs, fixtureClock, 0); err != nil { + t.Fatalf("Apply: %v", err) + } + if n := countRows(t, db, `SELECT count(*) FROM affected WHERE distro_backport = 1`); n != 1 { + t.Errorf("%d backported ranges, want 1", n) + } + // The licence columns come from A.4's DECISION, never from the feed row's + // own claim: a writer that re-read the YAML would launder an unverified + // assertion into the cache. + if n := countRows(t, db, `SELECT count(*) FROM advisory WHERE license_tier = ? AND license_spdx = ?`, + decision.Tier.Int(), decision.EffectiveSPDX); n != 1 { + t.Errorf("the advisory row does not carry the gate's tier %d and spdx %q", + decision.Tier.Int(), decision.EffectiveSPDX) + } +} + +// --------------------------------------------------------------------------- +// Assertion helpers +// --------------------------------------------------------------------------- + +func assertNoFullTableStatement(t *testing.T, statements []string) { + t.Helper() + if len(statements) == 0 { + t.Fatal("the trace is empty, so 'no full-table statement' would pass vacuously") + } + for _, f := range fullTableFindings(statements) { + t.Errorf("a delta batch issued a full-table statement:\n\t%s\n"+ + "A 200-record delta costs 200 row upserts and NOT a rebuild (internal/ingest/cache), "+ + "and A.14 forbids it regardless of batch size.", f) + } +} + +// fullTableFindings is the detector as a pure function, so its own negative +// control can call it instead of asserting through a testing.T. +func fullTableFindings(statements []string) []string { + var out []string + for _, q := range statements { + one := condense(q) + for _, re := range fullTablePatterns { + if re.MatchString(one) { + out = append(out, re.String()+" :: "+one) + } + } + if unscopedFTSDelete.MatchString(one) && !rowidScoped.MatchString(one) { + out = append(out, "DELETE FROM advisory_fts not scoped to one rowid :: "+one) + } + } + return out +} + +// TestTheFullTableDetectorActuallyDetects is the negative control. +// +// A detector that matched nothing would make every batch look clean, which is +// exactly how a guard passes for years while enforcing nothing. +func TestTheFullTableDetectorActuallyDetects(t *testing.T) { + corpus := []string{ + `DROP TABLE advisory_fts`, + `CREATE VIRTUAL TABLE advisory_fts USING fts5(description)`, + `CREATE TABLE advisory (source TEXT)`, + `ALTER TABLE advisory RENAME TO x`, + `INSERT INTO advisory_fts(advisory_fts) VALUES('rebuild')`, + `INSERT INTO advisory_fts(advisory_fts) VALUES('optimize')`, + `VACUUM`, + `DELETE FROM advisory_fts`, + `DELETE FROM advisory_fts WHERE source = ?`, + } + for _, q := range corpus { + if got := fullTableFindings([]string{q}); len(got) == 0 { + t.Errorf("the full-table detector accepted %q", q) + } + } + // And it must accept the statements a delta batch really issues, or every + // test above would be failing for the wrong reason. + legitimate := []string{ + cache.UpsertAdvisorySQL, cache.UpsertAdvisoryFTSSQL, cache.DeleteAdvisoryFTSSQL, + insertAffectedSQL, deleteAffectedSQL, insertAliasSQL, deleteAliasSQL, + selectModifiedSQL, cache.SelectFeedStateSQL, cache.UpsertFeedStateSQL, + } + if got := fullTableFindings(legitimate); len(got) > 0 { + t.Errorf("the full-table detector rejects statements a delta batch legitimately issues: %v", got) + } +} + +func countStatement(statements []string, want string) int { + n := 0 + target := strings.TrimSpace(want) + for _, q := range statements { + if strings.TrimSpace(q) == target { + n++ + } + } + return n +} + +func countRows(t *testing.T, db *sql.DB, query string, args ...any) int { + t.Helper() + var n int + if err := db.QueryRowContext(t.Context(), query, args...).Scan(&n); err != nil { + t.Fatalf("counting (%s): %v", query, err) + } + return n +} + +func ftsHits(t *testing.T, db *sql.DB, match string) int { + t.Helper() + var n int + if err := db.QueryRowContext(t.Context(), + `SELECT count(*) FROM advisory_fts WHERE advisory_fts MATCH ?`, match).Scan(&n); err != nil { + t.Fatalf("MATCH %q: %v", match, err) + } + return n +} + +func snapshotAdvisories(t *testing.T, db *sql.DB) []string { + t.Helper() + return dumpRows(t, db, ` +SELECT source, source_id, ifnull(cve_id,''), state, ifnull(modified,''), hex(raw_json) +FROM advisory ORDER BY source, source_id`) +} + +func dumpRows(t *testing.T, db *sql.DB, query string) []string { + t.Helper() + rows, err := db.QueryContext(t.Context(), query) + if err != nil { + t.Fatalf("querying (%s): %v", query, err) + } + defer func() { _ = rows.Close() }() + cols, err := rows.Columns() + if err != nil { + t.Fatalf("columns: %v", err) + } + var out []string + for rows.Next() { + vals := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + t.Fatalf("scan: %v", err) + } + parts := make([]string, len(cols)) + for i, v := range vals { + parts[i] = fmt.Sprintf("%s=%v", cols[i], v) + } + out = append(out, strings.Join(parts, " ")) + } + if err := rows.Err(); err != nil { + t.Fatalf("rows: %v", err) + } + return out +} + +func rowKeys(rows []string) []string { + out := make([]string, 0, len(rows)) + for _, r := range rows { + fields := strings.Fields(r) + if len(fields) >= 2 { + out = append(out, fields[0]+" "+fields[1]) + } + } + return out +} + +// isDeltaError reports whether err carries the given sentinel AND the +// package-wide one. Both matter: a caller that switches on ErrDelta must not +// have a refusal leak past it wearing a different sentinel. +func isDeltaError(err error, sentinel error) bool { + return err != nil && errors.Is(err, sentinel) && errors.Is(err, ErrDelta) +} + +// --------------------------------------------------------------------------- +// Test infrastructure +// --------------------------------------------------------------------------- + +// sqlTrace records every statement handed to the driver layer. +// +// It is the same shape internal/ingest/cache's own test uses, and for the same +// reason: "no code path may rebuild the index" is a claim about what reaches +// SQLite, so it is checked at SQLite's door rather than by reading the code +// that was supposed to obey it. +type sqlTrace struct { + mu sync.Mutex + statements []string +} + +func (l *sqlTrace) record(q string) { + l.mu.Lock() + defer l.mu.Unlock() + l.statements = append(l.statements, q) +} + +func (l *sqlTrace) reset() { + l.mu.Lock() + defer l.mu.Unlock() + l.statements = nil +} + +func (l *sqlTrace) snapshot() []string { + l.mu.Lock() + defer l.mu.Unlock() + return append([]string(nil), l.statements...) +} + +// globalTrace is process-wide because a database/sql driver is. The tests that +// read it do not run in parallel, and each resets it immediately before the +// batch it measures. +var globalTrace = &sqlTrace{} + +type traceDriver struct{ base driver.Driver } + +func (d traceDriver) Open(name string) (driver.Conn, error) { + c, err := d.base.Open(name) + if err != nil { + return nil, err + } + return traceConn{Conn: c}, nil +} + +// traceConn forwards every statement-carrying method to the real connection +// after recording the text. +type traceConn struct{ driver.Conn } + +func (c traceConn) Prepare(query string) (driver.Stmt, error) { + globalTrace.record(query) + return c.Conn.Prepare(query) +} + +func (c traceConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + globalTrace.record(query) + if p, ok := c.Conn.(driver.ConnPrepareContext); ok { + return p.PrepareContext(ctx, query) + } + return c.Conn.Prepare(query) +} + +func (c traceConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + globalTrace.record(query) + e, ok := c.Conn.(driver.ExecerContext) + if !ok { + return nil, driver.ErrSkip + } + return e.ExecContext(ctx, query, args) +} + +func (c traceConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + globalTrace.record(query) + q, ok := c.Conn.(driver.QueryerContext) + if !ok { + return nil, driver.ErrSkip + } + return q.QueryContext(ctx, query, args) +} + +func (c traceConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if b, ok := c.Conn.(driver.ConnBeginTx); ok { + return b.BeginTx(ctx, opts) + } + return c.Conn.Begin() +} + +func (c traceConn) ResetSession(ctx context.Context) error { + if r, ok := c.Conn.(driver.SessionResetter); ok { + return r.ResetSession(ctx) + } + return nil +} + +func (c traceConn) IsValid() bool { + if v, ok := c.Conn.(driver.Validator); ok { + return v.IsValid() + } + return true +} + +// newFixtureServer serves one body over TLS. +// +// It sends NO ETag and NO Last-Modified, so every poll is unconditional and +// returns 200. The 304 path has its own test, which sets a validator on +// purpose; a shared helper that quietly enabled conditional requests would make +// the other tests pass by never fetching anything. +func newFixtureServer(t *testing.T, body []byte, contentType string) *httptest.Server { + t.Helper() + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", contentType) + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv +} + +type zipMember struct { + name string + body []byte +} + +func buildZip(t *testing.T, members []zipMember) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, m := range members { + w, err := zw.Create(m.name) + if err != nil { + t.Fatalf("creating zip member %q: %v", m.name, err) + } + if _, err := w.Write(m.body); err != nil { + t.Fatalf("writing zip member %q: %v", m.name, err) + } + } + if err := zw.Close(); err != nil { + t.Fatalf("closing zip: %v", err) + } + return buf.Bytes() +} diff --git a/internal/ingest/delta/upsert.go b/internal/ingest/delta/upsert.go new file mode 100644 index 0000000..4d648a3 --- /dev/null +++ b/internal/ingest/delta/upsert.go @@ -0,0 +1,1188 @@ +// upsert.go is A.14's WRITE PATH: the decoded delta batch, and the row-scoped +// statements that put it in the A.2 cache. +// +// =========================================================================== +// THE ONE RULE THIS FILE EXISTS TO KEEP +// =========================================================================== +// +// A DELTA BATCH COSTS ONE UPSERT PER CHANGED ROW AND NOTHING ELSE. No DROP, no +// CREATE, no DELETE FROM advisory_fts that is not scoped to a single rowid, no +// `INSERT INTO advisory_fts(advisory_fts) VALUES('rebuild')`. +// +// internal/ingest/cache's own package comment gives the reason: "FTS5 accepts +// incremental INSERT/DELETE, so an hourly delta touching 200 records costs 200 +// row upserts and NOT a rebuild. That is why no code path in Anvil may DROP or +// rebuild `advisory_fts`." A.14's packet repeats it as a forbidden action, and +// adds "regardless of batch size" — because the tempting version of this defect +// is a size threshold above which somebody decides a rebuild is cheaper. +// +// It is enforced by allowedStatements, an ALLOWLIST of the exact statement +// texts this package may hand to the driver. A denylist of forbidden verbs is +// the shape this project has already lost three guards to: `REBUILD` is not a +// verb, `advisory_fts(advisory_fts)` is not a DDL keyword, and a writer that +// composed its DROP from two concatenated fragments would defeat any pattern +// nobody thought to list. An allowlist has the opposite failure mode — a new +// statement fails loudly until somebody adds it on purpose. +// +// =========================================================================== +// WHY THIS PACKAGE HAS ITS OWN DECODER, AND WHAT THAT COSTS +// =========================================================================== +// +// internal/ingest/bootstrap decodes the same three JSON shapes and its +// decoders are unexported, so this file re-derives them. That is a REAL +// cross-area hazard of exactly the kind plan/00-SPINE.md S6 names for the +// fingerprint: two producers writing the same table from the same bytes may +// drift, and the drift shows up as A.15's weekly self-heal "restoring" rows +// forever with nothing surfacing why. +// +// It is not left to inspection. delta_test.go's +// TestDeltaAndBootstrapDecodeTheSameBytesIntoTheSameRows runs A.8's importer +// and this package's over the SAME fixture documents and compares every +// written column of `advisory`, `affected` and `cve_alias`. A divergence is a +// red test in this package, which is the only place the two can be compared at +// all. The permanent fix — one decode package both import — is reported to the +// orchestrator rather than taken here, because internal/ingest/bootstrap is +// merged and frozen and this packet may not edit it. +// +// =========================================================================== +// SANITIZE IS A PRECONDITION, AND IT IS CHECKED +// =========================================================================== +// +// Apply does not sanitize; Decode does, field by field, and Apply REFUSES a +// batch whose strings do not survive sanitize.AssertAllSanitized. That split is +// deliberate: A.15 builds Records from its own baseline read and must not be +// able to reach these statements with raw feed text just because it skipped a +// call. internal/ingest/sanitize's writer guard sees the assertion in the same +// function as the bind, which is what it can check; the assertion is what makes +// the claim true rather than merely visible. +package delta + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/config" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/license" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/sanitize" +) + +// --------------------------------------------------------------------------- +// The statement allowlist +// --------------------------------------------------------------------------- + +// The four statements internal/ingest/cache does not export. +// +// They are byte-identical to the ones internal/ingest/bootstrap composes for +// the same tables, and that is on purpose: cache/schema.go exports the advisory +// and FTS write shapes precisely so two writers cannot disagree about them, and +// it exports nothing for `affected` or `cve_alias`. A second SHAPE here would +// be the defect that exporting the first four was meant to prevent, so these +// copy A.8's text exactly rather than improving on it. Reported to the +// orchestrator as the same gap A.8 reported. +const ( + deleteAffectedSQL = `DELETE FROM affected WHERE source = ? AND source_id = ?` + + insertAffectedSQL = ` +INSERT INTO affected (source, source_id, ecosystem, package, purl, introduced, fixed, distro_backport) +VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + + deleteAliasSQL = `DELETE FROM cve_alias WHERE source = ? AND source_id = ?` + + insertAliasSQL = ` +INSERT INTO cve_alias (cve_id, source, source_id) VALUES (?, ?, ?) +ON CONFLICT (cve_id, source, source_id) DO NOTHING` +) + +// selectModifiedSQL is the per-record freshness probe that makes the deltaLog +// route cheap: a record whose delta-log `dateUpdated` is not newer than what +// the cache already holds is NOT FETCHED. It is the difference between "poll +// every 15 minutes" and "download every changed record every 15 minutes". +const selectModifiedSQL = `SELECT modified FROM advisory WHERE source = ? AND source_id = ?` + +// selectFTSRowidSQL reads the rowid an advisory currently occupies, which is +// the only address advisory_fts has for it. It is used on the tombstone path, +// where the FTS entry is deleted by rowid. +const selectFTSRowidSQL = `SELECT rowid FROM advisory WHERE source = ? AND source_id = ?` + +// allowedStatements is THE GUARD. Every statement this package hands to the +// database driver must be a member, compared as exact text after trimming. +// +// It is a package-level var and not a function on purpose: internal/ingest/ +// sanitize's writer guard walks FUNCTION bodies looking for the names of the +// cache's advisory write shapes, and a function that named them only to build +// this set would be flagged as an unsanitised writer. A var initialiser is not +// a function body, so the guard sees the real write site and not this one. +// +// The value is a human-readable reason. It is not decoration: a future reader +// deciding whether to add an entry needs to see what the existing entries had +// to justify, and "it seemed necessary" is not one of the reasons here. +var allowedStatements = map[string]string{ + strings.TrimSpace(cache.UpsertAdvisorySQL): "one advisory row, ON CONFLICT DO UPDATE, RETURNING the rowid " + + "advisory_fts is addressed by. Never INSERT OR REPLACE: REPLACE re-inserts under a new rowid and " + + "silently orphans the FTS entry.", + strings.TrimSpace(cache.UpsertAdvisoryFTSSQL): "one FTS row by rowid. This is the row-scoped index write " + + "that makes a 200-record delta cost 200 writes.", + strings.TrimSpace(cache.DeleteAdvisoryFTSSQL): "one FTS row by rowid, for a tombstoned advisory. The " + + "`advisory` row itself is never deleted (A.2 exit criterion 22).", + strings.TrimSpace(deleteAffectedSQL): "the version ranges of ONE advisory. `affected` has a surrogate key " + + "and no unique natural key, so ranges are replaced per advisory rather than merged.", + strings.TrimSpace(insertAffectedSQL): "one version range of one advisory.", + strings.TrimSpace(deleteAliasSQL): "the CVE aliases of ONE advisory, replaced for the same reason.", + strings.TrimSpace(insertAliasSQL): "one CVE alias of one advisory.", + strings.TrimSpace(selectModifiedSQL): "read-only freshness probe for one (source, source_id).", + strings.TrimSpace(selectFTSRowidSQL): "read-only rowid lookup for one (source, source_id).", + strings.TrimSpace(cache.SelectFeedStateSQL): "read-only feed_state read, used to decide what is DUE before " + + "any poll is made.", +} + +// checkStatement is the allowlist gate. Every database call in this package +// goes through it, and nothing else in this package may call the driver. +// +// The error names the statement so an operator sees WHAT was refused, and the +// message says what to do about it, because the correct response to a refusal +// here is nearly always "this statement is fine, add it deliberately" and the +// wrong response is to route around the check. +func checkStatement(q string) error { + if _, ok := allowedStatements[strings.TrimSpace(q)]; ok { + return nil + } + return refuse(ErrStatementNotAllowed, + "this package may only execute statements on its allowlist and this one is not on it:\n\t%s\n"+ + "If it is a legitimate row-scoped write, add it to allowedStatements with the reason. "+ + "If it rebuilds, drops or re-creates advisory_fts, it is the thing A.14's packet forbids: "+ + "a delta batch costs one upsert per changed row, regardless of batch size.", + condense(q)) +} + +// condense renders a statement on one line for an error message. +func condense(q string) string { + return strings.Join(strings.Fields(q), " ") +} + +// execTx runs one allowlisted statement inside a transaction. +func execTx(ctx context.Context, tx *sql.Tx, q string, args ...any) error { + if err := checkStatement(q); err != nil { + return err + } + _, err := tx.ExecContext(ctx, q, args...) + return err +} + +// queryRowTx runs one allowlisted single-row query inside a transaction. +func queryRowTx(ctx context.Context, tx *sql.Tx, q string, args ...any) (*sql.Row, error) { + if err := checkStatement(q); err != nil { + return nil, err + } + return tx.QueryRowContext(ctx, q, args...), nil +} + +// queryRowDB runs one allowlisted single-row query outside a transaction. +func queryRowDB(ctx context.Context, db *sql.DB, q string, args ...any) (*sql.Row, error) { + if err := checkStatement(q); err != nil { + return nil, err + } + return db.QueryRowContext(ctx, q, args...), nil +} + +// --------------------------------------------------------------------------- +// The record model +// --------------------------------------------------------------------------- + +// AffectedRange is one row of the cache's `affected` table: a package, an +// ecosystem, and the version window a comparator answers against. +// +// plan/00-SPINE.md S1 is why these rows are the point of Lane A at all: +// "CVE/OSV/GHSA describe vulnerable PACKAGE VERSIONS, and a version comparator +// answers that exactly and for free." +type AffectedRange struct { + Ecosystem string + Package string + PURL string + + // Introduced and Fixed bound the vulnerable window. Either may be empty: + // an OSV entry with a `last_affected` event and no `fixed` has no fixed + // version, and that is a fact about the advisory, not a parse failure. + Introduced string + Fixed string + + // DistroBackport marks a range that came from a vendor or distro advisory + // rather than from upstream. research/12 §3: a distro backports a fix + // without moving the upstream version, so an upstream range calls a + // patched package vulnerable. This column is what A.17 needs to not do + // that. + DistroBackport bool +} + +// Record is one decoded advisory, already sanitized, ready to bind. +// +// It is EXPORTED because A.15's reconciliation writes the same rows through +// the same path. A second write path for the same table is how a schema +// invariant survives in one writer and not the other. +type Record struct { + // Source is the feed id and SourceID the native id within it. Together + // they are the primary key, and it is NEVER the CVE id: research/06 Risk + // #2 requires a cvelistV5 outage to be survivable by swapping sources, + // and GHSA advisories frequently carry no CVE at all. + Source string + SourceID string + + // CVEID is the nullable, indexed alias. Aliases carries the one-to-many. + CVEID string + Aliases []string + + Published string + Modified string + + // State is one of cache.AdvisoryPublished / AdvisoryWithdrawn / + // AdvisoryRejected. A non-published state MUST carry TombstonedAt, and + // the schema's advisory_tombstone_paired CHECK refuses the row otherwise + // — withdrawn advisories are tombstoned, never deleted, so that findings + // that depended on them become invalidated rather than vanishing. + State string + TombstonedAt string + + Severity string + CVSSVector string + + // CVSSScore and EPSSScore are `any` so that "no score" is SQL NULL rather + // than 0.0. A zero CVSS base score is a real value, and a comparator that + // cannot tell it from "absent" ranks an unscored advisory as harmless. + CVSSScore any + EPSSScore any + EPSSAsOf string + + KEV bool + + Description string + References []string + + Affected []AffectedRange + + // DataVersion is the record schema version the document declared, and + // ParseDegraded is spine S6's field for "this was persisted anyway". + // A.2 exit criterion 23: an unknown CVE dataVersion is PERSISTED with + // parse_degraded = 1, never dropped, because silently discarding a record + // from a newer schema is how a vulnerability disappears from a security + // tool with no error anywhere. + DataVersion string + ParseDegraded bool + + // StalenessSeconds overrides the batch-wide value when the record carries + // its own age. Zero means "use the batch's". + StalenessSeconds int + + // Raw is the document verbatim, stored in advisory.raw_json. It is never + // sanitized: the column is the publisher's bytes, and CVE-TOU requires + // records be stored byte-verbatim (research/06 §"License"). + Raw []byte +} + +// ReferencesText is the `references_text` column of advisory_fts: the +// references as one newline-separated string, each element already sanitized. +func (r Record) ReferencesText() string { return strings.Join(r.References, "\n") } + +// BatchStats counts what one Apply wrote. Every field counts WRITES, not net +// growth: `affected` and `cve_alias` are replaced per advisory, so a re-upsert +// of an unchanged advisory still counts its rows. +type BatchStats struct { + // Upserts is advisory rows written, and is the number A.14's validation + // asserts equals the batch size: 200 changed records, 200 upserts. + Upserts int + + // FTSUpserts is row-scoped writes to advisory_fts, and FTSDeletes is + // row-scoped deletes for tombstoned advisories. Their sum is the total + // number of statements that touched the index — there is no other one. + FTSUpserts int + FTSDeletes int + + AffectedRows int + AliasRows int + + // Degraded counts rows persisted with parse_degraded = 1. + Degraded int + + // Tombstoned counts rows written in a non-published state. + Tombstoned int +} + +// Merge folds o into s. +func (s *BatchStats) Merge(o BatchStats) { + s.Upserts += o.Upserts + s.FTSUpserts += o.FTSUpserts + s.FTSDeletes += o.FTSDeletes + s.AffectedRows += o.AffectedRows + s.AliasRows += o.AliasRows + s.Degraded += o.Degraded + s.Tombstoned += o.Tombstoned +} + +// MaxBatchRecords bounds one Apply. +// +// A delta batch is small by construction — research/06 measures the largest +// cvelistV5 hour at ~16.5 MiB of cumulative changes and the deltaLog names a +// few dozen records per fetch — and Apply is ONE TRANSACTION, so a batch that +// arrived here with hundreds of thousands of records is not a delta. It is a +// bulk import taking the wrong door, and the right answer is A.8's resumable, +// cursor-tracked path rather than a transaction that either commits a day's +// work or loses it. +const MaxBatchRecords = 50_000 + +// --------------------------------------------------------------------------- +// Apply +// --------------------------------------------------------------------------- + +// Apply upserts one decoded delta batch, row by row, in a single transaction. +// +// It is the whole write path. Nothing else in this package writes. +// +// THE LICENCE DECISION IS A PARAMETER AND NOT A LOOKUP. A caller has to hold +// an admitted license.Decision to reach this function at all, and the licence +// columns are bound from the DECISION rather than from the feed row's own +// claim — A.4 owns what a feed's licence is, and a writer that re-read the +// YAML would be laundering an unverified assertion into the cache. A refusal +// is refused here rather than defaulted, because Decision's zero value carries +// Tier 0, the most permissive tier this system has. +// +// asOf is stamped into every row and staleness is spine S6's staleness_seconds +// for the batch: the age of the DATA, not the age of the write. +func Apply( + ctx context.Context, + db *sql.DB, + feed config.FeedConfig, + d license.Decision, + batch []Record, + asOf time.Time, + staleness int, +) (BatchStats, error) { + var stats BatchStats + if db == nil { + return stats, refuse(ErrNoCache, "Apply needs the A.2 ingestion cache") + } + if d.Refused() { + return stats, refuse(license.ErrLicenseRefused, + "feed %q: the licence decision is a refusal (tier %d, dir %q), so no row may be written", + feed.ID, d.Tier.Int(), d.Dir) + } + if len(batch) > MaxBatchRecords { + return stats, refuse(ErrBatchTooLarge, + "feed %q: %d records in one delta batch exceeds %d; a batch that size is a bulk import and "+ + "belongs on A.8's resumable path, not in one transaction", + feed.ID, len(batch), MaxBatchRecords) + } + if staleness < 0 { + staleness = 0 + } + if len(batch) == 0 { + return stats, nil + } + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return stats, fmt.Errorf("delta: opening transaction for feed %q: %w", feed.ID, err) + } + defer func() { _ = tx.Rollback() }() + + for _, rec := range batch { + one, err := writeRecord(ctx, tx, feed, d, rec, asOf, staleness) + if err != nil { + return stats, err + } + stats.Merge(one) + } + if err := tx.Commit(); err != nil { + return stats, fmt.Errorf("delta: committing %d records for feed %q: %w", len(batch), feed.ID, err) + } + return stats, nil +} + +// writeRecord is the per-row write: one advisory upsert, one FTS write, and +// the advisory's own `affected` and `cve_alias` rows replaced. +// +// It writes exactly ONE advisory row and touches advisory_fts EXACTLY ONCE. +// That is the property A.14's validation measures, and it is a property of +// this function rather than of the loop above it. +func writeRecord( + ctx context.Context, + tx *sql.Tx, + feed config.FeedConfig, + d license.Decision, + rec Record, + asOf time.Time, + staleness int, +) (BatchStats, error) { + var stats BatchStats + + if strings.TrimSpace(rec.Source) == "" || strings.TrimSpace(rec.SourceID) == "" { + return stats, refuse(ErrBadRecord, + "feed %q: a record with source %q and source_id %q has no primary key", + feed.ID, rec.Source, rec.SourceID) + } + if len(rec.Raw) == 0 { + return stats, refuse(ErrBadRecord, + "feed %q record %q: raw_json is NOT NULL and the column stores the publisher's bytes verbatim", + feed.ID, rec.SourceID) + } + + // A.3 IS A PRECONDITION AND THIS IS WHERE IT IS CHECKED. Every string + // below is bound to a column; every one of them originated outside Anvil. + // AssertAllSanitized fails on a value Sanitize would have changed, so a + // caller that skipped the sanitizer cannot reach the bind. + fields := map[string]string{ + "source_id": rec.SourceID, + "cve_id": rec.CVEID, + "published": rec.Published, + "modified": rec.Modified, + "tombstoned_at": rec.TombstonedAt, + "severity": rec.Severity, + "cvss_vector": rec.CVSSVector, + "epss_as_of": rec.EPSSAsOf, + "description": rec.Description, + "references_text": rec.ReferencesText(), + "data_version": rec.DataVersion, + "license_manual_note": d.ManualNote, + } + for i, a := range rec.Affected { + fields["affected["+strconv.Itoa(i)+"].ecosystem"] = a.Ecosystem + fields["affected["+strconv.Itoa(i)+"].package"] = a.Package + fields["affected["+strconv.Itoa(i)+"].purl"] = a.PURL + fields["affected["+strconv.Itoa(i)+"].introduced"] = a.Introduced + fields["affected["+strconv.Itoa(i)+"].fixed"] = a.Fixed + } + for i, a := range rec.Aliases { + fields["alias["+strconv.Itoa(i)+"]"] = a + } + if err := sanitize.AssertAllSanitized(fields); err != nil { + return stats, fmt.Errorf("%w: feed %q record %q: %w", ErrUnsanitized, feed.ID, rec.SourceID, err) + } + + state := rec.State + if state == "" { + state = cache.AdvisoryPublished + } + var tombstone any + if state != cache.AdvisoryPublished { + ts := rec.TombstonedAt + if ts == "" { + // The schema pairs a non-published state with a non-null + // tombstone. A publisher that withdrew a record without saying + // when still has to be recorded as withdrawn, so the batch's own + // clock stands in — losing the "when" is better than losing the + // withdrawal, and dropping the row is not an option at all. + ts = asOf.UTC().Format(time.RFC3339) + } + tombstone = ts + stats.Tombstoned++ + } + + rowStaleness := rec.StalenessSeconds + if rowStaleness <= 0 { + rowStaleness = staleness + } + + row, err := queryRowTx(ctx, tx, cache.UpsertAdvisorySQL, + rec.Source, rec.SourceID, nullable(rec.CVEID), nullable(rec.Published), nullable(rec.Modified), + state, tombstone, nullable(rec.Severity), nullable(rec.CVSSVector), rec.CVSSScore, + rec.EPSSScore, nullable(rec.EPSSAsOf), boolInt(rec.KEV), + nullable(d.EffectiveSPDX), nullable(d.ManualNote), d.Tier.Int(), + string(cache.AdvisoryTrustDefault), + asOf.UTC().Format(time.RFC3339), rowStaleness, boolInt(rec.ParseDegraded), + nullable(rec.DataVersion), rec.Raw, + ) + if err != nil { + return stats, err + } + var rowid int64 + if err := row.Scan(&rowid); err != nil { + return stats, fmt.Errorf("delta: upserting %s/%s: %w", rec.Source, rec.SourceID, err) + } + stats.Upserts++ + if rec.ParseDegraded { + stats.Degraded++ + } + + // THE INDEX IS TOUCHED ONCE, BY ROWID. A tombstoned advisory leaves the + // index (its text must stop matching) while its `advisory` row stays, which + // is exit criterion 22's "tombstoned, never deleted" seen from the FTS + // side. + if state == cache.AdvisoryPublished { + if err := execTx(ctx, tx, cache.UpsertAdvisoryFTSSQL, rowid, rec.Description, rec.ReferencesText()); err != nil { + return stats, fmt.Errorf("delta: indexing %s/%s: %w", rec.Source, rec.SourceID, err) + } + stats.FTSUpserts++ + } else { + if err := execTx(ctx, tx, cache.DeleteAdvisoryFTSSQL, rowid); err != nil { + return stats, fmt.Errorf("delta: unindexing tombstoned %s/%s: %w", rec.Source, rec.SourceID, err) + } + stats.FTSDeletes++ + } + + // Replace, never append. `affected` has a surrogate primary key and no + // unique constraint over its natural key, so an advisory upserted twice + // would otherwise carry every version range twice and A.17's comparator + // would see one advisory as several. + if err := execTx(ctx, tx, deleteAffectedSQL, rec.Source, rec.SourceID); err != nil { + return stats, fmt.Errorf("delta: clearing affected for %s/%s: %w", rec.Source, rec.SourceID, err) + } + for _, a := range rec.Affected { + if err := execTx(ctx, tx, insertAffectedSQL, + rec.Source, rec.SourceID, a.Ecosystem, a.Package, nullable(a.PURL), + nullable(a.Introduced), nullable(a.Fixed), boolInt(a.DistroBackport)); err != nil { + return stats, fmt.Errorf("delta: writing affected for %s/%s: %w", rec.Source, rec.SourceID, err) + } + stats.AffectedRows++ + } + + if err := execTx(ctx, tx, deleteAliasSQL, rec.Source, rec.SourceID); err != nil { + return stats, fmt.Errorf("delta: clearing cve_alias for %s/%s: %w", rec.Source, rec.SourceID, err) + } + for _, alias := range rec.Aliases { + if alias == "" { + continue + } + if err := execTx(ctx, tx, insertAliasSQL, alias, rec.Source, rec.SourceID); err != nil { + return stats, fmt.Errorf("delta: writing cve_alias for %s/%s: %w", rec.Source, rec.SourceID, err) + } + stats.AliasRows++ + } + return stats, nil +} + +// nullable renders an empty string as SQL NULL. An empty TEXT and a NULL are +// different facts, and `advisory.cve_id IS NULL` is the one the alias design +// depends on. +func nullable(s string) any { + if strings.TrimSpace(s) == "" { + return nil + } + return s +} + +func boolInt(v bool) int { + if v { + return 1 + } + return 0 +} + +// --------------------------------------------------------------------------- +// Freshness probe — the reason the deltaLog route is cheap +// --------------------------------------------------------------------------- + +// CachedModified returns the `modified` timestamp the cache currently holds for +// one advisory, and whether the row exists at all. +// +// This is A.14's cursor, and it is a QUERY rather than a stored column ON +// PURPOSE. feed_state has exactly one cursor column, `watermark`, and A.8 +// already owns it: it stores a bootstrap Progress token there and A.14 reads +// the handover through bootstrap.Handoff. A second delta cursor squeezed into +// the same column would be two writers on one value, which is the failure A.8's +// own watermark doc comment describes. +// +// Deriving the cursor from the rows themselves has a property a stored cursor +// does not: it cannot disagree with the data. A record the cache is missing has +// no `modified` at all, so it is always fetched; a record the cache holds at or +// past the delta log's `dateUpdated` is never fetched. A cursor that skipped +// ahead of a failed write would silently lose the record forever. +func CachedModified(ctx context.Context, db *sql.DB, source, sourceID string) (string, bool, error) { + row, err := queryRowDB(ctx, db, selectModifiedSQL, source, sourceID) + if err != nil { + return "", false, err + } + var modified sql.NullString + switch err := row.Scan(&modified); { + case err == sql.ErrNoRows: + return "", false, nil + case err != nil: + return "", false, fmt.Errorf("delta: reading cached modified for %s/%s: %w", source, sourceID, err) + } + return modified.String, true, nil +} + +// isNewer reports whether a delta log's claimed update time is strictly newer +// than what the cache holds. +// +// IT FAILS TOWARD FETCHING. An unparseable timestamp on either side, or an +// absent row, returns true: re-fetching a record Anvil already has costs one +// small request, and skipping a record it does not have costs a missed +// vulnerability. Those are not comparable errors and the comparison must not +// pretend they are. +func isNewer(claimed, cached string) bool { + c, okClaimed := parseTimestamp(claimed) + h, okCached := parseTimestamp(cached) + if !okClaimed || !okCached { + return true + } + return c.After(h) +} + +// parseTimestamp accepts the two shapes advisory feeds actually emit: RFC3339 +// with an offset, and RFC3339 with fractional seconds. A value in neither shape +// is reported as unparseable rather than coerced, so isNewer can fail toward +// fetching instead of toward a wrong comparison. +func parseTimestamp(s string) (time.Time, bool) { + t := strings.TrimSpace(s) + if t == "" { + return time.Time{}, false + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} { + if v, err := time.Parse(layout, t); err == nil { + return v.UTC(), true + } + } + return time.Time{}, false +} + +// --------------------------------------------------------------------------- +// Decoding +// --------------------------------------------------------------------------- + +// MaxDocumentBytes bounds a single advisory document. +// +// It is the same order as A.8's own record cap and exists for the same reason: +// a feed that answers a record request with a gigabyte is a memory-exhaustion +// payload, and the bound has to be on what ARRIVED rather than on what a header +// claimed. +const MaxDocumentBytes = 16 << 20 + +// decoder is A.3 applied field by field, accumulating what was removed. +// +// The `s` method is named rather than inlined for the same reason A.8 names +// its own: internal/ingest/sanitize's writer guard resolves the package-local +// call graph by NAME, so a decoder that reaches the sanitizer through one +// method is visible to the guard, and there is exactly one place a field could +// be bound without passing through it — which is none. +type decoder struct { + feedID string + stats sanitize.SanitizeStats +} + +func (dc *decoder) s(raw string) string { + clean, st := sanitize.Sanitize(raw) + dc.stats.Merge(st) + return clean +} + +// Decode turns one fetched document into advisory records. +// +// THE FORMAT IS DECIDED BY LOOKING AT THE BYTES, never by which feed asked. +// A.1's rule — no feed fact compiled into Go — applies to a format mapping just +// as much as to a URL: a feed-id-to-parser table breaks the moment an operator +// points a row at a mirror, and what a document IS is a property of the +// document. +// +// A shape this decoder does not recognise is an ERROR and not a silent skip. +// That differs from A.8, which skips unrecognised archive members because a +// bulk archive is 300,000 files written by strangers and one bad member must +// not cost the other 299,999. A delta document is different: it was fetched +// because something said it changed, so "we do not understand it" means a +// change was dropped, and the caller needs to route the feed to a path that +// does understand it. SyncDelta does exactly that. +func Decode(feedID string, raw []byte) ([]Record, sanitize.SanitizeStats, error) { + dc := &decoder{feedID: feedID} + recs, err := dc.decode(raw, 0) + return recs, dc.stats, err +} + +// maxDecodeDepth bounds the array-of-documents recursion. One level of nesting +// is real (a JSON array of advisories); two is a document lying about its +// shape. +const maxDecodeDepth = 2 + +func (dc *decoder) decode(raw []byte, depth int) ([]Record, error) { + if len(raw) > MaxDocumentBytes { + return nil, refuse(ErrDocumentTooLarge, + "feed %q: a %d-byte document exceeds the %d-byte cap", dc.feedID, len(raw), MaxDocumentBytes) + } + if depth > maxDecodeDepth { + return nil, refuse(ErrUnrecognisedShape, + "feed %q: nested arrays of documents beyond depth %d", dc.feedID, maxDecodeDepth) + } + + // A UTF-8 BOM is written as an escape rather than as a literal: a literal + // BOM in a Go source file is a compile error and is exactly the kind of + // invisible character internal/ingest/invisible keeps out of this tree. + trimmed := bytes.TrimLeft(raw, " \t\r\n\ufeff") + if len(trimmed) == 0 { + return nil, refuse(ErrUnrecognisedShape, "feed %q: the document is empty", dc.feedID) + } + + head := trimmed + if len(head) > 4096 { + head = head[:4096] + } + + switch { + case trimmed[0] == '[': + var elems []json.RawMessage + if err := json.Unmarshal(trimmed, &elems); err != nil { + return nil, refuse(ErrUnrecognisedShape, "feed %q: the document opens as a JSON array but does not parse as one: %v", dc.feedID, err) + } + var out []Record + for _, e := range elems { + recs, err := dc.decode(e, depth+1) + if err != nil { + return nil, err + } + out = append(out, recs...) + } + return out, nil + + case trimmed[0] != '{': + return nil, refuse(ErrUnrecognisedShape, + "feed %q: the document is not JSON. CSV, XML and per-branch distro formats reach the cache "+ + "through A.8's bulk path; SyncDelta routes such a feed there rather than guessing here.", + dc.feedID) + + case bytes.Contains(head, []byte(`"fetchTime"`)) && bytes.Contains(head, []byte(`"numberOfChanges"`)): + return nil, refuse(ErrUnrecognisedShape, + "feed %q: this is a delta LOG, not an advisory. It names what changed; it does not carry it. "+ + "Parse it with ParseDeltaLog and fetch the records it names.", dc.feedID) + + case bytes.Contains(head, []byte(`"vulnerabilities"`)) && + (bytes.Contains(head, []byte(`"catalogVersion"`)) || bytes.Contains(head, []byte(`"cveID"`))): + return dc.decodeKEV(trimmed) + + case bytes.Contains(head, []byte(`"CVE_RECORD"`)): + rec, ok, err := dc.decodeCVE5(trimmed) + if err != nil || !ok { + return nil, refuse(ErrUnrecognisedShape, "feed %q: a CVE_RECORD document did not decode: %v", dc.feedID, err) + } + return []Record{rec}, nil + + default: + rec, ok, err := dc.decodeOSV(trimmed) + if err != nil || !ok { + return nil, refuse(ErrUnrecognisedShape, + "feed %q: the document is a JSON object in no shape this decoder recognises "+ + "(not OSV, not CVE 5.x, not KEV): %v", dc.feedID, err) + } + return []Record{rec}, nil + } +} + +// --- OSV, and therefore GHSA: github/advisory-database is OSV format --- + +type osvDoc struct { + SchemaVersion string `json:"schema_version"` + ID string `json:"id"` + Withdrawn string `json:"withdrawn"` + Published string `json:"published"` + Modified string `json:"modified"` + Summary string `json:"summary"` + Details string `json:"details"` + Aliases []string `json:"aliases"` + Related []string `json:"related"` + Severity []osvSeverity `json:"severity"` + References []osvReference `json:"references"` + Affected []osvAffected `json:"affected"` +} + +type osvSeverity struct { + Type string `json:"type"` + Score string `json:"score"` +} + +type osvReference struct { + Type string `json:"type"` + URL string `json:"url"` +} + +type osvAffected struct { + Package struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + PURL string `json:"purl"` + } `json:"package"` + Ranges []struct { + Type string `json:"type"` + Events []struct { + Introduced string `json:"introduced"` + Fixed string `json:"fixed"` + LastAffect string `json:"last_affected"` + } `json:"events"` + } `json:"ranges"` + Versions []string `json:"versions"` +} + +func (dc *decoder) decodeOSV(raw []byte) (Record, bool, error) { + var d osvDoc + if err := json.Unmarshal(raw, &d); err != nil { + return Record{}, false, err + } + if d.ID == "" { + return Record{}, false, nil + } + + rec := Record{ + Source: dc.feedID, + SourceID: dc.s(d.ID), + State: cache.AdvisoryPublished, + Raw: raw, + } + if d.Withdrawn != "" { + rec.State = cache.AdvisoryWithdrawn + rec.TombstonedAt = dc.s(d.Withdrawn) + } + rec.Published = dc.s(d.Published) + rec.Modified = dc.s(d.Modified) + rec.Description = dc.s(strings.TrimSpace(d.Summary + "\n\n" + d.Details)) + for _, s := range d.Severity { + if strings.HasPrefix(strings.ToUpper(s.Type), "CVSS") { + rec.CVSSVector = dc.s(s.Score) + break + } + } + for _, r := range d.References { + if r.URL != "" { + rec.References = append(rec.References, dc.s(r.URL)) + } + } + + for _, a := range append(append([]string{}, d.Aliases...), d.Related...) { + if IsCVEID(a) { + rec.Aliases = appendUnique(rec.Aliases, dc.s(a)) + } + } + if IsCVEID(d.ID) { + rec.CVEID = rec.SourceID + rec.Aliases = appendUnique(rec.Aliases, rec.SourceID) + } else if len(rec.Aliases) > 0 { + rec.CVEID = rec.Aliases[0] + } + + for _, a := range d.Affected { + eco := dc.s(a.Package.Ecosystem) + pkg := dc.s(a.Package.Name) + if eco == "" || pkg == "" { + continue + } + purl := dc.s(a.Package.PURL) + backport := isDistroEcosystem(eco) + emitted := false + for _, rg := range a.Ranges { + var introduced string + for _, ev := range rg.Events { + switch { + case ev.Introduced != "": + introduced = dc.s(ev.Introduced) + case ev.Fixed != "": + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: eco, Package: pkg, PURL: purl, + Introduced: introduced, Fixed: dc.s(ev.Fixed), DistroBackport: backport, + }) + emitted = true + case ev.LastAffect != "": + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: eco, Package: pkg, PURL: purl, + Introduced: introduced, DistroBackport: backport, + }) + emitted = true + } + } + } + if !emitted { + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: eco, Package: pkg, PURL: purl, DistroBackport: backport, + }) + } + } + return rec, true, nil +} + +// isDistroEcosystem marks the ecosystems whose advisories carry BACKPORTED +// fixes. research/12 §3, the CVE-2023-32681 / RHSA-2023:4520 class: a distro +// patches without moving the upstream version, so an upstream range calls a +// fixed package vulnerable. +// +// The list matches A.8's exactly. It is duplicated for the same reason the +// decoders are, and the same conformance test covers it: a divergence changes +// `affected.distro_backport` for the same bytes depending on which importer ran. +func isDistroEcosystem(eco string) bool { + lower := strings.ToLower(eco) + for _, p := range []string{ + "ubuntu", "debian", "alpine", "red hat", "redhat", "rocky", "almalinux", + "suse", "photon", "chainguard", "wolfi", "mageia", + } { + if strings.HasPrefix(lower, p) { + return true + } + } + return false +} + +// --- CVE 5.x, the shape the deltaLog names --- + +type cve5Doc struct { + DataType string `json:"dataType"` + DataVersion string `json:"dataVersion"` + CVEMetadata struct { + CVEID string `json:"cveId"` + State string `json:"state"` + Published string `json:"datePublished"` + Updated string `json:"dateUpdated"` + Rejected string `json:"dateRejected"` + } `json:"cveMetadata"` + Containers struct { + CNA cve5Container `json:"cna"` + ADP []cve5Container `json:"adp"` + } `json:"containers"` +} + +type cve5Container struct { + Descriptions []struct { + Lang string `json:"lang"` + Value string `json:"value"` + } `json:"descriptions"` + References []struct { + URL string `json:"url"` + } `json:"references"` + Metrics []struct { + CVSSv31 *cve5CVSS `json:"cvssV3_1"` + CVSSv30 *cve5CVSS `json:"cvssV3_0"` + CVSSv40 *cve5CVSS `json:"cvssV4_0"` + } `json:"metrics"` + Affected []struct { + Vendor string `json:"vendor"` + Product string `json:"product"` + PackageN string `json:"packageName"` + Repo string `json:"repo"` + CPEs []string `json:"cpes"` + Versions []struct { + Version string `json:"version"` + LessThan string `json:"lessThan"` + LessOrEqual string `json:"lessThanOrEqual"` + Status string `json:"status"` + } `json:"versions"` + } `json:"affected"` +} + +type cve5CVSS struct { + VectorString string `json:"vectorString"` + BaseScore float64 `json:"baseScore"` + BaseSeverity string `json:"baseSeverity"` +} + +// knownCVEDataVersions are the record schema versions this decoder was written +// against. An UNKNOWN one is PERSISTED with parse_degraded = 1 and never +// dropped (A.2 exit criterion 23, spine S6): silently discarding a record from +// a newer schema is how a vulnerability disappears from a security tool with no +// error anywhere. +// +// It matches A.8's list, and the conformance test is what keeps it matching. +var knownCVEDataVersions = map[string]bool{"5.0": true, "5.1": true, "5.2": true} + +func (dc *decoder) decodeCVE5(raw []byte) (Record, bool, error) { + var d cve5Doc + if err := json.Unmarshal(raw, &d); err != nil { + return Record{}, false, err + } + if d.CVEMetadata.CVEID == "" { + return Record{}, false, nil + } + + rec := Record{ + Source: dc.feedID, + SourceID: dc.s(d.CVEMetadata.CVEID), + CVEID: dc.s(d.CVEMetadata.CVEID), + Published: dc.s(d.CVEMetadata.Published), + Modified: dc.s(d.CVEMetadata.Updated), + State: cache.AdvisoryPublished, + DataVersion: dc.s(d.DataVersion), + ParseDegraded: !knownCVEDataVersions[strings.TrimSpace(d.DataVersion)], + Raw: raw, + } + rec.Aliases = append(rec.Aliases, rec.CVEID) + if strings.EqualFold(d.CVEMetadata.State, "REJECTED") { + rec.State = cache.AdvisoryRejected + rec.TombstonedAt = dc.s(firstNonEmpty(d.CVEMetadata.Rejected, d.CVEMetadata.Updated)) + } + + containers := append([]cve5Container{d.Containers.CNA}, d.Containers.ADP...) + for _, c := range containers { + for _, desc := range c.Descriptions { + if rec.Description == "" && (desc.Lang == "" || strings.HasPrefix(strings.ToLower(desc.Lang), "en")) { + rec.Description = dc.s(desc.Value) + } + } + for _, ref := range c.References { + if ref.URL != "" { + rec.References = appendUnique(rec.References, dc.s(ref.URL)) + } + } + for _, m := range c.Metrics { + for _, v := range []*cve5CVSS{m.CVSSv40, m.CVSSv31, m.CVSSv30} { + if v == nil || v.VectorString == "" || rec.CVSSVector != "" { + continue + } + rec.CVSSVector = dc.s(v.VectorString) + rec.Severity = dc.s(v.BaseSeverity) + score := v.BaseScore + rec.CVSSScore = score + } + } + for _, a := range c.Affected { + pkg := dc.s(firstNonEmpty(a.PackageN, a.Product)) + if pkg == "" { + continue + } + eco := dc.s(firstNonEmpty(a.Vendor, "cpe")) + for _, v := range a.Versions { + if strings.EqualFold(v.Status, "unaffected") { + continue + } + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: eco, + Package: pkg, + Introduced: dc.s(v.Version), + Fixed: dc.s(firstNonEmpty(v.LessThan, v.LessOrEqual)), + }) + } + } + } + return rec, true, nil +} + +// --- CISA KEV --- + +// kevDoc keeps its entries as RAW MESSAGES rather than as decoded structs. +// +// That is not a style choice: `advisory.raw_json` stores the publisher's bytes +// verbatim, and re-marshalling a decoded struct would store Anvil's rendering +// of the entry instead — different key order, dropped unknown fields, and a +// different digest from the one A.8 writes for the same catalogue entry. The +// conformance test compares those bytes. +type kevDoc struct { + CatalogVersion string `json:"catalogVersion"` + Vulnerabilities []json.RawMessage `json:"vulnerabilities"` +} + +type kevEntry struct { + CVEID string `json:"cveID"` + VendorProject string `json:"vendorProject"` + Product string `json:"product"` + VulnerabilityName string `json:"vulnerabilityName"` + DateAdded string `json:"dateAdded"` + ShortDescription string `json:"shortDescription"` + RequiredAction string `json:"requiredAction"` + DueDate string `json:"dueDate"` + Notes string `json:"notes"` +} + +// decodeKEV reads the whole catalogue into memory rather than streaming it, +// which is the one place this decoder deliberately differs from A.8's. +// +// The reason is the caller. A.8 streams because it walks a 570 MB archive whose +// members it has not seen; this decoder is handed a body A.7 already read into +// memory under Options.MaxBodyBytes, so a streaming parse here would buy +// nothing and would need a second bounded reader to enforce a bound that has +// already been enforced. MaxDocumentBytes is the backstop. +func (dc *decoder) decodeKEV(raw []byte) ([]Record, error) { + var d kevDoc + if err := json.Unmarshal(raw, &d); err != nil { + return nil, refuse(ErrUnrecognisedShape, "feed %q: a KEV-shaped document did not parse: %v", dc.feedID, err) + } + out := make([]Record, 0, len(d.Vulnerabilities)) + for _, entryRaw := range d.Vulnerabilities { + var e kevEntry + if err := json.Unmarshal(entryRaw, &e); err != nil || e.CVEID == "" { + // A single malformed catalogue entry is skipped, not fatal: the + // KEV catalogue is one document holding every entry, and one bad + // entry must not cost the rest. This is the one place a skip is + // right, and it is bounded to a single element. + continue + } + rec := Record{ + Source: dc.feedID, + SourceID: dc.s(e.CVEID), + CVEID: dc.s(e.CVEID), + Published: dc.s(e.DateAdded), + State: cache.AdvisoryPublished, + KEV: true, + Description: dc.s(strings.TrimSpace(e.VulnerabilityName + "\n\n" + e.ShortDescription + "\n\n" + e.RequiredAction)), + Raw: append([]byte(nil), entryRaw...), + } + rec.Aliases = append(rec.Aliases, rec.CVEID) + if pkg := dc.s(e.Product); pkg != "" { + rec.Affected = append(rec.Affected, AffectedRange{ + Ecosystem: dc.s(firstNonEmpty(e.VendorProject, "vendor")), + Package: pkg, + }) + } + if e.Notes != "" { + rec.References = append(rec.References, dc.s(e.Notes)) + } + out = append(out, rec) + } + return out, nil +} + +// --------------------------------------------------------------------------- +// Small shared helpers +// --------------------------------------------------------------------------- + +// IsCVEID is the ALLOWLIST that decides whether a string from a feed may be +// treated as a CVE identifier. +// +// It is exported because it is not only a parsing convenience: the deltaLog +// route in delta.go uses it as the ONLY thing that may cross from feed content +// into a fetch, so its strictness is a security property and not a nicety. See +// checkRecordName. +// +// The shape is CVE-<4+ digits>-<1+ digits> and nothing else. It is an allowlist +// of characters and structure, not a denylist of dangerous ones: this project +// has lost three guards to a symbol, a verb or a wording nobody listed, and +// `CVE-2024-0001/../../etc/passwd` is precisely the string a denylist misses. +func IsCVEID(s string) bool { + t := strings.TrimSpace(s) + if !strings.HasPrefix(t, "CVE-") || len(t) < 8 { + return false + } + rest := t[4:] + dash := strings.Index(rest, "-") + if dash < 4 { + return false + } + for _, r := range rest[:dash] { + if r < '0' || r > '9' { + return false + } + } + tail := rest[dash+1:] + if tail == "" { + return false + } + for _, r := range tail { + if r < '0' || r > '9' { + return false + } + } + return true +} + +func appendUnique(list []string, v string) []string { + if v == "" { + return list + } + for _, e := range list { + if e == v { + return list + } + } + return append(list, v) +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} diff --git a/internal/ingest/drift/drift.go b/internal/ingest/drift/drift.go new file mode 100644 index 0000000..2b54c92 --- /dev/null +++ b/internal/ingest/drift/drift.go @@ -0,0 +1,1056 @@ +// Package drift is A.16: what Anvil does when a feed changes shape underneath +// it, and what it does when a publisher takes an advisory back. +// +// =========================================================================== +// THE ONE RULE THIS PACKAGE EXISTS TO KEEP +// =========================================================================== +// +// A RECORD IS NEVER DROPPED FOR BEING UNFAMILIAR, AND NEVER EMITTED AS THOUGH +// IT WERE WHOLE WHEN IT IS NOT. +// +// Those are the same rule seen from two sides. research/06 Risk #3 states the +// first half — "on an unknown minor version, ingest raw and set +// parse_degraded=1 rather than dropping the record" — and plan/00-SPINE.md S6 +// states the second by making `parse_degraded` a REQUIRED field of the record +// rather than an optional diagnostic. A parser that skips a field it does not +// recognise and emits the record anyway produces a finding that looks complete +// and is not, and a finding that looks complete is one nobody re-checks. +// +// The failure this guards against is not hypothetical and does not announce +// itself. research/06's worked example is the deltaLog.json retention window +// being cut from 30 days to 15 in February 2026 with no announcement: the feed +// did not break, it did not error, it simply started meaning something +// slightly different. A version bump is the polite form of the same event. +// +// =========================================================================== +// WHAT "BRANCH ON dataVersion" MEANS HERE, AND WHAT IT DOES NOT MEAN +// =========================================================================== +// +// It does NOT mean a second CVE decoder. internal/ingest/delta already has one +// and internal/ingest/bootstrap has another, and delta's own package comment +// records that the second one is a real cross-area hazard which its +// conformance test exists to contain. A THIRD would be the same defect with a +// third name on it, so this package extracts NOTHING itself: it decides which +// parse profile a document's `dataVersion` selects, delegates the extraction +// to delta.Decode, and adds the one thing delta cannot express — WHICH FIELDS +// WERE NOT UNDERSTOOD. +// +// For the same reason drift.Record is a Go type ALIAS for delta.Record and not +// a struct of its own. A parallel record type would be a parallel write path a +// week later. +// +// THE FIELD CHECK IS AN ALLOWLIST. Three guards on this project were defeated +// by a symbol, a verb or a wording nobody thought to list, and a denylist of +// "fields we know are dangerous" cannot be written at all for a schema whose +// next version has not been published. Each profile enumerates the paths this +// parser HAS been written against — including the ones it deliberately does +// not extract, because "I know that field exists and I do not use it" is a +// decision, while "I have never heard of that field" is drift. Anything not on +// the list is reported by path. +// +// =========================================================================== +// DEGRADED IS LOUD; REPORTED IS NOT THE SAME AS DEGRADED +// =========================================================================== +// +// Two outcomes, deliberately distinguished, because a report nobody can act on +// is a report nobody reads: +// +// - An UNKNOWN dataVersion is degraded, always. The document may mean +// something this parser cannot see, anywhere in it. +// - A KNOWN dataVersion carrying an unrecognised field is degraded ONLY when +// that field sits in a LOAD-BEARING path: `/cveMetadata`, which decides +// identity and retraction, or an `affected` subtree, which is the version +// range Lane A's whole reason for existing is answered from +// (plan/00-SPINE.md S1). An unrecognised field in prose, credits or +// taxonomy mappings is REPORTED, and does not by itself make the version +// comparator's answer wrong. +// +// Both lists ride on the Report. Only the first sets `parse_degraded` on the +// row, and Report.Degraded and Record.ParseDegraded are asserted equal on +// every path — the one thing worse than a degraded record is a record whose +// flag and whose report disagree about whether it is degraded. +// +// =========================================================================== +// WHAT THIS PACKAGE DOES NOT DO +// =========================================================================== +// +// - It does not fetch, and it holds no clock of its own. A.7 polls, A.14 +// syncs, and Tombstoner takes its clock as a field so a test does not have +// to sleep. +// - It does not sanitize on the parse path. delta.Decode does that field by +// field; this package sanitizes only the strings it originates itself (the +// fallback record's identifiers, and the values it reads back out of the +// cache on the tombstone path). +// - It does not compute a fingerprint, derive one, or compare against one. +// anvil-fp/v1 is internal/record's and is the only one (S6). +package drift + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/delta" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/sanitize" +) + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +var ( + // ErrDrift is satisfied by every error this package originates, so a + // caller can tell "A.16 declined" from "the database failed" without + // listing every sentinel below. + ErrDrift = errors.New("drift") + + // ErrDriftRefused is satisfied by every refusal: a decision Anvil made, as + // opposed to something that went wrong. + ErrDriftRefused = fmt.Errorf("%w: refused", ErrDrift) + + // ErrDocumentTooLarge bounds one document. The bound is on what ARRIVED, + // never on what a header claimed, and it is delta.MaxDocumentBytes rather + // than a second number, because two caps for one document is how a + // document becomes acceptable to one half of the pipeline and not the + // other. + ErrDocumentTooLarge = fmt.Errorf("%w: document too large", ErrDriftRefused) + + // ErrNotAnObject is a document that is not a single JSON object. An array + // of advisories is a legitimate feed shape and delta.Decode handles it; + // this entry point answers about ONE record and says so rather than + // silently taking the first element. + ErrNotAnObject = fmt.Errorf("%w: document is not a single JSON object", ErrDriftRefused) + + // ErrNoPrimaryKey is a document from which no (source, source_id) can be + // formed even by the fallback. It is the ONLY case in this package where a + // document does not become a persistable record, and it is an error rather + // than a silent drop for exactly that reason: the cache is keyed on + // (source, source_id) and a row without one cannot be written, re-found or + // re-opened. + ErrNoPrimaryKey = fmt.Errorf("%w: no primary key in the document", ErrDriftRefused) +) + +// refuse builds a refusal: a decision Anvil made. +func refuse(sentinel error, format string, args ...any) error { + return fmt.Errorf("%w: %s", sentinel, fmt.Sprintf(format, args...)) +} + +// --------------------------------------------------------------------------- +// The record model, aliased and not redefined +// --------------------------------------------------------------------------- + +// Record is internal/ingest/delta's Record, ALIASED. +// +// It is `=` and not a new struct on purpose. A.14 owns the decoded-advisory +// shape and the only sanctioned write path for it (delta.Apply); a struct +// declared here would be convertible, plausible, and one refactor away from +// being written to the cache by a second route with a subtly different set of +// invariants. The alias means a drift-parsed record IS a delta record, so +// there is exactly one write path and no conversion in which a field can be +// dropped. +type Record = delta.Record + +// --------------------------------------------------------------------------- +// Versions and branches +// --------------------------------------------------------------------------- + +// Branch names the parse profile a document's `dataVersion` selects. +// +// It is Lane-A-local vocabulary with no counterpart among the record +// contract's six frozen enums, so declaring it here does not violate +// plan/IMPLEMENTATION-PLAN.md §6's single-owner rule. It exists so that a +// caller switches on a constant rather than comparing version strings it +// re-derived. +type Branch string + +const ( + // BranchCVE50, BranchCVE51 and BranchCVE52 are the CVE Record Format + // versions this parser has been written against. A.16's packet names + // exactly these three as known. + BranchCVE50 Branch = "cve-5.0" + BranchCVE51 Branch = "cve-5.1" + BranchCVE52 Branch = "cve-5.2" + + // BranchUnknown is every other value, INCLUDING an absent one. It is not + // an error and it is not a drop: it is the loud degraded state this + // package exists to produce. + BranchUnknown Branch = "unknown" +) + +// branchByVersion is the ALLOWLIST of understood `dataVersion` values. +// +// It must agree with internal/ingest/delta's own knownCVEDataVersions, which +// is unexported and therefore cannot be compared to this map directly. +// drift_test.go compares them BEHAVIOURALLY instead — it runs delta.Decode +// over a synthetic record at each version and asserts the decoder's own +// parse_degraded matches this table — so the two cannot drift apart without a +// red test in this package. A third copy of this list is in +// internal/ingest/bootstrap; the same behavioural comparison is the only tool +// available for it, and delta's conformance test already binds bootstrap to +// delta. +var branchByVersion = map[string]Branch{ + "5.0": BranchCVE50, + "5.1": BranchCVE51, + "5.2": BranchCVE52, +} + +// KnownVersions returns the understood `dataVersion` values in ascending +// order. It exists so a caller (or an operator-facing status page) can print +// what this build understands without reaching into the table. +// +// The order is NUMERIC PER COMPONENT and not lexical. Lexically, "5.10" sorts +// before "5.2", which would make newestBranch pick the wrong profile the first +// time CVE reaches a two-digit minor — a latent, silent, off-by-one-schema +// defect that would surface as a partial parse and nothing else. +func KnownVersions() []string { + out := make([]string, 0, len(branchByVersion)) + for v := range branchByVersion { + out = append(out, v) + } + return sortVersions(out) +} + +// sortVersions orders dotted numeric versions ascending, in place, and returns +// the slice. +// +// It is a named function rather than a sort.Slice call inside KnownVersions so +// that it can be tested against a list this build's table does not contain. +// With only 5.0, 5.1 and 5.2 known, a lexical sort and a numeric one give the +// same answer, so a test that could only look at KnownVersions() would pass +// against the wrong implementation — which is the "a guard that has never +// failed has not been tested" shape, arrived at by accident. +func sortVersions(versions []string) []string { + sort.Slice(versions, func(i, j int) bool { return compareVersions(versions[i], versions[j]) < 0 }) + return versions +} + +// compareVersions orders dotted numeric versions component by component. A +// non-numeric component compares as -1, below every real one, so a malformed +// entry can never become "newest". +func compareVersions(a, b string) int { + as, bs := strings.Split(a, "."), strings.Split(b, ".") + n := len(as) + if len(bs) > n { + n = len(bs) + } + for i := 0; i < n; i++ { + av, bv := versionComponent(as, i), versionComponent(bs, i) + switch { + case av < bv: + return -1 + case av > bv: + return 1 + } + } + return strings.Compare(a, b) +} + +func versionComponent(parts []string, i int) int { + if i >= len(parts) { + return 0 + } + n := 0 + for _, c := range parts[i] { + if c < '0' || c > '9' { + return -1 + } + n = n*10 + int(c-'0') + } + return n +} + +// BranchFor maps a declared `dataVersion` to its parse profile. +// +// The comparison is on the TRIMMED value and nothing else: no prefix match, no +// "5.x means 5.anything", no major-version fallback. A prefix rule is how "5.3 +// looks close enough to 5.2" becomes a silently partial parse, which is the +// entire failure this package was written to prevent. +func BranchFor(dataVersion string) Branch { + if b, ok := branchByVersion[strings.TrimSpace(dataVersion)]; ok { + return b + } + return BranchUnknown +} + +// Known reports whether b is a profile this parser implements. +func (b Branch) Known() bool { return b != BranchUnknown && b != "" } + +// --------------------------------------------------------------------------- +// Report codes +// --------------------------------------------------------------------------- + +// Code is one reason a document was reported on. Codes are stable strings so +// an operator can grep a log for one, and they are constants so a producer +// cannot invent a fourth spelling of "unknown version". +type Code string + +const ( + // CodeUnknownDataVersion is the packet's central case: a `dataVersion` + // this parser has no profile for. ALWAYS degrading. + CodeUnknownDataVersion Code = "unknown-data-version" + + // CodeMissingDataVersion is a CVE record that declares no version at all. + // It is treated exactly as an unknown one — a document that will not say + // what it is has not earned the benefit of the doubt. + CodeMissingDataVersion Code = "missing-data-version" + + // CodeUnknownField is a path outside the selected profile's allowlist, + // sitting outside every load-bearing subtree. Reported, not degrading. + CodeUnknownField Code = "unknown-field" + + // CodeUnknownFieldLoadBearing is a path outside the allowlist INSIDE + // `/cveMetadata` or an `affected` subtree. Degrading: identity, retraction + // and version ranges are what Lane A's answers are made of. + CodeUnknownFieldLoadBearing Code = "unknown-field-in-load-bearing-path" + + // CodeUndecodable is a document delta.Decode refused. The record is + // preserved by the fallback — raw bytes and primary key only — and is + // degraded, because a record with no parsed content is the most incomplete + // a record can be. + CodeUndecodable Code = "undecodable-document" + + // CodeDecoderDegraded is delta.Decode reporting parse_degraded when this + // package's own rules did not. It exists so the two can never disagree + // silently: if the decoder degrades a record, the report says why it is + // degraded rather than claiming everything was understood. + CodeDecoderDegraded Code = "decoder-reported-degraded" + + // CodeFieldsTruncated means the walk hit its node or field cap and the + // field lists are incomplete. It is degrading on its own: an incomplete + // answer about whether anything was missed is not an answer. + CodeFieldsTruncated Code = "field-scan-truncated" +) + +// codeIsDegrading is the ALLOWLIST of codes that set parse_degraded. +// +// A codes-that-are-harmless denylist would have the wrong default: a code +// added later and forgotten would silently NOT degrade. This way a new code +// degrades until somebody says otherwise on purpose. +var codeIsDegrading = map[Code]bool{ + CodeUnknownDataVersion: true, + CodeMissingDataVersion: true, + CodeUnknownFieldLoadBearing: true, + CodeUndecodable: true, + CodeDecoderDegraded: true, + CodeFieldsTruncated: true, + CodeUnknownField: false, +} + +// Degrading reports whether this code sets parse_degraded on the row. +// +// A code that is not in the table degrades. That default is the whole reason +// the table is written as "which codes are harmless" rather than "which codes +// are dangerous": the entry somebody forgets to add then fails loudly, and the +// alternative fails silently. +func (c Code) Degrading() bool { + deg, known := codeIsDegrading[c] + return deg || !known +} + +// --------------------------------------------------------------------------- +// The report +// --------------------------------------------------------------------------- + +// MaxReportedFields bounds each field list. A document is attacker-adjacent +// input and a report is a log line: an advisory carrying fifty thousand +// unrecognised keys must not be able to turn one log line into fifty thousand. +// Hitting the cap sets Truncated, which is itself degrading. +const MaxReportedFields = 64 + +// maxWalkNodes bounds the structural walk for the same reason. It is generous +// enough for the largest real CVE record (a few thousand nodes) and small +// enough that a hostile document cannot make the walk the expensive part of an +// ingest. +const maxWalkNodes = 20000 + +// Report is what the parser understood about one document and what it did not. +// +// It is the "carrying which fields were not understood" half of A.16. A bare +// degraded bool would say a record is incomplete without saying in what way, +// which is a status nobody can act on and therefore a status everybody learns +// to ignore. +type Report struct { + // DataVersion is the value the document declared, sanitized, verbatim + // otherwise. Empty means the document declared none. + DataVersion string + + // Branch is the profile DataVersion selected, and KnownVersion is + // Branch.Known() recorded at the time of the parse. + Branch Branch + KnownVersion bool + + // Degraded is the value written to `advisory.parse_degraded`. It equals + // "any code on Codes is degrading", and Parse refuses to return a Record + // whose ParseDegraded disagrees with it. + Degraded bool + + // Codes are the reasons, deduplicated, in a stable order. + Codes []Code + + // UnknownFields are the JSON paths outside the profile's allowlist, as + // "/containers/cna/affected[]/versions[]/lessThan" — array subscripts are + // collapsed to "[]" so a thousand affected entries with the same new key + // report it once. Sorted, deduplicated, capped at MaxReportedFields. + UnknownFields []string + + // DegradingFields is the subset of UnknownFields inside a load-bearing + // path. It is the list an operator acts on first. + DegradingFields []string + + // Truncated is set when the walk hit maxWalkNodes or MaxReportedFields, so + // a short list is never mistaken for a clean one. + Truncated bool + + // DecodeError is delta.Decode's refusal, when the fallback record was used + // instead. Empty otherwise. + DecodeError string +} + +// Clean reports whether the document raised nothing at all. +func (r Report) Clean() bool { return len(r.Codes) == 0 && !r.Degraded } + +// Has reports whether the report carries the given code. +func (r Report) Has(c Code) bool { + for _, got := range r.Codes { + if got == c { + return true + } + } + return false +} + +// maxRenderedVersion clips the declared version in rendered output. The COLUMN +// stores what the publisher sent; a log line does not have to. +const maxRenderedVersion = 48 + +// String renders the report as one operator-readable line. +func (r Report) String() string { + var b strings.Builder + state := "ok" + if r.Degraded { + state = "DEGRADED" + } + version := r.DataVersion + if version == "" { + version = "(absent)" + } + if len(version) > maxRenderedVersion { + version = version[:maxRenderedVersion] + "..." + } + fmt.Fprintf(&b, "drift: %s dataVersion=%q branch=%s", state, version, r.Branch) + if len(r.Codes) > 0 { + codes := make([]string, 0, len(r.Codes)) + for _, c := range r.Codes { + codes = append(codes, string(c)) + } + fmt.Fprintf(&b, " codes=[%s]", strings.Join(codes, " ")) + } + if len(r.DegradingFields) > 0 { + fmt.Fprintf(&b, " not-understood-in-load-bearing-path=[%s]", strings.Join(r.DegradingFields, " ")) + } + if n := len(r.UnknownFields) - len(r.DegradingFields); n > 0 { + fmt.Fprintf(&b, " other-fields-not-understood=%d", n) + } + if r.Truncated { + b.WriteString(" (field scan truncated)") + } + if r.DecodeError != "" { + fmt.Fprintf(&b, " decode-error=%q", r.DecodeError) + } + return b.String() +} + +// add records a code once. +func (r *Report) add(c Code) { + if !r.Has(c) { + r.Codes = append(r.Codes, c) + } + if c.Degrading() { + r.Degraded = true + } +} + +// --------------------------------------------------------------------------- +// Parse +// --------------------------------------------------------------------------- + +// ParseVersioned is the narrow entry point A.16's packet names: bytes in, one +// record and the degraded flag out. +// +// It cannot report WHICH fields it did not understand and it cannot report an +// error, so it is a convenience over Parse and not the primary API. Use Parse +// when either matters, which on an ingest path is always. +// +// TWO THINGS THE RETURNED RECORD DOES NOT HAVE, both by construction: +// +// - NO Source. Bytes do not know which feed delivered them, and the cache is +// keyed on (source, source_id). A caller writing the record must stamp +// rec.Source with the feed id — or call Parse(feedID, raw), which does it. +// delta.Apply refuses a record with no Source (ErrBadRecord); it does not +// invent one. +// - NO parsed content, when the document could not be decoded at all. The +// record still carries Raw verbatim and ParseDegraded true, because +// persisting the publisher's bytes under a degraded flag keeps the record +// re-parseable by a later build, and dropping it does not. +// +// The returned bool is Record.ParseDegraded, which Parse guarantees equals +// Report.Degraded. +func ParseVersioned(raw []byte) (Record, bool) { + rec, rep, err := Parse("", raw) + if err != nil { + // The error cannot be returned through this signature, so the record + // carries the only two facts that survive it: the publisher's bytes, + // and "this is degraded". Such a record has no primary key, and + // delta.Apply refuses it by name rather than writing a keyless row. + return Record{Raw: raw, ParseDegraded: true}, true + } + return rec, rep.Degraded +} + +// Parse decodes one advisory document, branching on its declared dataVersion, +// and reports what it did not understand. +// +// THE EXTRACTION IS delta.Decode's, NOT THIS PACKAGE'S. See the package +// comment: a third CVE decoder in this tree would be the cross-area drift the +// second one already has a conformance test to contain. +// +// feedID becomes Record.Source and is the feed's config id. It may be empty, +// in which case the caller must stamp it before writing. +func Parse(feedID string, raw []byte) (Record, Report, error) { + var rep Report + + if len(raw) > delta.MaxDocumentBytes { + return Record{}, rep, refuse(ErrDocumentTooLarge, + "feed %q: a %d-byte document exceeds the %d-byte cap", + feedID, len(raw), delta.MaxDocumentBytes) + } + + doc, err := decodeObject(raw) + if err != nil { + return Record{}, rep, refuse(ErrNotAnObject, "feed %q: %v", feedID, err) + } + + // --- branch on dataVersion, explicitly and without a prefix rule --- + declared, _ := sanitize.Sanitize(stringField(doc, "dataVersion")) + rep.DataVersion = declared + rep.Branch = BranchFor(declared) + rep.KnownVersion = rep.Branch.Known() + switch { + case strings.TrimSpace(declared) == "": + rep.add(CodeMissingDataVersion) + case !rep.KnownVersion: + rep.add(CodeUnknownDataVersion) + } + + // --- what, in this document, is outside the profile --- + scanBranch := rep.Branch + if !rep.KnownVersion { + // An unknown version is scanned against the NEWEST profile this build + // has, because that is the profile whose extraction actually ran. The + // resulting field list answers the question an operator has — "what is + // in this document that our newest parser has never seen?" — rather + // than the one nobody asked. + scanBranch = newestBranch() + } + w := newWalker(scanBranch) + w.walk("", doc) + rep.UnknownFields = w.unknownFields() + rep.DegradingFields = w.degradingFields() + rep.Truncated = w.truncated + if rep.Truncated { + rep.add(CodeFieldsTruncated) + } + if len(rep.DegradingFields) > 0 { + rep.add(CodeUnknownFieldLoadBearing) + } + if len(rep.UnknownFields) > len(rep.DegradingFields) { + rep.add(CodeUnknownField) + } + + // --- extraction, delegated --- + rec, decodeErr := decodeOne(feedID, raw) + if decodeErr != nil { + rep.add(CodeUndecodable) + rep.DecodeError = decodeErr.Error() + fallback, err := fallbackRecord(feedID, doc, declared, raw) + if err != nil { + return Record{}, rep, err + } + rec = fallback + } + if rec.ParseDegraded && !rep.Degraded { + // delta's own table said degraded and ours did not. That is a + // divergence between two lists that must agree, and the conformance + // test in drift_test.go exists to catch it before a feed does; here it + // is resolved in the only safe direction. + rep.add(CodeDecoderDegraded) + } + + // THE ROW'S FLAG IS THE REPORT'S CONCLUSION, never the other way round, + // and this single assignment is what makes the two unable to disagree. + // + // There is deliberately no `if rec.ParseDegraded != rep.Degraded { ... }` + // after it. A check on the line below its own assignment can never fire, + // and a guard that can never fire is worse than none: it reads as + // verification. The invariant is asserted where it can actually be + // observed — drift_test.go runs every fixture class through this function + // and compares the two fields. + rec.ParseDegraded = rep.Degraded + return rec, rep, nil +} + +// decodeOne runs delta.Decode and insists on exactly one record. +func decodeOne(feedID string, raw []byte) (Record, error) { + recs, _, err := delta.Decode(feedID, raw) + if err != nil { + return Record{}, err + } + switch len(recs) { + case 1: + return recs[0], nil + case 0: + return Record{}, refuse(ErrNotAnObject, "the document decoded to no records at all") + default: + return Record{}, refuse(ErrNotAnObject, + "the document decoded to %d records; this entry point answers about one", len(recs)) + } +} + +// fallbackRecord is the never-drop path: delta.Decode refused the document, so +// the record is reduced to the two things that can still be persisted — a +// primary key and the publisher's bytes. +// +// It is deliberately minimal. Reconstructing severity or version ranges here +// would be the third decoder the package comment refuses to write, and a +// half-reconstructed record is exactly the "looks complete and is not" outcome +// A.16 exists to prevent. What survives is enough for the record to be found +// again and re-parsed by a later build that understands the shape. +func fallbackRecord(feedID string, doc map[string]any, dataVersion string, raw []byte) (Record, error) { + meta, _ := doc["cveMetadata"].(map[string]any) + id := "" + if meta != nil { + id = stringField(meta, "cveId") + } + if strings.TrimSpace(id) == "" { + id = stringField(doc, "id") // OSV-shaped documents key their id at the root. + } + clean, _ := sanitize.Sanitize(strings.TrimSpace(id)) + if clean == "" { + return Record{}, refuse(ErrNoPrimaryKey, + "feed %q: the document could not be decoded and carries no cveMetadata.cveId or id, "+ + "so no (source, source_id) exists to store it under. The bytes are NOT discarded by "+ + "this package; they are returned to the caller, which must route them to a path that "+ + "can name them.", feedID) + } + rec := Record{ + Source: feedID, + SourceID: clean, + DataVersion: dataVersion, + ParseDegraded: true, + Raw: raw, + } + if delta.IsCVEID(clean) { + rec.CVEID = clean + rec.Aliases = []string{clean} + } + return rec, nil +} + +// decodeObject parses the document as one JSON object. +func decodeObject(raw []byte) (map[string]any, error) { + // The UTF-8 BOM is written as an ESCAPE and never as a literal. A literal + // BOM in a Go source file is exactly the kind of invisible character + // internal/ingest/invisible exists to keep out of this tree, and + // internal/ingest/delta writes it the same way for the same reason. + trimmed := strings.TrimLeft(string(raw), " \t\r\n\ufeff") + if strings.TrimSpace(trimmed) == "" { + return nil, errors.New("the document is empty") + } + dec := json.NewDecoder(strings.NewReader(trimmed)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, fmt.Errorf("the document does not parse as JSON: %w", err) + } + obj, ok := v.(map[string]any) + if !ok { + return nil, fmt.Errorf("the document is a %T at its root, not a JSON object", v) + } + // Trailing content after the object is refused rather than ignored. Two + // concatenated advisories parse as "the first one" to a streaming decoder, + // and "the first one" is a dropped record wearing a successful parse. + if dec.More() { + return nil, errors.New("the document carries content after its first JSON object; " + + "a second advisory would be silently dropped") + } + return obj, nil +} + +// stringField reads a string-valued key, or "" for anything else. A key whose +// value is not a string is not silently coerced: the walk reports the type +// change as drift, which is the loud half of the same observation. +func stringField(doc map[string]any, key string) string { + s, _ := doc[key].(string) + return s +} + +// --------------------------------------------------------------------------- +// The structural walk +// --------------------------------------------------------------------------- + +// walker compares a document's structure against one profile's allowlist. +type walker struct { + profile map[string]bool + opaque map[string]bool + unknown map[string]bool + degrading map[string]bool + nodes int + truncated bool +} + +func newWalker(b Branch) *walker { + return &walker{ + profile: profileFor(b), + opaque: opaquePaths(), + unknown: map[string]bool{}, + degrading: map[string]bool{}, + } +} + +// walk descends one value. path is the profile path of the value itself: "" at +// the root, and already carrying "[]" for an array-valued key, so that array +// ELEMENTS do not each add a segment and a thousand `affected` entries report +// one path rather than a thousand. +func (w *walker) walk(path string, v any) { + if w.truncated { + return + } + switch t := v.(type) { + case map[string]any: + keys := make([]string, 0, len(t)) + for k := range t { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + w.nodes++ + if w.nodes > maxWalkNodes { + w.truncated = true + return + } + child := path + "/" + k + if _, isArray := t[k].([]any); isArray { + child += "[]" + } + if !w.profile[child] { + // The children of a field nobody recognises are not separately + // reported: one unknown subtree is one finding, not a hundred. + w.note(child) + continue + } + if w.opaque[child] { + continue + } + w.walk(child, t[k]) + } + case []any: + for _, e := range t { + w.nodes++ + if w.nodes > maxWalkNodes { + w.truncated = true + return + } + w.walk(path, e) + } + } +} + +func (w *walker) note(path string) { + if len(w.unknown) >= MaxReportedFields && !w.unknown[path] { + w.truncated = true + return + } + w.unknown[path] = true + if isLoadBearing(path) { + w.degrading[path] = true + } +} + +func (w *walker) unknownFields() []string { return sortedKeys(w.unknown) } + +func (w *walker) degradingFields() []string { return sortedKeys(w.degrading) } + +func sortedKeys(m map[string]bool) []string { + if len(m) == 0 { + return nil + } + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// loadBearingPrefixes are the subtrees whose contents Lane A's answers are +// made of. +// +// - /cveMetadata decides WHICH advisory this is and whether it has been +// retracted. A field nobody understands there can change the identity or +// the state of the row. +// - the `affected` subtrees are the version ranges. plan/00-SPINE.md S1: +// "CVE/OSV/GHSA describe vulnerable PACKAGE VERSIONS, and a version +// comparator answers that exactly and for free." An unrecognised key there +// is a range that may not mean what the comparator read. +// +// Prose, credits, taxonomy mappings and provider metadata are deliberately NOT +// here. Degrading every record that carries a new prose field would make +// `parse_degraded` mean "this is a CVE record", and a flag that is always set +// is a flag nobody reads. +var loadBearingPrefixes = []string{ + "/cveMetadata", + "/containers/cna/affected[]", + "/containers/adp[]/affected[]", +} + +// isLoadBearing reports whether a path lies in or names a load-bearing +// subtree. +// +// The prefix match is SEGMENT-AWARE. A plain strings.HasPrefix would make +// "/cveMetadataExtra" load-bearing because it starts with "/cveMetadata", +// which is the same class of near-miss that has defeated three guards on this +// project. +func isLoadBearing(path string) bool { + for _, p := range loadBearingPrefixes { + if path == p { + return true + } + // The array form of a load-bearing key is the same key. + if path == strings.TrimSuffix(p, "[]") || path == p+"[]" { + return true + } + for _, base := range []string{p, strings.TrimSuffix(p, "[]")} { + if strings.HasPrefix(path, base+"/") || strings.HasPrefix(path, base+"[]/") { + return true + } + } + } + return false +} + +// --------------------------------------------------------------------------- +// The profiles: what each version's parser has been written against +// --------------------------------------------------------------------------- +// +// EACH ENTRY IS A CLAIM THAT SOMEBODY LOOKED AT THAT FIELD, not that Anvil +// extracts it. delta.Decode reads a handful of these; the rest are here +// because "this parser knows the field exists and does not use it" is a +// decision that can be reviewed, while "this parser has never heard of that +// field" is drift that cannot. +// +// HONESTY ABOUT WHERE THESE COME FROM, because a list presented as a +// transcription and assembled from memory is the worse of the two: this is the +// CVE Record Format as this parser was written against it, not a mechanical +// export of the published JSON Schema. Two consequences, both chosen because +// they fail loudly rather than silently: +// +// - A field that really belongs to 5.0 but is listed only under 5.1 costs a +// REPORT LINE on a 5.0 document. It never costs a dropped record, and +// outside a load-bearing subtree it never even sets parse_degraded. +// - 5.2's additive set is EMPTY. This parser was written against 5.1's key +// set and accepts 5.2 as a known version on A.16's packet's authority; a +// key that exists only in 5.2 is therefore reported as unrecognised. That +// is the loud outcome and it is the intended one — the alternative, +// accepting an unenumerated key set for a version nobody enumerated, is +// the silent one. + +// mediaPaths enumerates a `supportingMedia` array. prefix names the array key. +func mediaPaths(prefix string) []string { + return []string{prefix, prefix + "/type", prefix + "/base64", prefix + "/value"} +} + +// prosePaths enumerates an array of {lang, value, supportingMedia[]} objects, +// which is the shape CVE 5.x reuses for descriptions, workarounds, solutions, +// exploits, configurations and rejected reasons. +func prosePaths(prefix string) []string { + out := []string{prefix, prefix + "/lang", prefix + "/value"} + return append(out, mediaPaths(prefix+"/supportingMedia[]")...) +} + +// referencePaths enumerates a `references` array. +func referencePaths(prefix string) []string { + return []string{prefix, prefix + "/url", prefix + "/name", prefix + "/tags[]"} +} + +// affectedPaths enumerates the `affected` array: THE load-bearing subtree. +func affectedPaths(prefix string) []string { + return []string{ + prefix, + prefix + "/vendor", + prefix + "/product", + prefix + "/collectionURL", + prefix + "/packageName", + prefix + "/repo", + prefix + "/defaultStatus", + prefix + "/cpes[]", + prefix + "/modules[]", + prefix + "/platforms[]", + prefix + "/programFiles[]", + prefix + "/programRoutines[]", + prefix + "/programRoutines[]/name", + prefix + "/versions[]", + prefix + "/versions[]/version", + prefix + "/versions[]/status", + prefix + "/versions[]/versionType", + prefix + "/versions[]/lessThan", + prefix + "/versions[]/lessThanOrEqual", + prefix + "/versions[]/changes[]", + prefix + "/versions[]/changes[]/at", + prefix + "/versions[]/changes[]/status", + } +} + +// containerPaths enumerates one container (`cna`, or one element of `adp`). +// Both share a schema, so they share this list rather than two copies of it. +func containerPaths(prefix string, withCPEApplicability bool) []string { + out := []string{ + prefix, + prefix + "/providerMetadata", + prefix + "/dateAssigned", + prefix + "/datePublic", + prefix + "/title", + prefix + "/source", + prefix + "/tags[]", + prefix + "/taxonomyMappings[]", + prefix + "/replacedBy[]", + prefix + "/problemTypes[]", + prefix + "/problemTypes[]/descriptions[]", + prefix + "/problemTypes[]/descriptions[]/type", + prefix + "/problemTypes[]/descriptions[]/lang", + prefix + "/problemTypes[]/descriptions[]/description", + prefix + "/problemTypes[]/descriptions[]/cweId", + prefix + "/impacts[]", + prefix + "/impacts[]/capecId", + prefix + "/metrics[]", + prefix + "/metrics[]/format", + prefix + "/metrics[]/cvssV2_0", + prefix + "/metrics[]/cvssV3_0", + prefix + "/metrics[]/cvssV3_1", + prefix + "/metrics[]/cvssV4_0", + prefix + "/metrics[]/other", + prefix + "/timeline[]", + prefix + "/timeline[]/time", + prefix + "/timeline[]/lang", + prefix + "/timeline[]/value", + prefix + "/credits[]", + prefix + "/credits[]/lang", + prefix + "/credits[]/value", + prefix + "/credits[]/type", + prefix + "/credits[]/user", + } + out = append(out, referencePaths(prefix+"/references[]")...) + out = append(out, referencePaths(prefix+"/problemTypes[]/descriptions[]/references[]")...) + out = append(out, prosePaths(prefix+"/descriptions[]")...) + out = append(out, prosePaths(prefix+"/impacts[]/descriptions[]")...) + out = append(out, prosePaths(prefix+"/metrics[]/scenarios[]")...) + out = append(out, prosePaths(prefix+"/workarounds[]")...) + out = append(out, prosePaths(prefix+"/solutions[]")...) + out = append(out, prosePaths(prefix+"/exploits[]")...) + out = append(out, prosePaths(prefix+"/configurations[]")...) + out = append(out, prosePaths(prefix+"/rejectedReasons[]")...) + out = append(out, affectedPaths(prefix+"/affected[]")...) + if withCPEApplicability { + out = append(out, prefix+"/cpeApplicability[]") + } + return out +} + +// basePaths is the CVE 5.0 profile. +func basePaths(withCPEApplicability bool) []string { + out := []string{ + "/dataType", + "/dataVersion", + "/cveMetadata", + "/cveMetadata/cveId", + "/cveMetadata/assignerOrgId", + "/cveMetadata/assignerShortName", + "/cveMetadata/requesterUserId", + "/cveMetadata/serial", + "/cveMetadata/state", + "/cveMetadata/dateReserved", + "/cveMetadata/datePublished", + "/cveMetadata/dateUpdated", + "/cveMetadata/dateRejected", + "/containers", + "/containers/adp[]", + } + out = append(out, containerPaths("/containers/cna", withCPEApplicability)...) + out = append(out, containerPaths("/containers/adp[]", withCPEApplicability)...) + return out +} + +// opaquePaths are the subtrees the walk does NOT descend into. +// +// Each is a nested schema of its own — CVSS vectors, CPE applicability node +// trees, taxonomy mappings, provider and source metadata — whose internals are +// not what Lane A matches on. Enumerating three CVSS schemas here would import +// their revision history into this file and would report a new CVSS field as +// CVE drift, which is a false alarm about the wrong feed. The node is on the +// allowlist, so its PRESENCE is understood; only its contents go unexamined, +// and that is a decision recorded here rather than an omission. +func opaquePaths() map[string]bool { + out := map[string]bool{} + for _, prefix := range []string{"/containers/cna", "/containers/adp[]"} { + for _, p := range []string{ + "/providerMetadata", + "/source", + "/taxonomyMappings[]", + "/cpeApplicability[]", + "/metrics[]/cvssV2_0", + "/metrics[]/cvssV3_0", + "/metrics[]/cvssV3_1", + "/metrics[]/cvssV4_0", + "/metrics[]/other", + } { + out[prefix+p] = true + } + } + return out +} + +// profiles is the per-branch allowlist, built once. +var profiles = map[Branch]map[string]bool{ + BranchCVE50: pathSet(basePaths(false)), + BranchCVE51: pathSet(basePaths(true)), + BranchCVE52: pathSet(basePaths(true)), +} + +func pathSet(paths []string) map[string]bool { + out := make(map[string]bool, len(paths)) + for _, p := range paths { + out[p] = true + } + return out +} + +// profileFor returns the allowlist for a branch. An unknown branch is scanned +// against the newest profile; see Parse. +func profileFor(b Branch) map[string]bool { + if p, ok := profiles[b]; ok { + return p + } + return profiles[newestBranch()] +} + +// newestBranch is the highest known version's branch. It is derived from +// branchByVersion rather than written twice, so adding a version cannot leave a +// stale "newest" behind. +func newestBranch() Branch { + versions := KnownVersions() + if len(versions) == 0 { + return BranchUnknown + } + return branchByVersion[versions[len(versions)-1]] +} diff --git a/internal/ingest/drift/drift_test.go b/internal/ingest/drift/drift_test.go new file mode 100644 index 0000000..cf09704 --- /dev/null +++ b/internal/ingest/drift/drift_test.go @@ -0,0 +1,1400 @@ +// drift_test.go is A.16's evidence. +// +// The two claims A.16's packet asks to be measured are measured END TO END, +// against a real migrated A.2 cache and through A.14's real write path, not +// asserted against a struct field this package filled in itself: +// +// 1. "A synthetic dataVersion: 5.9 record is PERSISTED with parse_degraded=1, +// not dropped." Read back out of the database, including `raw_json` +// compared byte for byte against the document that went in. +// 2. "Tombstoning a previously published row makes a query for active +// findings referencing that advisory return it as INVALIDATED rather than +// silently vanishing." The finding is inserted before the tombstone and +// counted before and after. +// +// Everything else here follows the rules this project has already paid for: +// +// - EVERY GUARD IS VERIFIED RED. The statement allowlist is run against a +// hand-written DELETE, the reason allowlist against reasons nobody +// listed, and the load-bearing rule against a PAIR of documents that +// differ in exactly one key — so a green result is a difference the guard +// produced and not a fixture that could never have failed. +// - NO CORPUS COMES FROM THE IMPLEMENTATION. The known-version list is +// re-stated here from A.16's packet text ("5.0/5.1/5.2 known") and +// compared against the table; the field fixtures are hand-written CVE +// documents; the branch table is checked against internal/ingest/delta's +// DECODER BEHAVIOUR rather than against a copy of delta's own list. +// - NO NETWORK, no credentials, no environment reads, and no t.Skip. A skip +// here would hide exactly the control the packet asks for. +package drift + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/config" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/delta" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/license" +) + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const testFeedID = "cvelistv5" + +var fixtureClock = time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC) + +// packetKnownVersions is A.16's packet, quoted: "5.0/5.1/5.2 known; anything +// else sets degraded=true and stores raw verbatim". +// +// It is written out HERE, from the packet, precisely so that the table in +// drift.go is compared against something other than itself. A test whose +// corpus is the implementation certified a defect on this project rather than +// catching one. +var packetKnownVersions = []string{"5.0", "5.1", "5.2"} + +// cveDoc builds a CVE 5.x record as a map so a test can add, remove or rename +// exactly one key and compare the two outcomes. +func cveDoc(id, dataVersion string) map[string]any { + return map[string]any{ + "dataType": "CVE_RECORD", + "dataVersion": dataVersion, + "cveMetadata": map[string]any{ + "cveId": id, + "state": "PUBLISHED", + "assignerOrgId": "00000000-0000-4000-8000-000000000000", + "assignerShortName": "example", + "datePublished": "2026-01-01T00:00:00Z", + "dateUpdated": "2026-02-01T00:00:00Z", + }, + "containers": map[string]any{ + "cna": map[string]any{ + "providerMetadata": map[string]any{ + "orgId": "00000000-0000-4000-8000-000000000000", + "shortName": "example", + }, + "descriptions": []any{map[string]any{ + "lang": "en", + "value": "A synthetic advisory about " + id + " affecting quorumwidget.", + }}, + "references": []any{map[string]any{"url": "https://example.invalid/adv/" + id}}, + "metrics": []any{map[string]any{"cvssV3_1": map[string]any{ + "version": "3.1", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL", + }}}, + "affected": []any{map[string]any{ + "vendor": "example", + "product": "quorumwidget", + "packageName": "quorumwidget", + "versions": []any{map[string]any{ + "version": "1.0.0", + "status": "affected", + "versionType": "semver", + "lessThan": "1.2.3", + }}, + }}, + }, + }, + } +} + +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + raw, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshalling a fixture: %v", err) + } + return raw +} + +// cna reaches into a fixture's CNA container so a test can inject one key. +func cna(t *testing.T, doc map[string]any) map[string]any { + t.Helper() + containers, ok := doc["containers"].(map[string]any) + if !ok { + t.Fatal("fixture has no containers object") + } + c, ok := containers["cna"].(map[string]any) + if !ok { + t.Fatal("fixture has no cna container") + } + return c +} + +// firstAffectedVersion reaches the load-bearing subtree: the version range the +// comparator answers from. +func firstAffectedVersion(t *testing.T, doc map[string]any) map[string]any { + t.Helper() + affected, ok := cna(t, doc)["affected"].([]any) + if !ok || len(affected) == 0 { + t.Fatal("fixture has no affected entries") + } + entry, ok := affected[0].(map[string]any) + if !ok { + t.Fatal("fixture's first affected entry is not an object") + } + versions, ok := entry["versions"].([]any) + if !ok || len(versions) == 0 { + t.Fatal("fixture's first affected entry has no versions") + } + v, ok := versions[0].(map[string]any) + if !ok { + t.Fatal("fixture's first version is not an object") + } + return v +} + +// --------------------------------------------------------------------------- +// Cache fixtures +// --------------------------------------------------------------------------- + +func openCache(t *testing.T) *sql.DB { + t.Helper() + db, err := cache.Open(t.Context(), filepath.Join(t.TempDir(), "anvil-cache.sqlite")) + if err != nil { + t.Fatalf("cache.Open: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if _, err := cache.Migrate(t.Context(), db); err != nil { + t.Fatalf("cache.Migrate: %v", err) + } + return db +} + +// admittedDecision is a NON-REFUSED licence decision. +// +// It is a literal rather than a run of A.4's real gate, and that is a +// deliberate, narrow choice: this file measures A.16, and the gate has its own +// suite. What matters here is only that the decision is admitted — a refusal +// writes nothing, and a suite in which every write silently did nothing would +// look green for the worst possible reason. Decision.Refused() is asserted +// below so the fixture cannot rot into a refusal unnoticed. +func admittedDecision(t *testing.T) license.Decision { + t.Helper() + d := license.Decision{ + FeedID: testFeedID, + Tier: config.LicenseTier0, + Dir: "mirror/tier0/" + testFeedID, + EffectiveSPDX: "CC0-1.0", + DeclaredSPDX: "CC0-1.0", + } + if d.Refused() { + t.Fatal("the fixture licence decision is a refusal; every write below would be a no-op " + + "and this suite would be green for the wrong reason") + } + return d +} + +func testFeed() config.FeedConfig { + return config.FeedConfig{ID: testFeedID} +} + +// applyOne writes one parsed record through A.14's real write path. There is +// deliberately no second write path in this file: a test that inserted rows +// with its own INSERT would be measuring its own SQL. +func applyOne(t *testing.T, db *sql.DB, rec Record) delta.BatchStats { + t.Helper() + stats, err := delta.Apply(t.Context(), db, testFeed(), admittedDecision(t), + []Record{rec}, fixtureClock, 0) + if err != nil { + t.Fatalf("delta.Apply: %v", err) + } + return stats +} + +// parseAndApply is the whole ingest path this packet sits in: bytes in, one +// row in the cache. +func parseAndApply(t *testing.T, db *sql.DB, raw []byte) Report { + t.Helper() + rec, rep, err := Parse(testFeedID, raw) + if err != nil { + t.Fatalf("Parse: %v", err) + } + applyOne(t, db, rec) + return rep +} + +// insertFinding seeds a Lane A finding. +// +// It uses raw SQL because there is no sanctioned writer for `finding` yet — +// A.9 and A.10 own that table and neither exports a write path. The column +// list is copied from internal/ingest/cache's own test so the two cannot +// disagree about the shape, and `finding.id` is a LANE-LOCAL identifier and +// never a fingerprint (schema.go says so at the table). +func insertFinding(t *testing.T, db *sql.DB, id, source, sourceID string) { + t.Helper() + _, err := db.ExecContext(t.Context(), ` + INSERT INTO finding ( + id, collector, source, source_id, package, installed_version, ecosystem, + remediable_by_agent, as_of, staleness_seconds, anvil_trust, detected_at + ) VALUES (?, ?, ?, ?, 'quorumwidget', '1.0.0', 'deb', 0, ?, 0, ?, ?)`, + id, cache.CollectorHost, source, sourceID, + fixtureClock.Format(time.RFC3339), string(cache.FindingTrustDefault), + fixtureClock.Format(time.RFC3339)) + if err != nil { + t.Fatalf("seeding finding %q: %v", id, err) + } +} + +func scalar[T any](t *testing.T, db *sql.DB, query string, args ...any) T { + t.Helper() + var out T + if err := db.QueryRowContext(t.Context(), query, args...).Scan(&out); err != nil { + t.Fatalf("query %q: %v", condense(query), err) + } + return out +} + +// --------------------------------------------------------------------------- +// A.16's first stop condition: an unknown dataVersion round-trips +// --------------------------------------------------------------------------- + +// TestUnknownDataVersionIsPersistedDegradedAndNotDropped is the packet's +// headline validation, measured out of the database rather than out of a +// return value. +func TestUnknownDataVersionIsPersistedDegradedAndNotDropped(t *testing.T) { + db := openCache(t) + raw := mustJSON(t, cveDoc("CVE-2026-9999", "5.9")) + + rec, rep, err := Parse(testFeedID, raw) + if err != nil { + t.Fatalf("an unknown dataVersion produced an ERROR; the packet requires it to produce a "+ + "degraded record: %v", err) + } + if !rep.Degraded { + t.Fatal("dataVersion 5.9 was not reported as degraded") + } + if !rep.Has(CodeUnknownDataVersion) { + t.Errorf("the report does not carry %s; codes are %v", CodeUnknownDataVersion, rep.Codes) + } + if rep.Branch != BranchUnknown || rep.KnownVersion { + t.Errorf("dataVersion 5.9 selected branch %q (known=%v); want the unknown branch", + rep.Branch, rep.KnownVersion) + } + if rec.SourceID != "CVE-2026-9999" { + t.Errorf("the degraded record lost its primary key: source_id = %q", rec.SourceID) + } + if !bytes.Equal(rec.Raw, raw) { + t.Error("the degraded record does not carry the publisher's bytes verbatim") + } + + applyOne(t, db, rec) + + if got := scalar[int](t, db, + `SELECT count(*) FROM advisory WHERE source_id = 'CVE-2026-9999' AND parse_degraded = 1 + AND data_version = '5.9'`); got != 1 { + t.Fatalf("expected exactly one degraded row for the 5.9 record, found %d. "+ + "research/06 Risk #3: ingest raw and set parse_degraded=1 rather than dropping the record", got) + } + // "Round-trips WITHOUT DATA LOSS": the publisher's bytes and the version + // ranges a comparator would answer from both survived. + stored := scalar[[]byte](t, db, `SELECT raw_json FROM advisory WHERE source_id = 'CVE-2026-9999'`) + if !bytes.Equal(stored, raw) { + t.Errorf("raw_json is not byte-identical to the document that went in:\n stored %s\n input %s", + stored, raw) + } + if got := scalar[int](t, db, + `SELECT count(*) FROM affected WHERE source_id = 'CVE-2026-9999'`); got == 0 { + t.Error("the degraded record's version ranges were dropped; a degraded record is still a record") + } +} + +// TestKnownDataVersionsAreNotDegraded is the other half of the same claim: if +// every record came back degraded, parse_degraded would carry no information +// and the test above would pass for the wrong reason. +func TestKnownDataVersionsAreNotDegraded(t *testing.T) { + for _, version := range packetKnownVersions { + t.Run(version, func(t *testing.T) { + db := openCache(t) + id := "CVE-2026-" + strings.ReplaceAll(version, ".", "") + raw := mustJSON(t, cveDoc(id, version)) + + rep := parseAndApply(t, db, raw) + if rep.Degraded { + t.Fatalf("dataVersion %s was degraded: %s", version, rep) + } + if !rep.Clean() { + t.Fatalf("dataVersion %s raised codes on a document built from the known key set: %s", + version, rep) + } + if got := scalar[int](t, db, + `SELECT parse_degraded FROM advisory WHERE source_id = ?`, id); got != 0 { + t.Errorf("parse_degraded = %d for a known version", got) + } + }) + } +} + +// TestTheBranchTableAgreesWithTheDeltaDecoder is the cross-package conformance +// test. +// +// internal/ingest/delta keeps its own knownCVEDataVersions and it is +// unexported, so the two lists cannot be compared directly. They are compared +// BEHAVIOURALLY instead: delta.Decode's own parse_degraded is read for the +// same bytes this package branches on. A divergence — delta learning 5.3 while +// this table does not, or the reverse — is a red test here, which is the only +// place the two can be observed together at all. +func TestTheBranchTableAgreesWithTheDeltaDecoder(t *testing.T) { + versions := append([]string{}, packetKnownVersions...) + versions = append(versions, "5.3", "5.9", "6.0", "4.0", "", "5", "5.1.0", "v5.1", " 5.1 ") + + for _, version := range versions { + t.Run("dataVersion="+version, func(t *testing.T) { + raw := mustJSON(t, cveDoc("CVE-2026-1000", version)) + recs, _, err := delta.Decode(testFeedID, raw) + if err != nil { + t.Fatalf("delta.Decode refused a well-formed CVE record: %v", err) + } + if len(recs) != 1 { + t.Fatalf("delta.Decode returned %d records for one document", len(recs)) + } + deltaSaysDegraded := recs[0].ParseDegraded + driftSaysKnown := BranchFor(version).Known() + if deltaSaysDegraded == driftSaysKnown { + t.Fatalf("the two version tables disagree about %q: delta degraded=%v, "+ + "drift known=%v. One of the two lists has moved; they describe the same fact "+ + "and a silent divergence changes what gets flagged incomplete.", + version, deltaSaysDegraded, driftSaysKnown) + } + }) + } +} + +// TestKnownVersionsAreExactlyThePacketsThree compares the table against A.16's +// packet rather than against itself. +func TestKnownVersionsAreExactlyThePacketsThree(t *testing.T) { + got := KnownVersions() + if !reflect.DeepEqual(got, packetKnownVersions) { + t.Fatalf("KnownVersions() = %v; A.16's packet names %v", got, packetKnownVersions) + } + for _, v := range packetKnownVersions { + if !BranchFor(v).Known() { + t.Errorf("%q is named as known in the packet and is not in the table", v) + } + } +} + +// TestNoPrefixOrFuzzyRuleOnDataVersion holds the one rule that makes the +// branch meaningful: "5.3 looks close enough to 5.2" is how a partial parse +// becomes silent. +func TestNoPrefixOrFuzzyRuleOnDataVersion(t *testing.T) { + unknown := []string{"5", "5.", "5.1.0", "5.10", "5.11", "v5.1", "V5.1", "5.1-beta", "05.1", "", " "} + for _, v := range unknown { + if BranchFor(v).Known() { + t.Errorf("BranchFor(%q) reports a known branch; only an EXACT match on a listed "+ + "version may be known", v) + } + } + // Surrounding whitespace IS trimmed, and that is deliberate: a feed that + // pads its value has not changed its schema. + for _, v := range []string{" 5.1", "5.1 ", "\t5.1\n"} { + if BranchFor(v) != BranchCVE51 { + t.Errorf("BranchFor(%q) = %q; padding is not a schema change", v, BranchFor(v)) + } + } +} + +// TestVersionOrderIsNumericPerComponent guards newestBranch against the day +// CVE reaches a two-digit minor. Lexically "5.10" < "5.2", which would silently +// pick the wrong profile for every unknown version. +func TestVersionOrderIsNumericPerComponent(t *testing.T) { + cases := []struct { + a, b string + want int + }{ + {"5.2", "5.10", -1}, + {"5.10", "5.2", 1}, + {"5.1", "5.1", 0}, + {"5.0", "5.1", -1}, + {"6.0", "5.99", 1}, + {"5.1", "5.1.1", -1}, + {"x", "5.0", -1}, + } + for _, c := range cases { + if got := compareVersions(c.a, c.b); got != c.want { + t.Errorf("compareVersions(%q, %q) = %d, want %d", c.a, c.b, got, c.want) + } + } + if got := newestBranch(); got != BranchCVE52 { + t.Errorf("newestBranch() = %q; the newest known version is 5.2", got) + } + + // The ordering is exercised against a list THIS BUILD'S TABLE DOES NOT + // CONTAIN. With only 5.0, 5.1 and 5.2 known, lexical and numeric sorting + // agree, so a test that only looked at KnownVersions() would pass against + // a lexical implementation and go on passing until the day CVE ships a + // two-digit minor — at which point newestBranch would quietly select the + // wrong profile for every unknown version. + if got := sortVersions([]string{"5.10", "5.2", "5.1", "6.0", "5.9"}); !reflect.DeepEqual( + got, []string{"5.1", "5.2", "5.9", "5.10", "6.0"}) { + t.Fatalf("sortVersions = %v; want numeric-per-component order. Lexically \"5.10\" sorts "+ + "before \"5.2\", which would make the newest known profile the wrong one.", got) + } + if got := sortVersions(append([]string{}, KnownVersions()...)); !reflect.DeepEqual(got, KnownVersions()) { + t.Errorf("KnownVersions() is not in the order sortVersions produces: %v", KnownVersions()) + } +} + +// --------------------------------------------------------------------------- +// Which fields were not understood +// --------------------------------------------------------------------------- + +// TestAnUnknownFieldInALoadBearingPathDegradesAKnownVersion is the within- +// version drift case: the feed does not bump its version, it just starts +// carrying something new where the version ranges live. +// +// IT IS RUN AS A PAIR. The control is the same document without the injected +// key and must come back clean, so a green result is the difference the guard +// found rather than a fixture that could never have been clean. +func TestAnUnknownFieldInALoadBearingPathDegradesAKnownVersion(t *testing.T) { + control := cveDoc("CVE-2026-1111", "5.1") + if _, rep, err := Parse(testFeedID, mustJSON(t, control)); err != nil || !rep.Clean() { + t.Fatalf("the CONTROL document is not clean, so this test could not have failed: %s (err %v)", + rep, err) + } + + drifted := cveDoc("CVE-2026-1111", "5.1") + firstAffectedVersion(t, drifted)["rangeSemanticsV2"] = "closed-open" + + rec, rep, err := Parse(testFeedID, mustJSON(t, drifted)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if !rep.Degraded { + t.Fatal("an unrecognised key inside affected[].versions[] did not degrade the record. " + + "That subtree is the version range Lane A's answer is made of (spine S1); a field " + + "nobody understands there is a range that may not mean what the comparator read.") + } + if !rep.Has(CodeUnknownFieldLoadBearing) { + t.Errorf("codes = %v; want %s", rep.Codes, CodeUnknownFieldLoadBearing) + } + want := "/containers/cna/affected[]/versions[]/rangeSemanticsV2" + if !contains(rep.DegradingFields, want) { + t.Errorf("DegradingFields = %v; want it to name %q — a degraded flag that does not say "+ + "WHICH field is a status nobody can act on", rep.DegradingFields, want) + } + if !rec.ParseDegraded { + t.Error("the record's parse_degraded does not match the report") + } +} + +// TestAnUnknownFieldOutsideALoadBearingPathIsReportedNotDegraded is the other +// side of the same rule. Degrading every record that carries a new prose field +// would make parse_degraded mean "this is a CVE record", and a flag that is +// always set is a flag nobody reads. +func TestAnUnknownFieldOutsideALoadBearingPathIsReportedNotDegraded(t *testing.T) { + doc := cveDoc("CVE-2026-2222", "5.1") + cna(t, doc)["x_generatorNotes"] = "produced by a tool nobody told us about" + + _, rep, err := Parse(testFeedID, mustJSON(t, doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if rep.Degraded { + t.Fatalf("a new key in a non-load-bearing path degraded the record: %s", rep) + } + want := "/containers/cna/x_generatorNotes" + if !contains(rep.UnknownFields, want) { + t.Fatalf("UnknownFields = %v; want it to name %q. An `x_` prefix is NOT an exemption: "+ + "the CVE format permits vendor extensions, and a parser that stopped looking at them "+ + "would have a documented place to hide drift.", rep.UnknownFields, want) + } + if contains(rep.DegradingFields, want) { + t.Error("a non-load-bearing field was listed as degrading") + } + if !rep.Has(CodeUnknownField) { + t.Errorf("codes = %v; want %s", rep.Codes, CodeUnknownField) + } +} + +// TestAnUnknownFieldInCVEMetadataDegrades covers the other load-bearing +// subtree: identity and retraction. +func TestAnUnknownFieldInCVEMetadataDegrades(t *testing.T) { + doc := cveDoc("CVE-2026-3333", "5.1") + meta, ok := doc["cveMetadata"].(map[string]any) + if !ok { + t.Fatal("fixture has no cveMetadata") + } + meta["supersededBy"] = "CVE-2026-4444" + + _, rep, err := Parse(testFeedID, mustJSON(t, doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if !rep.Degraded || !contains(rep.DegradingFields, "/cveMetadata/supersededBy") { + t.Fatalf("an unrecognised key in /cveMetadata did not degrade: %s", rep) + } +} + +// TestTheLoadBearingPrefixMatchIsSegmentAware verifies the guard against the +// near-miss that has defeated three guards on this project: a plain +// strings.HasPrefix would call "/cveMetadataExtra" load-bearing. +// +// The naive answer is computed here and asserted to DIFFER, so the test fails +// if the implementation is ever simplified back into it. +func TestTheLoadBearingPrefixMatchIsSegmentAware(t *testing.T) { + nearMisses := []string{ + "/cveMetadataExtra", + "/cveMetadata2", + "/containers/cna/affectedProducts", + "/containers/cna/affected2[]/thing", + } + for _, p := range nearMisses { + if isLoadBearing(p) { + t.Errorf("isLoadBearing(%q) = true; a sibling key that merely starts with the same "+ + "letters is not the same subtree", p) + } + naive := false + for _, prefix := range loadBearingPrefixes { + base := strings.TrimSuffix(prefix, "[]") + if strings.HasPrefix(p, base) { + naive = true + } + } + if !naive { + t.Errorf("%q is not a near-miss at all, so it verifies nothing; pick a fixture a "+ + "naive prefix match would have accepted", p) + } + } + hits := []string{ + "/cveMetadata", + "/cveMetadata/cveId", + "/containers/cna/affected[]", + "/containers/cna/affected[]/versions[]/lessThan", + "/containers/adp[]/affected[]/vendor", + } + for _, p := range hits { + if !isLoadBearing(p) { + t.Errorf("isLoadBearing(%q) = false; that path IS a load-bearing subtree", p) + } + } +} + +// TestUnknownFieldsAreCappedAndTruncationIsItselfDegrading: a report is a log +// line, and a document is attacker-adjacent input. +func TestUnknownFieldsAreCappedAndTruncationIsItselfDegrading(t *testing.T) { + doc := cveDoc("CVE-2026-5555", "5.1") + container := cna(t, doc) + for i := 0; i < MaxReportedFields*4; i++ { + container[fmt.Sprintf("unlisted_%03d", i)] = i + } + + _, rep, err := Parse(testFeedID, mustJSON(t, doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if len(rep.UnknownFields) > MaxReportedFields { + t.Errorf("UnknownFields has %d entries; the cap is %d", len(rep.UnknownFields), MaxReportedFields) + } + if !rep.Truncated { + t.Fatal("the field list was capped and Truncated was not set; a short list must never be " + + "mistaken for a clean one") + } + if !rep.Degraded || !rep.Has(CodeFieldsTruncated) { + t.Fatalf("truncation did not degrade the record: %s. An incomplete answer about whether "+ + "anything was missed is not an answer.", rep) + } +} + +// TestAKeyChangingTypeIsDrift: a field that was an object and is now an array +// is a schema change, and it is one nothing else in this pipeline would notice. +func TestAKeyChangingTypeIsDrift(t *testing.T) { + doc := cveDoc("CVE-2026-6666", "5.1") + cna(t, doc)["title"] = []any{"now", "an", "array"} + + _, rep, err := Parse(testFeedID, mustJSON(t, doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if !contains(rep.UnknownFields, "/containers/cna/title[]") { + t.Fatalf("a scalar key that became an array was not reported: %s", rep) + } +} + +// --------------------------------------------------------------------------- +// Never dropped, even when nothing decodes +// --------------------------------------------------------------------------- + +// TestAnUndecodableDocumentIsPreservedNotDropped covers the case the packet's +// forbidden action is really about: not "a version we do not know" but "bytes +// we cannot parse at all". Those are the ones a parser is most tempted to skip. +func TestAnUndecodableDocumentIsPreservedNotDropped(t *testing.T) { + db := openCache(t) + // dataType is not CVE_RECORD and the document is in no shape delta + // recognises, so delta.Decode refuses it outright. + raw := mustJSON(t, map[string]any{ + "dataType": "CVE_RECORD_V6", + "dataVersion": "6.0", + "cveMetadata": map[string]any{"cveId": "CVE-2026-7777", "state": "PUBLISHED"}, + "payload": map[string]any{"somethingEntirelyNew": true}, + }) + + if _, _, err := delta.Decode(testFeedID, raw); err == nil { + t.Fatal("delta.Decode accepted the fixture, so this test does not exercise the fallback " + + "path at all; pick a document the decoder genuinely refuses") + } + + rec, rep, err := Parse(testFeedID, raw) + if err != nil { + t.Fatalf("an undecodable document produced an error instead of a degraded record: %v", err) + } + if !rep.Has(CodeUndecodable) || !rep.Degraded { + t.Fatalf("the fallback record was not reported as degraded: %s", rep) + } + if rep.DecodeError == "" { + t.Error("the report does not carry the decoder's refusal; 'it did not parse' without a " + + "reason is not something an operator can act on") + } + applyOne(t, db, rec) + + if got := scalar[int](t, db, + `SELECT count(*) FROM advisory WHERE source_id = 'CVE-2026-7777' AND parse_degraded = 1`); got != 1 { + t.Fatalf("the undecodable document was not persisted (%d rows). The bytes are the whole "+ + "point: a later build that understands the shape can re-parse them, and a dropped "+ + "record can never be recovered.", got) + } + stored := scalar[[]byte](t, db, `SELECT raw_json FROM advisory WHERE source_id = 'CVE-2026-7777'`) + if !bytes.Equal(stored, raw) { + t.Error("the fallback did not store the publisher's bytes verbatim") + } +} + +// TestADocumentWithNoPrimaryKeyIsRefusedLoudly is the one case that is not a +// degraded record, and it must not be a silent drop either. +func TestADocumentWithNoPrimaryKeyIsRefusedLoudly(t *testing.T) { + raw := mustJSON(t, map[string]any{ + "dataType": "SOMETHING_ELSE", + "dataVersion": "9.9", + "payload": map[string]any{"no": "identifier anywhere"}, + }) + _, _, err := Parse(testFeedID, raw) + if !errors.Is(err, ErrNoPrimaryKey) { + t.Fatalf("err = %v; want ErrNoPrimaryKey. The cache is keyed on (source, source_id) and a "+ + "row without one cannot be written, re-found or re-opened, so this has to be an error "+ + "the caller sees.", err) + } + if !errors.Is(err, ErrDrift) { + t.Error("the refusal is not tagged with ErrDrift, so a caller cannot tell it from a " + + "database failure") + } +} + +// TestParseRefusesADocumentThatIsNotOneObject: an array of advisories is a +// real feed shape, and taking its first element silently would drop the rest. +func TestParseRefusesADocumentThatIsNotOneObject(t *testing.T) { + for name, raw := range map[string]string{ + "array": `[{"dataType":"CVE_RECORD","dataVersion":"5.1","cveMetadata":{"cveId":"CVE-2026-1"}}]`, + "string": `"just a string"`, + "empty": ``, + } { + t.Run(name, func(t *testing.T) { + if _, _, err := Parse(testFeedID, []byte(raw)); !errors.Is(err, ErrNotAnObject) { + t.Fatalf("err = %v; want ErrNotAnObject", err) + } + }) + } +} + +// TestTheRecordAndTheReportNeverDisagreeAboutDegradation. A record whose flag +// and whose explanation disagree is worse than either alone: one of the two is +// what an operator reads and the other is what the comparator trusts. +func TestTheRecordAndTheReportNeverDisagreeAboutDegradation(t *testing.T) { + clean := cveDoc("CVE-2026-8001", "5.1") + + unknownVersion := cveDoc("CVE-2026-8002", "5.9") + + loadBearing := cveDoc("CVE-2026-8003", "5.1") + firstAffectedVersion(t, loadBearing)["newRangeKey"] = "x" + + cosmetic := cveDoc("CVE-2026-8004", "5.1") + cna(t, cosmetic)["newProseKey"] = "x" + + for name, doc := range map[string]map[string]any{ + "clean": clean, + "unknown-version": unknownVersion, + "load-bearing-drift": loadBearing, + "non-load-bearing": cosmetic, + "unknown-version-both": mergeDoc(cveDoc("CVE-2026-8005", "7.7"), "extraTopLevel", "x"), + } { + t.Run(name, func(t *testing.T) { + rec, rep, err := Parse(testFeedID, mustJSON(t, doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if rec.ParseDegraded != rep.Degraded { + t.Fatalf("record.ParseDegraded = %v but report.Degraded = %v (%s)", + rec.ParseDegraded, rep.Degraded, rep) + } + if rep.Degraded && len(rep.Codes) == 0 { + t.Fatal("a degraded record with no code says it is incomplete without saying why") + } + if rep.Degraded && rep.String() == "" { + t.Fatal("Report.String() is empty on a degraded record") + } + }) + } +} + +func mergeDoc(doc map[string]any, key string, value any) map[string]any { + doc[key] = value + return doc +} + +// TestParseVersionedAgreesWithParse holds the two entry points together. The +// narrow one is the packet's signature and the wide one is what an ingest path +// should call; they must not be able to disagree about the same bytes. +func TestParseVersionedAgreesWithParse(t *testing.T) { + for _, version := range []string{"5.1", "5.9"} { + t.Run(version, func(t *testing.T) { + raw := mustJSON(t, cveDoc("CVE-2026-4242", version)) + + narrow, degraded := ParseVersioned(raw) + wide, rep, err := Parse(testFeedID, raw) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if degraded != rep.Degraded { + t.Fatalf("ParseVersioned says degraded=%v, Parse says %v", degraded, rep.Degraded) + } + if narrow.Source != "" { + t.Errorf("ParseVersioned stamped a Source (%q); bytes do not know which feed "+ + "delivered them", narrow.Source) + } + // Stamping the feed id is the only difference between the two. + narrow.Source = testFeedID + if !reflect.DeepEqual(narrow, wide) { + t.Fatalf("the two entry points produced different records for the same bytes:\n"+ + " narrow %+v\n wide %+v", narrow, wide) + } + }) + } +} + +// TestParseVersionedOnAKeylessDocumentReturnsARecordTheWritePathRefuses. The +// narrow signature cannot return an error, so the failure has to arrive +// somewhere a caller cannot ignore. +func TestParseVersionedOnAKeylessDocumentReturnsARecordTheWritePathRefuses(t *testing.T) { + db := openCache(t) + raw := []byte(`{"dataVersion":"9.9","payload":{"no":"identifier"}}`) + + rec, degraded := ParseVersioned(raw) + if !degraded { + t.Fatal("a document that could not be parsed at all came back not degraded") + } + if !bytes.Equal(rec.Raw, raw) { + t.Error("the returned record does not carry the publisher's bytes") + } + _, err := delta.Apply(t.Context(), db, testFeed(), admittedDecision(t), + []Record{rec}, fixtureClock, 0) + if err == nil { + t.Fatal("delta.Apply accepted a record with no primary key; the refusal is what makes " + + "ParseVersioned's error-free signature safe") + } +} + +func contains(list []string, want string) bool { + for _, got := range list { + if got == want { + return true + } + } + return false +} + +// --------------------------------------------------------------------------- +// Tombstones +// --------------------------------------------------------------------------- + +// seedPublished writes one published advisory and one finding that rests on +// it, and returns the advisory's rowid. +func seedPublished(t *testing.T, db *sql.DB, id string) int64 { + t.Helper() + rep := parseAndApply(t, db, mustJSON(t, cveDoc(id, "5.1"))) + if rep.Degraded { + t.Fatalf("the seed advisory came back degraded: %s", rep) + } + insertFinding(t, db, "finding-"+id, testFeedID, id) + return scalar[int64](t, db, `SELECT rowid FROM advisory WHERE source = ? AND source_id = ?`, + testFeedID, id) +} + +func newTombstoner(t *testing.T, db *sql.DB) *Tombstoner { + t.Helper() + ts, err := NewTombstoner(db, func() time.Time { return fixtureClock }) + if err != nil { + t.Fatalf("NewTombstoner: %v", err) + } + return ts +} + +// TestTombstoneFlipsDependentFindingVisibility is A.16's second stop +// condition, and the reason exit criterion 22 exists at all. +func TestTombstoneFlipsDependentFindingVisibility(t *testing.T) { + db := openCache(t) + const id = "CVE-2026-1234" + seedPublished(t, db, id) + + before, err := FindingsReferencing(t.Context(), db, testFeedID, id) + if err != nil { + t.Fatalf("FindingsReferencing: %v", err) + } + if len(before) != 1 || before[0].Invalidated { + t.Fatalf("before the tombstone: %d findings, invalidated=%v; want one live finding", + len(before), len(before) > 0 && before[0].Invalidated) + } + + res, err := newTombstoner(t, db).Tombstone(t.Context(), testFeedID, id, string(ReasonWithdrawn)) + if err != nil { + t.Fatalf("Tombstone: %v", err) + } + if res.State != cache.AdvisoryWithdrawn || res.PreviousState != cache.AdvisoryPublished { + t.Errorf("state went %q -> %q; want %q -> %q", + res.PreviousState, res.State, cache.AdvisoryPublished, cache.AdvisoryWithdrawn) + } + if res.InvalidatedFindings != 1 { + t.Errorf("InvalidatedFindings = %d, want 1", res.InvalidatedFindings) + } + + after, err := FindingsReferencing(t.Context(), db, testFeedID, id) + if err != nil { + t.Fatalf("FindingsReferencing: %v", err) + } + if len(after) != 1 { + t.Fatalf("after the tombstone the finding VANISHED (%d rows). research/06 Risk #4 and "+ + "exit criterion 22: a prior finding must be re-openable and invalidated, which it "+ + "cannot be if the query stops returning it.", len(after)) + } + if !after[0].Invalidated { + t.Fatal("the finding is still reported as live although its advisory was withdrawn") + } + if after[0].AdvisoryState != cache.AdvisoryWithdrawn || after[0].TombstonedAt == "" { + t.Errorf("the invalidated finding does not carry the retraction: state=%q tombstoned_at=%q", + after[0].AdvisoryState, after[0].TombstonedAt) + } + if after[0].FindingID != "finding-"+id { + t.Errorf("FindingID = %q; the finding's identity must survive invalidation", after[0].FindingID) + } + + all, err := InvalidatedFindings(t.Context(), db) + if err != nil { + t.Fatalf("InvalidatedFindings: %v", err) + } + if len(all) != 1 || all[0].FindingID != "finding-"+id { + t.Fatalf("the re-open work list does not contain the invalidated finding: %+v", all) + } +} + +// TestTombstoneNeverDeletesARow is the forbidden action, measured. +func TestTombstoneNeverDeletesARow(t *testing.T) { + db := openCache(t) + const id = "CVE-2026-2345" + rowid := seedPublished(t, db, id) + rawBefore := scalar[[]byte](t, db, `SELECT raw_json FROM advisory WHERE source_id = ?`, id) + affectedBefore := scalar[int](t, db, `SELECT count(*) FROM affected WHERE source_id = ?`, id) + aliasBefore := scalar[int](t, db, `SELECT count(*) FROM cve_alias WHERE source_id = ?`, id) + if affectedBefore == 0 || aliasBefore == 0 { + t.Fatal("the seed advisory has no affected or alias rows, so their survival proves nothing") + } + + res, err := newTombstoner(t, db).Tombstone(t.Context(), testFeedID, id, string(ReasonRejected)) + if err != nil { + t.Fatalf("Tombstone: %v", err) + } + + if got := scalar[int](t, db, `SELECT count(*) FROM advisory WHERE source_id = ?`, id); got != 1 { + t.Fatalf("the advisory row is gone (%d rows). A withdrawn advisory is TOMBSTONED, never "+ + "deleted: a finding that referenced it must still find it.", got) + } + state := scalar[string](t, db, `SELECT state FROM advisory WHERE source_id = ?`, id) + stamp := scalar[string](t, db, `SELECT ifnull(tombstoned_at, '') FROM advisory WHERE source_id = ?`, id) + if state != cache.AdvisoryRejected || stamp == "" { + t.Errorf("state=%q tombstoned_at=%q; the schema pairs a non-published state with a "+ + "non-null timestamp", state, stamp) + } + if got := scalar[int64](t, db, `SELECT rowid FROM advisory WHERE source_id = ?`, id); got != rowid { + t.Errorf("the rowid moved from %d to %d; an INSERT OR REPLACE would do that and would "+ + "orphan the FTS entry with no error", rowid, got) + } + if res.RowID != rowid { + t.Errorf("TombstoneResult.RowID = %d, want %d", res.RowID, rowid) + } + if got := scalar[[]byte](t, db, `SELECT raw_json FROM advisory WHERE source_id = ?`, id); !bytes.Equal(got, rawBefore) { + t.Error("raw_json changed during a tombstone; the column holds the publisher's bytes verbatim") + } + if got := scalar[int](t, db, `SELECT count(*) FROM affected WHERE source_id = ?`, id); got != affectedBefore { + t.Errorf("affected rows went from %d to %d", affectedBefore, got) + } + if got := scalar[int](t, db, `SELECT count(*) FROM cve_alias WHERE source_id = ?`, id); got != aliasBefore { + t.Errorf("cve_alias rows went from %d to %d", aliasBefore, got) + } + if got := scalar[int](t, db, `SELECT count(*) FROM finding WHERE source_id = ?`, id); got != 1 { + t.Errorf("the dependent finding was removed (%d rows)", got) + } +} + +// TestTheRowidGuardFiresWhenARowWouldMove exercises the one check in this +// package that the public API cannot reach. +// +// The rowid cannot move today: the write is ON CONFLICT DO UPDATE and the +// statement allowlist admits nothing else. The check exists against the day +// somebody reaches for INSERT OR REPLACE, which fails exactly this way and +// fails SILENTLY — the row is re-inserted under a new rowid and its FTS entry +// is orphaned with no error anywhere. A defensive check nothing ever fires is +// a check nobody knows works, so this test lies to the writer about the rowid +// it read and requires the refusal. +func TestTheRowidGuardFiresWhenARowWouldMove(t *testing.T) { + db := openCache(t) + const id = "CVE-2026-9001" + seedPublished(t, db, id) + ts := newTombstoner(t, db) + + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + defer func() { _ = tx.Rollback() }() + + row, err := ts.readRow(t.Context(), tx, testFeedID, id) + if err != nil { + t.Fatalf("readRow: %v", err) + } + row.rowid++ // the row the writer thinks it read is not the row it will write + + _, _, err = ts.writeTombstonedRow(t.Context(), tx, testFeedID, id, + cache.AdvisoryWithdrawn, fixtureClock.Format(time.RFC3339), row) + if err == nil { + t.Fatal("the writer accepted a rowid that had moved; that is the INSERT OR REPLACE " + + "failure mode, and its whole danger is that it is silent") + } + if !strings.Contains(err.Error(), "orphan") { + t.Errorf("the refusal does not say what goes wrong: %v", err) + } +} + +// TestTombstoneRemovesTheAdvisoryFromTheSearchIndex: exit criterion 22 seen +// from the FTS side. The row stays addressable; its text stops matching. +func TestTombstoneRemovesTheAdvisoryFromTheSearchIndex(t *testing.T) { + db := openCache(t) + const id = "CVE-2026-3456" + seedPublished(t, db, id) + + if got := scalar[int](t, db, + `SELECT count(*) FROM advisory_fts WHERE advisory_fts MATCH 'quorumwidget'`); got != 1 { + t.Fatalf("the seed advisory is not in the index (%d hits), so its removal proves nothing", got) + } + if _, err := newTombstoner(t, db).Tombstone(t.Context(), testFeedID, id, string(ReasonWithdrawn)); err != nil { + t.Fatalf("Tombstone: %v", err) + } + if got := scalar[int](t, db, + `SELECT count(*) FROM advisory_fts WHERE advisory_fts MATCH 'quorumwidget'`); got != 0 { + t.Errorf("a withdrawn advisory still matches a search (%d hits)", got) + } + if got := scalar[int](t, db, `SELECT count(*) FROM advisory WHERE source_id = ?`, id); got != 1 { + t.Errorf("the advisory row went with its index entry (%d rows)", got) + } +} + +// TestTombstoneIsIdempotentAndKeepsTheFirstRetractionTime. A nightly +// reconcile re-runs this; a second call must not make a year-old withdrawal +// look like today's news. +func TestTombstoneIsIdempotentAndKeepsTheFirstRetractionTime(t *testing.T) { + db := openCache(t) + const id = "CVE-2026-4567" + seedPublished(t, db, id) + + first, err := NewTombstoner(db, func() time.Time { return fixtureClock }) + if err != nil { + t.Fatalf("NewTombstoner: %v", err) + } + res1, err := first.Tombstone(t.Context(), testFeedID, id, string(ReasonWithdrawn)) + if err != nil { + t.Fatalf("first Tombstone: %v", err) + } + if res1.AlreadyTombstoned { + t.Error("the first tombstone reported the row as already tombstoned") + } + + later := fixtureClock.Add(365 * 24 * time.Hour) + second, err := NewTombstoner(db, func() time.Time { return later }) + if err != nil { + t.Fatalf("NewTombstoner: %v", err) + } + res2, err := second.Tombstone(t.Context(), testFeedID, id, string(ReasonWithdrawn)) + if err != nil { + t.Fatalf("second Tombstone: %v", err) + } + if !res2.AlreadyTombstoned { + t.Error("the second tombstone did not report the row as already tombstoned") + } + if res2.TombstonedAt != res1.TombstonedAt { + t.Errorf("the retraction time moved from %q to %q on a repeat call", + res1.TombstonedAt, res2.TombstonedAt) + } + stored := scalar[string](t, db, `SELECT tombstoned_at FROM advisory WHERE source_id = ?`, id) + if stored != res1.TombstonedAt { + t.Errorf("tombstoned_at in the database is %q; the first retraction recorded %q", + stored, res1.TombstonedAt) + } + + // A state CHANGE is still applied, and still does not move the clock. + res3, err := second.Tombstone(t.Context(), testFeedID, id, string(ReasonRejected)) + if err != nil { + t.Fatalf("withdrawn -> rejected: %v", err) + } + if res3.State != cache.AdvisoryRejected { + t.Errorf("state = %q, want %q", res3.State, cache.AdvisoryRejected) + } + if res3.TombstonedAt != res1.TombstonedAt { + t.Errorf("a state change moved the retraction time to %q", res3.TombstonedAt) + } + if got := scalar[string](t, db, `SELECT state FROM advisory WHERE source_id = ?`, id); got != cache.AdvisoryRejected { + t.Errorf("stored state = %q, want %q", got, cache.AdvisoryRejected) + } +} + +// TestTombstoningAnAdvisoryTheCacheDoesNotHoldIsRefused. "Applied to nothing" +// and "applied successfully" are different facts. +func TestTombstoningAnAdvisoryTheCacheDoesNotHoldIsRefused(t *testing.T) { + db := openCache(t) + _, err := newTombstoner(t, db).Tombstone(t.Context(), testFeedID, "CVE-2026-0000", string(ReasonWithdrawn)) + if !errors.Is(err, ErrNoSuchAdvisory) { + t.Fatalf("err = %v; want ErrNoSuchAdvisory", err) + } +} + +// TestTombstoneRefusesAnIncompleteKey. +func TestTombstoneRefusesAnIncompleteKey(t *testing.T) { + db := openCache(t) + ts := newTombstoner(t, db) + for name, key := range map[string][2]string{ + "no source": {"", "CVE-2026-1"}, + "no source id": {testFeedID, ""}, + "whitespace": {" ", " "}, + } { + t.Run(name, func(t *testing.T) { + if _, err := ts.Tombstone(t.Context(), key[0], key[1], string(ReasonWithdrawn)); !errors.Is(err, ErrBadKey) { + t.Fatalf("err = %v; want ErrBadKey", err) + } + }) + } +} + +// TestNewTombstonerRefusesWithoutACache. +func TestNewTombstonerRefusesWithoutACache(t *testing.T) { + if _, err := NewTombstoner(nil, nil); !errors.Is(err, ErrNoCache) { + t.Fatalf("err = %v; want ErrNoCache", err) + } +} + +// --------------------------------------------------------------------------- +// The two allowlists, verified RED +// --------------------------------------------------------------------------- + +// TestTheReasonAllowlistRefusesWhatItDoesNotKnow. There is deliberately no +// default, because the default would be 'published' — the state that means NOT +// tombstoned. +func TestTheReasonAllowlistRefusesWhatItDoesNotKnow(t *testing.T) { + refused := []string{ + "", " ", "deleted", "removed", "disputed", "WITHDRAWN", "Withdrawn", + "withdrawn ", " withdrawn", "withdrawn\n", "published", "retracted", + "withdrawn; drop table advisory", + } + for _, r := range refused { + if _, err := StateForReason(r); !errors.Is(err, ErrReasonNotAllowed) { + t.Errorf("StateForReason(%q) was accepted; only an exact member of the allowlist may be", r) + } + } + for _, r := range Reasons() { + state, err := StateForReason(string(r)) + if err != nil { + t.Fatalf("StateForReason(%q): %v", r, err) + } + switch state { + case cache.AdvisoryWithdrawn, cache.AdvisoryRejected: + case cache.AdvisoryPublished: + t.Errorf("reason %q maps to %q, which is the state that means NOT tombstoned", r, state) + default: + t.Errorf("reason %q maps to %q, which is not a state the schema admits", r, state) + } + } +} + +// TestPoisonedIsRecordedAsWithdrawnAndTheReasonSurvives. research/06 Risk #4 +// names poisoned advisories separately from withdrawn ones; the schema's three +// states cannot, so the distinction lives on the result. +func TestPoisonedIsRecordedAsWithdrawnAndTheReasonSurvives(t *testing.T) { + db := openCache(t) + const id = "CVE-2026-5678" + seedPublished(t, db, id) + + res, err := newTombstoner(t, db).Tombstone(t.Context(), testFeedID, id, string(ReasonPoisoned)) + if err != nil { + t.Fatalf("Tombstone: %v", err) + } + if res.State != cache.AdvisoryWithdrawn { + t.Errorf("state = %q, want %q", res.State, cache.AdvisoryWithdrawn) + } + if res.Reason != ReasonPoisoned { + t.Errorf("Reason = %q; the reason as given must survive the mapping to a schema state", res.Reason) + } +} + +// TestTheStatementAllowlistRefusesADelete is the guard, VERIFIED RED against +// the statement it exists to stop. +func TestTheStatementAllowlistRefusesADelete(t *testing.T) { + forbidden := []string{ + `DELETE FROM advisory WHERE source = ? AND source_id = ?`, + `delete from advisory`, + `DELETE FROM finding WHERE source_id = ?`, + `DELETE FROM affected WHERE source_id = ?`, + `DROP TABLE advisory_fts`, + `INSERT INTO advisory_fts(advisory_fts) VALUES('rebuild')`, + `UPDATE advisory SET state = 'withdrawn'`, + strings.TrimSpace(selectAdvisoryRowSQL) + " LIMIT 1", + } + for _, q := range forbidden { + if err := checkStatement(q); !errors.Is(err, ErrStatementNotAllowed) { + t.Errorf("checkStatement(%q) = %v; want ErrStatementNotAllowed", condense(q), err) + } + } + // And the members really are members, or the guard above would pass by + // refusing everything. + for q := range allowedStatements { + if err := checkStatement(q); err != nil { + t.Errorf("checkStatement refused its own allowlist member %q: %v", condense(q), err) + } + } +} + +// TestTheStatementAllowlistCarriesNoRowRemoval closes the other route: not +// "somebody ran a DELETE" but "somebody ADDED one to the allowlist". +// +// advisory_fts is exempt by name and only by name: its DELETE is scoped to a +// rowid, removes an index entry rather than a record, and is required — a +// withdrawn advisory must stop matching a search. +func TestTheStatementAllowlistCarriesNoRowRemoval(t *testing.T) { + if len(allowedStatements) == 0 { + t.Fatal("the allowlist is empty, so this scan verifies nothing") + } + for q, reason := range allowedStatements { + if strings.TrimSpace(reason) == "" { + t.Errorf("allowlist member %q has no reason; an allowlist without reasons is not one", + condense(q)) + } + flat := strings.ToLower(condense(q)) + if !strings.Contains(flat, "delete") && !strings.Contains(flat, "drop") { + continue + } + if flat == strings.ToLower(condense(cache.DeleteAdvisoryFTSSQL)) { + continue + } + t.Errorf("allowlist member %q removes rows. A withdrawn or REJECTED advisory is "+ + "TOMBSTONED, never deleted (A.2 exit criterion 22).", condense(q)) + } +} + +// TestEveryStatementThisPackageRunsIsOnTheAllowlist walks this package's own +// source and requires every SQL-shaped string constant to be an allowlist +// member. +// +// It is the complement of checkStatement: the gate proves that what reaches +// the driver was checked, and this proves nobody added a statement that +// bypasses the gate by not being a constant it knows about. +func TestEveryStatementThisPackageRunsIsOnTheAllowlist(t *testing.T) { + for _, q := range []string{ + selectAdvisoryRowSQL, + selectFindingsForAdvisorySQL, + selectInvalidatedFindingsSQL, + cache.UpsertAdvisorySQL, + cache.DeleteAdvisoryFTSSQL, + } { + if err := checkStatement(q); err != nil { + t.Errorf("a statement this package executes is not allowlisted: %v", err) + } + } + if _, ok := allowedStatements[strings.TrimSpace(cache.UpsertAdvisorySQL)]; !ok { + t.Fatal("the shared advisory write shape is not on the allowlist, so the tombstone would " + + "have to compose its own UPDATE — which is the second write shape " + + "internal/ingest/cache/schema.go exports the first one to prevent") + } +} + +// --------------------------------------------------------------------------- +// Cross-cutting: this package writes through A.14 and invents no vocabulary +// --------------------------------------------------------------------------- + +// TestDriftRecordIsDeltaRecord holds the alias. A parallel record type would +// be a parallel write path a week later, and the two would drift. +func TestDriftRecordIsDeltaRecord(t *testing.T) { + var r Record + var d delta.Record + if reflect.TypeOf(r) != reflect.TypeOf(d) { + t.Fatalf("drift.Record is %v and delta.Record is %v; they must be the same type", + reflect.TypeOf(r), reflect.TypeOf(d)) + } +} + +// TestAdvisoryStatesComeFromTheCachePackage: the states this package writes +// are internal/ingest/cache's constants, and the schema's own CHECK is the +// arbiter. A bare string literal for an enum value is how ten cross-area +// defects happened on this project. +func TestAdvisoryStatesComeFromTheCachePackage(t *testing.T) { + literals, err := cache.CheckLiterals("advisory_state") + if err != nil { + t.Fatalf("cache.CheckLiterals: %v", err) + } + legal := map[string]bool{} + for _, l := range literals { + legal[l] = true + } + if len(legal) != 3 { + t.Fatalf("the schema's advisory_state CHECK admits %d values (%v); this package was "+ + "written against three", len(legal), literals) + } + for _, r := range Reasons() { + state, err := StateForReason(string(r)) + if err != nil { + t.Fatalf("StateForReason(%q): %v", r, err) + } + if !legal[state] { + t.Errorf("reason %q writes state %q, which the schema's CHECK does not admit (%v)", + r, state, literals) + } + } + + // The one state literal this package embeds in SQL is bound to the same + // constant. A bare literal inside a query string is invisible to the Go + // compiler and is how an enum value drifts from its owner. + want := "'" + cache.AdvisoryPublished + "'" + if !strings.Contains(selectInvalidatedFindingsSQL, want) { + t.Errorf("the invalidated-findings query does not compare against %s; its literal has "+ + "drifted from internal/ingest/cache's constant:\n%s", want, condense(selectInvalidatedFindingsSQL)) + } +} + +// TestADocumentWithTrailingContentIsRefused: two concatenated advisories parse +// as "the first one" to a streaming decoder, and "the first one" is a dropped +// record wearing a successful parse. +func TestADocumentWithTrailingContentIsRefused(t *testing.T) { + first := mustJSON(t, cveDoc("CVE-2026-9101", "5.1")) + second := mustJSON(t, cveDoc("CVE-2026-9102", "5.1")) + joined := append(append([]byte{}, first...), second...) + + if _, _, err := Parse(testFeedID, joined); !errors.Is(err, ErrNotAnObject) { + t.Fatalf("err = %v; want ErrNotAnObject", err) + } +} + +// TestTheDegradingCodeTableDefaultsToDegrading. A code added later and +// forgotten must degrade, not sail through. +func TestTheDegradingCodeTableDefaultsToDegrading(t *testing.T) { + if !Code("a-code-nobody-listed").Degrading() { + t.Fatal("an unlisted code does not degrade; the default has to be the safe one, because " + + "the unsafe one is invisible") + } + if CodeUnknownField.Degrading() { + t.Error("CodeUnknownField degrades; a flag that is always set is a flag nobody reads") + } + for _, c := range []Code{CodeUnknownDataVersion, CodeMissingDataVersion, + CodeUnknownFieldLoadBearing, CodeUndecodable, CodeDecoderDegraded, CodeFieldsTruncated} { + if !c.Degrading() { + t.Errorf("%s does not degrade", c) + } + } +} + +// TestAMissingDataVersionIsTreatedAsUnknown. A document that will not say what +// it is has not earned the benefit of the doubt. +func TestAMissingDataVersionIsTreatedAsUnknown(t *testing.T) { + doc := cveDoc("CVE-2026-6789", "5.1") + delete(doc, "dataVersion") + + rec, rep, err := Parse(testFeedID, mustJSON(t, doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if !rep.Degraded || !rep.Has(CodeMissingDataVersion) { + t.Fatalf("a record with no dataVersion was not degraded: %s", rep) + } + if !rec.ParseDegraded { + t.Error("the record's flag disagrees with the report") + } +} + +// TestReportStringNamesTheFieldsItDegradedOn. "Degraded" without "which" is a +// status nobody can act on. +func TestReportStringNamesTheFieldsItDegradedOn(t *testing.T) { + doc := cveDoc("CVE-2026-7890", "5.1") + firstAffectedVersion(t, doc)["someNewRangeKey"] = "x" + + _, rep, err := Parse(testFeedID, mustJSON(t, doc)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + line := rep.String() + for _, want := range []string{"DEGRADED", "someNewRangeKey", string(CodeUnknownFieldLoadBearing)} { + if !strings.Contains(line, want) { + t.Errorf("Report.String() does not mention %q:\n%s", want, line) + } + } +} + +// TestOversizedDocumentsAreRefusedOnWhatArrived. +func TestOversizedDocumentsAreRefusedOnWhatArrived(t *testing.T) { + raw := bytes.Repeat([]byte("a"), delta.MaxDocumentBytes+1) + if _, _, err := Parse(testFeedID, raw); !errors.Is(err, ErrDocumentTooLarge) { + t.Fatalf("err = %v; want ErrDocumentTooLarge", err) + } +} + +// TestFindingsReferencingRefusesWithoutACache. +func TestFindingsReferencingRefusesWithoutACache(t *testing.T) { + if _, err := FindingsReferencing(context.Background(), nil, "s", "i"); !errors.Is(err, ErrNoCache) { + t.Fatalf("err = %v; want ErrNoCache", err) + } + if _, err := InvalidatedFindings(context.Background(), nil); !errors.Is(err, ErrNoCache) { + t.Fatalf("err = %v; want ErrNoCache", err) + } +} + +// TestFindingsAreOrderedAndComplete. A read path that returns rows in an +// arbitrary order makes a diff between two runs unreadable, which is how a +// missing row stops being noticed. +func TestFindingsAreOrderedAndComplete(t *testing.T) { + db := openCache(t) + const id = "CVE-2026-8899" + parseAndApply(t, db, mustJSON(t, cveDoc(id, "5.1"))) + for _, n := range []string{"c", "a", "b"} { + insertFinding(t, db, "finding-"+n, testFeedID, id) + } + + got, err := FindingsReferencing(t.Context(), db, testFeedID, id) + if err != nil { + t.Fatalf("FindingsReferencing: %v", err) + } + ids := make([]string, 0, len(got)) + for _, f := range got { + ids = append(ids, f.FindingID) + } + want := []string{"finding-a", "finding-b", "finding-c"} + if !reflect.DeepEqual(ids, want) { + t.Fatalf("findings = %v, want %v", ids, want) + } + if !sort.StringsAreSorted(ids) { + t.Error("the result is not ordered") + } +} diff --git a/internal/ingest/drift/tombstone.go b/internal/ingest/drift/tombstone.go new file mode 100644 index 0000000..5234fec --- /dev/null +++ b/internal/ingest/drift/tombstone.go @@ -0,0 +1,701 @@ +// tombstone.go is the other half of A.16: what happens when a publisher takes +// an advisory back. +// +// =========================================================================== +// THE ROW IS NEVER DELETED, AND THIS FILE CANNOT DELETE IT +// =========================================================================== +// +// research/06 Risk #4: "Withdrawn and poisoned advisories... Anvil must +// propagate retractions as tombstones... must be able to re-open and +// invalidate a prior finding when its advisory is withdrawn." A.2 exit +// criterion 22 says the same thing from the schema's side, and +// internal/ingest/cache enforces the pairing with a named CHECK: a state that +// is not 'published' must carry a `tombstoned_at`. +// +// A DELETE would satisfy neither. A finding that referenced the advisory would +// lose the row it points at, and "this advisory was retracted, re-open the +// finding" would become indistinguishable from "this advisory never existed" — +// which is the silent-vanishing outcome the spine's regression checking is +// specifically written against. +// +// The rule is not left to care. Every statement this package hands to the +// driver must be on allowedStatements, an ALLOWLIST of exact texts. There is +// no DELETE against `advisory` on it and no way to add one by accident: a +// statement that is not a member is refused with ErrStatementNotAllowed before +// it reaches the database. The shape is A.14's, deliberately — a denylist of +// forbidden verbs is what this project has already lost three guards to. +// +// =========================================================================== +// WHY THE WRITE GOES THROUGH cache.UpsertAdvisorySQL AND NOT THROUGH AN UPDATE +// =========================================================================== +// +// internal/ingest/cache/schema.go names this step explicitly: the advisory +// write shape is exported "because the alternative — each of A.7, A.8, A.14, +// A.15 and A.16 composing its own upsert — is how a second, subtly different +// write shape enters a schema and breaks the FTS linkage or the tombstone +// invariant with no error message." +// +// So the tombstone is a READ-MODIFY-WRITE through the shared statement: the +// row is read back, two columns are replaced, and the whole row is bound to +// the same ON CONFLICT DO UPDATE every other writer uses. It costs one extra +// SELECT and buys the property that there is exactly one statement in this +// system that writes an `advisory` row. +// +// Two consequences worth stating, because both would otherwise look like +// oversights: +// +// - THE LICENCE COLUMNS ARE RE-BOUND FROM THE ROW, never re-derived. A +// retraction is not a licence decision. A.4's Gate owns that, and a second +// gate invoked from here would be an unreviewed one. +// - THE ROWID IS PRESERVED, because ON CONFLICT DO UPDATE updates in place. +// INSERT OR REPLACE would delete and re-insert under a NEW rowid and +// orphan the FTS entry silently — the exact defect cache/schema.go +// documents at the `advisory` table. +package drift + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/sanitize" +) + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +var ( + // ErrNoCache is a Tombstoner built without the A.2 ingestion cache. + ErrNoCache = fmt.Errorf("%w: no ingestion cache", ErrDriftRefused) + + // ErrNoSuchAdvisory is a tombstone for a (source, source_id) the cache + // does not hold. It is a REFUSAL and not a no-op: "the retraction was + // applied" and "there was nothing to apply it to" are different facts, and + // a caller that cannot tell them apart will report the second as the + // first. + ErrNoSuchAdvisory = fmt.Errorf("%w: no such advisory", ErrDriftRefused) + + // ErrReasonNotAllowed is the reason allowlist refusing a retraction reason + // it does not know. See Reason. + ErrReasonNotAllowed = fmt.Errorf("%w: retraction reason not on the allowlist", ErrDriftRefused) + + // ErrStatementNotAllowed is the SQL allowlist refusing a statement. It is + // what stands between a retraction and a DELETE. + ErrStatementNotAllowed = fmt.Errorf("%w: statement not on the allowlist", ErrDriftRefused) + + // ErrBadKey is an empty source or source_id. + ErrBadKey = fmt.Errorf("%w: incomplete primary key", ErrDriftRefused) +) + +// --------------------------------------------------------------------------- +// The reason allowlist +// --------------------------------------------------------------------------- + +// Reason is why an advisory was retracted. It is an ALLOWLIST: a value not +// named below is refused, and no reason is ever defaulted. +// +// The default is what matters here. `advisory.state` has three legal values +// and 'published' is one of them, so a reason that fell through to a default +// would tombstone the row into the state that means NOT tombstoned — and the +// schema's advisory_tombstone_paired CHECK would then refuse the row for +// carrying a `tombstoned_at`, which reads at the call site as a database fault +// rather than as a missing case. +type Reason string + +const ( + // ReasonWithdrawn is a publisher retracting an advisory: OSV's + // `withdrawn`, a GHSA withdrawal. + ReasonWithdrawn Reason = "withdrawn" + + // ReasonRejected is a CVE record in the REJECTED state. + ReasonRejected Reason = "rejected" + + // ReasonPoisoned is research/06 Risk #4's other half — an advisory Anvil + // itself distrusts rather than one the publisher pulled. + // + // It maps to the WITHDRAWN state, and the mapping is a fact about the + // schema and not a judgement: `advisory.state`'s CHECK admits exactly + // three values, A.2 is merged and frozen, and inventing a fourth here + // would be a write the database refuses. The distinction survives on the + // returned TombstoneResult, which carries the Reason as given. + ReasonPoisoned Reason = "poisoned" +) + +// reasonState maps each allowlisted reason to the `advisory.state` value it +// writes. The values are internal/ingest/cache's CONSTANTS and never string +// literals: a bare literal for an enum value is how ten cross-area defects +// happened on this project. +var reasonState = map[Reason]string{ + ReasonWithdrawn: cache.AdvisoryWithdrawn, + ReasonRejected: cache.AdvisoryRejected, + ReasonPoisoned: cache.AdvisoryWithdrawn, +} + +// Reasons returns the allowlisted retraction reasons in a stable order. +func Reasons() []Reason { + out := make([]Reason, 0, len(reasonState)) + for r := range reasonState { + out = append(out, r) + } + sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) + return out +} + +// StateForReason resolves a retraction reason to the `advisory.state` it +// writes, or refuses. +// +// THE COMPARISON IS EXACT. No trimming, no case folding, no "withdrawn " with +// a trailing space quietly accepted: a caller passing a reason it built by +// string concatenation should find out here rather than have the value +// normalised into something it did not mean. +func StateForReason(reason string) (string, error) { + state, ok := reasonState[Reason(reason)] + if !ok { + allowed := make([]string, 0, len(reasonState)) + for _, r := range Reasons() { + allowed = append(allowed, string(r)) + } + return "", refuse(ErrReasonNotAllowed, + "%q is not a retraction reason this package knows; the allowlist is [%s]. "+ + "If a new retraction kind is real, add it with the advisory state it maps to — "+ + "there is deliberately no default, because the default would be 'published'.", + clip(reason, 64), strings.Join(allowed, " ")) + } + return state, nil +} + +// --------------------------------------------------------------------------- +// The statement allowlist +// --------------------------------------------------------------------------- + +// selectAdvisoryRowSQL reads back every column the shared upsert binds, plus +// the rowid advisory_fts is addressed by. It is a read: nothing here can +// change a row. +const selectAdvisoryRowSQL = ` +SELECT rowid, cve_id, published, modified, state, tombstoned_at, + severity, cvss_vector, cvss_score, epss_score, epss_as_of, kev, + license_spdx, license_manual_note, license_tier, anvil_trust, + as_of, staleness_seconds, parse_degraded, data_version, raw_json +FROM advisory WHERE source = ? AND source_id = ?` + +// selectFindingsForAdvisorySQL is the "active findings referencing this +// advisory" query A.16's validation names. +// +// IT IS A LEFT JOIN, and that is the load-bearing detail. An inner join would +// drop a finding whose advisory row had gone missing — which is precisely the +// silent vanishing this whole file exists to prevent — so a missing advisory +// surfaces as a row with no state, marked invalidated, rather than as a +// shorter result set nobody can see the shape of. +const selectFindingsForAdvisorySQL = ` +SELECT f.id, f.collector, f.source, f.source_id, f.package, f.installed_version, + f.ecosystem, f.remediable_by_agent, f.detected_at, a.state, a.tombstoned_at +FROM finding f +LEFT JOIN advisory a ON a.source = f.source AND a.source_id = f.source_id +WHERE f.source = ? AND f.source_id = ? +ORDER BY f.id` + +// selectInvalidatedFindingsSQL is the same read across the whole cache: every +// finding whose advisory is no longer published. It is the work list for the +// re-open pass the spine's regression checking requires. +const selectInvalidatedFindingsSQL = ` +SELECT f.id, f.collector, f.source, f.source_id, f.package, f.installed_version, + f.ecosystem, f.remediable_by_agent, f.detected_at, a.state, a.tombstoned_at +FROM finding f +LEFT JOIN advisory a ON a.source = f.source AND a.source_id = f.source_id +WHERE a.state IS NULL OR a.state <> 'published' +ORDER BY f.source, f.source_id, f.id` + +// allowedStatements is THE GUARD. Every statement this package hands to the +// database driver must be a member, compared as exact text after trimming. +// +// THERE IS NO DELETE AGAINST `advisory` HERE AND THERE MUST NEVER BE ONE. Two +// tests in drift_test.go hold that: one runs a hand-written +// `DELETE FROM advisory ...` through this package's own exec helper and +// requires the refusal, and one scans the allowlist's KEYS for any statement +// that deletes from `advisory` or `finding`, so the guard cannot be defeated +// by adding a member rather than by routing around the check. +// +// It is a package-level var and not a function on purpose, for A.14's reason: +// internal/ingest/sanitize's writer guard walks FUNCTION bodies looking for +// the names of the cache's advisory write shapes, and a function that named +// them only to build this map would be flagged as an unsanitised writer. A var +// initialiser is not a function body, so the guard sees the real write site +// and not this one. +// +// The value is the reason the member is allowed. "It seemed necessary" is not +// one of them. +var allowedStatements = map[string]string{ + strings.TrimSpace(cache.UpsertAdvisorySQL): "the ONE shared advisory write shape, ON CONFLICT DO UPDATE, " + + "RETURNING the rowid advisory_fts is addressed by. The tombstone is a read-modify-write through " + + "this statement rather than an UPDATE of its own, so there is exactly one statement in this " + + "system that writes an advisory row.", + strings.TrimSpace(cache.DeleteAdvisoryFTSSQL): "one FTS row by rowid, for a tombstoned advisory. The " + + "`advisory` row itself is never deleted (A.2 exit criterion 22); its TEXT stops matching, which " + + "is the same requirement seen from the index side.", + strings.TrimSpace(selectAdvisoryRowSQL): "read-only: the row about to be re-bound, by primary key.", + strings.TrimSpace(selectFindingsForAdvisorySQL): "read-only: findings referencing one advisory, with its state.", + strings.TrimSpace(selectInvalidatedFindingsSQL): "read-only: every finding whose advisory is tombstoned.", +} + +// checkStatement is the allowlist gate. Every database call in this package +// goes through it, and nothing else in this package may call the driver. +func checkStatement(q string) error { + if _, ok := allowedStatements[strings.TrimSpace(q)]; ok { + return nil + } + return refuse(ErrStatementNotAllowed, + "this package may only execute statements on its allowlist and this one is not on it:\n\t%s\n"+ + "If it removes an `advisory` row, it is the thing A.16's packet forbids outright: a withdrawn "+ + "or REJECTED advisory is TOMBSTONED so a prior finding referencing it can be re-opened and "+ + "invalidated, and a removed row cannot be referenced at all.", + condense(q)) +} + +// condense renders a statement on one line for an error message. +func condense(q string) string { return strings.Join(strings.Fields(q), " ") } + +// clip bounds a value quoted back into an error message. +func clip(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "..." +} + +func execTx(ctx context.Context, tx *sql.Tx, q string, args ...any) error { + if err := checkStatement(q); err != nil { + return err + } + _, err := tx.ExecContext(ctx, q, args...) + return err +} + +func queryRowTx(ctx context.Context, tx *sql.Tx, q string, args ...any) (*sql.Row, error) { + if err := checkStatement(q); err != nil { + return nil, err + } + return tx.QueryRowContext(ctx, q, args...), nil +} + +func queryDB(ctx context.Context, db *sql.DB, q string, args ...any) (*sql.Rows, error) { + if err := checkStatement(q); err != nil { + return nil, err + } + return db.QueryContext(ctx, q, args...) +} + +func queryTx(ctx context.Context, tx *sql.Tx, q string, args ...any) (*sql.Rows, error) { + if err := checkStatement(q); err != nil { + return nil, err + } + return tx.QueryContext(ctx, q, args...) +} + +// --------------------------------------------------------------------------- +// Tombstoner +// --------------------------------------------------------------------------- + +// Tombstoner applies retractions to the A.2 ingestion cache. +// +// A NOTE ON THE SHAPE, reported rather than silently applied. A.16's packet +// specifies `func Tombstone(source, sourceID string, reason string) error`. +// That signature has nowhere to put the database handle, the context or the +// clock, so implementing it literally would mean a package-level database +// global — an ambient, untestable, unclosable handle, and a worse outcome than +// the deviation. The METHOD's name and its (source, sourceID, reason) +// arguments are exactly the packet's; the receiver carries the handle and the +// clock, and the return adds a result value so a caller can see what actually +// changed instead of inferring it from a nil error. +type Tombstoner struct { + db *sql.DB + now func() time.Time +} + +// NewTombstoner binds a Tombstoner to the A.2 cache. +// +// now may be nil, in which case time.Now is used. A test supplies its own so +// that "the retraction timestamp did not move on the second call" is an +// assertion about the code rather than about how fast the machine ran. +func NewTombstoner(db *sql.DB, now func() time.Time) (*Tombstoner, error) { + if db == nil { + return nil, refuse(ErrNoCache, "a Tombstoner needs the A.2 ingestion cache") + } + if now == nil { + now = time.Now + } + return &Tombstoner{db: db, now: now}, nil +} + +// TombstoneResult is what one retraction did. Every field is an observation, +// so a caller never has to infer the outcome from a nil error. +type TombstoneResult struct { + Source string + SourceID string + + // Reason is the reason as given, unmodified. It is carried because the + // schema's three states cannot express all of them: ReasonPoisoned and + // ReasonWithdrawn both write 'withdrawn'. + Reason Reason + + // PreviousState is what `advisory.state` held before, and State is what it + // holds now. + PreviousState string + State string + + // TombstonedAt is the retraction timestamp now on the row. On a row that + // was ALREADY tombstoned it is the ORIGINAL one: a retraction happened + // when it happened, and a second call must not move the clock forward and + // make the advisory look freshly withdrawn. + TombstonedAt string + + // AlreadyTombstoned is true when the row was not published when it + // arrived. The write still ran if the state changed (withdrawn -> rejected + // is a real transition); it did not if nothing would have changed. + AlreadyTombstoned bool + + // RowID is the `advisory` rowid, which is the only address advisory_fts + // has for the row, and FTSDeleted records that its text was removed from + // the index. The `advisory` row itself is never removed. + RowID int64 + FTSDeleted bool + + // InvalidatedFindings is how many `finding` rows now reference a + // tombstoned advisory. They are NOT deleted and NOT hidden: they become + // visible as invalidated, which is what lets a prior finding be re-opened. + InvalidatedFindings int + + // Sanitized reports what A.3 removed from the values read back out of the + // cache. It should be zero on every row this system wrote. A non-zero + // value means something reached the cache unsanitized, which is worth + // surfacing loudly — and worth surfacing WITHOUT blocking the retraction, + // because refusing to record a withdrawal on account of a dirty prose + // field would leave a retracted advisory live. + Sanitized sanitize.SanitizeStats +} + +// Tombstone marks one advisory as retracted WITHOUT deleting its row. +// +// What it does, in one transaction: +// +// 1. reads the row back by primary key, refusing if there is none; +// 2. replaces `state` and `tombstoned_at` and re-binds every other column as +// it was, through cache.UpsertAdvisorySQL — the one shared write shape; +// 3. removes the advisory's TEXT from advisory_fts by rowid, so a retracted +// advisory stops matching a search while its row stays addressable; +// 4. counts the findings that now reference a tombstoned advisory. +// +// It never issues a DELETE against `advisory`, `affected`, `cve_alias` or +// `finding`, and allowedStatements is what makes that a property rather than a +// promise. +func (t *Tombstoner) Tombstone(ctx context.Context, source, sourceID, reason string) (TombstoneResult, error) { + res := TombstoneResult{Source: source, SourceID: sourceID, Reason: Reason(reason)} + + if strings.TrimSpace(source) == "" || strings.TrimSpace(sourceID) == "" { + return res, refuse(ErrBadKey, + "a tombstone needs both halves of the primary key; got source %q and source_id %q", + clip(source, 64), clip(sourceID, 64)) + } + state, err := StateForReason(reason) + if err != nil { + return res, err + } + res.State = state + + tx, err := t.db.BeginTx(ctx, nil) + if err != nil { + return res, fmt.Errorf("drift: opening transaction to tombstone %s/%s: %w", source, sourceID, err) + } + defer func() { _ = tx.Rollback() }() + + row, err := t.readRow(ctx, tx, source, sourceID) + if err != nil { + return res, err + } + res.PreviousState = row.state + res.RowID = row.rowid + res.AlreadyTombstoned = row.state != cache.AdvisoryPublished + + // A retraction happened when it happened. Re-running the pass must not + // move the timestamp forward, or a nightly reconcile would keep making a + // year-old withdrawal look like today's news. + stamp := strings.TrimSpace(row.tombstonedAt) + if stamp == "" { + stamp = t.now().UTC().Format(time.RFC3339) + } + res.TombstonedAt = stamp + + if res.AlreadyTombstoned && row.state == state { + // Nothing would change. Commit nothing and say so; the finding count + // is still reported, because that is what the caller asked about. + n, err := countFindings(ctx, tx, source, sourceID) + if err != nil { + return res, err + } + res.InvalidatedFindings = n + return res, nil + } + + rowid, stats, err := t.writeTombstonedRow(ctx, tx, source, sourceID, state, stamp, row) + res.Sanitized = stats + if err != nil { + return res, err + } + res.RowID = rowid + + if err := execTx(ctx, tx, cache.DeleteAdvisoryFTSSQL, rowid); err != nil { + return res, fmt.Errorf("drift: unindexing tombstoned %s/%s: %w", source, sourceID, err) + } + res.FTSDeleted = true + + n, err := countFindings(ctx, tx, source, sourceID) + if err != nil { + return res, err + } + res.InvalidatedFindings = n + + if err := tx.Commit(); err != nil { + return res, fmt.Errorf("drift: committing the tombstone for %s/%s: %w", source, sourceID, err) + } + return res, nil +} + +// advisoryRow is one row read back, ready to be re-bound. +type advisoryRow struct { + rowid int64 + cveID sql.NullString + published sql.NullString + modified sql.NullString + state string + tombstonedAt string + severity sql.NullString + cvssVector sql.NullString + cvssScore sql.NullFloat64 + epssScore sql.NullFloat64 + epssAsOf sql.NullString + kev int + licenseSPDX sql.NullString + licenseNote sql.NullString + licenseTier int + anvilTrust string + asOf string + stalenessSeconds int + parseDegraded int + dataVersion sql.NullString + rawJSON []byte +} + +// readRow reads one advisory back by primary key. It does not clean and does +// not write; see writeTombstonedRow for where A.3 runs and why it runs there. +func (t *Tombstoner) readRow(ctx context.Context, tx *sql.Tx, source, sourceID string) (advisoryRow, error) { + var r advisoryRow + var tombstonedAt sql.NullString + + q, err := queryRowTx(ctx, tx, selectAdvisoryRowSQL, source, sourceID) + if err != nil { + return r, err + } + err = q.Scan(&r.rowid, &r.cveID, &r.published, &r.modified, &r.state, &tombstonedAt, + &r.severity, &r.cvssVector, &r.cvssScore, &r.epssScore, &r.epssAsOf, &r.kev, + &r.licenseSPDX, &r.licenseNote, &r.licenseTier, &r.anvilTrust, + &r.asOf, &r.stalenessSeconds, &r.parseDegraded, &r.dataVersion, &r.rawJSON) + switch { + case errors.Is(err, sql.ErrNoRows): + return r, refuse(ErrNoSuchAdvisory, + "the cache holds no advisory %s/%s, so there is nothing to tombstone. This is reported "+ + "rather than treated as a no-op: a retraction applied to nothing and a retraction "+ + "applied successfully are different facts.", source, sourceID) + case err != nil: + return r, fmt.Errorf("drift: reading advisory %s/%s: %w", source, sourceID, err) + } + r.tombstonedAt = tombstonedAt.String + return r, nil +} + +// writeTombstonedRow re-binds the row with its state and tombstone replaced. +// +// A.3 RUNS IN THIS FUNCTION, in the same body as the bind, and not one call +// further up. That is deliberate and it is the shape A.14 chose for the same +// reason: internal/ingest/sanitize's writer guard resolves the package-local +// call graph by NAME and can therefore check "the function that binds is the +// function that cleans". A sanitiser two frames away is a claim the guard +// cannot verify, and a claim nothing verifies is the state this project has +// already paid to leave. +// +// SANITIZE RATHER THAN ASSERT, and the direction matters. AssertAllSanitized +// would REFUSE the tombstone on a row that reached the cache dirty, leaving a +// retracted advisory live because some prose field carried a zero-width +// character. Sanitize cleans the value, lets the retraction land, and reports +// what it removed on TombstoneResult.Sanitized — loud, without being able to +// block a withdrawal. +// +// `raw_json` is bound BYTE-FOR-BYTE as it was read and is NOT sanitized. It is +// the publisher's own document, CVE-TOU requires records be stored verbatim +// (research/06 "License"), and a retraction is a fact about the row rather +// than an edit to the publisher's bytes. +func (t *Tombstoner) writeTombstonedRow( + ctx context.Context, + tx *sql.Tx, + source, sourceID, state, stamp string, + r advisoryRow, +) (int64, sanitize.SanitizeStats, error) { + var stats sanitize.SanitizeStats + clean := func(v string) string { + out, st := sanitize.Sanitize(v) + stats.Merge(st) + return out + } + cleanNull := func(v sql.NullString) any { + if !v.Valid { + return nil + } + return clean(v.String) + } + + row, err := queryRowTx(ctx, tx, cache.UpsertAdvisorySQL, + source, sourceID, cleanNull(r.cveID), cleanNull(r.published), cleanNull(r.modified), + state, clean(stamp), + cleanNull(r.severity), cleanNull(r.cvssVector), nullFloat(r.cvssScore), nullFloat(r.epssScore), + cleanNull(r.epssAsOf), r.kev, + cleanNull(r.licenseSPDX), cleanNull(r.licenseNote), r.licenseTier, clean(r.anvilTrust), + clean(r.asOf), r.stalenessSeconds, r.parseDegraded, cleanNull(r.dataVersion), r.rawJSON) + if err != nil { + return 0, stats, err + } + var rowid int64 + if err := row.Scan(&rowid); err != nil { + return 0, stats, fmt.Errorf("drift: writing the tombstone for %s/%s: %w", source, sourceID, err) + } + if rowid != r.rowid { + // ON CONFLICT DO UPDATE updates in place, so this cannot happen. It is + // checked because the version that CAN happen — INSERT OR REPLACE — + // fails exactly this way and fails silently, orphaning the FTS entry + // under the old rowid. + return 0, stats, fmt.Errorf("drift: %s/%s moved from rowid %d to %d during a tombstone; "+ + "the FTS entry under the old rowid would be orphaned", source, sourceID, r.rowid, rowid) + } + return rowid, stats, nil +} + +// nullFloat renders a nullable score as the `any` the shared upsert binds. A +// zero CVSS base score is a real value and must not become NULL, and "absent" +// must not become 0.0 — a comparator that cannot tell them apart ranks an +// unscored advisory as harmless. +func nullFloat(v sql.NullFloat64) any { + if !v.Valid { + return nil + } + return v.Float64 +} + +// --------------------------------------------------------------------------- +// The read path: findings that referenced a retracted advisory +// --------------------------------------------------------------------------- + +// FindingStatus is one Lane A finding together with the current state of the +// advisory it rests on. +// +// FindingID IS NOT A FINGERPRINT. `finding.id` is a Lane-A-local identifier; +// internal/ingest/cache/schema.go says so at the table, and plan/00-SPINE.md +// S6 allows exactly one fingerprint algorithm, anvil-fp/v1, owned by +// internal/record. This field must never be presented as, derived into, or +// compared against one. +type FindingStatus struct { + FindingID string + Collector string + Source string + SourceID string + Package string + InstalledVersion string + Ecosystem string + RemediableByAgent bool + DetectedAt string + + // AdvisoryState is `advisory.state`, or "" when no advisory row exists. + AdvisoryState string + + // TombstonedAt is when the advisory was retracted, empty when it was not. + TombstonedAt string + + // Invalidated is true when the finding's advisory is no longer published — + // including the case where the advisory row is missing entirely. It is a + // FLAG AND NOT A FILTER: the row is returned either way, because a prior + // finding must be re-openable and invalidated rather than silently gone. + Invalidated bool +} + +// FindingsReferencing returns every finding that rests on one advisory, +// invalidated or not. +// +// This is the query A.16's validation names: after a tombstone, a finding that +// referenced the advisory comes back marked invalidated instead of +// disappearing from the result set. +func FindingsReferencing(ctx context.Context, db *sql.DB, source, sourceID string) ([]FindingStatus, error) { + if db == nil { + return nil, refuse(ErrNoCache, "FindingsReferencing needs the A.2 ingestion cache") + } + rows, err := queryDB(ctx, db, selectFindingsForAdvisorySQL, source, sourceID) + if err != nil { + return nil, err + } + return scanFindings(rows) +} + +// InvalidatedFindings returns every finding in the cache whose advisory is no +// longer published: the work list for the re-open pass the spine's regression +// checking requires. +func InvalidatedFindings(ctx context.Context, db *sql.DB) ([]FindingStatus, error) { + if db == nil { + return nil, refuse(ErrNoCache, "InvalidatedFindings needs the A.2 ingestion cache") + } + rows, err := queryDB(ctx, db, selectInvalidatedFindingsSQL) + if err != nil { + return nil, err + } + return scanFindings(rows) +} + +func scanFindings(rows *sql.Rows) ([]FindingStatus, error) { + defer func() { _ = rows.Close() }() + var out []FindingStatus + for rows.Next() { + var f FindingStatus + var remediable int + var state, tombstoned sql.NullString + if err := rows.Scan(&f.FindingID, &f.Collector, &f.Source, &f.SourceID, &f.Package, + &f.InstalledVersion, &f.Ecosystem, &remediable, &f.DetectedAt, &state, &tombstoned); err != nil { + return nil, fmt.Errorf("drift: reading findings: %w", err) + } + f.RemediableByAgent = remediable != 0 + f.AdvisoryState = state.String + f.TombstonedAt = tombstoned.String + // A missing advisory row counts as invalidated. It should be + // unreachable under the schema's foreign key, and if it ever happens + // the finding must surface as unsupported rather than as healthy. + f.Invalidated = !state.Valid || state.String != cache.AdvisoryPublished + out = append(out, f) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("drift: reading findings: %w", err) + } + return out, nil +} + +// countFindings counts the findings resting on one advisory, inside the +// transaction that is retracting it. +func countFindings(ctx context.Context, tx *sql.Tx, source, sourceID string) (int, error) { + rows, err := queryTx(ctx, tx, selectFindingsForAdvisorySQL, source, sourceID) + if err != nil { + return 0, fmt.Errorf("drift: counting findings for %s/%s: %w", source, sourceID, err) + } + found, err := scanFindings(rows) + if err != nil { + return 0, err + } + return len(found), nil +} diff --git a/internal/ingest/reconcile/baseline.go b/internal/ingest/reconcile/baseline.go new file mode 100644 index 0000000..313157d --- /dev/null +++ b/internal/ingest/reconcile/baseline.go @@ -0,0 +1,1483 @@ +// Package reconcile owns A.15: the WEEKLY FULL-BASELINE SELF-HEAL. +// +// This is step A.15 of plan/20-lane-a-ingestion-sca.md. Lane A is the +// zero-inference half of Anvil (plan/00-SPINE.md S1): CVE/OSV/GHSA describe +// vulnerable PACKAGE VERSIONS and a version comparator answers that exactly +// and for free. Nothing here infers anything, calls a model, ranks anything, +// or emits a fingerprint. Every decision below is a documented rule over two +// row sets. +// +// # Why a periodic full baseline exists at all +// +// research/06 Recommendation §3: "cvelistV5 full baseline weekly as the +// self-heal (catches anything the delta pipeline dropped)." +// +// A delta stream DRIFTS. A missed delta, a malformed record, a 200 that +// arrived truncated, a batch that failed after the cursor moved — any one of +// them leaves the cache subtly wrong FOREVER, with nothing anywhere surfacing +// it. Nothing in the incremental path can detect its own gap: the cursor is +// derived from the rows, so a row that never arrived has no cursor entry to +// look wrong. That is the same failure shape as fingerprint drift (spine S6), +// and the answer is the same one: periodically rebuild from ground truth and +// DIFF, rather than trusting the incremental path to have been complete. +// +// # This package REPORTS. A silent self-heal is indistinguishable from one +// that never ran. +// +// If the fresh baseline disagrees with the delta-built cache, that +// disagreement is a signal ABOUT THE DELTA PATH and it must be visible. +// ReconcileReport therefore carries, separately and by name: how many rows +// matched, how many the live cache was missing, how many it held at an older +// version, how many diverged at the SAME version, how many it held that ground +// truth does not have, and how many were actually written back. It also +// carries a bounded sample of the disagreeing keys, because a count tells an +// operator that the delta path is dropping records and a sample tells them +// which ones, which is the difference between an alert and a diagnosis. +// +// # The baseline is built in a SCRATCH DATABASE, never into the live cache +// +// This is the design decision the whole package rests on. A.8's Bootstrap +// writes rows; if it wrote them into the live cache, the repair would BE the +// import and there would be nothing left to diff — a self-heal that cannot +// report what it healed, which is precisely the thing this package exists to +// avoid. So the fresh baseline is imported into a throwaway cache file, the +// two row sets are merge-joined, and only then are repairs applied to the live +// cache through A.14's row-scoped write path. +// +// # What it will and will not write +// +// The repair is RESTORE-ONLY and NEVER-REGRESS: +// +// - a key ground truth has and the live cache lacks is RESTORED. This is the +// packet's headline case: the record the delta pipeline silently dropped. +// - a key both hold, where ground truth's `modified` is strictly newer, is +// RESTORED. The delta path missed an update. +// - a key both hold at the SAME (or an unparseable) `modified` whose bytes +// differ is RESTORED. The live row is corrupt at a version ground truth +// can state exactly. +// - a key both hold where THE LIVE CACHE IS NEWER is LEFT ALONE and +// reported. A bulk artifact is cut at an instant; the delta stream is +// legitimately ahead of it, and overwriting a newer row with the +// baseline's older bytes would make the self-heal a data-loss event. +// - a key only the live cache holds is REPORTED AND NEVER DELETED. +// Withdrawn and REJECTED advisories are tombstoned rather than deleted +// (A.2 exit criterion 22) and that is A.16's job, not this one. A row that +// ground truth no longer carries may also simply post-date the artifact. +// +// # Nothing here composes a write shape +// +// Repairs go through delta.Apply — A.14's row-scoped upsert path, behind its +// own statement allowlist. That is deliberate: a second writer for `advisory` +// / `affected` / `advisory_fts` is exactly how a schema invariant survives in +// one writer and not the other, and delta.Record is exported for this reason +// ("A.15's reconciliation writes the same rows through the same path"). It +// also means the FTS index stays query-consistent after a repair for the same +// reason it does after a delta batch, rather than for a new reason nobody +// tested. This package therefore issues NO INSERT, REPLACE or UPDATE against +// those three tables, and no DDL of any kind: its own statement allowlist +// holds four statements, three of them SELECTs. +// +// # The two gates, unchanged +// +// - A.4's licence gate runs FIRST, before a byte is fetched, exactly as it +// does in A.8. A refusal ends the pass with no request made and no row +// written. The gate is resolved HERE as well as inside Bootstrap because +// delta.Apply takes the decision as a parameter rather than looking it up; +// the two resolutions are cross-checked against each other afterwards and +// a disagreement is a refusal, so "two answers to one question" is a +// detected condition rather than a latent one. +// - A.3's sanitizer runs inside delta.Decode on every string projected out +// of a baseline document, and delta.Apply re-checks the whole bind set +// with sanitize.AssertAllSanitized immediately before the parameters reach +// the driver. raw_json is the one deliberate exception, as it is +// everywhere else in Lane A: CVE-TOU obliges Anvil to store records +// byte-verbatim. +// +// # A failed self-heal is LOUD +// +// A.15's packet: "Do not skip reconciliation silently on a bootstrap failure — +// a failed weekly self-heal must increment feed_state.consecutive_failures and +// surface via A.16's staleness mechanism, not fail closed and disappear." So a +// bootstrap that errored, an import that did not complete, and a baseline that +// imported zero records all increment that counter in the LIVE cache and are +// named in the report. See recordFailure for the one case that deliberately +// does not increment it, and why. +package reconcile + +import ( + "context" + "crypto/sha256" + "database/sql" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/bootstrap" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/config" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/delta" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/license" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/sanitize" +) + +// --------------------------------------------------------------------------- +// Bounds +// --------------------------------------------------------------------------- + +const ( + // DefaultMaxSamples bounds how many disagreeing keys the report carries. + // + // A report is read by a person. Sixty-four keys is enough to see the shape + // of a drift — one ecosystem, one date range, one publisher — and small + // enough that a catastrophic diff produces a report rather than a second + // copy of the cache. The COUNTS are never truncated; only the sample is, + // and Truncated says so. + DefaultMaxSamples = 64 + + // DefaultRepairBatch is how many restored records are committed per + // delta.Apply transaction. Apply is one transaction, so this is the unit + // of work a crash can lose; every write in it is an idempotent upsert + // keyed on (source, source_id), so re-running loses nothing but time. + DefaultRepairBatch = 200 + + // MaxRepairBatch is the ceiling on Options.RepairBatch. delta.Apply + // refuses a batch over its own MaxBatchRecords on the grounds that a batch + // that size is a bulk import taking the wrong door; this keeps the refusal + // from ever being reached by a configuration mistake here. + MaxRepairBatch = 5000 +) + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +var ( + // ErrReconcile is satisfied by errors.Is for every refusal this package + // raises, so a caller that only needs "did the self-heal happen" needs one + // check. + ErrReconcile = errors.New("reconcile: refused") + + // ErrNotConfigured reports a Healer missing something it cannot invent: + // the live cache handle, the feed row, a working directory, or the + // baseline factory. + ErrNotConfigured = errors.New("reconcile: healer is not configured") + + // ErrNoBaselineMechanism reports a feed whose bootstrap_mechanism imports + // nothing — `none` or `incremental_api`. Running a "full baseline" against + // one of those produces an EMPTY ground truth, against which every row the + // live cache holds looks like a row ground truth does not have. That is + // not a drift report, it is a false alarm the size of the cache, so it is + // refused rather than reported. + ErrNoBaselineMechanism = errors.New("reconcile: the feed's bootstrap_mechanism imports no bulk baseline") + + // ErrEmptyBaseline reports a bootstrap that completed and wrote no rows. + // A full baseline of a CVE feed that holds zero advisories is a broken + // artifact, not ground truth about an empty world, and diffing against it + // would report the entire live cache as unexplained. + ErrEmptyBaseline = errors.New("reconcile: the fresh baseline imported no records") + + // ErrIncompleteBaseline reports a bootstrap that stopped part-way. A + // partial import is a PREFIX of ground truth: the keys it is missing are + // indistinguishable from keys the publisher dropped, so the "only in the + // live cache" count would be fiction. The pass refuses to diff rather than + // publish a number it cannot defend. + ErrIncompleteBaseline = errors.New("reconcile: the fresh baseline did not complete") + + // ErrDecisionMismatch reports that the licence decision this package + // resolved and the one A.8 resolved inside the bootstrap disagree about + // the tier or the output directory. Both read the same feed row through + // the same function, so a disagreement means the two mirrors differ — and + // writing rows under whichever answer happened to be in hand is how an + // unadmitted body reaches the cache. + ErrDecisionMismatch = errors.New("reconcile: the licence gate answered differently for the same feed") + + // ErrBaselineFailed reports a bootstrap that returned an error. The wrapped + // error is A.8's own. + ErrBaselineFailed = errors.New("reconcile: building the fresh baseline failed") + + // ErrStatementNotAllowed reports a statement this package tried to execute + // that is not on its allowlist. See allowedStatements. + ErrStatementNotAllowed = errors.New("reconcile: statement is not on this package's allowlist") +) + +func refuse(sentinel error, format string, args ...any) error { + return fmt.Errorf("%w: %w", ErrReconcile, fmt.Errorf("%w: "+format, append([]any{sentinel}, args...)...)) +} + +// --------------------------------------------------------------------------- +// The statement allowlist +// --------------------------------------------------------------------------- +// +// A DENYLIST LOSES. This package names every statement it may execute and +// refuses everything else, so that a future edit reaching for a DELETE, a +// DROP, or a rebuild of advisory_fts fails at the call rather than at review. +// Three of the four are read-only; the fourth writes feed_state, whose +// parameters are a feed id, timestamps and counters Anvil itself computes. +// +// NOTE WHAT IS NOT HERE: nothing that writes `advisory`, `affected` or +// `advisory_fts`. Those writes belong to delta.Apply, behind ITS allowlist, +// which is the whole reason this package has no second write shape. + +const ( + // scanAdvisorySQL streams one feed's advisory rows in key order, which is + // what makes the diff a MERGE JOIN with bounded memory rather than two + // 300,000-entry maps. `source` equals the feed id in both writers (A.8's + // and A.14's decoders both bind decodeCtx.feedID), so it is the scope of + // the comparison: the live cache holds many feeds and the baseline holds + // exactly one. + scanAdvisorySQL = `SELECT source_id, modified, state, raw_json FROM advisory WHERE source = ? ORDER BY source_id` + + // selectBaselineRecordSQL reads ONE record's bytes back out of the fresh + // baseline at repair time. + // + // It is a single-row statement rather than an `IN (?, ?, ...)` because a + // variable placeholder count cannot be allowlisted as exact text, and an + // allowlist that has to accept a statement PREFIX is not an allowlist. The + // cost is one query per restored row, which is bounded by the number of + // records the delta path actually dropped. + // + // staleness_seconds comes back with it because it is the age A.8 computed + // from the artifact's own Last-Modified. Re-deriving it here would be a + // second answer to "how old is this data", and spine S6 requires the field + // to mean the age of the DATA rather than the age of the write. + selectBaselineRecordSQL = `SELECT staleness_seconds, raw_json FROM advisory WHERE source = ? AND source_id = ?` +) + +var allowedStatements = map[string]string{ + strings.TrimSpace(scanAdvisorySQL): "read-only, feed-scoped, key-ordered scan of one side of the diff. " + + "ORDER BY source_id is load-bearing: it is what lets the comparison be a streaming merge join.", + strings.TrimSpace(selectBaselineRecordSQL): "read-only single-record read from the SCRATCH baseline, for a " + + "record the diff decided to restore.", + strings.TrimSpace(cache.SelectFeedStateSQL): "read-only feed_state read: the cadence input, and the current " + + "consecutive_failures a failed pass has to increment.", + strings.TrimSpace(cache.UpsertFeedStateSQL): "the ONLY write this package issues. It moves " + + "consecutive_failures and nothing else: etag, last_modified, watermark and last_ok_at are read back " + + "and written unchanged, because they belong to A.7 and A.8 and a self-heal has no business moving them.", +} + +func checkStatement(q string) error { + if _, ok := allowedStatements[strings.TrimSpace(q)]; ok { + return nil + } + return refuse(ErrStatementNotAllowed, + "this package may only execute statements on its allowlist and this one is not on it:\n\t%s\n"+ + "If it is a legitimate read, add it to allowedStatements with the reason. If it writes "+ + "advisory, affected or advisory_fts, it does NOT belong here at all: those writes go through "+ + "delta.Apply so that one write path holds the schema invariants for both A.14 and A.15.", + strings.Join(strings.Fields(q), " ")) +} + +func queryAllowed(ctx context.Context, db *sql.DB, q string, args ...any) (*sql.Rows, error) { + if err := checkStatement(q); err != nil { + return nil, err + } + return db.QueryContext(ctx, q, args...) +} + +func queryRowAllowed(ctx context.Context, db *sql.DB, q string, args ...any) (*sql.Row, error) { + if err := checkStatement(q); err != nil { + return nil, err + } + return db.QueryRowContext(ctx, q, args...), nil +} + +func execAllowed(ctx context.Context, db *sql.DB, q string, args ...any) error { + if err := checkStatement(q); err != nil { + return err + } + _, err := db.ExecContext(ctx, q, args...) + return err +} + +// --------------------------------------------------------------------------- +// The vocabulary of a disagreement +// --------------------------------------------------------------------------- + +// DisagreementKind names one way the fresh baseline and the live cache can +// disagree about a single key. +// +// These are LANE-A-LOCAL vocabulary with no counterpart in the record +// contract's six frozen enums, so declaring them here does not violate +// plan/IMPLEMENTATION-PLAN.md §6's single-owner rule — the same reasoning +// cache.CollectorHost is declared under. They exist so that a caller switches +// on a Go constant rather than on a string literal. +type DisagreementKind string + +const ( + // KindMissingInLive is a key the fresh baseline has and the live cache + // does not hold at all. This is the packet's headline case: the record the + // delta pipeline silently dropped. It is RESTORED. + KindMissingInLive DisagreementKind = "missing-in-live" + + // KindStaleInLive is a key both hold where ground truth's `modified` is + // strictly newer. The delta path missed an update. It is RESTORED. + KindStaleInLive DisagreementKind = "stale-in-live" + + // KindDivergent is a key both hold at the same — or an unparseable — + // `modified` whose stored bytes differ. The live row is corrupt at a + // version ground truth can state exactly. It is RESTORED. + // + // An unparseable timestamp on either side lands here rather than in + // KindAheadInLive ON PURPOSE: a comparison that cannot be made must fail + // toward the publisher's bytes, never toward keeping a row Anvil cannot + // date. + KindDivergent DisagreementKind = "divergent-at-same-version" + + // KindAheadInLive is a key both hold where the LIVE cache's `modified` is + // strictly newer. The delta stream is legitimately ahead of an artifact + // cut at an earlier instant. It is REPORTED AND LEFT ALONE: restoring it + // would overwrite a newer record with older bytes, which would make the + // self-heal a data-loss event. + KindAheadInLive DisagreementKind = "ahead-in-live" + + // KindOnlyInLive is a key the live cache holds that the fresh baseline + // does not. It is REPORTED AND NEVER DELETED — withdrawn and REJECTED + // advisories are tombstoned rather than deleted (A.2 exit criterion 22) + // and that is A.16's pass, and a row may also simply post-date the + // artifact. A rising count here is a real signal and it is the operator's + // to interpret, not this package's to act on. + KindOnlyInLive DisagreementKind = "only-in-live" +) + +// DisagreementKinds returns every kind, in report order. It exists so a test +// or a dashboard enumerates them from one place. +func DisagreementKinds() []DisagreementKind { + return []DisagreementKind{ + KindMissingInLive, KindStaleInLive, KindDivergent, KindAheadInLive, KindOnlyInLive, + } +} + +// Valid reports whether k is one of the declared kinds. +func (k DisagreementKind) Valid() bool { + for _, v := range DisagreementKinds() { + if k == v { + return true + } + } + return false +} + +// Repairable reports whether this kind is one the self-heal writes back. +// +// It is a property of the KIND rather than a branch inside the diff loop, so +// that the diff, the repair and the report cannot disagree about which +// disagreements get fixed. +func (k DisagreementKind) Repairable() bool { + switch k { + case KindMissingInLive, KindStaleInLive, KindDivergent: + return true + default: + return false + } +} + +// Disagreement is one sampled key the two row sets disagree about. +type Disagreement struct { + // SourceID is the advisory's native id within the feed. It is NOT + // necessarily a CVE id: research/06 Risk #2 keeps the cache off CVE ids as + // a primary key. + SourceID string + + // Kind is what kind of disagreement it is. + Kind DisagreementKind + + // BaselineModified and LiveModified are the two `modified` values as + // stored, unparsed. They are carried verbatim because the interesting case + // is usually the one where a timestamp did not parse. + BaselineModified string + LiveModified string + + // Repaired is whether this key was written back. It is false for every + // non-repairable kind, for every key in a report-only pass, and for a key + // whose baseline record failed to decode. + Repaired bool + + // Note carries the reason when a repairable key was not repaired. + Note string +} + +// --------------------------------------------------------------------------- +// The report +// --------------------------------------------------------------------------- + +// ReconcileReport is what one self-heal found. It is returned on every path +// including the refusals, because "the licence gate said no", "the baseline +// failed to build" and "the caches agree" must not look alike to a caller +// reading a repair count. +// +// A.15's packet names the counts as {new, updated, matched, +// missing-in-live-cache}. Two of those four are ONE quantity: a key the fresh +// baseline has and the live cache does not is both "new to the live cache" and +// "missing in the live cache". This report keeps ONE field for it — +// MissingInLive — rather than two that could drift apart, and adds the counts +// the packet did not name but that a drift report is useless without: the +// direction of a mismatch (stale versus ahead), and the keys the live cache +// holds that ground truth does not. Updated is the packet's fourth name and is +// a METHOD over the two directional counts, so it cannot be set to something +// they do not add up to. +type ReconcileReport struct { + // FeedID echoes the row this ran for, and RanAt is the pass's own clock. + FeedID string + RanAt time.Time + + // Duration is measured on the injected clock, so a test with a fixed clock + // reports zero rather than a number that varies per run. + Duration time.Duration + + // Plan is delta.Due's answer for this feed at RanAt. BaselineDue and + // BaselineInterval are the fields that decided whether this pass ran, and + // they come from the feed row's baseline_interval_seconds — there is no + // weekly constant in this package, because a cadence written as a Go + // constant is exactly what A.1 forbids. + Plan delta.Plan + + // Skipped is true when the baseline window has not turned over and Force + // was not set. It is not an error and the returned error is nil. + Skipped bool + + // Refused is true when A.4's licence gate declined the feed, and + // RefusedBecause carries the gate's own sentence. No request was made and + // no row was written or read. + Refused bool + RefusedBecause string + + // Tier and Dir are A.4's decision as this package resolved it, and they + // are cross-checked against the one A.8 resolved inside the bootstrap. + Tier int + Dir string + + // Bootstrap is A.8's own result for the fresh baseline: what it + // transferred, how many entries it read and how many records it wrote. + // It is the cost side of the pass. + Bootstrap bootstrap.BootstrapResult + + // BaselinePath is the scratch database the baseline was built in, and is + // set only when Options.KeepBaseline asked for it to survive the pass. + BaselinePath string + + // Failed is true when the baseline could not be built or could not be + // trusted, and FailedBecause says which. FailureRecorded is whether + // feed_state.consecutive_failures was successfully incremented, and + // ConsecutiveFailures is the value after the increment — the number A.16's + // staleness mechanism reads. + Failed bool + FailedBecause string + FailureRecorded bool + ConsecutiveFailures int + + // BaselineRows and LiveRows are the two row sets' sizes, scoped to this + // feed's `source`. Their difference is not the drift: a key can be present + // on both sides and still disagree. + BaselineRows int + LiveRows int + + // The diff, by kind. Every key on either side lands in exactly one of + // these five, so Matched + MissingInLive + StaleInLive + Divergent + + // AheadInLive == BaselineRows, and Matched + StaleInLive + Divergent + + // AheadInLive + OnlyInLive == LiveRows. CheckTotals asserts both. + Matched int + MissingInLive int + StaleInLive int + Divergent int + AheadInLive int + OnlyInLive int + + // Restored is advisory rows actually written back, as counted BY THE WRITE + // PATH rather than by the loop that chose them: it is Batch.Upserts. A + // number produced by the thing being measured is evidence; a number + // produced by the thing doing the asking is a hope. + Restored int + + // Batch is delta.Apply's own accounting for every repair transaction: + // advisory upserts, row-scoped FTS writes, affected and alias rows. Its + // FTSUpserts + FTSDeletes is the TOTAL number of statements that touched + // advisory_fts, and there is no other one. + Batch delta.BatchStats + + // RepairFailures counts repairable keys whose baseline record could not be + // decoded. One malformed record in ground truth must not block restoring + // the rest, so it is counted, sampled and carried on. + RepairFailures int + + // Sanitize is the merged A.3 report over every baseline document decoded + // during the repair. A non-zero count is not an error; it is the ordinary + // state of text written by strangers. + Sanitize sanitize.SanitizeStats + + // ReportOnly echoes the option: the pass diffed and deliberately wrote + // nothing. + ReportOnly bool + + // Samples is a bounded sample of disagreeing keys, in scan order, and + // SamplesTruncated says the sample stopped before the disagreements did. + // The COUNTS above are never truncated. + Samples []Disagreement + SamplesTruncated bool + + // Note is a sentence for an operator when the outcome needs one. + Note string +} + +// Updated is A.15's packet's fourth count: keys present on both sides that the +// self-heal had to rewrite. It is derived rather than stored so that it cannot +// disagree with the two directional counts it is made of. +func (r ReconcileReport) Updated() int { return r.StaleInLive + r.Divergent } + +// Disagreements is every key that was not an exact match, in either direction, +// including the ones this pass deliberately did not touch. +func (r ReconcileReport) Disagreements() int { + return r.MissingInLive + r.StaleInLive + r.Divergent + r.AheadInLive + r.OnlyInLive +} + +// Drifted reports whether the delta-built cache and ground truth disagreed +// about anything at all. +// +// It is the one boolean a daemon should alert on. A self-heal that repaired +// three records is not a success story; it is evidence that the delta path +// dropped three records and will drop more. +func (r ReconcileReport) Drifted() bool { return r.Disagreements() > 0 } + +// CheckTotals verifies that every key on either side was classified into +// exactly one bucket. +// +// It is exported because it is the arithmetic that makes the report readable +// as a partition rather than as five loosely related counters, and a caller +// logging the report can assert it for free. A failure means the diff has a +// bug, not that the caches disagree. +func (r ReconcileReport) CheckTotals() error { + if got := r.Matched + r.MissingInLive + r.StaleInLive + r.Divergent + r.AheadInLive; got != r.BaselineRows { + return fmt.Errorf("reconcile: the baseline side of the diff accounts for %d of %d rows "+ + "(matched %d, missing-in-live %d, stale-in-live %d, divergent %d, ahead-in-live %d)", + got, r.BaselineRows, r.Matched, r.MissingInLive, r.StaleInLive, r.Divergent, r.AheadInLive) + } + if got := r.Matched + r.StaleInLive + r.Divergent + r.AheadInLive + r.OnlyInLive; got != r.LiveRows { + return fmt.Errorf("reconcile: the live side of the diff accounts for %d of %d rows "+ + "(matched %d, stale-in-live %d, divergent %d, ahead-in-live %d, only-in-live %d)", + got, r.LiveRows, r.Matched, r.StaleInLive, r.Divergent, r.AheadInLive, r.OnlyInLive) + } + return nil +} + +// Summary is one line for an operator's log. +// +// It always names the disagreement counts, including when they are zero: "the +// weekly self-heal ran and found nothing" is the observation that makes every +// other week's number meaningful, and a log line that appears only on drift +// cannot distinguish a healthy pipeline from a pass that stopped running. +func (r ReconcileReport) Summary() string { + switch { + case r.Skipped: + return fmt.Sprintf("self-heal %s: skipped (%s)", r.FeedID, r.Note) + case r.Refused: + return fmt.Sprintf("self-heal %s: refused by the licence gate (%s)", r.FeedID, r.RefusedBecause) + case r.Failed: + return fmt.Sprintf("self-heal %s: FAILED (%s); consecutive_failures now %d", + r.FeedID, r.FailedBecause, r.ConsecutiveFailures) + } + mode := "repaired" + if r.ReportOnly { + mode = "report-only, wrote nothing;" + } + return fmt.Sprintf( + "self-heal %s: baseline %d rows, live %d rows; matched %d, missing-in-live %d, stale-in-live %d, "+ + "divergent %d, ahead-in-live %d, only-in-live %d; %s %d rows (%d repair failures)", + r.FeedID, r.BaselineRows, r.LiveRows, r.Matched, r.MissingInLive, r.StaleInLive, + r.Divergent, r.AheadInLive, r.OnlyInLive, mode, r.Restored, r.RepairFailures) +} + +// --------------------------------------------------------------------------- +// Baseliner — A.8, as a seam +// --------------------------------------------------------------------------- + +// Baseliner builds a full baseline into whatever cache it was constructed +// around. *bootstrap.Bootstrapper satisfies it. +// +// It is an interface for the same reason delta.FeedPoller is one: so that the +// daemon supplies ONE configured bootstrapper — with its HTTP client, its git +// runner, its credential lookup and its mirror — rather than this package +// constructing a second one, which would be a second implementation of A.8's +// size caps, redirect scope and credential rules. +type Baseliner interface { + Bootstrap(ctx context.Context, feed config.FeedConfig) (bootstrap.BootstrapResult, error) +} + +// BaselineFactory hands back a Baseliner bound to the SCRATCH cache handle +// this pass opened. +// +// The factory shape is what keeps the fresh baseline out of the live cache: +// the caller never gets to choose the database, because the whole design +// depends on the import landing somewhere the diff can still see it as +// separate. +type BaselineFactory func(scratch *sql.DB) (Baseliner, error) + +// FromBootstrapper turns a configured A.8 bootstrapper into a BaselineFactory +// by copying it and replacing only its DB. +// +// tmpl is taken BY VALUE and copied again per call, so the caller's +// bootstrapper is never mutated and two concurrent self-heals cannot share a +// cache handle. +func FromBootstrapper(tmpl bootstrap.Bootstrapper) BaselineFactory { + return func(scratch *sql.DB) (Baseliner, error) { + if scratch == nil { + return nil, refuse(ErrNotConfigured, "the baseline factory was handed a nil scratch cache") + } + b := tmpl + b.DB = scratch + return &b, nil + } +} + +// --------------------------------------------------------------------------- +// Healer +// --------------------------------------------------------------------------- + +// Options configures a Healer. Live, Feed, WorkDir and Baseline are required. +type Options struct { + // Live is the A.2 ingestion cache the delta pipeline has been writing to. + // It is NOT internal/store: that is the audit store of record and nothing + // here may touch it. + Live *sql.DB + + // Feed is the row to self-heal. cvelistV5 is the worked example + // (research/06 Recommendation §3) but nothing here is specific to it. + Feed config.FeedConfig + + // Mirror is the filesystem A.4 reads pinned licence evidence from. Nil + // means the process working directory, which is what a daemon wants and + // what a test must never rely on. + // + // It must be the same mirror the Baseline factory's bootstrapper reads, + // and WeeklySelfHeal cross-checks the two decisions rather than assuming + // it. + Mirror fs.FS + + // WorkDir is where the scratch baseline database is created. It must be a + // real directory on disk: the baseline is the same 570 MB import A.8 does, + // and it is not held in memory. + WorkDir string + + // Baseline builds the fresh ground truth into the scratch cache. + Baseline BaselineFactory + + // Now is the clock. Nil means time.Now. + Now func() time.Time + + // MaxSamples overrides DefaultMaxSamples. Negative means zero samples. + MaxSamples int + + // RepairBatch overrides DefaultRepairBatch. + RepairBatch int + + // Force runs the pass even when the baseline window has not turned over. + // It is how an operator says "self-heal this feed now", and it is the only + // thing that overrides the feed row's cadence. + Force bool + + // ReportOnly diffs and writes nothing. It exists so that an operator can + // see what a self-heal WOULD do before letting it do it, and so that a + // drift alarm can run more often than a repair. + ReportOnly bool + + // KeepBaseline leaves the scratch database on disk and names it in the + // report. It is for investigating a drift the counts do not explain. + KeepBaseline bool +} + +// Healer runs the weekly full-baseline self-heal for one feed. +// +// It holds no mutable state across passes and is safe for concurrent use +// across DIFFERENT feeds. Two concurrent passes over the SAME feed are not +// useful — they would each build a full baseline — but they are not unsafe, +// because every repair is an upsert keyed on (source, source_id). +type Healer struct { + live *sql.DB + feed config.FeedConfig + mirror fs.FS + workDir string + newBaseline BaselineFactory + now func() time.Time + maxSamples int + repairBatch int + force bool + reportOnly bool + keepBaseline bool +} + +// New builds a Healer, refusing anything it cannot invent. +func New(opts Options) (*Healer, error) { + if opts.Live == nil { + return nil, refuse(ErrNotConfigured, + "a self-heal diffs against the live A.2 ingestion cache and needs its handle") + } + if strings.TrimSpace(opts.Feed.ID) == "" { + return nil, refuse(ErrNotConfigured, "a self-heal needs the feed row it is healing") + } + if strings.TrimSpace(opts.WorkDir) == "" { + return nil, refuse(ErrNotConfigured, + "a self-heal builds the fresh baseline in a scratch database on disk and needs a directory "+ + "for it; the baseline is the same bulk import A.8 does and is not held in memory") + } + if opts.Baseline == nil { + return nil, refuse(ErrNotConfigured, + "a self-heal needs A.8's bootstrap to build ground truth. This package will not construct "+ + "one: an HTTP client, a credential lookup and a mirror built here would be a second "+ + "implementation of A.8's size caps, redirect scope and licence gate") + } + h := &Healer{ + live: opts.Live, + feed: opts.Feed, + mirror: opts.Mirror, + workDir: opts.WorkDir, + newBaseline: opts.Baseline, + now: opts.Now, + maxSamples: opts.MaxSamples, + repairBatch: opts.RepairBatch, + force: opts.Force, + reportOnly: opts.ReportOnly, + keepBaseline: opts.KeepBaseline, + } + if h.now == nil { + h.now = time.Now + } + if opts.MaxSamples == 0 { + h.maxSamples = DefaultMaxSamples + } + if h.maxSamples < 0 { + h.maxSamples = 0 + } + if h.repairBatch <= 0 { + h.repairBatch = DefaultRepairBatch + } + if h.repairBatch > MaxRepairBatch { + return nil, refuse(ErrNotConfigured, + "RepairBatch %d exceeds %d; delta.Apply is one transaction per batch and a batch that size "+ + "is a bulk import taking the wrong door", h.repairBatch, MaxRepairBatch) + } + return h, nil +} + +// WeeklySelfHeal is A.15's entry point: `WeeklySelfHeal(ctx) (ReconcileReport, +// error)`. +// +// THE ORDER OF WHAT FOLLOWS IS THE CONTRACT: +// +// 1. the feed row's baseline cadence is consulted (no network, pure) +// 2. the bootstrap mechanism is checked for one that imports a baseline +// 3. A.4's LICENCE GATE runs — before a byte is fetched, so a feed with no +// acquired licence body costs no bytes at all +// 4. a SCRATCH cache is opened and migrated +// 5. A.8 imports the full baseline INTO THE SCRATCH CACHE +// 6. the baseline is checked for being trustworthy at all: complete, and not +// empty. A prefix of ground truth is not ground truth. +// 7. the two row sets are merge-joined in key order and classified +// 8. repairable disagreements are written back through A.14's row-scoped +// upsert path, in batches +// +// Steps 1-3 and 6 all end the pass without writing. Step 6's failures — and a +// failure in step 5 — increment feed_state.consecutive_failures so that A.16's +// staleness mechanism sees them; see recordFailure. +// +// A non-nil error is returned WITH a populated ReconcileReport, never instead +// of one. +// The results are NAMED so that the deferred Duration write lands in the value +// the caller receives. With unnamed results the deferred assignment would +// mutate a local nobody ever reads again, and every report would carry a zero +// duration that looked like a measurement. +func (h *Healer) WeeklySelfHeal(ctx context.Context) (rep ReconcileReport, err error) { + started := h.now().UTC() + rep = ReconcileReport{ + FeedID: h.feed.ID, + RanAt: started, + ReportOnly: h.reportOnly, + Tier: license.NoTier, + } + defer func() { rep.Duration = h.now().UTC().Sub(started) }() + + // --- 1. Is it due? The cadence comes from the feed row and nowhere else. + lastOK, err := h.lastSuccess(ctx) + if err != nil { + return rep, err + } + rep.Plan = delta.Due(h.feed, lastOK, started) + if !rep.Plan.BaselineDue && !h.force { + rep.Skipped = true + switch { + case !h.feed.Enabled: + rep.Note = "the feed row is disabled; nothing about it is scheduled" + case rep.Plan.BaselineInterval <= 0: + rep.Note = "the feed row schedules no full-baseline self-heal " + + "(baseline_interval_seconds is zero)" + default: + rep.Note = fmt.Sprintf( + "the last success at %s falls in the same %s baseline window as %s", + lastOK.UTC().Format(time.RFC3339), rep.Plan.BaselineInterval, + started.Format(time.RFC3339)) + } + return rep, nil + } + + // --- 2. Does this feed HAVE a baseline to re-import? + switch h.feed.BootstrapMechanism { + case config.BootstrapBulkArchive, config.BootstrapBloblessClone: + default: + return rep, refuse(ErrNoBaselineMechanism, + "feed %q declares bootstrap_mechanism %q, which imports no records. A full-baseline diff "+ + "against an empty ground truth would report every row the live cache holds as "+ + "unexplained, which is a false alarm the size of the cache", + h.feed.ID, h.feed.BootstrapMechanism) + } + + // --- 3. The licence gate, before a byte is fetched. + decision, err := license.Resolve(license.FromFeed(h.feed, "", h.mirror)) + if err != nil { + rep.Refused, rep.RefusedBecause = true, err.Error() + return rep, fmt.Errorf("%w: feed %q: %w", ErrReconcile, h.feed.ID, err) + } + if decision.Refused() { + rep.Refused = true + rep.RefusedBecause = "the licence gate returned a refusal without an error" + return rep, refuse(license.ErrLicenseRefused, "feed %q: %s", h.feed.ID, rep.RefusedBecause) + } + rep.Tier, rep.Dir = decision.Tier.Int(), decision.Dir + + // --- 4. The scratch cache. The fresh baseline never touches the live one. + scratchDir, err := os.MkdirTemp(h.workDir, "anvil-baseline-") + if err != nil { + return rep, fmt.Errorf("reconcile: creating a scratch directory for feed %q: %w", h.feed.ID, err) + } + keep := h.keepBaseline + defer func() { + if !keep { + _ = os.RemoveAll(scratchDir) + } + }() + scratchPath := filepath.Join(scratchDir, "anvil-baseline.sqlite") + if keep { + rep.BaselinePath = scratchPath + } + scratch, err := cache.Open(ctx, scratchPath) + if err != nil { + return rep, fmt.Errorf("reconcile: opening the scratch baseline cache for feed %q: %w", h.feed.ID, err) + } + defer func() { _ = scratch.Close() }() + if _, err := cache.Migrate(ctx, scratch); err != nil { + return rep, fmt.Errorf("reconcile: migrating the scratch baseline cache for feed %q: %w", h.feed.ID, err) + } + + // --- 5. Build ground truth. + builder, err := h.newBaseline(scratch) + if err != nil { + return rep, err + } + if builder == nil { + return rep, refuse(ErrNotConfigured, "the baseline factory returned no Baseliner for feed %q", h.feed.ID) + } + res, bootErr := builder.Bootstrap(ctx, h.feed) + rep.Bootstrap = res + + // --- 6. Is the baseline trustworthy enough to diff against? + switch { + case bootErr != nil: + return h.fail(ctx, &rep, fmt.Errorf("%w: feed %q: %w", ErrBaselineFailed, h.feed.ID, bootErr)) + case !res.Complete: + return h.fail(ctx, &rep, refuse(ErrIncompleteBaseline, + "feed %q: the import stopped after %d entries and %d records. A partial baseline is a PREFIX "+ + "of ground truth: the keys it is missing cannot be told apart from keys the publisher "+ + "dropped, so every 'only in the live cache' count derived from it would be fiction", + h.feed.ID, res.EntriesRead, res.RecordsUpserted)) + case res.RecordsUpserted == 0: + return h.fail(ctx, &rep, refuse(ErrEmptyBaseline, + "feed %q: the import completed and wrote no records. That is a broken artifact, not ground "+ + "truth about an empty world", h.feed.ID)) + case res.Tier != decision.Tier.Int() || res.Dir != decision.Dir: + return h.fail(ctx, &rep, refuse(ErrDecisionMismatch, + "feed %q: this package resolved tier %d dir %q and the bootstrap resolved tier %d dir %q. "+ + "Both read the same feed row through license.Resolve, so they are reading different "+ + "mirrors, and neither answer may be used to write a row", + h.feed.ID, decision.Tier.Int(), decision.Dir, res.Tier, res.Dir)) + } + + // --- 7. The diff. + repairs, err := h.diff(ctx, scratch, &rep) + if err != nil { + return rep, err + } + if err := rep.CheckTotals(); err != nil { + return rep, err + } + + // --- 8. The repair. + if h.reportOnly { + rep.Note = "report-only: the disagreements above were found and deliberately not written back" + return rep, nil + } + if err := h.repair(ctx, scratch, decision, repairs, started, &rep); err != nil { + return rep, err + } + // The repair may already have left a note about records it could not + // restore. It is kept: the closing sentence is context, and the thing that + // went wrong outranks it. + if closing := h.closingNote(rep); rep.Note == "" { + rep.Note = closing + } else { + rep.Note = closing + "; " + rep.Note + } + return rep, nil +} + +// closingNote is the sentence an operator reads when nothing went wrong, and it +// deliberately does not congratulate a pass that repaired rows. +func (h *Healer) closingNote(rep ReconcileReport) string { + switch { + case !rep.Drifted(): + return "the delta-built cache and the fresh baseline agree on every row" + case rep.Restored > 0: + return fmt.Sprintf( + "the delta pipeline had dropped or staled %d of %d records for this feed; they were restored "+ + "from the fresh baseline. This is a defect in the delta path, not a success of the "+ + "self-heal", rep.Restored, rep.BaselineRows) + default: + return "the two row sets disagree but nothing was repairable: see the ahead-in-live and " + + "only-in-live counts, which this pass reports and never acts on" + } +} + +// --------------------------------------------------------------------------- +// The diff +// --------------------------------------------------------------------------- + +// sideRow is one advisory row of one side, reduced to what the comparison +// needs. raw_json is hashed and discarded as it is read, so peak memory is one +// record and not one cache. +type sideRow struct { + sourceID string + modified string + digest [sha256.Size]byte +} + +// cursor is a one-row-lookahead reader over an ordered scan. +type cursor struct { + rows *sql.Rows + cur sideRow + valid bool + err error + seen int + // last is the previous key, kept so the scan can prove the ORDER BY it + // depends on actually held. A merge join over an unordered scan silently + // produces nonsense, and "the statement says ORDER BY" is not evidence + // that the rows arrived that way. + last string +} + +func newCursor(rows *sql.Rows) *cursor { + c := &cursor{rows: rows} + c.advance() + return c +} + +func (c *cursor) advance() { + if c.err != nil { + c.valid = false + return + } + if !c.rows.Next() { + c.valid = false + c.err = c.rows.Err() + return + } + var ( + sourceID string + modified sql.NullString + state sql.NullString + raw []byte + ) + if err := c.rows.Scan(&sourceID, &modified, &state, &raw); err != nil { + c.valid, c.err = false, err + return + } + if c.seen > 0 && sourceID <= c.last { + c.valid = false + c.err = fmt.Errorf( + "reconcile: the advisory scan returned %q after %q; the merge join depends on the "+ + "ORDER BY holding and it did not", sourceID, c.last) + return + } + c.cur = sideRow{sourceID: sourceID, modified: modified.String, digest: contentDigest(state.String, modified.String, raw)} + c.last = sourceID + c.seen++ + c.valid = true +} + +// contentDigest is the comparison key for "are these two rows the same +// record". +// +// IT IS NOT A FINGERPRINT AND MUST NEVER BE PRESENTED AS ONE. anvil-fp/v1 is +// the one fingerprint algorithm in this system, it is owned by +// internal/record, and FINGERPRINT-SPEC.md is authoritative for it (spine S6: +// two producers emitting different digests under one name breaks regression +// matching forever). This value never leaves this package, is never stored, is +// never compared against anything a record produced, and would be just as +// correct if it were any other collision-resistant hash. +// +// Fields are length-prefixed so that no concatenation of one row can be +// confused with a different concatenation of another. +func contentDigest(state, modified string, raw []byte) [sha256.Size]byte { + h := sha256.New() + writeField := func(b []byte) { + var n [8]byte + v := uint64(len(b)) + for i := 7; i >= 0; i-- { + n[i] = byte(v) + v >>= 8 + } + _, _ = h.Write(n[:]) + _, _ = h.Write(b) + } + writeField([]byte(state)) + writeField([]byte(modified)) + writeField(raw) + var out [sha256.Size]byte + copy(out[:], h.Sum(nil)) + return out +} + +// diff merge-joins the two ordered row sets and classifies every key on either +// side into exactly one bucket. +// +// It returns the source_ids to restore, in scan order. It collects KEYS and +// not records on purpose: the repair reads each record's bytes back out of the +// scratch cache afterwards, so that no cursor is open on either database while +// the live cache is being written. +func (h *Healer) diff(ctx context.Context, scratch *sql.DB, rep *ReconcileReport) ([]string, error) { + baselineRows, err := queryAllowed(ctx, scratch, scanAdvisorySQL, h.feed.ID) + if err != nil { + return nil, fmt.Errorf("reconcile: scanning the fresh baseline for feed %q: %w", h.feed.ID, err) + } + defer func() { _ = baselineRows.Close() }() + + liveRows, err := queryAllowed(ctx, h.live, scanAdvisorySQL, h.feed.ID) + if err != nil { + return nil, fmt.Errorf("reconcile: scanning the live cache for feed %q: %w", h.feed.ID, err) + } + defer func() { _ = liveRows.Close() }() + + base := newCursor(baselineRows) + live := newCursor(liveRows) + + var repairs []string + for base.valid || live.valid { + if err := ctx.Err(); err != nil { + return nil, err + } + switch { + case live.valid && (!base.valid || live.cur.sourceID < base.cur.sourceID): + rep.LiveRows++ + rep.OnlyInLive++ + h.sample(rep, Disagreement{ + SourceID: live.cur.sourceID, + Kind: KindOnlyInLive, + LiveModified: live.cur.modified, + Note: "ground truth does not carry this key; it is reported and never deleted " + + "(tombstoning is A.16's pass, and the row may simply post-date the artifact)", + }) + live.advance() + + case base.valid && (!live.valid || base.cur.sourceID < live.cur.sourceID): + rep.BaselineRows++ + rep.MissingInLive++ + repairs = append(repairs, base.cur.sourceID) + h.sample(rep, Disagreement{ + SourceID: base.cur.sourceID, + Kind: KindMissingInLive, + BaselineModified: base.cur.modified, + }) + base.advance() + + default: + rep.BaselineRows++ + rep.LiveRows++ + switch kind := classify(base.cur, live.cur); kind { + case "": + rep.Matched++ + case KindStaleInLive: + rep.StaleInLive++ + repairs = append(repairs, base.cur.sourceID) + h.sample(rep, Disagreement{ + SourceID: base.cur.sourceID, Kind: kind, + BaselineModified: base.cur.modified, LiveModified: live.cur.modified, + }) + case KindDivergent: + rep.Divergent++ + repairs = append(repairs, base.cur.sourceID) + h.sample(rep, Disagreement{ + SourceID: base.cur.sourceID, Kind: kind, + BaselineModified: base.cur.modified, LiveModified: live.cur.modified, + Note: "the two sides carry different bytes at the same (or an undatable) version", + }) + case KindAheadInLive: + rep.AheadInLive++ + h.sample(rep, Disagreement{ + SourceID: base.cur.sourceID, Kind: kind, + BaselineModified: base.cur.modified, LiveModified: live.cur.modified, + Note: "the live cache is newer than the artifact; restoring would overwrite a newer " + + "record with older bytes, so this pass leaves it alone", + }) + } + base.advance() + live.advance() + } + } + if base.err != nil { + return nil, fmt.Errorf("reconcile: reading the fresh baseline for feed %q: %w", h.feed.ID, base.err) + } + if live.err != nil { + return nil, fmt.Errorf("reconcile: reading the live cache for feed %q: %w", h.feed.ID, live.err) + } + return repairs, nil +} + +// classify decides what kind of disagreement one shared key has, or "" when +// the two sides carry the same record. +// +// The direction test is on the two `modified` timestamps and it is +// deliberately three-armed rather than two: +// +// - both parse and the live one is later -> the live cache is AHEAD. Leave +// it alone. A bulk artifact is cut at an instant and the delta stream is +// legitimately past it. +// - both parse and the baseline is later -> the live cache is STALE. The +// delta path missed an update. +// - anything else (equal timestamps, or one that will not parse) -> +// DIVERGENT, and ground truth wins. A comparison that cannot be made must +// fail toward the publisher's bytes, because keeping a row Anvil cannot +// date over a row it fetched from the publisher is the wrong risk to take. +func classify(base, live sideRow) DisagreementKind { + if base.digest == live.digest { + return "" + } + bt, bok := parseModified(base.modified) + lt, lok := parseModified(live.modified) + switch { + case bok && lok && lt.After(bt): + return KindAheadInLive + case bok && lok && bt.After(lt): + return KindStaleInLive + default: + return KindDivergent + } +} + +// parseModified accepts the shapes advisory feeds actually emit and reports +// anything else as undatable rather than coercing it. +// +// The layout list matches internal/ingest/delta's own parseTimestamp, which is +// unexported. That duplication is deliberate and narrow: it is a shared FORMAT +// list rather than a shared judgement, and the two functions answer different +// questions — delta's fails toward re-fetching a record, this one fails toward +// restoring from ground truth. Merging them would give one policy to two +// decisions that legitimately differ. +func parseModified(s string) (time.Time, bool) { + v := strings.TrimSpace(s) + if v == "" { + return time.Time{}, false + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, v); err == nil { + return t.UTC(), true + } + } + return time.Time{}, false +} + +// sample records a disagreement in the bounded sample, and marks the report +// truncated once the bound is reached. The COUNTS are never affected. +func (h *Healer) sample(rep *ReconcileReport, d Disagreement) { + if len(rep.Samples) >= h.maxSamples { + rep.SamplesTruncated = true + return + } + rep.Samples = append(rep.Samples, d) +} + +// --------------------------------------------------------------------------- +// The repair +// --------------------------------------------------------------------------- + +// repair writes the chosen records back into the live cache through +// delta.Apply, in batches. +// +// Nothing here composes a statement against `advisory`, `affected` or +// `advisory_fts`. delta.Apply owns those, behind its own allowlist, and that +// is what keeps the FTS index query-consistent after a repair for the same +// reason it is after a delta batch. +func (h *Healer) repair( + ctx context.Context, + scratch *sql.DB, + decision license.Decision, + ids []string, + asOf time.Time, + rep *ReconcileReport, +) error { + if len(ids) == 0 { + return nil + } + failed := map[string]string{} + batch := make([]delta.Record, 0, h.repairBatch) + flush := func() error { + if len(batch) == 0 { + return nil + } + // staleness is passed as zero because every record carries its own, + // read from the row A.8 wrote: that is the age of the DATA the + // artifact's Last-Modified declared, which is what spine S6's + // staleness_seconds means. delta.Apply's per-record override wins over + // the batch value, so the batch value is only ever a fallback nothing + // here needs. + stats, err := delta.Apply(ctx, h.live, h.feed, decision, batch, asOf, 0) + rep.Batch.Merge(stats) + rep.Restored = rep.Batch.Upserts + batch = batch[:0] + return err + } + + for _, id := range ids { + if err := ctx.Err(); err != nil { + return err + } + recs, staleness, err := h.readBaselineRecord(ctx, scratch, id, &rep.Sanitize) + if err != nil { + // ONE malformed record in ground truth must not block restoring + // the rest. It is counted, sampled, and carried past. + failed[id] = err.Error() + rep.RepairFailures++ + continue + } + for i := range recs { + if recs[i].StalenessSeconds <= 0 { + recs[i].StalenessSeconds = staleness + } + batch = append(batch, recs[i]) + } + if len(batch) >= h.repairBatch { + if err := flush(); err != nil { + h.markRepairs(rep, failed) + return err + } + } + } + if err := flush(); err != nil { + h.markRepairs(rep, failed) + return err + } + h.markRepairs(rep, failed) + if rep.RepairFailures > 0 { + rep.Note = fmt.Sprintf("%d baseline records could not be decoded and were not restored; "+ + "see the samples carrying a note", rep.RepairFailures) + } + return nil +} + +// markRepairs stamps the sample entries with what actually happened to them. +func (h *Healer) markRepairs(rep *ReconcileReport, failed map[string]string) { + for i := range rep.Samples { + if !rep.Samples[i].Kind.Repairable() { + continue + } + if why, bad := failed[rep.Samples[i].SourceID]; bad { + rep.Samples[i].Note = "the baseline record did not decode and was not restored: " + why + continue + } + rep.Samples[i].Repaired = true + } +} + +// readBaselineRecord reads one record's verbatim bytes out of the scratch +// baseline and decodes them through A.14's decoder — the same decoder the +// delta path uses, so a restored row is byte-for-byte the row a working delta +// path would have written. +// +// A decoded batch that does not contain the key we asked about is refused: a +// repair that writes rows nobody asked for is not a repair. +func (h *Healer) readBaselineRecord( + ctx context.Context, + scratch *sql.DB, + id string, + acc *sanitize.SanitizeStats, +) ([]delta.Record, int, error) { + row, err := queryRowAllowed(ctx, scratch, selectBaselineRecordSQL, h.feed.ID, id) + if err != nil { + return nil, 0, err + } + var ( + staleness int + raw []byte + ) + if err := row.Scan(&staleness, &raw); err != nil { + return nil, 0, fmt.Errorf("reading baseline record %q: %w", id, err) + } + recs, stats, err := delta.Decode(h.feed.ID, raw) + if err != nil { + return nil, 0, fmt.Errorf("decoding baseline record %q: %w", id, err) + } + found := false + for _, r := range recs { + if r.SourceID == id { + found = true + break + } + } + if !found { + return nil, 0, fmt.Errorf("the baseline document stored under %q decoded to %d record(s), none of "+ + "them that key; restoring them would write rows the diff never asked about", id, len(recs)) + } + // The sanitizer report is merged only for documents that decoded, because + // a document that did not decode contributed no field to any row. The + // accumulator is a parameter rather than a field on the Healer so that a + // pass carries no mutable state on the receiver at all. + acc.Merge(stats) + return recs, staleness, nil +} + +// --------------------------------------------------------------------------- +// feed_state +// --------------------------------------------------------------------------- + +// lastSuccess reads feed_state.last_ok_at from the LIVE cache. It is the same +// durable input delta.Due reads, and reading it from the same column is what +// keeps the two schedulers from disagreeing about when this feed last worked. +// +// A feed with no row has never been polled and everything about it is due. A +// row whose last_ok_at does not parse is treated the same way, deliberately: a +// clock we cannot read must not be allowed to postpone a self-heal forever. +func (h *Healer) lastSuccess(ctx context.Context) (time.Time, error) { + st, ok, err := h.readFeedState(ctx) + if err != nil || !ok || !st.lastOK.Valid { + return time.Time{}, err + } + v := strings.TrimSpace(st.lastOK.String) + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02T15:04:05.000000000Z"} { + if t, err := time.Parse(layout, v); err == nil { + return t.UTC(), nil + } + } + return time.Time{}, nil +} + +type feedStateRow struct { + etag, lastModified, watermark, lastOK sql.NullString + failures int + tier int +} + +func (h *Healer) readFeedState(ctx context.Context) (feedStateRow, bool, error) { + var st feedStateRow + row, err := queryRowAllowed(ctx, h.live, cache.SelectFeedStateSQL, h.feed.ID) + if err != nil { + return st, false, err + } + switch err := row.Scan(&st.etag, &st.lastModified, &st.watermark, &st.lastOK, &st.failures, &st.tier); { + case errors.Is(err, sql.ErrNoRows): + return st, false, nil + case err != nil: + return st, false, fmt.Errorf("reconcile: reading feed_state for %q: %w", h.feed.ID, err) + } + return st, true, nil +} + +// fail records a failed self-heal and returns the report and the error. +// +// A.15's packet: a failed weekly self-heal must increment +// feed_state.consecutive_failures and surface via A.16's staleness mechanism, +// "not fail closed and disappear". So the counter moves before the error is +// returned, and whether it moved is itself reported. +func (h *Healer) fail(ctx context.Context, rep *ReconcileReport, cause error) (ReconcileReport, error) { + rep.Failed = true + rep.FailedBecause = cause.Error() + if err := h.recordFailure(ctx, rep); err != nil { + // The self-heal failed AND the failure could not be recorded. Both are + // reported; the original cause is the one returned, because it is the + // one an operator has to fix first. + rep.Note = "the failure could not be recorded in feed_state either: " + err.Error() + } + return *rep, cause +} + +// recordFailure increments feed_state.consecutive_failures in the LIVE cache. +// +// It preserves etag, last_modified, watermark and last_ok_at exactly as read. +// Those are A.7's conditional-GET state and A.8's bootstrap cursor, and a +// self-heal has no business moving either: clearing an etag would cost a full +// re-download on the next poll, and clearing a watermark would cost A.8 its +// resume position. +// +// IT DOES NOT RESET THE COUNTER ON SUCCESS, and that is deliberate. The column +// is shared with A.7, which clears it when a poll succeeds. A self-heal that +// zeroed it would erase A.7's record of a failing poll, and a security tool +// that loses its own "this feed is not working" signal is exactly what A.16's +// staleness mechanism exists to prevent. +// +// A LICENCE REFUSAL IS NOT A FAILURE AND DOES NOT REACH HERE. A refusal means +// no publisher licence body has been acquired for this feed, which is the +// ORDINARY state of a fresh clone (see internal/ingest/license's known-limits +// note: the admission path admits nothing at all today). Counting it weekly +// would climb a failure counter on every feed in the table with nothing +// broken, which would make the counter useless for the case it exists for. The +// refusal is reported instead — loudly, on Refused and RefusedBecause, and as +// an error satisfying license.ErrLicenseRefused. +func (h *Healer) recordFailure(ctx context.Context, rep *ReconcileReport) error { + st, ok, err := h.readFeedState(ctx) + if err != nil { + return err + } + tier := st.tier + if !ok { + // No row yet. license_tier is NOT NULL with a CHECK over (0,1,2,3), so + // there is no honest value to invent when the gate never admitted this + // feed. Say so rather than writing a tier nobody decided. + if rep.Tier < 0 || rep.Tier > 3 { + return fmt.Errorf("reconcile: feed %q has no feed_state row and no admitted licence tier, "+ + "so there is no row to increment and no honest tier to create one with", h.feed.ID) + } + tier = rep.Tier + } + next := st.failures + 1 + if err := execAllowed(ctx, h.live, cache.UpsertFeedStateSQL, + h.feed.ID, nullOf(st.etag), nullOf(st.lastModified), nullOf(st.watermark), nullOf(st.lastOK), + next, tier); err != nil { + return fmt.Errorf("reconcile: recording a self-heal failure for %q: %w", h.feed.ID, err) + } + rep.FailureRecorded = true + rep.ConsecutiveFailures = next + return nil +} + +func nullOf(v sql.NullString) any { + if !v.Valid || strings.TrimSpace(v.String) == "" { + return nil + } + return v.String +} + +// SortedSamples returns the report's samples grouped by kind and then by +// source id, which is the order a human reads them in. The report itself keeps +// scan order, because scan order is what makes a clustered drift — one +// ecosystem, one date range — visible at a glance. +func SortedSamples(in []Disagreement) []Disagreement { + out := append([]Disagreement(nil), in...) + order := map[DisagreementKind]int{} + for i, k := range DisagreementKinds() { + order[k] = i + } + sort.SliceStable(out, func(i, j int) bool { + if order[out[i].Kind] != order[out[j].Kind] { + return order[out[i].Kind] < order[out[j].Kind] + } + return out[i].SourceID < out[j].SourceID + }) + return out +} diff --git a/internal/ingest/reconcile/reconcile_test.go b/internal/ingest/reconcile/reconcile_test.go new file mode 100644 index 0000000..59b3de9 --- /dev/null +++ b/internal/ingest/reconcile/reconcile_test.go @@ -0,0 +1,1963 @@ +// Tests for A.15, the weekly full-baseline self-heal. +// +// =========================================================================== +// WHAT THESE TESTS ARE FOR, AND WHAT A GREEN RUN DOES NOT PROVE +// =========================================================================== +// +// Four claims carry this package. Each has a test whose failure would be a +// real defect rather than a cosmetic one: +// +// 1. THE SELF-HEAL ACTUALLY HEALS, AND SAYS SO. +// TestSelfHealRestoresTheRecordsTheDeltaPathDropped is A.15's named +// validation: a live cache deliberately missing three records the fresh +// baseline has. It drives the REAL A.8 bootstrapper over an httptest- +// served zip, restores through A.14's real write path, and then asserts +// both halves — the three rows are back, AND every unrelated row is +// byte-identical including its as_of, which is what proves the repair was +// row-scoped rather than a re-import wearing a diff's clothes. +// +// 2. IT NEVER MAKES THE CACHE WORSE. A bulk artifact is cut at an instant and +// the delta stream is legitimately past it. +// TestANewerLiveRowIsNeverOverwrittenByAnOlderBaseline and +// TestARowGroundTruthLacksIsReportedAndNeverDeleted are the two directions +// of that, and both assert on the stored bytes rather than on a counter. +// +// 3. A FAILURE IS LOUD. TestABootstrapFailureIncrementsConsecutiveFailures, +// TestAnIncompleteBaselineIsRefusedAndCounted and +// TestAnEmptyBaselineIsRefusedAndCounted assert the packet's forbidden +// action directly: the counter A.16's staleness mechanism reads moves, and +// the live cache is not touched. +// +// 4. NOTHING FULL-TABLE REACHES THE CACHE. +// TestNoFullTableStatementReachesTheLiveCache opens the live cache through +// a tracing driver and inspects EVERY statement that reached the driver +// layer during a real repair. It is an observation, not an assertion about +// code someone read. +// +// THE GUARDS ARE VERIFIED RED. TestTheStatementAllowlistIsOnTheLivePath +// removes an entry from the allowlist and proves the production path then +// fails, so the allowlist is load-bearing rather than decorative; +// TestTheMergeJoinRefusesAnUnorderedScan feeds the cursor a deliberately +// descending scan and proves it refuses instead of silently producing a +// nonsense diff. +// +// THE CORPUS IS AUTHORED HERE, NOT DERIVED FROM THE IMPLEMENTATION. The CVE +// 5.1 documents below are written in this file. The live cache is then built +// by running the REAL delta writer over a subset of them, which is how the +// production system builds it — so a divergence between A.8's decoder and +// A.14's decoder would surface here as a wall of "divergent" rows rather than +// hiding until a real self-heal ran. +// +// NO TEST HERE REACHES THE NETWORK: the one bulk archive is served by +// httptest. NO TEST USES A REAL CREDENTIAL. +// +// A green run does NOT prove the self-heal is affordable against a 300,000 +// record cvelistV5 baseline; the fixtures here are tens of records. Nor does +// it prove anything about `go test -race`, which cannot run on the Windows dev +// host this was written on. +package reconcile + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "database/sql" + "database/sql/driver" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "net/http" + "net/http/httptest" + "path" + "path/filepath" + "strings" + "sync" + "testing" + "testing/fstest" + "time" + + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/bootstrap" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/cache" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/config" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/delta" + "github.com/Susquehanna-Syntax/Anvil/internal/ingest/license" + + _ "modernc.org/sqlite" +) + +// --------------------------------------------------------------------------- +// Clocks +// --------------------------------------------------------------------------- + +// seedAt is when the delta pipeline is pretended to have written the live +// cache, and healAt is when the self-heal runs. They differ so that `as_of` +// alone distinguishes a row this pass rewrote from one it left alone — which +// is the assertion that makes "without disturbing unrelated rows" checkable. +var ( + seedAt = time.Date(2026, 8, 1, 6, 0, 0, 0, time.UTC) + healAt = time.Date(2026, 8, 9, 12, 0, 0, 0, time.UTC) +) + +func fixedClock(t time.Time) func() time.Time { return func() time.Time { return t } } + +// --------------------------------------------------------------------------- +// The corpus. Authored here. +// --------------------------------------------------------------------------- + +// cveDoc renders one CVE 5.1 record. Everything a test needs to vary is a +// parameter, and nothing about it is read back out of the implementation. +func cveDoc(id, updated, description string) string { + doc := map[string]any{ + "dataType": "CVE_RECORD", + "dataVersion": "5.1", + "cveMetadata": map[string]any{ + "cveId": id, + "state": "PUBLISHED", + "datePublished": "2026-01-02T00:00:00.000Z", + "dateUpdated": updated, + "assignerShortName": "example", + }, + "containers": map[string]any{ + "cna": map[string]any{ + "descriptions": []any{ + map[string]any{"lang": "en", "value": description}, + }, + "references": []any{ + map[string]any{"url": "https://example.invalid/advisory/" + id}, + }, + "affected": []any{map[string]any{ + "vendor": "example", + "product": "widget", + "versions": []any{map[string]any{"version": "1.0.0", "status": "affected"}}, + }}, + }, + }, + } + b, err := json.Marshal(doc) + if err != nil { + panic("cveDoc: " + err.Error()) + } + return string(b) +} + +// corpus is n advisories with stable ids and a distinctive term per record, so +// an FTS round-trip can name exactly one of them. +func corpus(n int) []string { + out := make([]string, 0, n) + for i := 0; i < n; i++ { + id := fmt.Sprintf("CVE-2026-%05d", 10000+i) + out = append(out, cveDoc(id, "2026-03-01T00:00:00.000Z", + fmt.Sprintf("Synthetic advisory for %s concerning marker%04d in the widget parser.", id, i))) + } + return out +} + +func docID(t *testing.T, doc string) string { + t.Helper() + var d struct { + CVEMetadata struct { + CVEID string `json:"cveId"` + } `json:"cveMetadata"` + } + if err := json.Unmarshal([]byte(doc), &d); err != nil { + t.Fatalf("reading the id back out of a fixture document: %v", err) + } + if d.CVEMetadata.CVEID == "" { + t.Fatal("a fixture document carries no cveId") + } + return d.CVEMetadata.CVEID +} + +// --------------------------------------------------------------------------- +// The licence mirror +// --------------------------------------------------------------------------- + +// cc0Verbatim is the publisher licence text the synthetic mirror pins. A.4 +// classifies BODIES, so a fixture that wants an admission has to supply a real +// permissive one. +const cc0Verbatim = `Creative Commons Legal Code + +CC0 1.0 Universal + +The person who associated a work with this deed has dedicated the work to the +public domain by waiving all rights to the work worldwide under copyright law.` + +const cc0Notes = `SPDX-License-Identifier: CC0-1.0 + +Anvil's record: this source is public domain and carries no obligation.` + +func digestOf(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +func testFeed(id string) config.FeedConfig { + return config.FeedConfig{ + ID: id, + URL: "https://example.invalid/" + id, + Enabled: true, + AuthMode: config.AuthNone, + SyncMechanism: config.SyncConditionalGetETag, + IntervalSeconds: 900, + ReconcileIntervalSeconds: 86400, + BaselineIntervalSeconds: 604800, + FreshnessSLOSeconds: 86400, + OnFailure: config.OnFailureServeStale, + LicenseTier: config.LicenseTier0, + LicenseSPDX: "CC0-1.0", + MirrorDir: id, + BootstrapMechanism: config.BootstrapBulkArchive, + } +} + +// admittingMirror renders the mirror tree A.4 reads: a pinned manifest, the +// publisher's acquired text at the digest the pin names, and Anvil's own +// record. The gate's admission path is exacting and a mirror assembled by +// guesswork simply refuses, so this mirrors the shape A.4's own fixtures use. +func admittingMirror(t *testing.T, feeds ...config.FeedConfig) fs.FS { + t.Helper() + fsys := fstest.MapFS{} + var man strings.Builder + man.WriteString("# synthetic manifest, reconcile_test\n") + man.WriteString("schema_version = 1\n") + man.WriteString("generated_utc = \"2026-08-09\"\n") + man.WriteString("generated_by = \"reconcile_test\"\n") + + notes := map[config.LicenseTier]*strings.Builder{} + for _, f := range feeds { + dir := f.MirrorDir + if dir == "" { + dir = f.ID + } + fmt.Fprintf(&man, "\n[[body]]\nfeed_id = %q\ntier = %d\ndir = %q\n"+ + "spdx_id = %q\ntext_url = \"https://example.invalid/LICENSE\"\n"+ + "sha256 = %q\nclaim_source = \"reconcile_test fixture\"\n", + f.ID, f.LicenseTier.Int(), dir, f.LicenseSPDX, digestOf(cc0Verbatim)) + fsys[path.Join(license.TierDir(f.LicenseTier), dir, license.VerbatimFileName)] = + &fstest.MapFile{Data: []byte(cc0Verbatim)} + + b, ok := notes[f.LicenseTier] + if !ok { + b = &strings.Builder{} + b.WriteString("# fixture notes\n") + notes[f.LicenseTier] = b + } + fmt.Fprintf(b, "\n%s\n%s\n%s\n", + license.BodyBeginMarker(f.ID), cc0Notes, license.BodyEndMarker(f.ID)) + } + for tier, b := range notes { + fsys[path.Join(license.TierDir(tier), license.NotesFileName)] = &fstest.MapFile{Data: []byte(b.String())} + } + fsys[license.ManifestFileName] = &fstest.MapFile{Data: []byte(man.String())} + return fsys +} + +// emptyMirror pins nothing, so A.4 refuses every feed against it. It is what a +// fresh clone looks like. +func emptyMirror() fs.FS { return fstest.MapFS{} } + +func admittedDecision(t *testing.T, feed config.FeedConfig, mirror fs.FS) license.Decision { + t.Helper() + d, err := license.Resolve(license.FromFeed(feed, "", mirror)) + if err != nil { + t.Fatalf("the fixture mirror does not admit feed %q: %v", feed.ID, err) + } + if d.Refused() { + t.Fatalf("the fixture mirror returned a refusal for feed %q with no error", feed.ID) + } + return d +} + +// --------------------------------------------------------------------------- +// Caches +// --------------------------------------------------------------------------- + +func newCacheAt(t *testing.T, path string) *sql.DB { + t.Helper() + db, err := cache.Open(context.Background(), path) + if err != nil { + t.Fatalf("opening cache %s: %v", path, err) + } + t.Cleanup(func() { _ = db.Close() }) + if _, err := cache.Migrate(context.Background(), db); err != nil { + t.Fatalf("migrating cache %s: %v", path, err) + } + return db +} + +func newCache(t *testing.T) *sql.DB { + t.Helper() + return newCacheAt(t, filepath.Join(t.TempDir(), "anvil-cache.sqlite")) +} + +// seedLive writes documents into a cache THROUGH THE PRODUCTION DELTA PATH. +// +// That is the point: the "delta-built cache" a self-heal diffs against has to +// be built the way the delta pipeline builds it, or the test is comparing the +// baseline against a fixture nobody's code would ever produce. +func seedLive(t *testing.T, db *sql.DB, feed config.FeedConfig, d license.Decision, docs []string, at time.Time) { + t.Helper() + var batch []delta.Record + for _, doc := range docs { + recs, _, err := delta.Decode(feed.ID, []byte(doc)) + if err != nil { + t.Fatalf("decoding a fixture document: %v", err) + } + batch = append(batch, recs...) + } + if _, err := delta.Apply(context.Background(), db, feed, d, batch, at, 0); err != nil { + t.Fatalf("seeding the live cache: %v", err) + } +} + +type storedRow struct { + sourceID string + modified string + state string + asOf string + staleness int + raw string +} + +// snapshot reads back every row of one feed. It is used to assert that +// unrelated rows were not disturbed, which a repair count cannot show. +func snapshot(t *testing.T, db *sql.DB, feedID string) map[string]storedRow { + t.Helper() + rows, err := db.Query( + `SELECT source_id, modified, state, as_of, staleness_seconds, raw_json + FROM advisory WHERE source = ? ORDER BY source_id`, feedID) + if err != nil { + t.Fatalf("snapshotting %q: %v", feedID, err) + } + defer func() { _ = rows.Close() }() + out := map[string]storedRow{} + for rows.Next() { + var ( + r storedRow + modified sql.NullString + raw []byte + ) + if err := rows.Scan(&r.sourceID, &modified, &r.state, &r.asOf, &r.staleness, &raw); err != nil { + t.Fatalf("scanning a snapshot row: %v", err) + } + r.modified, r.raw = modified.String, string(raw) + out[r.sourceID] = r + } + if err := rows.Err(); err != nil { + t.Fatalf("reading the snapshot: %v", err) + } + return out +} + +func countRows(t *testing.T, db *sql.DB, query string, args ...any) int { + t.Helper() + var n int + if err := db.QueryRow(query, args...).Scan(&n); err != nil { + t.Fatalf("counting (%s): %v", query, err) + } + return n +} + +func failuresFor(t *testing.T, db *sql.DB, feedID string) (int, bool) { + t.Helper() + var n int + err := db.QueryRow(`SELECT consecutive_failures FROM feed_state WHERE feed_id = ?`, feedID).Scan(&n) + switch { + case errors.Is(err, sql.ErrNoRows): + return 0, false + case err != nil: + t.Fatalf("reading consecutive_failures for %q: %v", feedID, err) + } + return n, true +} + +// --------------------------------------------------------------------------- +// A fake baseliner, for the cases a real bulk archive cannot express +// --------------------------------------------------------------------------- + +// fakeBaseliner writes a chosen corpus into whatever scratch cache it is +// handed, and returns whatever BootstrapResult a test needs. +// +// It writes through delta.Apply for the same reason seedLive does: a scratch +// cache assembled by hand would not necessarily be a cache the real importer +// could produce. +type fakeBaseliner struct { + t *testing.T + db *sql.DB + feed config.FeedConfig + decision license.Decision + docs []string + at time.Time + + // err, incomplete and emptyResult drive the failure paths. + err error + incomplete bool + emptyResult bool + + // tierOverride and dirOverride drive the decision-mismatch path. A nil + // override means "echo the decision". + tierOverride *int + dirOverride *string + + calls *int + // afterWrite runs with the scratch handle once the corpus is in, so a test + // can plant a row the real importer would never write. + afterWrite func(scratch *sql.DB) +} + +func (f *fakeBaseliner) Bootstrap(ctx context.Context, feed config.FeedConfig) (bootstrap.BootstrapResult, error) { + if f.calls != nil { + *f.calls++ + } + res := bootstrap.BootstrapResult{ + FeedID: feed.ID, + Mechanism: feed.BootstrapMechanism, + Tier: f.decision.Tier.Int(), + Dir: f.decision.Dir, + Complete: !f.incomplete, + } + if f.tierOverride != nil { + res.Tier = *f.tierOverride + } + if f.dirOverride != nil { + res.Dir = *f.dirOverride + } + if f.err != nil { + return res, f.err + } + if !f.emptyResult { + var batch []delta.Record + for _, doc := range f.docs { + recs, _, err := delta.Decode(feed.ID, []byte(doc)) + if err != nil { + return res, err + } + batch = append(batch, recs...) + } + at := f.at + if at.IsZero() { + at = healAt + } + stats, err := delta.Apply(ctx, f.db, feed, f.decision, batch, at, 0) + if err != nil { + return res, err + } + res.RecordsUpserted = stats.Upserts + res.EntriesRead = len(f.docs) + } + if f.afterWrite != nil { + f.afterWrite(f.db) + var n int + if err := f.db.QueryRow(`SELECT count(*) FROM advisory WHERE source = ?`, feed.ID).Scan(&n); err == nil { + res.RecordsUpserted = n + } + } + return res, nil +} + +// fakeFactory binds a fakeBaseliner to whatever scratch handle the healer +// opens. The prototype's db field is ignored; the scratch handle wins, which +// is the property that keeps the baseline out of the live cache. +func fakeFactory(proto fakeBaseliner) BaselineFactory { + return func(scratch *sql.DB) (Baseliner, error) { + f := proto + f.db = scratch + return &f, nil + } +} + +func newHealer(t *testing.T, opts Options) *Healer { + t.Helper() + if opts.WorkDir == "" { + opts.WorkDir = t.TempDir() + } + if opts.Now == nil { + opts.Now = fixedClock(healAt) + } + h, err := New(opts) + if err != nil { + t.Fatalf("building the healer: %v", err) + } + return h +} + +// --------------------------------------------------------------------------- +// 1. The self-heal actually heals, and says so +// --------------------------------------------------------------------------- + +// TestSelfHealRestoresTheRecordsTheDeltaPathDropped is A.15's named +// validation: "a synthetic 'live cache is missing 3 records the fresh baseline +// has' fixture, asserting the reconcile pass restores all 3 without disturbing +// unrelated rows." +// +// It drives the REAL A.8 bootstrapper over an httptest-served zip, so the path +// under test is the production one end to end: bulk archive -> scratch cache +// -> merge-join diff -> A.14's row-scoped upsert. +func TestSelfHealRestoresTheRecordsTheDeltaPathDropped(t *testing.T) { + const total = 20 + const dropped = 3 + + docs := corpus(total) + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + // The archive the publisher serves: ground truth, all 20 records. + archive := buildZip(t, docs) + srv := serveArchive(t, archive) + feed.URL, feed.BootstrapURL = srv.URL+"/all.zip", srv.URL+"/all.zip" + + // The cache the delta pipeline built: the first three records never + // arrived. + live := newCache(t) + seedLive(t, live, feed, decision, docs[dropped:], seedAt) + + missing := make([]string, 0, dropped) + for _, d := range docs[:dropped] { + missing = append(missing, docID(t, d)) + } + before := snapshot(t, live, feed.ID) + if len(before) != total-dropped { + t.Fatalf("the fixture live cache holds %d rows, want %d", len(before), total-dropped) + } + + h := newHealer(t, Options{ + Live: live, + Feed: feed, + Mirror: mirror, + Baseline: FromBootstrapper(bootstrap.Bootstrapper{ + Mirror: mirror, + WorkDir: t.TempDir(), + Clock: fixedClock(healAt), + }), + }) + + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v\nreport: %s", err, rep.Summary()) + } + + // --- The counts the packet asks for. --- + if rep.BaselineRows != total { + t.Errorf("the fresh baseline held %d rows, want %d", rep.BaselineRows, total) + } + if rep.LiveRows != total-dropped { + t.Errorf("the live cache held %d rows, want %d", rep.LiveRows, total-dropped) + } + if rep.MissingInLive != dropped { + t.Errorf("missing-in-live is %d, want %d; the diff did not see the dropped records", + rep.MissingInLive, dropped) + } + if rep.Matched != total-dropped { + t.Errorf("matched is %d, want %d", rep.Matched, total-dropped) + } + if rep.Updated() != 0 || rep.AheadInLive != 0 || rep.OnlyInLive != 0 { + t.Errorf("nothing but the three drops should disagree, got updated=%d ahead=%d only-in-live=%d", + rep.Updated(), rep.AheadInLive, rep.OnlyInLive) + } + if rep.Restored != dropped { + t.Fatalf("restored %d rows, want %d. A.15's stop condition is a NON-ZERO restored count when "+ + "records were deliberately dropped beforehand", rep.Restored, dropped) + } + if err := rep.CheckTotals(); err != nil { + t.Errorf("the diff does not partition the two row sets: %v", err) + } + if !rep.Drifted() { + t.Error("Drifted() is false after three records were restored; the one boolean a daemon " + + "alerts on did not fire") + } + + // --- The three rows are actually back, and decode to the right content. + after := snapshot(t, live, feed.ID) + if len(after) != total { + t.Fatalf("the live cache holds %d rows after the self-heal, want %d", len(after), total) + } + for i, id := range missing { + row, ok := after[id] + if !ok { + t.Fatalf("%s is still missing after the self-heal", id) + } + if row.raw != docs[i] { + t.Errorf("%s was restored with bytes that are not the publisher's", id) + } + if row.asOf != healAt.Format(time.RFC3339) { + t.Errorf("%s carries as_of %q, want the self-heal's clock %q", + id, row.asOf, healAt.Format(time.RFC3339)) + } + } + + // --- NOTHING ELSE MOVED. This is the half a repair count cannot show. --- + for id, was := range before { + now, ok := after[id] + if !ok { + t.Fatalf("%s was in the live cache before the self-heal and is gone after it", id) + } + if now != was { + t.Errorf("%s was rewritten by the self-heal and should not have been:\n before %+v\n after %+v", + id, was, now) + } + } + + // --- The write path's own accounting agrees with the report. --- + if rep.Batch.Upserts != dropped || rep.Batch.FTSUpserts != dropped { + t.Errorf("delta.Apply reports %d upserts and %d FTS writes, want %d of each", + rep.Batch.Upserts, rep.Batch.FTSUpserts, dropped) + } + if rep.Batch.FTSDeletes != 0 { + t.Errorf("the repair deleted %d FTS rows; nothing here is tombstoned", rep.Batch.FTSDeletes) + } +} + +// TestFTSStaysQueryConsistentAfterARepair is the round-trip half of A.14's +// exit criterion, re-asserted for this pass: a restored row is findable by +// text immediately, and a row restored OVER a stale one stops matching the +// stale text. +func TestFTSStaysQueryConsistentAfterARepair(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + // Ground truth: two records. One the live cache never got, one the live + // cache got at an older version whose text says something different. + fresh := cveDoc("CVE-2026-30001", "2026-04-01T00:00:00.000Z", + "An advisory mentioning zebracrossing in the parser.") + updated := cveDoc("CVE-2026-30002", "2026-04-01T00:00:00.000Z", + "The corrected text mentioning pelicancrossing only.") + stale := cveDoc("CVE-2026-30002", "2026-01-01T00:00:00.000Z", + "The superseded text mentioning toucancrossing only.") + + live := newCache(t) + seedLive(t, live, feed, decision, []string{stale}, seedAt) + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{ + t: t, feed: feed, decision: decision, docs: []string{fresh, updated}, + }), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + if rep.MissingInLive != 1 || rep.StaleInLive != 1 || rep.Restored != 2 { + t.Fatalf("want one missing, one stale, two restored; got %+v / restored %d", + []int{rep.MissingInLive, rep.StaleInLive}, rep.Restored) + } + + matches := func(term string) []string { + t.Helper() + rows, err := live.Query( + `SELECT a.source_id FROM advisory_fts f JOIN advisory a ON a.rowid = f.rowid + WHERE advisory_fts MATCH ? ORDER BY a.source_id`, term) + if err != nil { + t.Fatalf("MATCH %q: %v", term, err) + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + t.Fatalf("scanning a MATCH row: %v", err) + } + out = append(out, id) + } + return out + } + + if got := matches("zebracrossing"); len(got) != 1 || got[0] != "CVE-2026-30001" { + t.Errorf("the restored record is not findable by text: MATCH zebracrossing gave %v", got) + } + if got := matches("pelicancrossing"); len(got) != 1 || got[0] != "CVE-2026-30002" { + t.Errorf("the updated record's new text is not indexed: MATCH pelicancrossing gave %v", got) + } + if got := matches("toucancrossing"); len(got) != 0 { + t.Errorf("the SUPERSEDED text still matches after the repair: MATCH toucancrossing gave %v. "+ + "That is the contentless-FTS phantom-hit failure A.2 carries contentless_delete=1 for", got) + } +} + +// --------------------------------------------------------------------------- +// 2. It never makes the cache worse +// --------------------------------------------------------------------------- + +// TestANewerLiveRowIsNeverOverwrittenByAnOlderBaseline is the data-loss guard. +// +// A bulk artifact is cut at an instant. The delta stream is legitimately past +// it, and a self-heal that restored "ground truth" over a newer row would +// throw away the very update the delta path exists to deliver. +func TestANewerLiveRowIsNeverOverwrittenByAnOlderBaseline(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + old := cveDoc("CVE-2026-40001", "2026-01-01T00:00:00.000Z", "The version the weekly archive was cut with.") + newer := cveDoc("CVE-2026-40001", "2026-06-01T00:00:00.000Z", "The version the delta stream already delivered.") + + live := newCache(t) + seedLive(t, live, feed, decision, []string{newer}, seedAt) + before := snapshot(t, live, feed.ID) + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: []string{old}}), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + + if rep.AheadInLive != 1 { + t.Fatalf("ahead-in-live is %d, want 1; the pass did not notice the live cache was newer", rep.AheadInLive) + } + if rep.Restored != 0 || rep.StaleInLive != 0 || rep.Divergent != 0 { + t.Fatalf("the pass wrote something: restored=%d stale=%d divergent=%d", rep.Restored, rep.StaleInLive, rep.Divergent) + } + if got := snapshot(t, live, feed.ID); !sameSnapshot(got, before) { + t.Errorf("the newer live row was overwritten with the archive's older bytes:\n before %+v\n after %+v", + before, got) + } + if !rep.Drifted() { + t.Error("a row the archive and the cache disagree about did not register as drift") + } + // The disagreement is REPORTED even though nothing was done about it. + if len(rep.Samples) != 1 || rep.Samples[0].Kind != KindAheadInLive || rep.Samples[0].Repaired { + t.Errorf("the ahead-in-live key was not reported as an untouched disagreement: %+v", rep.Samples) + } +} + +// TestARowGroundTruthLacksIsReportedAndNeverDeleted is the other direction. +// +// A.2 exit criterion 22 tombstones withdrawn and REJECTED advisories rather +// than deleting them, and that is A.16's pass. A self-heal that deleted a row +// because this week's archive did not carry it would destroy the row a prior +// finding references. +func TestARowGroundTruthLacksIsReportedAndNeverDeleted(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + shared := cveDoc("CVE-2026-50001", "2026-03-01T00:00:00.000Z", "Carried by both sides.") + orphan := cveDoc("CVE-2026-50002", "2026-03-01T00:00:00.000Z", "The live cache has it and the archive does not.") + + live := newCache(t) + seedLive(t, live, feed, decision, []string{shared, orphan}, seedAt) + before := snapshot(t, live, feed.ID) + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: []string{shared}}), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + if rep.OnlyInLive != 1 || rep.Matched != 1 { + t.Fatalf("want one only-in-live and one matched, got only-in-live=%d matched=%d", rep.OnlyInLive, rep.Matched) + } + if got := snapshot(t, live, feed.ID); !sameSnapshot(got, before) { + t.Errorf("the self-heal changed the live cache; it should have reported and done nothing") + } + if got := countRows(t, live, `SELECT count(*) FROM advisory WHERE source = ?`, feed.ID); got != 2 { + t.Errorf("the live cache holds %d rows, want 2: a row ground truth lacks was deleted", got) + } + if len(rep.Samples) != 1 || rep.Samples[0].Kind != KindOnlyInLive { + t.Errorf("the only-in-live key was not sampled: %+v", rep.Samples) + } +} + +// TestDivergentBytesAtTheSameVersionAreRestoredFromGroundTruth covers the row +// that is corrupt rather than stale: the two sides claim the same version and +// carry different bytes. +func TestDivergentBytesAtTheSameVersionAreRestoredFromGroundTruth(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + truth := cveDoc("CVE-2026-60001", "2026-03-01T00:00:00.000Z", "The publisher's own description.") + corrupt := cveDoc("CVE-2026-60001", "2026-03-01T00:00:00.000Z", "A description that lost half its text.") + + live := newCache(t) + seedLive(t, live, feed, decision, []string{corrupt}, seedAt) + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: []string{truth}}), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + if rep.Divergent != 1 || rep.Restored != 1 { + t.Fatalf("want one divergent and one restored, got divergent=%d restored=%d", rep.Divergent, rep.Restored) + } + if got := snapshot(t, live, feed.ID)["CVE-2026-60001"].raw; got != truth { + t.Errorf("the corrupt row was not replaced with the publisher's bytes") + } +} + +// TestAnUndatableRowFailsTowardGroundTruth pins the third arm of classify: a +// `modified` neither side can parse must not be read as "the live cache is +// ahead". Keeping a row Anvil cannot date over a row it fetched from the +// publisher is the wrong risk. +func TestAnUndatableRowFailsTowardGroundTruth(t *testing.T) { + base := sideRow{sourceID: "CVE-2026-1", modified: "whenever", digest: contentDigest("published", "whenever", []byte("a"))} + live := sideRow{sourceID: "CVE-2026-1", modified: "", digest: contentDigest("published", "", []byte("b"))} + if got := classify(base, live); got != KindDivergent { + t.Errorf("an undatable pair classified as %q, want %q", got, KindDivergent) + } + if !KindDivergent.Repairable() { + t.Error("KindDivergent is not repairable, so an undatable disagreement would never be fixed") + } + + same := contentDigest("published", "2026-01-01", []byte("a")) + if got := classify(sideRow{digest: same}, sideRow{digest: same}); got != "" { + t.Errorf("identical rows classified as %q, want a match", got) + } +} + +// TestEveryDisagreementKindIsClassifiedAndAccountedFor keeps the kind table +// and the repair rule from drifting apart. +func TestEveryDisagreementKindIsClassifiedAndAccountedFor(t *testing.T) { + kinds := DisagreementKinds() + if len(kinds) != 5 { + t.Fatalf("DisagreementKinds returns %d kinds; the report's five buckets are the partition", len(kinds)) + } + seen := map[DisagreementKind]bool{} + for _, k := range kinds { + if !k.Valid() { + t.Errorf("%q is enumerated and not Valid", k) + } + if seen[k] { + t.Errorf("%q is enumerated twice", k) + } + seen[k] = true + } + if DisagreementKind("something-else").Valid() { + t.Error("Valid admits a kind nobody declared") + } + repairable := 0 + for _, k := range kinds { + if k.Repairable() { + repairable++ + } + } + if repairable != 3 { + t.Errorf("%d kinds are repairable, want exactly three (missing, stale, divergent): the two "+ + "reported-only kinds are what stop the self-heal deleting or regressing rows", repairable) + } + if KindAheadInLive.Repairable() || KindOnlyInLive.Repairable() { + t.Error("a reported-only kind is marked repairable; that is the data-loss path") + } +} + +// --------------------------------------------------------------------------- +// 3. A failure is loud +// --------------------------------------------------------------------------- + +// TestABootstrapFailureIncrementsConsecutiveFailures is the packet's forbidden +// action, asserted directly: "a failed weekly self-heal must increment +// feed_state.consecutive_failures and surface via A.16's staleness mechanism, +// not fail closed and disappear." +func TestABootstrapFailureIncrementsConsecutiveFailures(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + docs := corpus(4) + live := newCache(t) + seedLive(t, live, feed, decision, docs, seedAt) + before := snapshot(t, live, feed.ID) + + boom := errors.New("the publisher returned 503 for six hours") + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, err: boom}), + }) + + if n, ok := failuresFor(t, live, feed.ID); ok && n != 0 { + t.Fatalf("the fixture starts with consecutive_failures = %d, want 0", n) + } + + rep, err := h.WeeklySelfHeal(context.Background()) + if err == nil { + t.Fatal("a bootstrap failure returned no error; the self-heal failed closed and disappeared") + } + if !errors.Is(err, ErrBaselineFailed) || !errors.Is(err, boom) { + t.Errorf("the error does not carry the cause: %v", err) + } + if !rep.Failed || rep.FailedBecause == "" { + t.Errorf("the report does not say the pass failed: %+v", rep) + } + if !rep.FailureRecorded { + t.Fatal("the failure was not recorded in feed_state") + } + if rep.ConsecutiveFailures != 1 { + t.Errorf("consecutive_failures reported as %d, want 1", rep.ConsecutiveFailures) + } + n, ok := failuresFor(t, live, feed.ID) + if !ok { + t.Fatal("no feed_state row was written at all; A.16's staleness mechanism has nothing to read") + } + if n != 1 { + t.Errorf("feed_state.consecutive_failures is %d, want 1", n) + } + + // A second failure keeps climbing. A counter that saturates at one cannot + // distinguish a blip from an outage. + if _, err := h.WeeklySelfHeal(context.Background()); err == nil { + t.Fatal("the second failure returned no error") + } + if n, _ := failuresFor(t, live, feed.ID); n != 2 { + t.Errorf("after two failures consecutive_failures is %d, want 2", n) + } + + // And the cache was not touched on the way past. + if got := snapshot(t, live, feed.ID); !sameSnapshot(got, before) { + t.Error("a failed self-heal modified the live cache") + } + if !strings.Contains(rep.Summary(), "FAILED") { + t.Errorf("the operator's one-line summary does not say the pass failed: %q", rep.Summary()) + } +} + +// TestAnIncompleteBaselineIsRefusedAndCounted: a partial import is a PREFIX of +// ground truth, so its "only in the live cache" count would be fiction. +func TestAnIncompleteBaselineIsRefusedAndCounted(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + docs := corpus(6) + live := newCache(t) + seedLive(t, live, feed, decision, docs, seedAt) + before := snapshot(t, live, feed.ID) + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{ + t: t, feed: feed, decision: decision, docs: docs[:2], incomplete: true, + }), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if !errors.Is(err, ErrIncompleteBaseline) { + t.Fatalf("an incomplete baseline gave %v, want ErrIncompleteBaseline", err) + } + if !rep.Failed || !rep.FailureRecorded || rep.ConsecutiveFailures != 1 { + t.Errorf("an incomplete baseline was not counted as a failure: %+v", rep) + } + if rep.MissingInLive != 0 || rep.OnlyInLive != 0 { + t.Errorf("the pass diffed against a partial baseline anyway: %+v", rep) + } + if got := snapshot(t, live, feed.ID); !sameSnapshot(got, before) { + t.Error("a refused self-heal modified the live cache") + } +} + +// TestAnEmptyBaselineIsRefusedAndCounted: a full baseline of a CVE feed that +// holds zero advisories is a broken artifact, not ground truth about an empty +// world. Diffing against it would report the whole cache as unexplained. +func TestAnEmptyBaselineIsRefusedAndCounted(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + docs := corpus(5) + live := newCache(t) + seedLive(t, live, feed, decision, docs, seedAt) + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, emptyResult: true}), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if !errors.Is(err, ErrEmptyBaseline) { + t.Fatalf("an empty baseline gave %v, want ErrEmptyBaseline", err) + } + if rep.OnlyInLive != 0 { + t.Errorf("the pass reported %d rows as unexplained against an empty baseline; that is the "+ + "false alarm the size of the cache", rep.OnlyInLive) + } + if n, _ := failuresFor(t, live, feed.ID); n != 1 { + t.Errorf("consecutive_failures is %d, want 1", n) + } + if got := countRows(t, live, `SELECT count(*) FROM advisory WHERE source = ?`, feed.ID); got != len(docs) { + t.Errorf("the live cache holds %d rows, want %d", got, len(docs)) + } +} + +// TestALicenceDecisionMismatchIsRefused: both sides call license.Resolve on +// the same feed row, so a disagreement means they are reading different +// mirrors and neither answer may be used to write a row. +func TestALicenceDecisionMismatchIsRefused(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + wrongDir := decision.Dir + "-somewhere-else" + live := newCache(t) + seedLive(t, live, feed, decision, corpus(2), seedAt) + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{ + t: t, feed: feed, decision: decision, docs: corpus(2), dirOverride: &wrongDir, + }), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if !errors.Is(err, ErrDecisionMismatch) { + t.Fatalf("a mismatched licence decision gave %v, want ErrDecisionMismatch", err) + } + if !rep.Failed || !rep.FailureRecorded { + t.Errorf("a licence decision mismatch was not counted as a failure: %+v", rep) + } +} + +// TestTheLicenceGateRunsBeforeAnythingIsFetched. A refusal must cost no +// request, and — deliberately — must NOT climb the failure counter: no +// publisher licence body has been acquired is the ORDINARY state of a fresh +// clone, and counting it weekly would make the counter useless for the case it +// exists for. +func TestTheLicenceGateRunsBeforeAnythingIsFetched(t *testing.T) { + feed := testFeed("cvelistv5") + live := newCache(t) + + calls := 0 + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: emptyMirror(), + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, docs: corpus(2), calls: &calls}), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err == nil { + t.Fatal("an unadmitted feed produced no refusal") + } + if !errors.Is(err, license.ErrLicenseRefused) && !strings.Contains(err.Error(), "licen") { + t.Errorf("the refusal does not come from the licence gate: %v", err) + } + if calls != 0 { + t.Errorf("the baseline was built %d times despite a licence refusal; the gate must run BEFORE "+ + "a byte is fetched", calls) + } + if !rep.Refused || rep.RefusedBecause == "" { + t.Errorf("the report does not name the refusal: %+v", rep) + } + if rep.Failed { + t.Error("a licence refusal was recorded as a self-heal failure") + } + if _, ok := failuresFor(t, live, feed.ID); ok { + t.Error("a licence refusal wrote a feed_state row; a fresh clone would climb a failure counter " + + "on every feed in the table with nothing broken") + } + if rep.Tier != license.NoTier { + t.Errorf("a refused pass reports tier %d; a refusal must never carry a valid tier, and 0 is the "+ + "most permissive tier this system has", rep.Tier) + } +} + +// TestAFeedWithNoBulkBaselineIsRefused. Running a "full baseline" against a +// mechanism that imports nothing yields an empty ground truth, against which +// the entire live cache looks unexplained. +func TestAFeedWithNoBulkBaselineIsRefused(t *testing.T) { + for _, mech := range []config.BootstrapMechanism{config.BootstrapNone, config.BootstrapIncrementalAPI} { + t.Run(string(mech), func(t *testing.T) { + feed := testFeed("kev") + feed.BootstrapMechanism = mech + mirror := admittingMirror(t, feed) + + calls := 0 + h := newHealer(t, Options{ + Live: newCache(t), Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, calls: &calls}), + }) + _, err := h.WeeklySelfHeal(context.Background()) + if !errors.Is(err, ErrNoBaselineMechanism) { + t.Fatalf("bootstrap_mechanism %q gave %v, want ErrNoBaselineMechanism", mech, err) + } + if calls != 0 { + t.Errorf("the baseline was built anyway (%d calls)", calls) + } + }) + } +} + +// --------------------------------------------------------------------------- +// 4. Nothing full-table reaches the cache +// --------------------------------------------------------------------------- + +// TestNoFullTableStatementReachesTheLiveCache opens the live cache through a +// tracing driver and inspects every statement that reached the driver layer +// during a repair. +// +// This is A.2's and A.14's shared rule, re-checked for this pass: FTS5 accepts +// incremental INSERT/DELETE, so a repair touching three records costs three +// row-scoped index writes and NOT a rebuild. It is checked as an observation +// of production statements, not as an assertion about code someone read. +func TestNoFullTableStatementReachesTheLiveCache(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + docs := corpus(12) + live := newTracedCache(t) + seedLive(t, live, feed, decision, docs[3:], seedAt) + + trace.reset() + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: docs}), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + if rep.Restored != 3 { + t.Fatalf("restored %d rows, want 3", rep.Restored) + } + + stmts := trace.snapshot() + if len(stmts) == 0 { + t.Fatal("the trace captured nothing; the driver seam is inert and would pass anything") + } + + // The forbidden shapes. This list is a DENYLIST only in its role as an + // alarm: the structural guarantee is reconcile's own allowlist plus + // delta.Apply's, and this is the observation that proves those two hold in + // combination on a real run. + forbidden := []string{ + "drop table", "drop view", "create virtual table", "create table", + "alter table", "vacuum", "reindex", "'rebuild'", + } + // Every DELETE that reaches the cache must be ROW-SCOPED. A.14 legitimately + // replaces one advisory's `affected` and `cve_alias` rows per upsert + // (surrogate key, no unique natural key) and deletes one FTS row by rowid; + // what must never appear is a delete whose scope is a table. + rowScopes := []string{ + "where rowid = ?", + "where source = ? and source_id = ?", + } + upserts, ftsWrites := 0, 0 + for _, q := range stmts { + flat := strings.ToLower(strings.Join(strings.Fields(q), " ")) + for _, bad := range forbidden { + if strings.Contains(flat, bad) { + t.Errorf("a full-table statement reached the live cache: %q (matched %q)", flat, bad) + } + } + if strings.HasPrefix(flat, "delete ") { + scoped := false + for _, s := range rowScopes { + if strings.HasSuffix(flat, s) { + scoped = true + } + } + if !scoped { + t.Errorf("a DELETE reached the live cache that is not row-scoped: %q", flat) + } + } + if strings.HasPrefix(flat, "insert into advisory (") { + upserts++ + } + if strings.Contains(flat, "advisory_fts") && + (strings.HasPrefix(flat, "insert") || strings.HasPrefix(flat, "delete")) { + ftsWrites++ + } + } + if upserts != 3 { + t.Errorf("%d advisory upserts reached the driver, want exactly 3 — one per restored record", upserts) + } + if ftsWrites != 3 { + t.Errorf("%d statements touched advisory_fts, want exactly 3", ftsWrites) + } +} + +// TestTheStatementAllowlistIsOnTheLivePath is the RED verification of the +// allowlist: an allowlist that has never refused anything has not been tested. +// +// It removes the scan statement from the allowlist and asserts the PRODUCTION +// path then fails — so the guard is load-bearing rather than a decorative map +// nothing consults. +func TestTheStatementAllowlistIsOnTheLivePath(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + docs := corpus(3) + + run := func() error { + live := newCache(t) + seedLive(t, live, feed, decision, docs[1:], seedAt) + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: docs}), + }) + _, err := h.WeeklySelfHeal(context.Background()) + return err + } + + if err := run(); err != nil { + t.Fatalf("the pass does not pass with the allowlist intact: %v", err) + } + + key := strings.TrimSpace(scanAdvisorySQL) + saved := allowedStatements[key] + delete(allowedStatements, key) + err := run() + allowedStatements[key] = saved + + if !errors.Is(err, ErrStatementNotAllowed) { + t.Fatalf("with the scan removed from the allowlist the pass returned %v; the guard is not on "+ + "the production path", err) + } + if err := run(); err != nil { + t.Fatalf("the allowlist was not restored: %v", err) + } +} + +// TestTheAllowlistHoldsNoWriteAgainstTheAdvisoryTables. This package must have +// no second write path for advisory / affected / advisory_fts; those writes +// belong to delta.Apply so that one writer holds the schema invariants for +// both A.14 and A.15. +func TestTheAllowlistHoldsNoWriteAgainstTheAdvisoryTables(t *testing.T) { + if len(allowedStatements) == 0 { + t.Fatal("the allowlist is empty; the guard would refuse everything and the tests above would not pass") + } + writesFeedState := 0 + for q, reason := range allowedStatements { + if strings.TrimSpace(reason) == "" { + t.Errorf("allowlist entry has no reason:\n\t%s", q) + } + flat := strings.ToLower(strings.Join(strings.Fields(q), " ")) + switch { + case strings.HasPrefix(flat, "select "): + // A read is fine whatever it names. + case strings.HasPrefix(flat, "insert into feed_state"): + writesFeedState++ + default: + t.Errorf("allowlist entry is neither a SELECT nor the feed_state upsert:\n\t%s", flat) + } + for _, table := range []string{"advisory ", "advisory(", "advisory_fts", "affected"} { + if strings.HasPrefix(flat, "select ") { + continue + } + if strings.Contains(flat, table) { + t.Errorf("an allowlisted write names %q; those writes belong to delta.Apply:\n\t%s", table, flat) + } + } + } + if writesFeedState != 1 { + t.Errorf("%d allowlisted statements write feed_state, want exactly one (the failure counter)", writesFeedState) + } + if err := checkStatement(`DELETE FROM advisory WHERE source = ?`); !errors.Is(err, ErrStatementNotAllowed) { + t.Errorf("the guard admitted a plausible-looking DELETE: %v", err) + } + if err := checkStatement(`INSERT INTO advisory_fts(advisory_fts) VALUES('rebuild')`); !errors.Is(err, ErrStatementNotAllowed) { + t.Errorf("the guard admitted an FTS rebuild: %v", err) + } + for q := range allowedStatements { + if err := checkStatement(q); err != nil { + t.Errorf("the guard refuses its own allowlist entry: %v", err) + } + if err := checkStatement(" \n" + q + "\n "); err != nil { + t.Errorf("the guard is whitespace-sensitive in a way that would refuse a real call: %v", err) + } + } +} + +// TestTheMergeJoinRefusesAnUnorderedScan is the RED verification of the second +// guard. A merge join over an unordered scan silently produces nonsense — every +// key looks missing on one side and unexplained on the other — and "the +// statement says ORDER BY" is not evidence that the rows arrived that way. +func TestTheMergeJoinRefusesAnUnorderedScan(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + db := newCache(t) + seedLive(t, db, feed, decision, corpus(5), seedAt) + + // Deliberately DESCENDING, issued straight at the driver so the allowlist + // is not the thing under test here. + rows, err := db.QueryContext(context.Background(), + `SELECT source_id, modified, state, raw_json FROM advisory WHERE source = ? ORDER BY source_id DESC`, feed.ID) + if err != nil { + t.Fatalf("querying: %v", err) + } + defer func() { _ = rows.Close() }() + + c := newCursor(rows) + for c.valid { + c.advance() + } + if c.err == nil { + t.Fatal("the cursor consumed a descending scan without complaint; the merge join would have " + + "reported every row as a disagreement in both directions") + } + if !strings.Contains(c.err.Error(), "ORDER BY") { + t.Errorf("the cursor's refusal does not say what went wrong: %v", c.err) + } + + // And the ascending scan the production path uses is accepted. + asc, err := db.QueryContext(context.Background(), scanAdvisorySQL, feed.ID) + if err != nil { + t.Fatalf("querying: %v", err) + } + defer func() { _ = asc.Close() }() + ok := newCursor(asc) + seen := 0 + for ok.valid { + seen++ + ok.advance() + } + if ok.err != nil { + t.Fatalf("the ordered scan was refused: %v", ok.err) + } + if seen != 5 { + t.Errorf("the cursor saw %d rows, want 5", seen) + } +} + +// --------------------------------------------------------------------------- +// Reporting, cadence and the remaining behaviour +// --------------------------------------------------------------------------- + +// TestReportOnlyDiffsAndWritesNothing. A drift alarm should be able to run +// more often than a repair, and an operator should be able to see what a +// self-heal WOULD do before letting it do it. +func TestReportOnlyDiffsAndWritesNothing(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + docs := corpus(8) + live := newCache(t) + seedLive(t, live, feed, decision, docs[3:], seedAt) + before := snapshot(t, live, feed.ID) + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, ReportOnly: true, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: docs}), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + if rep.MissingInLive != 3 { + t.Errorf("missing-in-live is %d, want 3", rep.MissingInLive) + } + if rep.Restored != 0 || rep.Batch.Upserts != 0 { + t.Errorf("a report-only pass wrote %d rows", rep.Restored) + } + if !rep.ReportOnly { + t.Error("the report does not say it was report-only") + } + if got := snapshot(t, live, feed.ID); !sameSnapshot(got, before) { + t.Error("a report-only pass changed the live cache") + } + for _, s := range rep.Samples { + if s.Repaired { + t.Errorf("a report-only pass marked %s as repaired", s.SourceID) + } + } +} + +// TestTheBaselineNeverLandsInTheLiveCache. The whole design rests on the fresh +// baseline being imported somewhere else: if it landed in the live cache, the +// repair would BE the import and there would be nothing left to diff. +func TestTheBaselineNeverLandsInTheLiveCache(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + docs := corpus(9) + + live := newCache(t) + seedLive(t, live, feed, decision, docs[4:], seedAt) + + var handedLive bool + factory := func(scratch *sql.DB) (Baseliner, error) { + if scratch == live { + handedLive = true + } + return &fakeBaseliner{t: t, db: scratch, feed: feed, decision: decision, docs: docs}, nil + } + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, ReportOnly: true, Baseline: factory, + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + if handedLive { + t.Fatal("the baseline factory was handed the LIVE cache handle") + } + // Report-only, so the live cache still holds only what the delta path put + // there — which is only observable because the baseline went elsewhere. + if got := countRows(t, live, `SELECT count(*) FROM advisory WHERE source = ?`, feed.ID); got != 5 { + t.Errorf("the live cache holds %d rows after a report-only pass over a 9-row baseline, want 5", got) + } + if rep.BaselineRows != 9 { + t.Errorf("the baseline held %d rows, want 9", rep.BaselineRows) + } +} + +// TestTheScratchBaselineIsCleanedUpUnlessAsked. A 570 MB import per feed per +// week is not something to leave behind by accident, and it IS something an +// operator investigating a drift needs to be able to keep. +func TestTheScratchBaselineIsCleanedUpUnlessAsked(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + docs := corpus(3) + + for _, keep := range []bool{false, true} { + t.Run(fmt.Sprintf("keep=%v", keep), func(t *testing.T) { + workDir := t.TempDir() + live := newCache(t) + seedLive(t, live, feed, decision, docs, seedAt) + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, WorkDir: workDir, KeepBaseline: keep, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: docs}), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + entries, err := filepath.Glob(filepath.Join(workDir, "anvil-baseline-*")) + if err != nil { + t.Fatalf("globbing the work directory: %v", err) + } + switch { + case keep && len(entries) != 1: + t.Errorf("KeepBaseline left %d scratch directories, want 1", len(entries)) + case keep && rep.BaselinePath == "": + t.Error("KeepBaseline did not name the scratch database in the report") + case !keep && len(entries) != 0: + t.Errorf("the scratch baseline was left behind: %v", entries) + case !keep && rep.BaselinePath != "": + t.Errorf("the report names a scratch path that was deleted: %q", rep.BaselinePath) + } + }) + } +} + +// TestTheCadenceComesFromTheFeedRowAndForceOverridesIt. There is no weekly +// constant in this package: A.1 puts every cadence in feeds.yaml so an +// operator can dial the pipeline down on a constrained host. +func TestTheCadenceComesFromTheFeedRowAndForceOverridesIt(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + docs := corpus(4) + + live := newCache(t) + seedLive(t, live, feed, decision, docs[1:], seedAt) + + // A success recorded inside the same weekly window as the clock. + lastOK := healAt.Add(-2 * time.Hour).Format(time.RFC3339) + if _, err := live.Exec(cache.UpsertFeedStateSQL, feed.ID, nil, nil, nil, lastOK, 0, 0); err != nil { + t.Fatalf("seeding feed_state: %v", err) + } + + calls := 0 + base := Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: docs, calls: &calls}), + } + + rep, err := newHealer(t, base).WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + if !rep.Skipped { + t.Fatalf("the pass ran inside the same baseline window: %+v", rep.Plan) + } + if calls != 0 { + t.Errorf("a skipped pass built the baseline anyway (%d calls)", calls) + } + if rep.Plan.BaselineInterval != 604800*time.Second { + t.Errorf("the plan reports a %s baseline interval; it must come from the feed row", + rep.Plan.BaselineInterval) + } + if rep.Note == "" || !strings.Contains(rep.Summary(), "skipped") { + t.Errorf("a skipped pass does not say why: %q", rep.Summary()) + } + + forced := base + forced.Force = true + rep, err = newHealer(t, forced).WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("forced WeeklySelfHeal: %v", err) + } + if rep.Skipped || calls != 1 { + t.Fatalf("Force did not override the cadence: skipped=%v calls=%d", rep.Skipped, calls) + } + if rep.Restored != 1 { + t.Errorf("the forced pass restored %d rows, want 1", rep.Restored) + } + + // A feed row with no baseline cadence schedules no self-heal at all. + noCadence := base + noCadence.Feed.BaselineIntervalSeconds = 0 + rep, err = newHealer(t, noCadence).WeeklySelfHeal(context.Background()) + if err != nil || !rep.Skipped { + t.Fatalf("a zero baseline_interval_seconds did not skip: err=%v skipped=%v", err, rep.Skipped) + } + if !strings.Contains(rep.Note, "baseline_interval_seconds") { + t.Errorf("the skip reason does not name the missing cadence: %q", rep.Note) + } +} + +// TestSamplesAreBoundedAndTheCountsAreNot. A report is read by a person; a +// catastrophic diff must produce a report and not a second copy of the cache. +func TestSamplesAreBoundedAndTheCountsAreNot(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + docs := corpus(30) + live := newCache(t) + seedLive(t, live, feed, decision, docs[20:], seedAt) + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, MaxSamples: 4, ReportOnly: true, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: docs}), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + if rep.MissingInLive != 20 { + t.Errorf("missing-in-live is %d, want 20; the COUNTS must not be truncated", rep.MissingInLive) + } + if len(rep.Samples) != 4 { + t.Errorf("the sample holds %d entries, want 4", len(rep.Samples)) + } + if !rep.SamplesTruncated { + t.Error("the report does not say the sample was truncated") + } + if err := rep.CheckTotals(); err != nil { + t.Errorf("the counts do not partition the row sets: %v", err) + } +} + +// TestCheckTotalsCatchesAMiscount. CheckTotals is only worth exporting if it +// can fail, so this drives it against a report that does not add up. +func TestCheckTotalsCatchesAMiscount(t *testing.T) { + good := ReconcileReport{BaselineRows: 5, LiveRows: 4, Matched: 3, MissingInLive: 1, StaleInLive: 1, OnlyInLive: 0} + if err := good.CheckTotals(); err != nil { + t.Fatalf("a consistent report was rejected: %v", err) + } + bad := good + bad.Matched = 2 + if err := bad.CheckTotals(); err == nil { + t.Error("CheckTotals accepted a report whose buckets do not account for every row") + } + if good.Updated() != 1 { + t.Errorf("Updated() is %d, want stale+divergent = 1", good.Updated()) + } + if good.Disagreements() != 2 || !good.Drifted() { + t.Errorf("Disagreements()=%d Drifted()=%v, want 2 and true", good.Disagreements(), good.Drifted()) + } + clean := ReconcileReport{BaselineRows: 3, LiveRows: 3, Matched: 3} + if clean.Drifted() { + t.Error("a report with no disagreements claims drift") + } + if !strings.Contains(clean.Summary(), "matched 3") { + t.Errorf("a clean summary does not state what it found: %q", clean.Summary()) + } +} + +// TestAnUndecodableBaselineRecordDoesNotBlockTheRest. One malformed record in +// ground truth must not cost the other restorations; it is counted, sampled +// and carried past. +func TestAnUndecodableBaselineRecordDoesNotBlockTheRest(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + docs := corpus(4) + live := newCache(t) + seedLive(t, live, feed, decision, docs[2:], seedAt) + + // A row the real importer would never write, planted straight into the + // scratch baseline: a key with bytes nothing can decode. + plant := func(scratch *sql.DB) { + var rowid int64 + err := scratch.QueryRow(cache.UpsertAdvisorySQL, + feed.ID, "CVE-2026-99999", "CVE-2026-99999", nil, "2026-05-01T00:00:00Z", + cache.AdvisoryPublished, nil, nil, nil, nil, nil, nil, 0, + decision.EffectiveSPDX, nil, decision.Tier.Int(), string(cache.AdvisoryTrustDefault), + healAt.Format(time.RFC3339), 0, 0, nil, []byte("this is not a document")).Scan(&rowid) + if err != nil { + t.Fatalf("planting an undecodable baseline row: %v", err) + } + } + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, + Baseline: fakeFactory(fakeBaseliner{ + t: t, feed: feed, decision: decision, docs: docs, afterWrite: plant, + }), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("one bad record aborted the whole self-heal: %v", err) + } + if rep.MissingInLive != 3 { + t.Fatalf("missing-in-live is %d, want 3 (two real drops plus the planted key)", rep.MissingInLive) + } + if rep.RepairFailures != 1 { + t.Errorf("repair failures is %d, want 1", rep.RepairFailures) + } + if rep.Restored != 2 { + t.Errorf("restored %d rows, want 2; the two decodable drops must still be repaired", rep.Restored) + } + if !strings.Contains(rep.Note, "could not be decoded") { + t.Errorf("the report does not mention the undecodable record: %q", rep.Note) + } + var bad *Disagreement + for i := range rep.Samples { + if rep.Samples[i].SourceID == "CVE-2026-99999" { + bad = &rep.Samples[i] + } + } + if bad == nil { + t.Fatal("the undecodable key was not sampled") + } + if bad.Repaired || !strings.Contains(bad.Note, "did not decode") { + t.Errorf("the sample does not say what happened to it: %+v", *bad) + } + if n := countRows(t, live, `SELECT count(*) FROM advisory WHERE source = ? AND source_id = ?`, + feed.ID, "CVE-2026-99999"); n != 0 { + t.Error("the undecodable record was written into the live cache anyway") + } +} + +// TestNewRefusesAnUnusableHealer. Every one of these is something the package +// cannot invent, and inventing any of them is how a self-heal ends up +// bootstrapping into the live cache or fetching without a licence. +func TestNewRefusesAnUnusableHealer(t *testing.T) { + feed := testFeed("cvelistv5") + ok := Options{Live: newCache(t), Feed: feed, WorkDir: t.TempDir(), Baseline: fakeFactory(fakeBaseliner{})} + if _, err := New(ok); err != nil { + t.Fatalf("a complete Options was refused: %v", err) + } + for name, mutate := range map[string]func(*Options){ + "no live cache": func(o *Options) { o.Live = nil }, + "no feed": func(o *Options) { o.Feed = config.FeedConfig{} }, + "no work directory": func(o *Options) { o.WorkDir = "" }, + "no baseline factory": func(o *Options) { o.Baseline = nil }, + } { + t.Run(name, func(t *testing.T) { + bad := ok + mutate(&bad) + if _, err := New(bad); !errors.Is(err, ErrNotConfigured) { + t.Errorf("New accepted %s: %v", name, err) + } + }) + } + over := ok + over.RepairBatch = MaxRepairBatch + 1 + if _, err := New(over); !errors.Is(err, ErrNotConfigured) { + t.Errorf("New accepted a repair batch over the cap: %v", err) + } + if f := FromBootstrapper(bootstrap.Bootstrapper{}); f != nil { + if _, err := f(nil); !errors.Is(err, ErrNotConfigured) { + t.Errorf("FromBootstrapper accepted a nil scratch handle: %v", err) + } + } +} + +// TestARepairBatchIsBoundedAndStillRestoresEverything drives the batching seam +// so that a self-heal larger than one transaction is exercised at all. +func TestARepairBatchIsBoundedAndStillRestoresEverything(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + + docs := corpus(17) + live := newCache(t) + seedLive(t, live, feed, decision, docs[:2], seedAt) + + h := newHealer(t, Options{ + Live: live, Feed: feed, Mirror: mirror, RepairBatch: 4, + Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: docs}), + }) + rep, err := h.WeeklySelfHeal(context.Background()) + if err != nil { + t.Fatalf("WeeklySelfHeal: %v", err) + } + if rep.Restored != 15 { + t.Fatalf("restored %d rows across batches of 4, want 15", rep.Restored) + } + if got := countRows(t, live, `SELECT count(*) FROM advisory WHERE source = ?`, feed.ID); got != 17 { + t.Errorf("the live cache holds %d rows, want 17", got) + } + if got := countRows(t, live, `SELECT count(*) FROM advisory_fts`); got != 17 { + t.Errorf("the FTS index holds %d rows, want 17", got) + } +} + +// TestTheReportCarriesTheDurationItMeasured. The duration is written by a +// deferred assignment, which reaches the caller's value only because +// WeeklySelfHeal's results are NAMED. With unnamed results every report would +// carry a zero that looked like a measurement, and nothing else in this file +// would have noticed. +func TestTheReportCarriesTheDurationItMeasured(t *testing.T) { + feed := testFeed("cvelistv5") + mirror := admittingMirror(t, feed) + decision := admittedDecision(t, feed, mirror) + docs := corpus(3) + + // A clock that advances one second per reading, so the measured span is a + // fixed non-zero number rather than a wall-clock race. + var ticks int + stepping := func() time.Time { + ticks++ + return healAt.Add(time.Duration(ticks-1) * time.Second) + } + + for _, tc := range []struct { + name string + opts Options + }{ + {"repaired", Options{Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, docs: docs})}}, + {"failed", Options{Baseline: fakeFactory(fakeBaseliner{t: t, feed: feed, decision: decision, err: errors.New("no")})}}, + } { + t.Run(tc.name, func(t *testing.T) { + ticks = 0 + live := newCache(t) + seedLive(t, live, feed, decision, docs[1:], seedAt) + o := tc.opts + o.Live, o.Feed, o.Mirror, o.Now = live, feed, mirror, stepping + rep, _ := newHealer(t, o).WeeklySelfHeal(context.Background()) + if rep.Duration <= 0 { + t.Errorf("the report carries a %s duration after %d clock readings", rep.Duration, ticks) + } + if rep.RanAt != healAt { + t.Errorf("RanAt is %s, want the first clock reading %s", rep.RanAt, healAt) + } + }) + } +} + +// TestSortedSamplesGroupsByKind is small, and exists because a report an +// operator cannot read is a report that does not get read. +func TestSortedSamplesGroupsByKind(t *testing.T) { + in := []Disagreement{ + {SourceID: "CVE-2", Kind: KindOnlyInLive}, + {SourceID: "CVE-3", Kind: KindMissingInLive}, + {SourceID: "CVE-1", Kind: KindMissingInLive}, + } + got := SortedSamples(in) + want := []string{"CVE-1", "CVE-3", "CVE-2"} + for i := range want { + if got[i].SourceID != want[i] { + t.Fatalf("SortedSamples gave %v, want %v", ids(got), want) + } + } + if in[0].SourceID != "CVE-2" { + t.Error("SortedSamples mutated its input") + } +} + +func ids(in []Disagreement) []string { + out := make([]string, 0, len(in)) + for _, d := range in { + out = append(out, d.SourceID) + } + return out +} + +// TestIntegrationNotesForTheManualRun records what a green run here does NOT +// establish, so that the gap is written down rather than assumed away. +func TestIntegrationNotesForTheManualRun(t *testing.T) { + t.Log(strings.Join([]string{ + "NOT PROVED BY THIS PACKAGE'S TESTS:", + " - affordability. The fixtures are tens of records; a real cvelistV5 baseline is ~300,000", + " records and ~570 MB, and neither the wall time of the merge join nor the disk cost of the", + " scratch database has been measured against one.", + " - go test -race. It cannot run on the Windows dev host (cgo.exe exit 2); CI runs it on Linux.", + " - that A.8's decoder and A.14's decoder agree about every REAL publisher document. They agree", + " about the synthetic CVE 5.1 documents in this file, which is what makes the 'matched' count", + " meaningful here; a disagreement on a real corpus would surface as a wall of 'divergent' rows", + " on the first real self-heal, which is a loud failure rather than a silent one.", + }, "\n")) +} + +// --------------------------------------------------------------------------- +// Helpers: the archive, the tracing driver, snapshot comparison +// --------------------------------------------------------------------------- + +func buildZip(t *testing.T, docs []string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for _, doc := range docs { + id := docID(t, doc) + w, err := zw.Create("cves/2026/" + id + ".json") + if err != nil { + t.Fatalf("creating zip member for %s: %v", id, err) + } + if _, err := w.Write([]byte(doc)); err != nil { + t.Fatalf("writing zip member for %s: %v", id, err) + } + } + if err := zw.Close(); err != nil { + t.Fatalf("closing zip: %v", err) + } + return buf.Bytes() +} + +func serveArchive(t *testing.T, body []byte) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Last-Modified", "Sun, 09 Aug 2026 00:00:00 GMT") + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv +} + +func sameSnapshot(a, b map[string]storedRow) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +// --- the tracing driver --- + +const traceDriverName = "sqlite-anvil-reconcile-trace" + +type sqlTrace struct { + mu sync.Mutex + stmts []string +} + +func (l *sqlTrace) record(q string) { + l.mu.Lock() + defer l.mu.Unlock() + l.stmts = append(l.stmts, q) +} + +func (l *sqlTrace) reset() { + l.mu.Lock() + defer l.mu.Unlock() + l.stmts = nil +} + +func (l *sqlTrace) snapshot() []string { + l.mu.Lock() + defer l.mu.Unlock() + return append([]string(nil), l.stmts...) +} + +var trace = &sqlTrace{} + +type traceDriver struct{ base driver.Driver } + +func (d traceDriver) Open(name string) (driver.Conn, error) { + c, err := d.base.Open(name) + if err != nil { + return nil, err + } + return traceConn{Conn: c}, nil +} + +// traceConn forwards every statement-carrying method to the real connection +// after recording the SQL. It implements every optional interface +// database/sql probes for, so no call can slip past by falling back to a path +// this wrapper does not cover. +type traceConn struct{ driver.Conn } + +func (c traceConn) Prepare(query string) (driver.Stmt, error) { + trace.record(query) + return c.Conn.Prepare(query) +} + +func (c traceConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + trace.record(query) + if p, ok := c.Conn.(driver.ConnPrepareContext); ok { + return p.PrepareContext(ctx, query) + } + return c.Conn.Prepare(query) +} + +func (c traceConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + trace.record(query) + e, ok := c.Conn.(driver.ExecerContext) + if !ok { + return nil, driver.ErrSkip + } + return e.ExecContext(ctx, query, args) +} + +func (c traceConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + trace.record(query) + q, ok := c.Conn.(driver.QueryerContext) + if !ok { + return nil, driver.ErrSkip + } + return q.QueryContext(ctx, query, args) +} + +func (c traceConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if b, ok := c.Conn.(driver.ConnBeginTx); ok { + return b.BeginTx(ctx, opts) + } + return c.Conn.Begin() +} + +func (c traceConn) ResetSession(ctx context.Context) error { + if r, ok := c.Conn.(driver.SessionResetter); ok { + return r.ResetSession(ctx) + } + return nil +} + +func (c traceConn) IsValid() bool { + if v, ok := c.Conn.(driver.Validator); ok { + return v.IsValid() + } + return true +} + +func init() { + probe, err := sql.Open("sqlite", "file:anvil-reconcile-driver-probe?mode=memory") + if err != nil { + panic("reconcile_test: cannot resolve the sqlite driver: " + err.Error()) + } + base := probe.Driver() + _ = probe.Close() + sql.Register(traceDriverName, traceDriver{base: base}) +} + +// newTracedCache opens a live cache through the tracing driver, using the +// cache package's OWN DSN so the connection pragmas are the production ones. +func newTracedCache(t *testing.T) *sql.DB { + t.Helper() + dsn, err := cache.DSN(filepath.Join(t.TempDir(), "anvil-cache.sqlite")) + if err != nil { + t.Fatalf("building the cache DSN: %v", err) + } + db, err := sql.Open(traceDriverName, dsn) + if err != nil { + t.Fatalf("opening the traced cache: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if err := cache.CheckWAL(context.Background(), db); err != nil { + t.Fatalf("the traced cache is not in WAL mode: %v", err) + } + if _, err := cache.Migrate(context.Background(), db); err != nil { + t.Fatalf("migrating the traced cache: %v", err) + } + return db +}