Add read-only daily data integrity verification

cmd/sippy/main.go

@@ -57,6 +57,7 @@ func main() {
 		NewVersionCommand(),
 		NewAnnotateJobRunsCommand(),
 		NewSeedDataCommand(),
+		NewVerifyCommand(),
 	)
 
 	rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info",

cmd/sippy/verify.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"cloud.google.com/go/civil"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	bqcachedclient "github.com/openshift/sippy/pkg/bigquery"
+	"github.com/openshift/sippy/pkg/db/verify"
+	"github.com/openshift/sippy/pkg/flags"
+	"github.com/openshift/sippy/pkg/flags/configflags"
+	"github.com/openshift/sippy/pkg/variantregistry"
+)
+
+type VerifyFlags struct {
+	Date             string
+	Checks           []string
+	Release          string
+	DBFlags          *flags.PostgresFlags
+	BigQueryFlags    *flags.BigQueryFlags
+	GoogleCloudFlags *flags.GoogleCloudFlags
+	ConfigFlags      *configflags.ConfigFlags
+}
+
+func NewVerifyFlags(now time.Time) *VerifyFlags {
+	return &VerifyFlags{
+		Date:             civil.DateOf(now.UTC()).AddDays(-2).String(),
+		DBFlags:          flags.NewPostgresDatabaseFlags(),
+		BigQueryFlags:    flags.NewBigQueryFlags(),
+		GoogleCloudFlags: flags.NewGoogleCloudFlags(),
+		ConfigFlags:      configflags.NewConfigFlags(),
+	}
+}
+
+func (f *VerifyFlags) BindFlags(fs *pflag.FlagSet) {
+	f.DBFlags.BindFlags(fs)
+	f.BigQueryFlags.BindFlags(fs)
+	f.GoogleCloudFlags.BindFlags(fs)
+	f.ConfigFlags.BindFlags(fs)
+	fs.StringVar(&f.Date, "date", f.Date, "UTC calendar date to verify (YYYY-MM-DD; defaults to the day before yesterday)")
+	fs.StringArrayVar(&f.Checks, "check", nil, "Check to run; repeat for multiple checks (bq-completeness, daily-totals, cumulative-summaries; defaults to all)")
+	fs.StringVar(&f.Release, "release", "", "Verify only this release (defaults to every configured and discovered release)")
+}
+
+type verifyCommandDependencies struct {
+	now time.Time
+	run func(context.Context, *VerifyFlags, civil.Date, []verify.Check) (verify.Result, error)
+}
+
+func NewVerifyCommand() *cobra.Command {
+	return newVerifyCommandWithDependencies(verifyCommandDependencies{now: time.Now(), run: runVerify})
+}
+
+func newVerifyCommandWithDependencies(dependencies verifyCommandDependencies) *cobra.Command {
+	f := NewVerifyFlags(dependencies.now)
+	cmd := &cobra.Command{
+		Use:          "verify",
+		Short:        "Verify daily data integrity without modifying storage",
+		Args:         cobra.NoArgs,
+		SilenceUsage: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			date, err := civil.ParseDate(f.Date)
+			if err != nil {
+				return fmt.Errorf("invalid --date %q: expected YYYY-MM-DD: %w", f.Date, err)
+			}
+			checks, err := verify.ParseChecks(f.Checks)
+			if err != nil {
+				return err
+			}
+			result, runErr := dependencies.run(cmd.Context(), f, date, checks)
+			if runErr != nil && len(result.Summaries) == 0 {
+				for _, check := range checks {
+					result.Summaries = append(result.Summaries, verify.Summary{
+						Check: check, Release: f.Release, Date: date, Passed: false, Error: runErr.Error(),
+					})
+				}
+			}
+			result.Sort()
+			result.Log(log.StandardLogger())
+			if runErr != nil {
+				return runErr
+			}
+			if !result.Passed() {
+				return fmt.Errorf("one or more verification checks failed")
+			}
+			return nil
+		},
+	}
+	f.BindFlags(cmd.Flags())
+	return cmd
+}
+
+func runVerify(ctx context.Context, verifyFlags *VerifyFlags, date civil.Date, checks []verify.Check) (verify.Result, error) {
+	dbc, err := verifyFlags.DBFlags.GetDBClient()
+	if err != nil {
+		return verify.Result{}, fmt.Errorf("getting PostgreSQL client: %w", err)
+	}
+
+	runner := verify.Runner{PostgreSQL: verify.NewPostgreSQL(dbc)}
+	var bqClient *bqcachedclient.Client
+	if verify.ContainsCheck(checks, verify.CheckBQCompleteness) {
+		var initializationErrors []error
+		config, configErr := verifyFlags.ConfigFlags.GetConfig()
+		if configErr != nil {
+			initializationErrors = append(initializationErrors, fmt.Errorf("loading Sippy config: %w", configErr))
+		} else {
+			runner.Config = config
+			overrides, overrideErr := variantregistry.BuildSyntheticReleaseJobOverrides(config.Releases)
+			if overrideErr != nil {
+				initializationErrors = append(initializationErrors, fmt.Errorf("building synthetic release overrides: %w", overrideErr))
+			} else {
+				runner.SyntheticReleaseOverrides = overrides
+			}
+		}
+
+		opCtx, queryCtx := bqcachedclient.OpCtxForCronEnv(ctx, "verify")
+		bqClient, err = verifyFlags.BigQueryFlags.GetBigQueryClient(
+			queryCtx, opCtx, nil, verifyFlags.GoogleCloudFlags.ServiceAccountCredentialFile,
+		)
+		if err != nil {
+			initializationErrors = append(initializationErrors, fmt.Errorf("initializing BigQuery client: %w", err))
+		} else {
+			runner.BigQuery = verify.NewBigQuery(bqClient)
+			defer func() {
+				if closeErr := bqClient.BQ.Close(); closeErr != nil {
+					log.WithError(closeErr).Warn("closing BigQuery client")
+				}
+			}()
+		}
+		runner.BigQueryInitializationError = errors.Join(initializationErrors...)
+	}
+
+	return runner.Run(ctx, verify.Options{Date: date, Checks: checks, Release: verifyFlags.Release}), nil
+}

cmd/sippy/verify_test.go

@@ -0,0 +1,131 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"testing"
+	"time"
+
+	"cloud.google.com/go/civil"
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/openshift/sippy/pkg/db/verify"
+)
+
+func TestVerifyCommandDefaultsAndSelection(t *testing.T) {
+	now := time.Date(2026, 8, 28, 1, 30, 0, 0, time.FixedZone("west", -7*60*60))
+	tests := []struct {
+		name       string
+		args       []string
+		wantDate   civil.Date
+		wantChecks []verify.Check
+		wantRel    string
+	}{
+		{
+			name:       "UTC day before yesterday and all checks",
+			wantDate:   civil.Date{Year: 2026, Month: 8, Day: 26},
+			wantChecks: verify.AllChecks,
+		},
+		{
+			name:       "explicit repeatable selection",
+			args:       []string{"--date=2024-02-29", "--check=daily-totals", "--check=bq-completeness", "--release=4.20"},
+			wantDate:   civil.Date{Year: 2024, Month: 2, Day: 29},
+			wantChecks: []verify.Check{verify.CheckBQCompleteness, verify.CheckDailyTotals},
+			wantRel:    "4.20",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			called := false
+			cmd := newVerifyCommandWithDependencies(verifyCommandDependencies{
+				now: now,
+				run: func(_ context.Context, flags *VerifyFlags, date civil.Date, checks []verify.Check) (verify.Result, error) {
+					called = true
+					assert.Equal(t, tt.wantDate, date)
+					assert.Equal(t, tt.wantChecks, checks)
+					assert.Equal(t, tt.wantRel, flags.Release)
+					return verify.Result{Summaries: []verify.Summary{{Check: checks[0], Date: date, Passed: true}}}, nil
+				},
+			})
+			cmd.SetArgs(tt.args)
+			require.NoError(t, cmd.Execute())
+			assert.True(t, called)
+		})
+	}
+}
+
+func TestVerifyCommandValidationAndExit(t *testing.T) {
+	tests := []struct {
+		name    string
+		args    []string
+		result  verify.Result
+		wantErr string
+		called  bool
+	}{
+		{name: "invalid date", args: []string{"--date=nope"}, wantErr: "invalid --date", called: false},
+		{name: "invalid check", args: []string{"--check=nope"}, wantErr: "invalid --check", called: false},
+		{name: "mismatch returns failure", result: verify.Result{Summaries: []verify.Summary{{Check: verify.CheckDailyTotals, Date: civil.Date{Year: 2026, Month: 1, Day: 1}, Passed: false}}}, wantErr: "one or more verification checks failed", called: true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			called := false
+			cmd := newVerifyCommandWithDependencies(verifyCommandDependencies{
+				now: time.Date(2026, 8, 27, 0, 0, 0, 0, time.UTC),
+				run: func(context.Context, *VerifyFlags, civil.Date, []verify.Check) (verify.Result, error) {
+					called = true
+					return tt.result, nil
+				},
+			})
+			cmd.SetArgs(tt.args)
+			err := cmd.Execute()
+			require.ErrorContains(t, err, tt.wantErr)
+			assert.Equal(t, tt.called, called)
+		})
+	}
+}
+
+func TestVerifyCommandHasNoFixFlag(t *testing.T) {
+	cmd := NewVerifyCommand()
+	assert.Nil(t, cmd.Flags().Lookup("fix"))
+	assert.Nil(t, cmd.PersistentFlags().Lookup("fix"))
+}
+
+func TestVerifyCommandLogsSummaryOnRunError(t *testing.T) {
+	logger := log.StandardLogger()
+	var output bytes.Buffer
+	previousOutput := logger.Out
+	previousFormatter := logger.Formatter
+	previousLevel := logger.Level
+	logger.SetOutput(&output)
+	logger.SetFormatter(&log.JSONFormatter{})
+	logger.SetLevel(log.InfoLevel)
+	t.Cleanup(func() {
+		logger.SetOutput(previousOutput)
+		logger.SetFormatter(previousFormatter)
+		logger.SetLevel(previousLevel)
+	})
+
+	runErr := errors.New("database unavailable")
+	cmd := newVerifyCommandWithDependencies(verifyCommandDependencies{
+		now: time.Date(2026, 8, 27, 0, 0, 0, 0, time.UTC),
+		run: func(context.Context, *VerifyFlags, civil.Date, []verify.Check) (verify.Result, error) {
+			return verify.Result{}, runErr
+		},
+	})
+	cmd.SetArgs([]string{"--check=daily-totals", "--release=4.20"})
+	require.ErrorIs(t, cmd.Execute(), runErr)
+
+	var record map[string]any
+	require.NoError(t, json.Unmarshal(output.Bytes(), &record))
+	assert.Equal(t, "verification summary", record["msg"])
+	assert.Equal(t, "error", record["level"])
+	assert.Equal(t, "daily-totals", record["check"])
+	assert.Equal(t, "4.20", record["release"])
+	assert.Equal(t, "2026-08-25", record["date"])
+	assert.Equal(t, false, record["passed"])
+	assert.Equal(t, runErr.Error(), record["error"])
+}

docs/features/daily-data-integrity-verification.md

@@ -0,0 +1,53 @@
+# Daily data integrity verification
+
+The `sippy verify` command performs read-only checks of one UTC calendar day in
+the Prow data pipeline. It reports discrepancies but never repairs data.
+
+## Usage
+
+```console
+sippy verify [--date YYYY-MM-DD] [--check CHECK]... [--release RELEASE]
+```
+
+`--date` defaults to the UTC calendar day before yesterday. `--check` may be
+repeated and accepts `bq-completeness`, `daily-totals`, and
+`cumulative-summaries`; omitting it runs all three. `--release` limits every
+selected check to one release. Without it, the command checks every release
+definition and every non-empty, non-deleted historical release discovered in
+`prow_jobs`. There is intentionally no active-release filter, so this can
+include pseudo-releases with no data on the selected day.
+
+## Checks
+
+- `bq-completeness` compares deduplicated numeric Prow build IDs attributed to
+  each release in BigQuery with `prow_job_runs`. Both sources use the Prow
+  start-time half-open interval `[date 00:00:00Z, next date 00:00:00Z)`.
+  BigQuery retains the loader's terminal-state and non-null URL filters.
+  Malformed BigQuery build IDs are failures.
+- `daily-totals` recomputes counts from `prow_job_run_tests` and compares them
+  in both directions with `test_daily_totals`. It uses the production composite
+  run join, normalizes a null suite to ID 0, separates lifecycle values, and
+  excludes runs labeled `InfraFailure`. Only successes, failures, flakes, and
+  runs are compared; timestamps are not compared.
+- `cumulative-summaries` checks that each target-day cumulative row equals the
+  previous day's prefix counters plus the target day's daily counters. Keys
+  without daily data must carry forward, and first-day keys equal their daily
+  counters. The four prefix counters checked are successes, failures, flakes,
+  and runs.
+
+## Credentials and output
+
+PostgreSQL uses the standard Sippy database flags. BigQuery and Google
+credential flags are only used when `bq-completeness` is selected. Selecting
+that check without usable service-account credentials is a failed check; it is
+not silently skipped. PostgreSQL-only selections require no Google
+credentials.
+
+The command emits one bounded structured summary record for every applicable
+`(check, release, date)` and separate deterministically ordered discrepancy
+records. It runs all selected checks before returning final status.
+
+Exit status is 0 only when every selected check passes. Mismatches, malformed
+IDs, missing selected BigQuery credentials, and operational errors return exit
+status 1. The command has no `--fix` mode and performs no writes, migrations,
+remediation, alerting, or `prow_job_run_test_outputs` verification.

pkg/bigquery/bqlabel/labels.go

@@ -102,6 +102,7 @@ const (
 	CacheLookup                         QueryValue = "cache-lookup"
 	GATestStatusLoader                  QueryValue = "ga-test-status-loader"
 	BackendDisruptionByRun              QueryValue = "backend-disruption-by-run"
+	VerifyProwJobs                      QueryValue = "verify-prow-jobs"
 )
 
 // sanitizeLabelValue sanitizes a label value to meet BigQuery requirements: