From 5fc05fa18c9549f10e2dff26714bd5af7c7a71c6 Mon Sep 17 00:00:00 2001 From: Andrii Rusanov Date: Tue, 16 Jun 2026 15:29:33 +0300 Subject: [PATCH] Add --exclude-table flag to ignore changes in certain tables --- cmd/pg-schema-diff/dump_cmd.go | 24 ++- cmd/pg-schema-diff/dump_cmd_test.go | 23 +++ cmd/pg-schema-diff/main_test.go | 5 + cmd/pg-schema-diff/plan_cmd.go | 24 ++- cmd/pg-schema-diff/plan_cmd_test.go | 19 ++ .../exclude_table_cases_test.go | 99 +++++++++++ internal/schema/filters.go | 84 +++++++++ internal/schema/filters_test.go | 166 ++++++++++++++++++ internal/schema/schema.go | 37 +++- internal/schema/schema_test.go | 62 +++++++ pkg/diff/plan_generator.go | 10 ++ pkg/schema/schema.go | 1 + 12 files changed, 544 insertions(+), 10 deletions(-) create mode 100644 internal/migration_acceptance_tests/exclude_table_cases_test.go diff --git a/cmd/pg-schema-diff/dump_cmd.go b/cmd/pg-schema-diff/dump_cmd.go index 79bfac5..07aa825 100644 --- a/cmd/pg-schema-diff/dump_cmd.go +++ b/cmd/pg-schema-diff/dump_cmd.go @@ -26,18 +26,28 @@ func buildDumpCmd() *cobra.Command { cmd.Flags().StringArrayVar(&includeSchemas, "include-schema", nil, "Include the specified schema in the dump") cmd.Flags().StringArrayVar(&excludeSchemas, "exclude-schema", nil, "Exclude the specified schema from the dump") + var excludeTablePatterns []string + cmd.Flags().StringArrayVar(&excludeTablePatterns, "exclude-table", nil, + "Exclude tables matching this Go regexp. The pattern is matched (fully anchored) against both the table "+ + "name and the schema-qualified name, e.g., 'tmp_.*' or 'public\\.tmp_.*'. Can be repeated.") + cmd.RunE = func(cmd *cobra.Command, args []string) error { connConfig, err := parseConnectionFlags(connFlags) if err != nil { return err } + if err := validateExcludeTablePatterns(excludeTablePatterns); err != nil { + return err + } + cmd.SilenceUsage = true plan, err := generateDump(cmd.Context(), generateDumpParams{ - connConfig: connConfig, - includeSchemas: includeSchemas, - excludeSchemas: excludeSchemas, + connConfig: connConfig, + includeSchemas: includeSchemas, + excludeSchemas: excludeSchemas, + excludeTablePatterns: excludeTablePatterns, }) if err != nil { return err @@ -51,9 +61,10 @@ func buildDumpCmd() *cobra.Command { } type generateDumpParams struct { - connConfig *pgx.ConnConfig - includeSchemas []string - excludeSchemas []string + connConfig *pgx.ConnConfig + includeSchemas []string + excludeSchemas []string + excludeTablePatterns []string } func generateDump(ctx context.Context, params generateDumpParams) (diff.Plan, error) { @@ -82,6 +93,7 @@ func generateDump(ctx context.Context, params generateDumpParams) (diff.Plan, er diff.WithTempDbFactory(tempDbFactory), diff.WithIncludeSchemas(params.includeSchemas...), diff.WithExcludeSchemas(params.excludeSchemas...), + diff.WithExcludeTablePatterns(params.excludeTablePatterns...), diff.WithDoNotValidatePlan(), diff.WithNoConcurrentIndexOps(), ) diff --git a/cmd/pg-schema-diff/dump_cmd_test.go b/cmd/pg-schema-diff/dump_cmd_test.go index 4d73a4b..456e013 100644 --- a/cmd/pg-schema-diff/dump_cmd_test.go +++ b/cmd/pg-schema-diff/dump_cmd_test.go @@ -8,6 +8,8 @@ func (suite *cmdTestSuite) TestDumpCmd() { // outputContains is a list of substrings that are expected to be contained in the stdout output of the command. outputContains []string + // outputNotContains is a list of substrings that are expected to NOT be contained in the stdout output. + outputNotContains []string // expectErrContains is a list of substrings that are expected to be contained in the error returned by // cmd.RunE. This is DISTINCT from stdErr. expectErrContains []string @@ -28,12 +30,33 @@ func (suite *cmdTestSuite) TestDumpCmd() { "name", }, }, + { + name: "dump with exclude-table", + args: []string{"--exclude-table", "tmp_.*"}, + dynamicArgs: []dArgGenerator{ + tempDsnDArg(suite.pgEngine, "dsn", []string{ + "CREATE TABLE foobar(id INT PRIMARY KEY)", + "CREATE TABLE tmp_foo(id INT PRIMARY KEY)", + }), + }, + outputContains: []string{"foobar"}, + outputNotContains: []string{"tmp_foo"}, + }, + { + name: "dump with invalid exclude-table pattern", + args: []string{"--exclude-table", "["}, + dynamicArgs: []dArgGenerator{ + tempDsnDArg(suite.pgEngine, "dsn", nil), + }, + expectErrContains: []string{"invalid --exclude-table pattern"}, + }, } { suite.Run(tc.name, func() { suite.runCmdWithAssertions(runCmdWithAssertionsParams{ args: append([]string{"dump"}, tc.args...), dynamicArgs: tc.dynamicArgs, outputContains: tc.outputContains, + outputNotContains: tc.outputNotContains, expectErrContains: tc.expectErrContains, }) }) diff --git a/cmd/pg-schema-diff/main_test.go b/cmd/pg-schema-diff/main_test.go index 6d18e7a..255876a 100644 --- a/cmd/pg-schema-diff/main_test.go +++ b/cmd/pg-schema-diff/main_test.go @@ -38,6 +38,8 @@ type runCmdWithAssertionsParams struct { outputEquals string // outputContains is a list of substrings that are expected to be contained in the stdout output of the command. outputContains []string + // outputNotContains is a list of substrings that are expected to NOT be contained in the stdout output. + outputNotContains []string // expectErrContains is a list of substrings that are expected to be contained in the error returned by // cmd.RunE. This is DISTINCT from stdErr. expectErrContains []string @@ -74,6 +76,9 @@ func (suite *cmdTestSuite) runCmdWithAssertions(tc runCmdWithAssertionsParams) { suite.Contains(stdOutStr, o) } } + for _, o := range tc.outputNotContains { + suite.NotContains(stdOutStr, o) + } if len(tc.outputEquals) > 0 { suite.Equal(tc.outputEquals, stdOutStr) } diff --git a/cmd/pg-schema-diff/plan_cmd.go b/cmd/pg-schema-diff/plan_cmd.go index 6c65722..3dbc54a 100644 --- a/cmd/pg-schema-diff/plan_cmd.go +++ b/cmd/pg-schema-diff/plan_cmd.go @@ -111,8 +111,9 @@ func buildPlanCmd() *cobra.Command { type ( // parsePlanOptionsFlags stores the flags that are parsed into planOptions. planOptionsFlags struct { - includeSchemas []string - excludeSchemas []string + includeSchemas []string + excludeSchemas []string + excludeTablePatterns []string dataPackNewTables bool disablePlanValidation bool @@ -221,6 +222,9 @@ func createPlanOptionsFlags(cmd *cobra.Command) *planOptionsFlags { cmd.Flags().StringArrayVar(&flags.includeSchemas, "include-schema", nil, "Include the specified schema in the plan") cmd.Flags().StringArrayVar(&flags.excludeSchemas, "exclude-schema", nil, "Exclude the specified schema in the plan") + cmd.Flags().StringArrayVar(&flags.excludeTablePatterns, "exclude-table", nil, + "Exclude tables matching this Go regexp. The pattern is matched (fully anchored) against both the table "+ + "name and the schema-qualified name, e.g., 'tmp_.*' or 'public\\.tmp_.*'. Can be repeated.") cmd.Flags().BoolVar(&flags.dataPackNewTables, "data-pack-new-tables", true, "If set, will data pack new tables in the plan to minimize table size (re-arranges columns).") cmd.Flags().BoolVar(&flags.disablePlanValidation, "disable-plan-validation", false, "If set, will disable plan validation. Plan validation runs the migration against a temporary"+ @@ -314,10 +318,26 @@ func dsnSchemaSource(connConfig *pgx.ConnConfig) schemaSourceFactory { } } +// validateExcludeTablePatterns fail-fast validates regexes before any database work is done. The patterns are +// compiled for real inside schema.GetSchema; this exists purely for a clean CLI error. +func validateExcludeTablePatterns(patterns []string) error { + for _, pattern := range patterns { + if _, err := regexp.Compile(pattern); err != nil { + return fmt.Errorf("invalid --exclude-table pattern %q: %w", pattern, err) + } + } + return nil +} + func parsePlanOptions(p planOptionsFlags) (planOptions, error) { + if err := validateExcludeTablePatterns(p.excludeTablePatterns); err != nil { + return planOptions{}, err + } + opts := []diff.PlanOpt{ diff.WithIncludeSchemas(p.includeSchemas...), diff.WithExcludeSchemas(p.excludeSchemas...), + diff.WithExcludeTablePatterns(p.excludeTablePatterns...), } if p.dataPackNewTables { diff --git a/cmd/pg-schema-diff/plan_cmd_test.go b/cmd/pg-schema-diff/plan_cmd_test.go index e27540d..aeab605 100644 --- a/cmd/pg-schema-diff/plan_cmd_test.go +++ b/cmd/pg-schema-diff/plan_cmd_test.go @@ -4,6 +4,8 @@ import ( "regexp" "testing" "time" + + "github.com/stretchr/testify/require" ) func (suite *cmdTestSuite) TestPlanCmd() { @@ -124,6 +126,15 @@ func (suite *cmdTestSuite) TestPlanCmd() { }, expectErrContains: []string{"invalid output format"}, }, + { + name: "invalid exclude-table pattern", + args: []string{"--exclude-table", "["}, + dynamicArgs: []dArgGenerator{ + tempDsnDArg(suite.pgEngine, "from-dsn", nil), + tempDsnDArg(suite.pgEngine, "to-dsn", nil), + }, + expectErrContains: []string{"invalid --exclude-table pattern"}, + }, } { suite.Run(tc.name, func() { suite.runCmdWithAssertions(runCmdWithAssertionsParams{ @@ -245,3 +256,11 @@ func (suite *cmdTestSuite) TestParseInsertStatementStr() { }) } } + +func TestParsePlanOptionsExcludeTablePatterns(t *testing.T) { + _, err := parsePlanOptions(planOptionsFlags{excludeTablePatterns: []string{"tmp_.*"}}) + require.NoError(t, err) + + _, err = parsePlanOptions(planOptionsFlags{excludeTablePatterns: []string{"["}}) + require.ErrorContains(t, err, `invalid --exclude-table pattern "["`) +} diff --git a/internal/migration_acceptance_tests/exclude_table_cases_test.go b/internal/migration_acceptance_tests/exclude_table_cases_test.go new file mode 100644 index 0000000..7f7f474 --- /dev/null +++ b/internal/migration_acceptance_tests/exclude_table_cases_test.go @@ -0,0 +1,99 @@ +package migration_acceptance_tests + +import ( + "testing" + + "github.com/stripe/pg-schema-diff/pkg/diff" +) + +var excludeTableAcceptanceTestCases = []acceptanceTestCase{ + { + name: "Excluded table only in old schema is not dropped", + oldSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY); + CREATE TABLE tmp_foo(id INT PRIMARY KEY); + CREATE INDEX tmp_foo_idx ON tmp_foo(id); + `}, + newSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY); + `}, + planOpts: []diff.PlanOpt{diff.WithExcludeTablePatterns("tmp_.*")}, + expectEmptyPlan: true, + expectedDBSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY); + CREATE TABLE tmp_foo(id INT PRIMARY KEY); + CREATE INDEX tmp_foo_idx ON tmp_foo(id); + `}, + }, + { + name: "Excluded table only in new schema is not created", + oldSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY); + `}, + newSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY); + CREATE TABLE tmp_foo(id INT PRIMARY KEY); + `}, + planOpts: []diff.PlanOpt{diff.WithExcludeTablePatterns("tmp_.*")}, + expectEmptyPlan: true, + expectedDBSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY); + `}, + }, + { + name: "Changes to non-excluded tables are still planned", + oldSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY); + CREATE TABLE tmp_foo(id INT PRIMARY KEY); + `}, + newSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY, new_col TEXT); + `}, + planOpts: []diff.PlanOpt{diff.WithExcludeTablePatterns("tmp_.*")}, + expectedDBSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY, new_col TEXT); + CREATE TABLE tmp_foo(id INT PRIMARY KEY); + `}, + }, + { + name: "Schema-qualified pattern only excludes tables in that schema", + oldSchemaDDL: []string{` + CREATE SCHEMA schema_1; + CREATE TABLE schema_1.tmp_foo(id INT PRIMARY KEY); + CREATE TABLE tmp_foo(id INT PRIMARY KEY); + `}, + newSchemaDDL: []string{` + CREATE SCHEMA schema_1; + CREATE TABLE tmp_foo(id INT PRIMARY KEY); + `}, + planOpts: []diff.PlanOpt{diff.WithExcludeTablePatterns(`schema_1\.tmp_foo`)}, + expectEmptyPlan: true, + expectedDBSchemaDDL: []string{` + CREATE SCHEMA schema_1; + CREATE TABLE schema_1.tmp_foo(id INT PRIMARY KEY); + CREATE TABLE tmp_foo(id INT PRIMARY KEY); + `}, + }, + { + name: "Partitions of an excluded partitioned table are also excluded", + oldSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY); + CREATE TABLE tmp_events(id INT) PARTITION BY RANGE (id); + CREATE TABLE events_p1 PARTITION OF tmp_events FOR VALUES FROM (0) TO (100); + `}, + newSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY); + `}, + planOpts: []diff.PlanOpt{diff.WithExcludeTablePatterns("tmp_.*")}, + expectEmptyPlan: true, + expectedDBSchemaDDL: []string{` + CREATE TABLE foobar(id INT PRIMARY KEY); + CREATE TABLE tmp_events(id INT) PARTITION BY RANGE (id); + CREATE TABLE events_p1 PARTITION OF tmp_events FOR VALUES FROM (0) TO (100); + `}, + }, +} + +func TestExcludeTableTestCases(t *testing.T) { + runTestCases(t, excludeTableAcceptanceTestCases) +} diff --git a/internal/schema/filters.go b/internal/schema/filters.go index 8c2fe68..67ebcde 100644 --- a/internal/schema/filters.go +++ b/internal/schema/filters.go @@ -1,5 +1,11 @@ package schema +import ( + "fmt" + "regexp" + "strings" +) + // nameFilter is one of the most generic of filters. We can use it to filter objects by their schema name or name. // In the future, it might be expanded to include a "type" field, e.g., to filter down to specific tables. type nameFilter func(name SchemaQualifiedName) bool @@ -51,3 +57,81 @@ func filterSliceByName[T any](objs []T, getNameFn func(T) SchemaQualifiedName, f } return filteredObjs } + +// unescapeIdentifier converts an escaped identifier (as produced by EscapeIdentifier) back to its raw form. +// Identifiers that are not wrapped in double quotes are returned as-is. +func unescapeIdentifier(escaped string) string { + if len(escaped) >= 2 && strings.HasPrefix(escaped, `"`) && strings.HasSuffix(escaped, `"`) { + return strings.ReplaceAll(escaped[1:len(escaped)-1], `""`, `"`) + } + return escaped +} + +// buildExcludeTablesFilter builds a nameFilter that excludes (returns false for) any table whose unescaped name or +// unescaped schema-qualified name (e.g., "public.foobar") fully matches any of the given regex patterns. Patterns +// are anchored, i.e., wrapped in ^(?:...)$, so "users" matches only a table named exactly "users". Returns nil if no +// patterns are provided. +func buildExcludeTablesFilter(patterns []string) (nameFilter, error) { + if len(patterns) == 0 { + return nil, nil + } + + var regexes []*regexp.Regexp + for _, pattern := range patterns { + regex, err := regexp.Compile(fmt.Sprintf("^(?:%s)$", pattern)) + if err != nil { + return nil, fmt.Errorf("compiling exclude table pattern %q: %w", pattern, err) + } + regexes = append(regexes, regex) + } + + return func(table SchemaQualifiedName) bool { + name := unescapeIdentifier(table.EscapedName) + qualifiedName := fmt.Sprintf("%s.%s", table.SchemaName, name) + for _, regex := range regexes { + if regex.MatchString(name) || regex.MatchString(qualifiedName) { + return false + } + } + return true + }, nil +} + +// excludeTables removes tables for which keepTable returns false from the schema, along with partitions of excluded +// tables (transitively) and any objects owned by excluded tables (indexes, foreign key constraints, triggers). Check +// constraints, policies, and privileges are stored on the Table struct, so they are removed with their table. +// +// Foreign keys owned by kept tables that reference an excluded table are kept, consistent with how cross-schema +// foreign keys behave with WithExcludeSchemas (see the nameFilter docstring on schemaFetcher about dependency +// validation). +func excludeTables(s Schema, keepTable nameFilter) Schema { + excludedTables := make(map[string]bool) + // Iterate until a fixed point is reached to handle multi-level partitioning, where a partition's parent is + // itself a partition of an excluded table. + for { + changed := false + for _, table := range s.Tables { + fqName := table.GetFQEscapedName() + if excludedTables[fqName] { + continue + } + parentIsExcluded := table.ParentTable != nil && excludedTables[table.ParentTable.GetFQEscapedName()] + if parentIsExcluded || !keepTable(table.SchemaQualifiedName) { + excludedTables[fqName] = true + changed = true + } + } + if !changed { + break + } + } + + keepOwningRel := func(owningRel SchemaQualifiedName) bool { + return !excludedTables[owningRel.GetFQEscapedName()] + } + s.Tables = filterSliceByName(s.Tables, func(t Table) SchemaQualifiedName { return t.SchemaQualifiedName }, keepOwningRel) + s.Indexes = filterSliceByName(s.Indexes, func(idx Index) SchemaQualifiedName { return idx.OwningRelName }, keepOwningRel) + s.ForeignKeyConstraints = filterSliceByName(s.ForeignKeyConstraints, func(fk ForeignKeyConstraint) SchemaQualifiedName { return fk.OwningTable }, keepOwningRel) + s.Triggers = filterSliceByName(s.Triggers, func(t Trigger) SchemaQualifiedName { return t.OwningTable }, keepOwningRel) + return s +} diff --git a/internal/schema/filters_test.go b/internal/schema/filters_test.go index 164055b..08ff266 100644 --- a/internal/schema/filters_test.go +++ b/internal/schema/filters_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type ( @@ -156,3 +157,168 @@ func TestAndNameFilters(t *testing.T) { }) } } + +func TestUnescapeIdentifier(t *testing.T) { + for _, tc := range []struct { + name string + input string + expected string + }{ + {name: "quoted", input: `"foobar"`, expected: "foobar"}, + {name: "quoted with inner quotes", input: `"foo""bar"`, expected: `foo"bar`}, + {name: "unquoted", input: "foobar", expected: "foobar"}, + {name: "empty", input: "", expected: ""}, + {name: "single quote char", input: `"`, expected: `"`}, + } { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, unescapeIdentifier(tc.input)) + }) + } +} + +func TestBuildExcludeTablesFilter(t *testing.T) { + for _, tc := range []struct { + name string + patterns []string + input SchemaQualifiedName + // expectedKeep is whether the filter should keep (true) or exclude (false) the input. + expectedKeep bool + }{ + { + name: "bare name match", + patterns: []string{"tmp_.*"}, + input: SchemaQualifiedName{SchemaName: "public", EscapedName: `"tmp_foo"`}, + expectedKeep: false, + }, + { + name: "bare name match in non-public schema", + patterns: []string{"tmp_.*"}, + input: SchemaQualifiedName{SchemaName: "schema_1", EscapedName: `"tmp_foo"`}, + expectedKeep: false, + }, + { + name: "no match", + patterns: []string{"tmp_.*"}, + input: SchemaQualifiedName{SchemaName: "public", EscapedName: `"foobar"`}, + expectedKeep: true, + }, + { + name: "anchored: no substring match", + patterns: []string{"tmp_.*"}, + input: SchemaQualifiedName{SchemaName: "public", EscapedName: `"my_tmp_foo"`}, + expectedKeep: true, + }, + { + name: "anchored: plain name does not match as prefix", + patterns: []string{"users"}, + input: SchemaQualifiedName{SchemaName: "public", EscapedName: `"users_audit"`}, + expectedKeep: true, + }, + { + name: "qualified name match", + patterns: []string{`schema_1\.tmp_.*`}, + input: SchemaQualifiedName{SchemaName: "schema_1", EscapedName: `"tmp_foo"`}, + expectedKeep: false, + }, + { + name: "qualified pattern does not match other schemas", + patterns: []string{`schema_1\.tmp_.*`}, + input: SchemaQualifiedName{SchemaName: "public", EscapedName: `"tmp_foo"`}, + expectedKeep: true, + }, + { + name: "multiple patterns", + patterns: []string{"foo", "bar"}, + input: SchemaQualifiedName{SchemaName: "public", EscapedName: `"bar"`}, + expectedKeep: false, + }, + { + name: "identifier with special characters", + patterns: []string{"some table"}, + input: SchemaQualifiedName{SchemaName: "public", EscapedName: `"some table"`}, + expectedKeep: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + filter, err := buildExcludeTablesFilter(tc.patterns) + require.NoError(t, err) + assert.Equal(t, tc.expectedKeep, filter(tc.input)) + }) + } +} + +func TestBuildExcludeTablesFilterInvalidPattern(t *testing.T) { + _, err := buildExcludeTablesFilter([]string{"["}) + require.ErrorContains(t, err, "compiling exclude table pattern") +} + +func TestBuildExcludeTablesFilterEmpty(t *testing.T) { + filter, err := buildExcludeTablesFilter(nil) + require.NoError(t, err) + require.Nil(t, filter) +} + +func TestExcludeTables(t *testing.T) { + fooTable := SchemaQualifiedName{SchemaName: "public", EscapedName: `"foo"`} + tmpTable := SchemaQualifiedName{SchemaName: "public", EscapedName: `"tmp_bar"`} + partitionedTable := SchemaQualifiedName{SchemaName: "public", EscapedName: `"tmp_events"`} + // The partition and sub-partition names do not match the exclude pattern; they must be excluded because their + // (transitive) parent is excluded. + partition := SchemaQualifiedName{SchemaName: "public", EscapedName: `"events_p1"`} + subPartition := SchemaQualifiedName{SchemaName: "public", EscapedName: `"events_p1_sub"`} + + input := Schema{ + // Children are listed before their parents so the fixed-point loop must take multiple passes to exclude the + // transitive partition chain. This guards against a regression that "optimizes" excludeTables to a single pass. + Tables: []Table{ + {SchemaQualifiedName: subPartition, ParentTable: &partition}, + {SchemaQualifiedName: partition, ParentTable: &partitionedTable, PartitionKeyDef: "RANGE (id)"}, + {SchemaQualifiedName: partitionedTable, PartitionKeyDef: "RANGE (id)"}, + {SchemaQualifiedName: fooTable}, + {SchemaQualifiedName: tmpTable}, + }, + Indexes: []Index{ + {Name: "foo_idx", OwningRelName: fooTable}, + {Name: "tmp_bar_idx", OwningRelName: tmpTable}, + {Name: "events_p1_idx", OwningRelName: partition}, + }, + ForeignKeyConstraints: []ForeignKeyConstraint{ + {EscapedName: `"foo_fk"`, OwningTable: fooTable, ForeignTable: tmpTable}, + {EscapedName: `"tmp_bar_fk"`, OwningTable: tmpTable, ForeignTable: fooTable}, + }, + Triggers: []Trigger{ + {EscapedName: `"foo_trigger"`, OwningTable: fooTable}, + {EscapedName: `"tmp_bar_trigger"`, OwningTable: tmpTable}, + }, + } + + filter, err := buildExcludeTablesFilter([]string{"tmp_.*"}) + require.NoError(t, err) + output := excludeTables(input, filter) + + var tableNames []string + for _, table := range output.Tables { + tableNames = append(tableNames, table.GetFQEscapedName()) + } + assert.ElementsMatch(t, []string{fooTable.GetFQEscapedName()}, tableNames) + + var indexNames []string + for _, idx := range output.Indexes { + indexNames = append(indexNames, idx.Name) + } + assert.ElementsMatch(t, []string{"foo_idx"}, indexNames) + + // The FK owned by the kept table is kept even though it references an excluded table. This is consistent with + // how cross-schema FKs behave with WithExcludeSchemas. + var fkNames []string + for _, fk := range output.ForeignKeyConstraints { + fkNames = append(fkNames, fk.EscapedName) + } + assert.ElementsMatch(t, []string{`"foo_fk"`}, fkNames) + + var triggerNames []string + for _, trigger := range output.Triggers { + triggerNames = append(triggerNames, trigger.EscapedName) + } + assert.ElementsMatch(t, []string{`"foo_trigger"`}, triggerNames) +} diff --git a/internal/schema/schema.go b/internal/schema/schema.go index b3945d5..6ec7251 100644 --- a/internal/schema/schema.go +++ b/internal/schema/schema.go @@ -566,6 +566,17 @@ func WithExcludeSchemas(schemas ...string) GetSchemaOpt { } } +// WithExcludeTables filters the schema to exclude tables whose unescaped name or schema-qualified name (e.g., +// "public.foobar") fully matches any of the given regex patterns (patterns are anchored, i.e., evaluated as +// ^(?:pattern)$). Objects owned by an excluded table (indexes, constraints, triggers, policies, privileges) and +// partitions of an excluded table are also excluded. This unions with any patterns already provided via +// WithExcludeTables. If empty, then no tables are excluded. +func WithExcludeTables(patterns ...string) GetSchemaOpt { + return func(o *getSchemaOptions) { + o.excludeTablePatterns = append(o.excludeTablePatterns, patterns...) + } +} + type getSchemaOptions struct { // includeSchemas is a list of schemas to include in the schema. If empty, then all schemas are included. // We could have built a more complex set of options using the nameFilter system (nested unions and intersections); @@ -573,6 +584,9 @@ type getSchemaOptions struct { includeSchemas []string // excludeSchemas is the exclude analog of includeSchemas. excludeSchemas []string + // excludeTablePatterns is a list of anchored regex patterns of tables to exclude from the schema, matched + // against both the bare table name and the schema-qualified name. + excludeTablePatterns []string } // GetSchema fetches the database schema. It is a non-atomic operation. @@ -599,10 +613,16 @@ func GetSchema(ctx context.Context, db queries.DBTX, opts ...GetSchemaOpt) (Sche return Schema{}, fmt.Errorf("building name filter: %w", err) } + excludeTablesFilter, err := buildExcludeTablesFilter(options.excludeTablePatterns) + if err != nil { + return Schema{}, fmt.Errorf("building exclude tables filter: %w", err) + } + return (&schemaFetcher{ q: queries.New(db), goroutineRunnerFactory: goroutineRunnerFactory, nameFilter: nameFilter, + excludeTablesFilter: excludeTablesFilter, }).getSchema(ctx) } @@ -675,6 +695,13 @@ type ( // Examples of dependencies that could be filtered out include the functions used by triggers and the parent // tables of partitions. nameFilter nameFilter + // excludeTablesFilter excludes tables (and the objects they own) from the fetched schema. It is applied to the + // assembled schema at the end of getSchema, rather than per-fetch, so that partitions of excluded tables and + // the objects owned by those partitions are excluded as well. Nil if no table exclusions were requested. + // + // The same dependency-validation caveat as nameFilter applies: e.g., a foreign key on a kept table that + // references an excluded table is kept, and plans involving it may be invalid. + excludeTablesFilter nameFilter } ) @@ -825,7 +852,7 @@ func (s *schemaFetcher) getSchema(ctx context.Context) (Schema, error) { return Schema{}, fmt.Errorf("getting materialized views: %w", err) } - return Schema{ + fetchedSchema := Schema{ NamedSchemas: schemas, Extensions: extensions, Enums: enums, @@ -838,7 +865,13 @@ func (s *schemaFetcher) getSchema(ctx context.Context) (Schema, error) { Triggers: triggers, Views: views, MaterializedViews: materializedViews, - }, nil + } + + if s.excludeTablesFilter != nil { + fetchedSchema = excludeTables(fetchedSchema, s.excludeTablesFilter) + } + + return fetchedSchema, nil } func (s *schemaFetcher) fetchNamedSchemas(ctx context.Context) ([]NamedSchema, error) { diff --git a/internal/schema/schema_test.go b/internal/schema/schema_test.go index c73bb17..af81935 100644 --- a/internal/schema/schema_test.go +++ b/internal/schema/schema_test.go @@ -1443,3 +1443,65 @@ func TestTriggerDefStmtToCreateOrReplace(t *testing.T) { }) } } + +func TestGetSchemaWithExcludeTables(t *testing.T) { + engine, err := pgengine.StartEngine() + require.NoError(t, err) + defer engine.Close() + + db, err := engine.CreateDatabase() + require.NoError(t, err) + defer db.DropDB() + + connPool, err := sql.Open("pgx", db.GetDSN()) + require.NoError(t, err) + defer connPool.Close() + + _, err = connPool.Exec(` + CREATE SCHEMA schema_1; + + CREATE TABLE foo (id INT PRIMARY KEY); + CREATE INDEX foo_idx ON foo(id); + + -- Excluded by the bare pattern, along with its index and foreign key + CREATE TABLE tmp_bar (id INT PRIMARY KEY, foo_id INT REFERENCES foo(id)); + CREATE INDEX tmp_bar_idx ON tmp_bar(foo_id); + + -- Excluded by the bare pattern, which matches tables in all schemas + CREATE TABLE schema_1.tmp_bar (id INT); + + -- Excluded by the schema-qualified pattern + CREATE TABLE schema_1.qualified_excluded (id INT); + -- Kept: the schema-qualified pattern only matches schema_1 + CREATE TABLE qualified_excluded (id INT); + + -- Kept: patterns are anchored, so tmp_.* must match the entire name + CREATE TABLE my_tmp_bar (id INT); + + -- The partition is excluded because its parent is excluded, even though its own name does not match + CREATE TABLE tmp_events (id INT) PARTITION BY RANGE (id); + CREATE TABLE events_p1 PARTITION OF tmp_events FOR VALUES FROM (0) TO (100); + `) + require.NoError(t, err) + + fetchedSchema, err := GetSchema(context.Background(), connPool, WithExcludeTables(`tmp_.*`, `schema_1\.qualified_excluded`)) + require.NoError(t, err) + + var tableNames []string + for _, table := range fetchedSchema.Tables { + tableNames = append(tableNames, table.GetFQEscapedName()) + } + assert.ElementsMatch(t, []string{`"public"."foo"`, `"public"."qualified_excluded"`, `"public"."my_tmp_bar"`}, tableNames) + + var indexNames []string + for _, idx := range fetchedSchema.Indexes { + indexNames = append(indexNames, idx.Name) + } + assert.ElementsMatch(t, []string{"foo_pkey", "foo_idx"}, indexNames) + + assert.Empty(t, fetchedSchema.ForeignKeyConstraints) + + // Invalid patterns error out before any introspection happens. + _, err = GetSchema(context.Background(), connPool, WithExcludeTables(`[`)) + require.ErrorContains(t, err, "compiling exclude table pattern") +} diff --git a/pkg/diff/plan_generator.go b/pkg/diff/plan_generator.go index 2c2c92b..aa7c912 100644 --- a/pkg/diff/plan_generator.go +++ b/pkg/diff/plan_generator.go @@ -90,6 +90,16 @@ func WithExcludeSchemas(schemas ...string) PlanOpt { } } +// WithExcludeTablePatterns excludes tables whose name or schema-qualified name (e.g., "public.foobar") fully matches +// any of the given regex patterns (patterns are anchored, i.e., evaluated as ^(?:pattern)$). Objects owned by an +// excluded table (indexes, constraints, triggers, policies, privileges) and partitions of an excluded table are also +// excluded. The exclusion applies to both the current and target schemas. +func WithExcludeTablePatterns(patterns ...string) PlanOpt { + return func(opts *planOptions) { + opts.getSchemaOpts = append(opts.getSchemaOpts, schema.WithExcludeTables(patterns...)) + } +} + func WithGetSchemaOpts(getSchemaOpts ...externalschema.GetSchemaOpt) PlanOpt { return func(opts *planOptions) { opts.getSchemaOpts = append(opts.getSchemaOpts, getSchemaOpts...) diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go index a7af346..94c5a07 100644 --- a/pkg/schema/schema.go +++ b/pkg/schema/schema.go @@ -13,6 +13,7 @@ type GetSchemaOpt = internalschema.GetSchemaOpt var ( WithIncludeSchemas = internalschema.WithIncludeSchemas WithExcludeSchemas = internalschema.WithExcludeSchemas + WithExcludeTables = internalschema.WithExcludeTables ) // GetSchemaHash hash gets the hash of the target schema. It can be used to compare against the hash in the migration