Add --exclude-table flag to ignore changes in certain tables

cmd/pg-schema-diff/dump_cmd.go

@@ -26,18 +26,28 @@ func buildDumpCmd() *cobra.Command {
 	cmd.Flags().StringArrayVar(&includeSchemas, "include-schema", nil, "Include the specified schema in the dump")
 	cmd.Flags().StringArrayVar(&excludeSchemas, "exclude-schema", nil, "Exclude the specified schema from the dump")
 
+	var excludeTablePatterns []string
+	cmd.Flags().StringArrayVar(&excludeTablePatterns, "exclude-table", nil,
+		"Exclude tables matching this Go regexp. The pattern is matched (fully anchored) against both the table "+
+			"name and the schema-qualified name, e.g., 'tmp_.*' or 'public\\.tmp_.*'. Can be repeated.")
+
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		connConfig, err := parseConnectionFlags(connFlags)
 		if err != nil {
 			return err
 		}
 
+		if err := validateExcludeTablePatterns(excludeTablePatterns); err != nil {
+			return err
+		}
+
 		cmd.SilenceUsage = true
 
 		plan, err := generateDump(cmd.Context(), generateDumpParams{
-			connConfig:     connConfig,
-			includeSchemas: includeSchemas,
-			excludeSchemas: excludeSchemas,
+			connConfig:           connConfig,
+			includeSchemas:       includeSchemas,
+			excludeSchemas:       excludeSchemas,
+			excludeTablePatterns: excludeTablePatterns,
 		})
 		if err != nil {
 			return err
@@ -51,9 +61,10 @@ func buildDumpCmd() *cobra.Command {
 }
 
 type generateDumpParams struct {
-	connConfig     *pgx.ConnConfig
-	includeSchemas []string
-	excludeSchemas []string
+	connConfig           *pgx.ConnConfig
+	includeSchemas       []string
+	excludeSchemas       []string
+	excludeTablePatterns []string
 }
 
 func generateDump(ctx context.Context, params generateDumpParams) (diff.Plan, error) {
@@ -82,6 +93,7 @@ func generateDump(ctx context.Context, params generateDumpParams) (diff.Plan, er
 		diff.WithTempDbFactory(tempDbFactory),
 		diff.WithIncludeSchemas(params.includeSchemas...),
 		diff.WithExcludeSchemas(params.excludeSchemas...),
+		diff.WithExcludeTablePatterns(params.excludeTablePatterns...),
 		diff.WithDoNotValidatePlan(),
 		diff.WithNoConcurrentIndexOps(),
 	)

cmd/pg-schema-diff/dump_cmd_test.go

@@ -8,6 +8,8 @@ func (suite *cmdTestSuite) TestDumpCmd() {
 
 		// outputContains is a list of substrings that are expected to be contained in the stdout output of the command.
 		outputContains []string
+		// outputNotContains is a list of substrings that are expected to NOT be contained in the stdout output.
+		outputNotContains []string
 		// expectErrContains is a list of substrings that are expected to be contained in the error returned by
 		// cmd.RunE. This is DISTINCT from stdErr.
 		expectErrContains []string
@@ -28,12 +30,33 @@ func (suite *cmdTestSuite) TestDumpCmd() {
 				"name",
 			},
 		},
+		{
+			name: "dump with exclude-table",
+			args: []string{"--exclude-table", "tmp_.*"},
+			dynamicArgs: []dArgGenerator{
+				tempDsnDArg(suite.pgEngine, "dsn", []string{
+					"CREATE TABLE foobar(id INT PRIMARY KEY)",
+					"CREATE TABLE tmp_foo(id INT PRIMARY KEY)",
+				}),
+			},
+			outputContains:    []string{"foobar"},
+			outputNotContains: []string{"tmp_foo"},
+		},
+		{
+			name: "dump with invalid exclude-table pattern",
+			args: []string{"--exclude-table", "["},
+			dynamicArgs: []dArgGenerator{
+				tempDsnDArg(suite.pgEngine, "dsn", nil),
+			},
+			expectErrContains: []string{"invalid --exclude-table pattern"},
+		},
 	} {
 		suite.Run(tc.name, func() {
 			suite.runCmdWithAssertions(runCmdWithAssertionsParams{
 				args:              append([]string{"dump"}, tc.args...),
 				dynamicArgs:       tc.dynamicArgs,
 				outputContains:    tc.outputContains,
+				outputNotContains: tc.outputNotContains,
 				expectErrContains: tc.expectErrContains,
 			})
 		})

cmd/pg-schema-diff/main_test.go

@@ -38,6 +38,8 @@ type runCmdWithAssertionsParams struct {
 	outputEquals string
 	// outputContains is a list of substrings that are expected to be contained in the stdout output of the command.
 	outputContains []string
+	// outputNotContains is a list of substrings that are expected to NOT be contained in the stdout output.
+	outputNotContains []string
 	// expectErrContains is a list of substrings that are expected to be contained in the error returned by
 	// cmd.RunE. This is DISTINCT from stdErr.
 	expectErrContains []string
@@ -74,6 +76,9 @@ func (suite *cmdTestSuite) runCmdWithAssertions(tc runCmdWithAssertionsParams) {
 			suite.Contains(stdOutStr, o)
 		}
 	}
+	for _, o := range tc.outputNotContains {
+		suite.NotContains(stdOutStr, o)
+	}
 	if len(tc.outputEquals) > 0 {
 		suite.Equal(tc.outputEquals, stdOutStr)
 	}

cmd/pg-schema-diff/plan_cmd.go

@@ -111,8 +111,9 @@ func buildPlanCmd() *cobra.Command {
 type (
 	// parsePlanOptionsFlags stores the flags that are parsed into planOptions.
 	planOptionsFlags struct {
-		includeSchemas []string
-		excludeSchemas []string
+		includeSchemas       []string
+		excludeSchemas       []string
+		excludeTablePatterns []string
 
 		dataPackNewTables     bool
 		disablePlanValidation bool
@@ -221,6 +222,9 @@ func createPlanOptionsFlags(cmd *cobra.Command) *planOptionsFlags {
 
 	cmd.Flags().StringArrayVar(&flags.includeSchemas, "include-schema", nil, "Include the specified schema in the plan")
 	cmd.Flags().StringArrayVar(&flags.excludeSchemas, "exclude-schema", nil, "Exclude the specified schema in the plan")
+	cmd.Flags().StringArrayVar(&flags.excludeTablePatterns, "exclude-table", nil,
+		"Exclude tables matching this Go regexp. The pattern is matched (fully anchored) against both the table "+
+			"name and the schema-qualified name, e.g., 'tmp_.*' or 'public\\.tmp_.*'. Can be repeated.")
 
 	cmd.Flags().BoolVar(&flags.dataPackNewTables, "data-pack-new-tables", true, "If set, will data pack new tables in the plan to minimize table size (re-arranges columns).")
 	cmd.Flags().BoolVar(&flags.disablePlanValidation, "disable-plan-validation", false, "If set, will disable plan validation. Plan validation runs the migration against a temporary"+
@@ -314,10 +318,26 @@ func dsnSchemaSource(connConfig *pgx.ConnConfig) schemaSourceFactory {
 	}
 }
 
+// validateExcludeTablePatterns fail-fast validates regexes before any database work is done. The patterns are
+// compiled for real inside schema.GetSchema; this exists purely for a clean CLI error.
+func validateExcludeTablePatterns(patterns []string) error {
+	for _, pattern := range patterns {
+		if _, err := regexp.Compile(pattern); err != nil {
+			return fmt.Errorf("invalid --exclude-table pattern %q: %w", pattern, err)
+		}
+	}
+	return nil
+}
+
 func parsePlanOptions(p planOptionsFlags) (planOptions, error) {
+	if err := validateExcludeTablePatterns(p.excludeTablePatterns); err != nil {
+		return planOptions{}, err
+	}
+
 	opts := []diff.PlanOpt{
 		diff.WithIncludeSchemas(p.includeSchemas...),
 		diff.WithExcludeSchemas(p.excludeSchemas...),
+		diff.WithExcludeTablePatterns(p.excludeTablePatterns...),
 	}
 
 	if p.dataPackNewTables {

cmd/pg-schema-diff/plan_cmd_test.go

@@ -4,6 +4,8 @@ import (
 	"regexp"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/require"
 )
 
 func (suite *cmdTestSuite) TestPlanCmd() {
@@ -124,6 +126,15 @@ func (suite *cmdTestSuite) TestPlanCmd() {
 			},
 			expectErrContains: []string{"invalid output format"},
 		},
+		{
+			name: "invalid exclude-table pattern",
+			args: []string{"--exclude-table", "["},
+			dynamicArgs: []dArgGenerator{
+				tempDsnDArg(suite.pgEngine, "from-dsn", nil),
+				tempDsnDArg(suite.pgEngine, "to-dsn", nil),
+			},
+			expectErrContains: []string{"invalid --exclude-table pattern"},
+		},
 	} {
 		suite.Run(tc.name, func() {
 			suite.runCmdWithAssertions(runCmdWithAssertionsParams{
@@ -245,3 +256,11 @@ func (suite *cmdTestSuite) TestParseInsertStatementStr() {
 		})
 	}
 }
+
+func TestParsePlanOptionsExcludeTablePatterns(t *testing.T) {
+	_, err := parsePlanOptions(planOptionsFlags{excludeTablePatterns: []string{"tmp_.*"}})
+	require.NoError(t, err)
+
+	_, err = parsePlanOptions(planOptionsFlags{excludeTablePatterns: []string{"["}})
+	require.ErrorContains(t, err, `invalid --exclude-table pattern "["`)
+}

internal/migration_acceptance_tests/exclude_table_cases_test.go

@@ -0,0 +1,99 @@
+package migration_acceptance_tests
+
+import (
+	"testing"
+
+	"github.com/stripe/pg-schema-diff/pkg/diff"
+)
+
+var excludeTableAcceptanceTestCases = []acceptanceTestCase{
+	{
+		name: "Excluded table only in old schema is not dropped",
+		oldSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY);
+			CREATE TABLE tmp_foo(id INT PRIMARY KEY);
+			CREATE INDEX tmp_foo_idx ON tmp_foo(id);
+		`},
+		newSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY);
+		`},
+		planOpts:        []diff.PlanOpt{diff.WithExcludeTablePatterns("tmp_.*")},
+		expectEmptyPlan: true,
+		expectedDBSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY);
+			CREATE TABLE tmp_foo(id INT PRIMARY KEY);
+			CREATE INDEX tmp_foo_idx ON tmp_foo(id);
+		`},
+	},
+	{
+		name: "Excluded table only in new schema is not created",
+		oldSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY);
+		`},
+		newSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY);
+			CREATE TABLE tmp_foo(id INT PRIMARY KEY);
+		`},
+		planOpts:        []diff.PlanOpt{diff.WithExcludeTablePatterns("tmp_.*")},
+		expectEmptyPlan: true,
+		expectedDBSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY);
+		`},
+	},
+	{
+		name: "Changes to non-excluded tables are still planned",
+		oldSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY);
+			CREATE TABLE tmp_foo(id INT PRIMARY KEY);
+		`},
+		newSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY, new_col TEXT);
+		`},
+		planOpts: []diff.PlanOpt{diff.WithExcludeTablePatterns("tmp_.*")},
+		expectedDBSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY, new_col TEXT);
+			CREATE TABLE tmp_foo(id INT PRIMARY KEY);
+		`},
+	},
+	{
+		name: "Schema-qualified pattern only excludes tables in that schema",
+		oldSchemaDDL: []string{`
+			CREATE SCHEMA schema_1;
+			CREATE TABLE schema_1.tmp_foo(id INT PRIMARY KEY);
+			CREATE TABLE tmp_foo(id INT PRIMARY KEY);
+		`},
+		newSchemaDDL: []string{`
+			CREATE SCHEMA schema_1;
+			CREATE TABLE tmp_foo(id INT PRIMARY KEY);
+		`},
+		planOpts:        []diff.PlanOpt{diff.WithExcludeTablePatterns(`schema_1\.tmp_foo`)},
+		expectEmptyPlan: true,
+		expectedDBSchemaDDL: []string{`
+			CREATE SCHEMA schema_1;
+			CREATE TABLE schema_1.tmp_foo(id INT PRIMARY KEY);
+			CREATE TABLE tmp_foo(id INT PRIMARY KEY);
+		`},
+	},
+	{
+		name: "Partitions of an excluded partitioned table are also excluded",
+		oldSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY);
+			CREATE TABLE tmp_events(id INT) PARTITION BY RANGE (id);
+			CREATE TABLE events_p1 PARTITION OF tmp_events FOR VALUES FROM (0) TO (100);
+		`},
+		newSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY);
+		`},
+		planOpts:        []diff.PlanOpt{diff.WithExcludeTablePatterns("tmp_.*")},
+		expectEmptyPlan: true,
+		expectedDBSchemaDDL: []string{`
+			CREATE TABLE foobar(id INT PRIMARY KEY);
+			CREATE TABLE tmp_events(id INT) PARTITION BY RANGE (id);
+			CREATE TABLE events_p1 PARTITION OF tmp_events FOR VALUES FROM (0) TO (100);
+		`},
+	},
+}
+
+func TestExcludeTableTestCases(t *testing.T) {
+	runTestCases(t, excludeTableAcceptanceTestCases)
+}

internal/schema/filters.go

@@ -1,5 +1,11 @@
 package schema
 
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
 // nameFilter is one of the most generic of filters. We can use it to filter objects by their schema name or name.
 // In the future, it might be expanded to include a "type" field, e.g., to filter down to specific tables.
 type nameFilter func(name SchemaQualifiedName) bool
@@ -51,3 +57,81 @@ func filterSliceByName[T any](objs []T, getNameFn func(T) SchemaQualifiedName, f
 	}
 	return filteredObjs
 }
+
+// unescapeIdentifier converts an escaped identifier (as produced by EscapeIdentifier) back to its raw form.
+// Identifiers that are not wrapped in double quotes are returned as-is.
+func unescapeIdentifier(escaped string) string {
+	if len(escaped) >= 2 && strings.HasPrefix(escaped, `"`) && strings.HasSuffix(escaped, `"`) {
+		return strings.ReplaceAll(escaped[1:len(escaped)-1], `""`, `"`)
+	}
+	return escaped
+}
+
+// buildExcludeTablesFilter builds a nameFilter that excludes (returns false for) any table whose unescaped name or
+// unescaped schema-qualified name (e.g., "public.foobar") fully matches any of the given regex patterns. Patterns
+// are anchored, i.e., wrapped in ^(?:...)$, so "users" matches only a table named exactly "users". Returns nil if no
+// patterns are provided.
+func buildExcludeTablesFilter(patterns []string) (nameFilter, error) {
+	if len(patterns) == 0 {
+		return nil, nil
+	}
+
+	var regexes []*regexp.Regexp
+	for _, pattern := range patterns {
+		regex, err := regexp.Compile(fmt.Sprintf("^(?:%s)$", pattern))
+		if err != nil {
+			return nil, fmt.Errorf("compiling exclude table pattern %q: %w", pattern, err)
+		}
+		regexes = append(regexes, regex)
+	}
+
+	return func(table SchemaQualifiedName) bool {
+		name := unescapeIdentifier(table.EscapedName)
+		qualifiedName := fmt.Sprintf("%s.%s", table.SchemaName, name)
+		for _, regex := range regexes {
+			if regex.MatchString(name) || regex.MatchString(qualifiedName) {
+				return false
+			}
+		}
+		return true
+	}, nil
+}
+
+// excludeTables removes tables for which keepTable returns false from the schema, along with partitions of excluded
+// tables (transitively) and any objects owned by excluded tables (indexes, foreign key constraints, triggers). Check
+// constraints, policies, and privileges are stored on the Table struct, so they are removed with their table.
+//
+// Foreign keys owned by kept tables that reference an excluded table are kept, consistent with how cross-schema
+// foreign keys behave with WithExcludeSchemas (see the nameFilter docstring on schemaFetcher about dependency
+// validation).
+func excludeTables(s Schema, keepTable nameFilter) Schema {
+	excludedTables := make(map[string]bool)
+	// Iterate until a fixed point is reached to handle multi-level partitioning, where a partition's parent is
+	// itself a partition of an excluded table.
+	for {
+		changed := false
+		for _, table := range s.Tables {
+			fqName := table.GetFQEscapedName()
+			if excludedTables[fqName] {
+				continue
+			}
+			parentIsExcluded := table.ParentTable != nil && excludedTables[table.ParentTable.GetFQEscapedName()]
+			if parentIsExcluded || !keepTable(table.SchemaQualifiedName) {
+				excludedTables[fqName] = true
+				changed = true
+			}
+		}
+		if !changed {
+			break
+		}
+	}
+
+	keepOwningRel := func(owningRel SchemaQualifiedName) bool {
+		return !excludedTables[owningRel.GetFQEscapedName()]
+	}
+	s.Tables = filterSliceByName(s.Tables, func(t Table) SchemaQualifiedName { return t.SchemaQualifiedName }, keepOwningRel)
+	s.Indexes = filterSliceByName(s.Indexes, func(idx Index) SchemaQualifiedName { return idx.OwningRelName }, keepOwningRel)
+	s.ForeignKeyConstraints = filterSliceByName(s.ForeignKeyConstraints, func(fk ForeignKeyConstraint) SchemaQualifiedName { return fk.OwningTable }, keepOwningRel)
+	s.Triggers = filterSliceByName(s.Triggers, func(t Trigger) SchemaQualifiedName { return t.OwningTable }, keepOwningRel)
+	return s
+}