Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,47 @@ dberd --source cockroach \
--format-to-file schema.d2 \
--render-to-file schema.svg \
--skip-tables goose_db_version,schema_migrations \
--add-logical-references \
--logical-references-file examples/logical-references.yaml \
--source-dsn "postgres://user@host:port/db?sslmode=disable"
```

Use `--skip-tables` to omit a comma-separated list of tables from the output.
Unqualified names, such as `goose_db_version`, match that table in every schema;
qualified names, such as `public.goose_db_version`, match only that exact table.

Use `--add-logical-references` to infer logical references from columns named
`<table>_id`. The column is linked to the `id` column of a uniquely matching
table in the same schema; singular and regular plural table names are matched.
Ambiguous matches are skipped, and existing references take precedence over
inferred ones. Inferred relationships are labelled `logical <source-column>` in
generated diagrams.

Use `--logical-references-file` to add explicit logical references that are not
Comment thread
ashumkin marked this conversation as resolved.
defined as database foreign keys. The file is YAML and declares fully qualified
source and target table-column pairs:

```yaml
references:
- source:
table: public.orders
column: billing_contact
target:
table: public.contacts
column: external_id
name: billing contact
```

Each endpoint must exist in the extracted schema. Conflicting mappings fail, while
an exact duplicate is ignored. Predefined references are applied after
`--skip-tables` and before optional `--add-logical-references` inference, so an
explicit mapping takes precedence over inferred references for the same source
column. `name` is optional and labels the relationship in generated diagrams;
unnamed predefined references keep the target format's existing endpoint-based
label. Inferred references are named `logical <source-column>`. See
[`examples/logical-references.yaml`](examples/logical-references.yaml) for a
copyable template.

Or using Docker:
```bash
docker run --rm -v $(pwd):/work ghcr.io/holydocs/dberd:latest \
Expand Down
32 changes: 32 additions & 0 deletions cmd/dberd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ func main() {
renderToFile := flag.String("render-to-file", "", "Output file for the rendered diagram")
sourceDSN := flag.String("source-dsn", "", "Connection string for source database")
skipTables := flag.String("skip-tables", "", "Comma-separated table names to omit from the output schema")
addLogicalReferences := flag.Bool("add-logical-references", false, "Infer references from <table>_id column names")
logicalReferencesFile := flag.String("logical-references-file", "", "YAML file containing predefined logical references")

help := flag.Bool("help", false, "Show help")

Expand Down Expand Up @@ -75,6 +77,21 @@ func main() {
}

schema.SkipTables(strings.Split(*skipTables, ",")...)
if *logicalReferencesFile != "" {
predefinedReferences, err := loadPredefinedLogicalReferences(*logicalReferencesFile)
if err != nil {
fmt.Fprintf(os.Stderr, "Error: Loading predefined logical references %v\n", err)
os.Exit(1)
}

if err := schema.AddPredefinedLogicalReferences(predefinedReferences); err != nil {
fmt.Fprintf(os.Stderr, "Error: Adding predefined logical references %v\n", err)
os.Exit(1)
}
}
if *addLogicalReferences {
schema.AddLogicalReferences()
}
schema.Sort()

fs, err := target.FormatSchema(ctx, schema)
Expand Down Expand Up @@ -106,6 +123,21 @@ func main() {
}
}

func loadPredefinedLogicalReferences(path string) ([]dberd.Reference, error) {
file, err := os.Open(path)
if err != nil {
return nil, fmt.Errorf("opening %s: %w", path, err)
}
defer file.Close()

references, err := dberd.ParsePredefinedLogicalReferences(file)
if err != nil {
return nil, fmt.Errorf("parsing %s: %w", path, err)
}

return references, nil
}

func pickSource(sourceType, sourceDSN string) (dberd.Source, error) {
switch sourceType {
case "postgres":
Expand Down
112 changes: 108 additions & 4 deletions dberd.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,109 @@ func (s *Schema) Sort() {
})
}

// AddLogicalReferences adds references inferred from <table>_id column names.
// Existing references take precedence, and targets must be in the same schema.
func (s *Schema) AddLogicalReferences() {
type tableAlias struct {
qualifier string
name string
}

referencedSources := make(map[TableColumn]struct{}, len(s.References))
for _, reference := range s.References {
referencedSources[reference.Source] = struct{}{}
}

targetsByAlias := make(map[tableAlias][]TableColumn)
for _, table := range s.Tables {
if !tableHasColumn(table, "id") {
continue
}

qualifier, name := splitTableName(table.Name)
target := TableColumn{Table: table.Name, Column: "id"}
for _, alias := range logicalTableAliases(name) {
key := tableAlias{qualifier: qualifier, name: alias}
targetsByAlias[key] = append(targetsByAlias[key], target)
}
}

for _, table := range s.Tables {
qualifier, _ := splitTableName(table.Name)
for _, column := range table.Columns {
stem, ok := strings.CutSuffix(column.Name, "_id")
if !ok || stem == "" {
continue
}

source := TableColumn{Table: table.Name, Column: column.Name}
if _, ok := referencedSources[source]; ok {
continue
}

targets := targetsByAlias[tableAlias{qualifier: qualifier, name: stem}]
if len(targets) != 1 {
continue
}

s.References = append(s.References, Reference{
Source: source,
Target: targets[0],
Name: "logical " + column.Name,
})
referencedSources[source] = struct{}{}
}
}
}

func splitTableName(name string) (string, string) {
separator := strings.LastIndexByte(name, '.')
if separator == -1 {
return "", name
}

return name[:separator], name[separator+1:]
}

func logicalTableAliases(name string) []string {
if name == "" {
return nil
}

aliases := []string{name}
var singular string
switch {
case strings.HasSuffix(name, "ies"):
singular = strings.TrimSuffix(name, "ies") + "y"
case strings.HasSuffix(name, "zzes"):
singular = strings.TrimSuffix(name, "zes")
case strings.HasSuffix(name, "ches"),
strings.HasSuffix(name, "shes"),
strings.HasSuffix(name, "sses"),
strings.HasSuffix(name, "xes"),
strings.HasSuffix(name, "zes"):
singular = strings.TrimSuffix(name, "es")
case strings.HasSuffix(name, "s"):
Comment thread
ashumkin marked this conversation as resolved.
singular = strings.TrimSuffix(name, "s")
}

if singular != "" && singular != name {
aliases = append(aliases, singular)
}

return aliases
}

func tableHasColumn(table Table, name string) bool {
for _, column := range table.Columns {
if column.Name == name {
return true
}
}

return false
}

// SkipTables removes tables with the given names and all references to them.
// An unqualified name matches tables in every schema, while a qualified name
// matches only the exact table name. Names are case-sensitive; surrounding
Expand Down Expand Up @@ -119,14 +222,15 @@ type Column struct {

// TableColumn represents a reference to a specific column in a table.
type TableColumn struct {
Table string `json:"table"`
Column string `json:"column"`
Table string `json:"table" yaml:"table"`
Column string `json:"column" yaml:"column"`
}

// Reference represents a foreign key relationship between two table columns.
type Reference struct {
Source TableColumn `json:"source"`
Target TableColumn `json:"target"`
Source TableColumn `json:"source" yaml:"source"`
Target TableColumn `json:"target" yaml:"target"`
Name string `json:"name,omitempty" yaml:"name,omitempty"`
}

// FormattedSchema represents a formatted database schema.
Expand Down
Loading
Loading