Pdok 18835/vervanging trex v2 merge gpkgs

config/mvtconfig.go

@@ -0,0 +1,46 @@
+package config
+
+import (
+	"fmt"
+
+	"github.com/BurntSushi/toml"
+)
+
+type DataSource struct {
+	Name string `toml:"name"`
+	Path string `toml:"path"`
+}
+
+type LayerConfig struct {
+	Name       string `toml:"name"`
+	MinZoom    uint   `toml:"minzoom"`
+	MaxZoom    uint   `toml:"maxzoom"`
+	DataSource string `toml:"datasource"`
+	TableName  string `toml:"table_name"`
+}
+
+// Tileset mirrors a `[[Tileset]]` array-of-tables entry, including its
+// nested `[[Tileset.layer]]` entries.
+type Tileset struct {
+	Name    string        `toml:"name"`
+	MinZoom uint          `toml:"minzoom"`
+	MaxZoom uint          `toml:"maxzoom"`
+	Layer   []LayerConfig `toml:"layer"`
+}
+
+// TomlConfig mirrors the top-level structure of an mvt config toml file,
+// e.g. example/NetherlandsRDNewQuad.toml.
+type TomlConfig struct {
+	DataSource []DataSource `toml:"datasource"`
+	Tileset    []Tileset    `toml:"tileset"`
+}
+
+// ParseMVTConfig reads an mvt config toml file and returns the resulting
+// TomlConfig.
+func ParseMVTConfig(path string) (TomlConfig, error) {
+	var cfg TomlConfig
+	if _, err := toml.DecodeFile(path, &cfg); err != nil {
+		return TomlConfig{}, fmt.Errorf("decoding mvt config %q: %w", path, err)
+	}
+	return cfg, nil
+}

go.mod

@@ -3,6 +3,7 @@ module github.com/pdok/texel
 go 1.26.2
 
 require (
+	github.com/BurntSushi/toml v1.5.0
 	github.com/carlmjohnson/versioninfo v0.22.5
 	github.com/creasty/defaults v1.8.0
 	github.com/go-playground/validator/v10 v10.30.3

go.sum

@@ -1,3 +1,5 @@
+github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
+github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/arolek/p v0.0.0-20191103215535-df3c295ed582 h1:DugKk4B3PpqfZ1QunDSPqNZDAErQmdCgwurloi1Axaw=
 github.com/arolek/p v0.0.0-20191103215535-df3c295ed582/go.mod h1:JPNItmi3yb44Q5QWM+Kh5n9oeRhfcJzPNS90mbLo25U=
 github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=

main.go

@@ -10,6 +10,7 @@ import (
 	"slices"
 	"syscall"
 
+	"github.com/pdok/texel/config"
 	"github.com/pdok/texel/pointindex"
 
 	"github.com/go-spatial/geom"
@@ -39,8 +40,9 @@ const (
 	TILEBUFFER          string = `tilebuffer`
 	USELINETRACE        string = `uselinetrace`
 
-	MVTSOURCE string = `mvtSourceGpkg`
-	MVTOUTDIR string = `mvtOutDir`
+	MVTCONFIG  string = `config`
+	MVTOUTDIR  string = `mvtOutDir`
+	TILEMATRIX string = `tilematrix`
 )
 
 //nolint:funlen
@@ -228,22 +230,29 @@ func main() {
 			Usage: "Build MVT tiles from an already-encoded target GeoPackage (see --" + ENCODETILES + ")",
 			Flags: []cli.Flag{
 				&cli.StringFlag{
-					Name:     MVTSOURCE,
-					Aliases:  []string{"s"},
-					Usage:    `GeoPackage containing the encoded ("<table>_encoded") and attribute tables`,
+					Name:     MVTCONFIG,
+					Aliases:  []string{"c"},
+					Usage:    "Config toml file describing datasources, tilesets and layers",
 					Required: true,
-					EnvVars:  []string{strcase.ToScreamingSnake(MVTSOURCE)},
+					EnvVars:  []string{strcase.ToScreamingSnake(MVTCONFIG)},
 				},
 				&cli.StringFlag{
 					Name:     MVTOUTDIR,
 					Aliases:  []string{"o"},
-					Usage:    "Directory to write the generated <tileX>/<tileY>.mvt files to",
+					Usage:    "Directory to write the generated <Z>/<tileX>/<tileY>.mvt files to",
 					Required: true,
 					EnvVars:  []string{strcase.ToScreamingSnake(MVTOUTDIR)},
 				},
+				&cli.UintFlag{
+					Name:     TILEMATRIX,
+					Aliases:  []string{"z"},
+					Usage:    "Target zoomlevel",
+					Required: true,
+					EnvVars:  []string{strcase.ToScreamingSnake(TILEMATRIX)},
+				},
 			},
 			Action: func(c *cli.Context) error {
-				return runBuildMVTTiles(c.String(MVTSOURCE), c.String(MVTOUTDIR))
+				return runMVT(c.String(MVTCONFIG), c.String(MVTOUTDIR), c.Uint(TILEMATRIX))
 			},
 		},
 	}
@@ -296,25 +305,87 @@ func processBySnapping(source processing.Source, targets map[tms20.TMID]processi
 	}, snapConfig.EncodeTiles, snapConfig.Buffer)
 }
 
-// Initialize resources for creating vecotrtiles and delegate to processing.
-func runBuildMVTTiles(sourcePath, outDir string) error {
-	if _, err := os.Stat(sourcePath); os.IsNotExist(err) {
-		return fmt.Errorf("source GeoPackage does not exist: %s", sourcePath)
+// Construct Layer info from config; init gpkg sources
+// Make sure to reuse gpkg handles
+// Also return helper that closes all sources
+func buildLayers(z uint, rawConfig config.TomlConfig) ([]processing.Layer, func()) {
+	type initedSource struct {
+		source gpkg.MVTSourceGeopackage
+		tables map[string]gpkg.Table
+	}
+	dataSourceDictionary := processing.DatasourceToDictionary(rawConfig.DataSource)
+	layers := make([]processing.Layer, 0)
+	sources := make(map[string]initedSource)
+	for _, tileset := range rawConfig.Tileset {
+		if z < tileset.MinZoom || z > tileset.MaxZoom {
+			continue
+		}
+		for _, rawLayer := range tileset.Layer {
+			if z < rawLayer.MinZoom || z > rawLayer.MaxZoom {
+				continue
+			}
+			// Only init gpkg sources once
+			if _, present := sources[rawLayer.DataSource]; !present {
+				source, tableMap := initMvtSource(rawLayer, dataSourceDictionary)
+				sources[rawLayer.DataSource] = initedSource{source, tableMap}
+			}
+			initedSource := sources[rawLayer.DataSource]
+			table, present := initedSource.tables[rawLayer.TableName]
+			if !present {
+				err := fmt.Errorf("layer %s requires table %s in datasource %s; not found", rawLayer.Name, rawLayer.TableName, rawLayer.DataSource)
+				panic(err)
+			}
+			// Pair gpkg handle with layer table
+			source := initedSource.source
+			source.Table = table
+			layer := processing.BuildLayer(rawLayer.Name, source)
+			layers = append(layers, layer)
+		}
+	}
+	closeSources := func() {
+		for _, s := range sources {
+			s.source.Close()
+		}
 	}
+	return layers, closeSources
+}
 
+func initMvtSource(rawLayer config.LayerConfig, dataSourceDicationary map[string]string) (gpkg.MVTSourceGeopackage, map[string]gpkg.Table) {
+	path, present := dataSourceDicationary[rawLayer.DataSource]
+	if !present {
+		err := fmt.Errorf("configuration error for layer %s, datasource not found: %s", rawLayer.Name, rawLayer.DataSource)
+		panic(err)
+	}
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		err = fmt.Errorf("source Geopackage does not exist: %s", path)
+		panic(err)
+	}
 	source := gpkg.MVTSourceGeopackage{}
-	source.Init(sourcePath)
-	defer source.Close()
-
-	mvtTarget := gpkg.MVTFileTarget{OutDir: outDir}
-
+	source.Init(path)
 	tables := source.GetTableInfo()
+	tableMap := make(map[string]gpkg.Table, len(tables))
 	for _, table := range tables {
-		source.Table = table
-		log.Printf("  building MVT tiles for %s", table.Name)
-		if err := processing.BuildAndWriteMVTTiles(&source, &mvtTarget, table.Name); err != nil {
-			return fmt.Errorf("building MVT tiles for %s: %w", table.Name, err)
-		}
+		tableMap[table.Name] = table
 	}
-	return nil
+	return source, tableMap
+}
+
+// runMVT wires the config, layers and target together to build
+// and write the MVT tiles for the requested zoomlevel.
+func runMVT(configPath, outDir string, zoomlevel uint) error {
+	if _, err := os.Stat(configPath); os.IsNotExist(err) {
+		return fmt.Errorf("config file does not exist: %s", configPath)
+	}
+
+	rawConfig, err := config.ParseMVTConfig(configPath)
+	if err != nil {
+		return err
+	}
+
+	layers, closeSources := buildLayers(zoomlevel, rawConfig)
+	defer closeSources()
+
+	mvtTarget := gpkg.MVTFileTarget{OutDir: outDir}
+
+	return processing.BuildAndWriteMVTTiles(layers, zoomlevel, &mvtTarget)
 }

processing/gpkg/gpkgmvt.go

@@ -100,21 +100,21 @@ func (source MVTSourceGeopackage) AttributeColumnNames() []string {
 
 // selectEncodedSQL builds the SELECT for fetching the encoded features of a single tile
 func (t Table) selectEncodedSQL() string {
-	return `SELECT feature_id, geometry_type, data FROM "` + t.EncodedName() + `" WHERE tile_x = ? AND tile_y = ?`
+	return `SELECT feature_id, geometry_type, data FROM "` + t.EncodedName() + `" WHERE tile_x = ? AND tile_y = ? AND tile_z = ?`
 }
 
 // selectDistinctTilesSQL builds the SELECT for enumerating every tile that has at
 // least one encoded feature.
 func (t Table) selectDistinctTilesSQL() string {
-	return `SELECT DISTINCT tile_x, tile_y FROM "` + t.EncodedName() + `" ORDER BY tile_x, tile_y`
+	return `SELECT DISTINCT tile_x, tile_y FROM "` + t.EncodedName() + `" WHERE tile_z = ? ORDER BY tile_x, tile_y`
 }
 
 // GetFeaturesForTile returns every encoded feature stored for the given tile,
 // i.e. the equivalent of the tile-features table's rows for (tileX, tileY).
-func (source MVTSourceGeopackage) GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error) {
-	rows, err := source.handle.Query(source.Table.selectEncodedSQL(), tileX, tileY)
+func (source MVTSourceGeopackage) GetFeaturesForTile(tileX, tileY, tileZ uint) ([]tile.EncodedFeatureRow, error) {
+	rows, err := source.handle.Query(source.Table.selectEncodedSQL(), tileX, tileY, tileZ)
 	if err != nil {
-		return nil, fmt.Errorf("querying encoded features for tile (%d,%d): %w", tileX, tileY, err)
+		return nil, fmt.Errorf("querying encoded features for tile (%d,%d,%d): %w", tileX, tileY, tileZ, err)
 	}
 	defer rows.Close()
 
@@ -153,25 +153,24 @@ func deserializeFromBytes(data []byte) []uint32 {
 }
 
 // ListTiles returns every distinct tile present in the "<table>_encoded" table.
-func (source MVTSourceGeopackage) ListTiles() ([]processing.TileCoord, error) {
-	rows, err := source.handle.Query(source.Table.selectDistinctTilesSQL())
+func (source MVTSourceGeopackage) ListTiles(z uint, tileSet map[processing.TileCoord]bool) error {
+	rows, err := source.handle.Query(source.Table.selectDistinctTilesSQL(), z)
 	if err != nil {
-		return nil, fmt.Errorf("listing tiles: %w", err)
+		return fmt.Errorf("listing tiles: %w", err)
 	}
 	defer rows.Close()
 
-	var tiles []processing.TileCoord
 	for rows.Next() {
 		var x, y int64
 		if err := rows.Scan(&x, &y); err != nil {
-			return nil, fmt.Errorf("scanning tile row: %w", err)
+			return fmt.Errorf("scanning tile row: %w", err)
 		}
-		tiles = append(tiles, processing.TileCoord{X: uint(x), Y: uint(y)}) //nolint:gosec // G115 Tile coords fit within uint
+		tileSet[processing.TileCoord{X: uint(x), Y: uint(y), Z: z}] = true //nolint:gosec // G115 Tile coord fit within uint
 	}
 	if err := rows.Err(); err != nil {
-		return nil, fmt.Errorf("iterating tile rows: %w", err)
+		return fmt.Errorf("iterating tile rows: %w", err)
 	}
-	return tiles, nil
+	return nil
 }
 
 // Read from source.Table, store rows as attributes[fid][columnName] = value for all fids
@@ -245,13 +244,13 @@ type MVTFileTarget struct {
 
 // WriteTile writes one tile's serialized bytes to <OutDir>/<tileX>/<tileY>.mvt,
 // creating directories as needed.
-func (t *MVTFileTarget) WriteTile(x, y uint, data []byte) error {
-	dir := filepath.Join(t.OutDir, strconv.FormatUint(uint64(x), 10))
+func (t *MVTFileTarget) WriteTile(x, y, z uint, data []byte) error {
+	dir := filepath.Join(t.OutDir, strconv.FormatUint(uint64(z), 10), strconv.FormatUint(uint64(x), 10))
 	if err := os.MkdirAll(dir, 0o755); err != nil {
 		return fmt.Errorf("creating tile directory %s: %w", dir, err)
 	}
 
-	path := filepath.Join(dir, strconv.FormatUint(uint64(y), 10)+".mvt")
+	path := filepath.Join(dir, strconv.FormatUint(uint64(y), 10)+".pbf")
 	if err := os.WriteFile(path, data, 0o644); err != nil { //nolint:gosec // G306 tile output does not need restrictive permissions
 		return fmt.Errorf("writing tile file %s: %w", path, err)
 	}

processing/gpkg/gpkgsnap.go

@@ -145,8 +145,8 @@ func (t Table) insertSQL() string {
 // insertSQLEncoded used for writing the encoded geometries when -enc is on
 func (t Table) insertSQLEncoded() string {
 	return `INSERT INTO "` + t.EncodedName() + `"` +
-		` (tile_x, tile_y, feature_id, geometry_type, data)` +
-		` VALUES (?, ?, ?, ?, ?)`
+		` (tile_x, tile_y, tile_z, feature_id, geometry_type, data)` +
+		` VALUES (?, ?, ?, ?, ?, ?)`
 }
 
 func serializeToBytes(intslice []uint32) []byte {
@@ -280,7 +280,7 @@ func (target *TargetGeopackage) writeEncodedFeatures(encFeature []processing.Fea
 		fid := cols[0]
 		for _, eg := range ef.EncodedGeoms() {
 			bytes := serializeToBytes(eg.Encoding)
-			_, err := stmt.Exec(eg.XTile, eg.YTile, fid, eg.GeometryType, bytes)
+			_, err := stmt.Exec(eg.XTile, eg.YTile, uint(ef.TileMatrixID()), fid, eg.GeometryType, bytes) //nolint:gosec // G115 integers < 40
 			if err != nil {
 				log.Fatalf("Could not get a result summary from the prepared statement for fid %s: %s", fid, err)
 			}
@@ -349,6 +349,7 @@ func buildEncodedTable(h *gpkg.Handle, t Table) error {
 			id INTEGER PRIMARY KEY,
 			tile_x INTEGER NOT NULL,
 			tile_y INTEGER NOT NULL,
+			tile_z INTEGER NOT NULL,
 			feature_id INTEGER NOT NULL,
 			geometry_type INTEGER NOT NULL,
 			data BLOB

processing/interface.go

@@ -24,7 +24,7 @@ type SnapResult struct {
 }
 
 type TileCoord struct {
-	X, Y uint
+	X, Y, Z uint
 }
 
 // Source and target for snapping
@@ -38,12 +38,12 @@ type Target interface {
 
 // Source and target for tile generation
 type MVTSource interface {
-	ListTiles() ([]TileCoord, error)
-	GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error)
+	ListTiles(zoomlevel uint, tileSet map[TileCoord]bool) error
+	GetFeaturesForTile(tileX, tileY, tileZ uint) ([]tile.EncodedFeatureRow, error)
 	GetAttributesForFeatures(featureIDs []int64) (tile.InternalAttributeTable, error)
 	AttributeColumnNames() []string
 }
 
 type MVTTarget interface {
-	WriteTile(x, y uint, data []byte) error
+	WriteTile(x, y, z uint, data []byte) error
 }