diff --git a/config/mvtconfig.go b/config/mvtconfig.go new file mode 100644 index 0000000..bc5ccee --- /dev/null +++ b/config/mvtconfig.go @@ -0,0 +1,46 @@ +package config + +import ( + "fmt" + + "github.com/BurntSushi/toml" +) + +type DataSource struct { + Name string `toml:"name"` + Path string `toml:"path"` +} + +type LayerConfig struct { + Name string `toml:"name"` + MinZoom uint `toml:"minzoom"` + MaxZoom uint `toml:"maxzoom"` + DataSource string `toml:"datasource"` + TableName string `toml:"table_name"` +} + +// Tileset mirrors a `[[Tileset]]` array-of-tables entry, including its +// nested `[[Tileset.layer]]` entries. +type Tileset struct { + Name string `toml:"name"` + MinZoom uint `toml:"minzoom"` + MaxZoom uint `toml:"maxzoom"` + Layer []LayerConfig `toml:"layer"` +} + +// TomlConfig mirrors the top-level structure of an mvt config toml file, +// e.g. example/NetherlandsRDNewQuad.toml. +type TomlConfig struct { + DataSource []DataSource `toml:"datasource"` + Tileset []Tileset `toml:"tileset"` +} + +// ParseMVTConfig reads an mvt config toml file and returns the resulting +// TomlConfig. +func ParseMVTConfig(path string) (TomlConfig, error) { + var cfg TomlConfig + if _, err := toml.DecodeFile(path, &cfg); err != nil { + return TomlConfig{}, fmt.Errorf("decoding mvt config %q: %w", path, err) + } + return cfg, nil +} diff --git a/go.mod b/go.mod index 24ab543..59eeaf7 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/pdok/texel go 1.26.2 require ( + github.com/BurntSushi/toml v1.5.0 github.com/carlmjohnson/versioninfo v0.22.5 github.com/creasty/defaults v1.8.0 github.com/go-playground/validator/v10 v10.30.3 diff --git a/go.sum b/go.sum index 1c9ce50..d474429 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/arolek/p v0.0.0-20191103215535-df3c295ed582 h1:DugKk4B3PpqfZ1QunDSPqNZDAErQmdCgwurloi1Axaw= github.com/arolek/p v0.0.0-20191103215535-df3c295ed582/go.mod h1:JPNItmi3yb44Q5QWM+Kh5n9oeRhfcJzPNS90mbLo25U= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= diff --git a/main.go b/main.go index d619b2f..c3d9e8b 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,7 @@ import ( "slices" "syscall" + "github.com/pdok/texel/config" "github.com/pdok/texel/pointindex" "github.com/go-spatial/geom" @@ -39,8 +40,9 @@ const ( TILEBUFFER string = `tilebuffer` USELINETRACE string = `uselinetrace` - MVTSOURCE string = `mvtSourceGpkg` - MVTOUTDIR string = `mvtOutDir` + MVTCONFIG string = `config` + MVTOUTDIR string = `mvtOutDir` + TILEMATRIX string = `tilematrix` ) //nolint:funlen @@ -228,22 +230,29 @@ func main() { Usage: "Build MVT tiles from an already-encoded target GeoPackage (see --" + ENCODETILES + ")", Flags: []cli.Flag{ &cli.StringFlag{ - Name: MVTSOURCE, - Aliases: []string{"s"}, - Usage: `GeoPackage containing the encoded ("_encoded") and attribute tables`, + Name: MVTCONFIG, + Aliases: []string{"c"}, + Usage: "Config toml file describing datasources, tilesets and layers", Required: true, - EnvVars: []string{strcase.ToScreamingSnake(MVTSOURCE)}, + EnvVars: []string{strcase.ToScreamingSnake(MVTCONFIG)}, }, &cli.StringFlag{ Name: MVTOUTDIR, Aliases: []string{"o"}, - Usage: "Directory to write the generated /.mvt files to", + Usage: "Directory to write the generated //.mvt files to", Required: true, EnvVars: []string{strcase.ToScreamingSnake(MVTOUTDIR)}, }, + &cli.UintFlag{ + Name: TILEMATRIX, + Aliases: []string{"z"}, + Usage: "Target zoomlevel", + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(TILEMATRIX)}, + }, }, Action: func(c *cli.Context) error { - return runBuildMVTTiles(c.String(MVTSOURCE), c.String(MVTOUTDIR)) + return runMVT(c.String(MVTCONFIG), c.String(MVTOUTDIR), c.Uint(TILEMATRIX)) }, }, } @@ -296,25 +305,87 @@ func processBySnapping(source processing.Source, targets map[tms20.TMID]processi }, snapConfig.EncodeTiles, snapConfig.Buffer) } -// Initialize resources for creating vecotrtiles and delegate to processing. -func runBuildMVTTiles(sourcePath, outDir string) error { - if _, err := os.Stat(sourcePath); os.IsNotExist(err) { - return fmt.Errorf("source GeoPackage does not exist: %s", sourcePath) +// Construct Layer info from config; init gpkg sources +// Make sure to reuse gpkg handles +// Also return helper that closes all sources +func buildLayers(z uint, rawConfig config.TomlConfig) ([]processing.Layer, func()) { + type initedSource struct { + source gpkg.MVTSourceGeopackage + tables map[string]gpkg.Table + } + dataSourceDictionary := processing.DatasourceToDictionary(rawConfig.DataSource) + layers := make([]processing.Layer, 0) + sources := make(map[string]initedSource) + for _, tileset := range rawConfig.Tileset { + if z < tileset.MinZoom || z > tileset.MaxZoom { + continue + } + for _, rawLayer := range tileset.Layer { + if z < rawLayer.MinZoom || z > rawLayer.MaxZoom { + continue + } + // Only init gpkg sources once + if _, present := sources[rawLayer.DataSource]; !present { + source, tableMap := initMvtSource(rawLayer, dataSourceDictionary) + sources[rawLayer.DataSource] = initedSource{source, tableMap} + } + initedSource := sources[rawLayer.DataSource] + table, present := initedSource.tables[rawLayer.TableName] + if !present { + err := fmt.Errorf("layer %s requires table %s in datasource %s; not found", rawLayer.Name, rawLayer.TableName, rawLayer.DataSource) + panic(err) + } + // Pair gpkg handle with layer table + source := initedSource.source + source.Table = table + layer := processing.BuildLayer(rawLayer.Name, source) + layers = append(layers, layer) + } + } + closeSources := func() { + for _, s := range sources { + s.source.Close() + } } + return layers, closeSources +} +func initMvtSource(rawLayer config.LayerConfig, dataSourceDicationary map[string]string) (gpkg.MVTSourceGeopackage, map[string]gpkg.Table) { + path, present := dataSourceDicationary[rawLayer.DataSource] + if !present { + err := fmt.Errorf("configuration error for layer %s, datasource not found: %s", rawLayer.Name, rawLayer.DataSource) + panic(err) + } + if _, err := os.Stat(path); os.IsNotExist(err) { + err = fmt.Errorf("source Geopackage does not exist: %s", path) + panic(err) + } source := gpkg.MVTSourceGeopackage{} - source.Init(sourcePath) - defer source.Close() - - mvtTarget := gpkg.MVTFileTarget{OutDir: outDir} - + source.Init(path) tables := source.GetTableInfo() + tableMap := make(map[string]gpkg.Table, len(tables)) for _, table := range tables { - source.Table = table - log.Printf(" building MVT tiles for %s", table.Name) - if err := processing.BuildAndWriteMVTTiles(&source, &mvtTarget, table.Name); err != nil { - return fmt.Errorf("building MVT tiles for %s: %w", table.Name, err) - } + tableMap[table.Name] = table } - return nil + return source, tableMap +} + +// runMVT wires the config, layers and target together to build +// and write the MVT tiles for the requested zoomlevel. +func runMVT(configPath, outDir string, zoomlevel uint) error { + if _, err := os.Stat(configPath); os.IsNotExist(err) { + return fmt.Errorf("config file does not exist: %s", configPath) + } + + rawConfig, err := config.ParseMVTConfig(configPath) + if err != nil { + return err + } + + layers, closeSources := buildLayers(zoomlevel, rawConfig) + defer closeSources() + + mvtTarget := gpkg.MVTFileTarget{OutDir: outDir} + + return processing.BuildAndWriteMVTTiles(layers, zoomlevel, &mvtTarget) } diff --git a/processing/gpkg/gpkgmvt.go b/processing/gpkg/gpkgmvt.go index c6ef7c5..43adbd1 100644 --- a/processing/gpkg/gpkgmvt.go +++ b/processing/gpkg/gpkgmvt.go @@ -100,21 +100,21 @@ func (source MVTSourceGeopackage) AttributeColumnNames() []string { // selectEncodedSQL builds the SELECT for fetching the encoded features of a single tile func (t Table) selectEncodedSQL() string { - return `SELECT feature_id, geometry_type, data FROM "` + t.EncodedName() + `" WHERE tile_x = ? AND tile_y = ?` + return `SELECT feature_id, geometry_type, data FROM "` + t.EncodedName() + `" WHERE tile_x = ? AND tile_y = ? AND tile_z = ?` } // selectDistinctTilesSQL builds the SELECT for enumerating every tile that has at // least one encoded feature. func (t Table) selectDistinctTilesSQL() string { - return `SELECT DISTINCT tile_x, tile_y FROM "` + t.EncodedName() + `" ORDER BY tile_x, tile_y` + return `SELECT DISTINCT tile_x, tile_y FROM "` + t.EncodedName() + `" WHERE tile_z = ? ORDER BY tile_x, tile_y` } // GetFeaturesForTile returns every encoded feature stored for the given tile, // i.e. the equivalent of the tile-features table's rows for (tileX, tileY). -func (source MVTSourceGeopackage) GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error) { - rows, err := source.handle.Query(source.Table.selectEncodedSQL(), tileX, tileY) +func (source MVTSourceGeopackage) GetFeaturesForTile(tileX, tileY, tileZ uint) ([]tile.EncodedFeatureRow, error) { + rows, err := source.handle.Query(source.Table.selectEncodedSQL(), tileX, tileY, tileZ) if err != nil { - return nil, fmt.Errorf("querying encoded features for tile (%d,%d): %w", tileX, tileY, err) + return nil, fmt.Errorf("querying encoded features for tile (%d,%d,%d): %w", tileX, tileY, tileZ, err) } defer rows.Close() @@ -153,25 +153,24 @@ func deserializeFromBytes(data []byte) []uint32 { } // ListTiles returns every distinct tile present in the "
_encoded" table. -func (source MVTSourceGeopackage) ListTiles() ([]processing.TileCoord, error) { - rows, err := source.handle.Query(source.Table.selectDistinctTilesSQL()) +func (source MVTSourceGeopackage) ListTiles(z uint, tileSet map[processing.TileCoord]bool) error { + rows, err := source.handle.Query(source.Table.selectDistinctTilesSQL(), z) if err != nil { - return nil, fmt.Errorf("listing tiles: %w", err) + return fmt.Errorf("listing tiles: %w", err) } defer rows.Close() - var tiles []processing.TileCoord for rows.Next() { var x, y int64 if err := rows.Scan(&x, &y); err != nil { - return nil, fmt.Errorf("scanning tile row: %w", err) + return fmt.Errorf("scanning tile row: %w", err) } - tiles = append(tiles, processing.TileCoord{X: uint(x), Y: uint(y)}) //nolint:gosec // G115 Tile coords fit within uint + tileSet[processing.TileCoord{X: uint(x), Y: uint(y), Z: z}] = true //nolint:gosec // G115 Tile coord fit within uint } if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating tile rows: %w", err) + return fmt.Errorf("iterating tile rows: %w", err) } - return tiles, nil + return nil } // Read from source.Table, store rows as attributes[fid][columnName] = value for all fids @@ -245,13 +244,13 @@ type MVTFileTarget struct { // WriteTile writes one tile's serialized bytes to //.mvt, // creating directories as needed. -func (t *MVTFileTarget) WriteTile(x, y uint, data []byte) error { - dir := filepath.Join(t.OutDir, strconv.FormatUint(uint64(x), 10)) +func (t *MVTFileTarget) WriteTile(x, y, z uint, data []byte) error { + dir := filepath.Join(t.OutDir, strconv.FormatUint(uint64(z), 10), strconv.FormatUint(uint64(x), 10)) if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("creating tile directory %s: %w", dir, err) } - path := filepath.Join(dir, strconv.FormatUint(uint64(y), 10)+".mvt") + path := filepath.Join(dir, strconv.FormatUint(uint64(y), 10)+".pbf") if err := os.WriteFile(path, data, 0o644); err != nil { //nolint:gosec // G306 tile output does not need restrictive permissions return fmt.Errorf("writing tile file %s: %w", path, err) } diff --git a/processing/gpkg/gpkgsnap.go b/processing/gpkg/gpkgsnap.go index 58cd563..ba1be5b 100644 --- a/processing/gpkg/gpkgsnap.go +++ b/processing/gpkg/gpkgsnap.go @@ -145,8 +145,8 @@ func (t Table) insertSQL() string { // insertSQLEncoded used for writing the encoded geometries when -enc is on func (t Table) insertSQLEncoded() string { return `INSERT INTO "` + t.EncodedName() + `"` + - ` (tile_x, tile_y, feature_id, geometry_type, data)` + - ` VALUES (?, ?, ?, ?, ?)` + ` (tile_x, tile_y, tile_z, feature_id, geometry_type, data)` + + ` VALUES (?, ?, ?, ?, ?, ?)` } func serializeToBytes(intslice []uint32) []byte { @@ -280,7 +280,7 @@ func (target *TargetGeopackage) writeEncodedFeatures(encFeature []processing.Fea fid := cols[0] for _, eg := range ef.EncodedGeoms() { bytes := serializeToBytes(eg.Encoding) - _, err := stmt.Exec(eg.XTile, eg.YTile, fid, eg.GeometryType, bytes) + _, err := stmt.Exec(eg.XTile, eg.YTile, uint(ef.TileMatrixID()), fid, eg.GeometryType, bytes) //nolint:gosec // G115 integers < 40 if err != nil { log.Fatalf("Could not get a result summary from the prepared statement for fid %s: %s", fid, err) } @@ -349,6 +349,7 @@ func buildEncodedTable(h *gpkg.Handle, t Table) error { id INTEGER PRIMARY KEY, tile_x INTEGER NOT NULL, tile_y INTEGER NOT NULL, + tile_z INTEGER NOT NULL, feature_id INTEGER NOT NULL, geometry_type INTEGER NOT NULL, data BLOB diff --git a/processing/interface.go b/processing/interface.go index 4345b29..0d422be 100644 --- a/processing/interface.go +++ b/processing/interface.go @@ -24,7 +24,7 @@ type SnapResult struct { } type TileCoord struct { - X, Y uint + X, Y, Z uint } // Source and target for snapping @@ -38,12 +38,12 @@ type Target interface { // Source and target for tile generation type MVTSource interface { - ListTiles() ([]TileCoord, error) - GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error) + ListTiles(zoomlevel uint, tileSet map[TileCoord]bool) error + GetFeaturesForTile(tileX, tileY, tileZ uint) ([]tile.EncodedFeatureRow, error) GetAttributesForFeatures(featureIDs []int64) (tile.InternalAttributeTable, error) AttributeColumnNames() []string } type MVTTarget interface { - WriteTile(x, y uint, data []byte) error + WriteTile(x, y, z uint, data []byte) error } diff --git a/processing/processingmvt.go b/processing/processingmvt.go index f90cb8f..c6f4dab 100644 --- a/processing/processingmvt.go +++ b/processing/processingmvt.go @@ -4,43 +4,104 @@ package processing import ( "fmt" + "log" + vectorTile "github.com/go-spatial/geom/encoding/mvt/vector_tile" + "github.com/pdok/texel/config" "github.com/pdok/texel/tile" ) // Orchestrator for the tile creation pipeline -func BuildAndWriteMVTTiles(source MVTSource, target MVTTarget, tableName string) error { - // Build key index (columns) just once. - keyIndex := tile.BuildKeyDictionary(source.AttributeColumnNames()) - - tiles, err := source.ListTiles() +func BuildAndWriteMVTTiles(layers []Layer, zoomlevel uint, target MVTTarget) error { + tiles, err := listTiles(zoomlevel, layers) if err != nil { return err } - for _, tc := range tiles { - encFeatRows, err := source.GetFeaturesForTile(tc.X, tc.Y) + for _, coord := range tiles { + log.Printf(" building (%d, %d, Z = %d)", coord.X, coord.Y, coord.Z) + data, err := processTile(coord, layers) if err != nil { - return fmt.Errorf("getting features for tile (%d,%d): %w", tc.X, tc.Y, err) + return fmt.Errorf("building tile (%d, %d): %w", coord.X, coord.Y, err) } - featIDs := make([]int64, len(encFeatRows)) - for i, encFeatRow := range encFeatRows { - featIDs[i] = encFeatRow.FeatureID + if err := target.WriteTile(coord.X, coord.Y, coord.Z, data); err != nil { + return fmt.Errorf("writing tile (%d, %d): %w", coord.X, coord.Y, err) } + } + return nil +} - attributes, err := source.GetAttributesForFeatures(featIDs) - if err != nil { - return fmt.Errorf("getting attributes for tile (%d,%d): %w", tc.X, tc.Y, err) - } +type Layer struct { + Source MVTSource + Name string + keyIndex map[string]uint32 +} + +func BuildLayer(name string, source MVTSource) Layer { + keydict := tile.BuildKeyDictionary(source.AttributeColumnNames()) + return Layer{ + Name: name, + Source: source, + keyIndex: keydict, + } +} + +type MvtConfig struct { + Layers []Layer + Zoomlevel uint +} + +func DatasourceToDictionary(sources []config.DataSource) map[string]string { + dataSourceDictionary := make(map[string]string, 0) + for _, dataSource := range sources { + dataSourceDictionary[dataSource.Name] = dataSource.Path + } + return dataSourceDictionary +} - data, err := tile.BuildMVTTile(tableName, keyIndex, encFeatRows, attributes) +func listTiles(zoomlevel uint, layers []Layer) ([]TileCoord, error) { + tileSet := make(map[TileCoord]bool, 0) + for _, layer := range layers { + err := layer.Source.ListTiles(zoomlevel, tileSet) if err != nil { - return fmt.Errorf("building tile (%d,%d): %w", tc.X, tc.Y, err) + return nil, err } + } + + tileList := make([]TileCoord, 0, len(tileSet)) + for tilecoord := range tileSet { + tileList = append(tileList, tilecoord) + } + return tileList, nil +} + +func processLayer(tileCoord TileCoord, layer Layer) (*vectorTile.Tile_Layer, error) { + encFeatRows, err := layer.Source.GetFeaturesForTile(tileCoord.X, tileCoord.Y, tileCoord.Z) + if err != nil { + return nil, fmt.Errorf("getting features for tile (%d,%d): %w", tileCoord.X, tileCoord.Y, err) + } + featIDs := make([]int64, len(encFeatRows)) + for i, encFeatRow := range encFeatRows { + featIDs[i] = encFeatRow.FeatureID + } - if err := target.WriteTile(tc.X, tc.Y, data); err != nil { - return fmt.Errorf("writing tile (%d,%d): %w", tc.X, tc.Y, err) + attributes, err := layer.Source.GetAttributesForFeatures(featIDs) + if err != nil { + return nil, fmt.Errorf("getting attributes for tile (%d,%d): %w", tileCoord.X, tileCoord.Y, err) + } + + return tile.BuildLayer(layer.Name, layer.keyIndex, encFeatRows, attributes) +} + +func processTile(coord TileCoord, layers []Layer) ([]byte, error) { + encLayers := make([]*vectorTile.Tile_Layer, 0, len(layers)) + for _, layer := range layers { + encLayer, err := processLayer(coord, layer) + if err != nil { + err = fmt.Errorf("during layer encoding %s: %w", layer.Name, err) + return nil, err } + encLayers = append(encLayers, encLayer) } - return nil + return tile.BuildEncodeTile(encLayers) } diff --git a/tile/assemble.go b/tile/assemble.go index 78106a7..7217a6b 100644 --- a/tile/assemble.go +++ b/tile/assemble.go @@ -180,8 +180,29 @@ func BuildFeature(featureID int64, geom EncodedGeometry, attributes map[string]a }, nil } -// Construct layer according to protobuf format -func BuildLayer(name string, features []*vectorTile.Tile_Feature, keyIndex map[string]uint32, valueIndex map[any]uint32) *vectorTile.Tile_Layer { +func BuildEncodeTile(layers []*vectorTile.Tile_Layer) ([]byte, error) { + tile := &vectorTile.Tile{Layers: layers} + data, err := oldproto.Marshal(tile) + if err != nil { + return nil, fmt.Errorf("marshaling tile: %w", err) + } + return data, nil +} + +func BuildLayer(name string, keyIndex map[string]uint32, encFeatrows []EncodedFeatureRow, attributes InternalAttributeTable) (*vectorTile.Tile_Layer, error) { + valueIndex := BuildValueDictionary(attributes) + + features := make([]*vectorTile.Tile_Feature, 0, len(encFeatrows)) + for _, encodedFeature := range encFeatrows { + featureID := encodedFeature.FeatureID + attrs := attributesForFeature(attributes, featureID) + feature, err := BuildFeature(featureID, encodedFeature.Geom, attrs, keyIndex, valueIndex) + if err != nil { + return nil, err + } + features = append(features, feature) + } + keys := mapslicehelp.OrderedByIndex(keyIndex) rawValues := mapslicehelp.OrderedByIndex(valueIndex) @@ -194,7 +215,7 @@ func BuildLayer(name string, features []*vectorTile.Tile_Feature, keyIndex map[s extent := uint32(precision) layerName := name - return &vectorTile.Tile_Layer{ + tile := &vectorTile.Tile_Layer{ Version: &version, Name: &layerName, Features: features, @@ -202,37 +223,7 @@ func BuildLayer(name string, features []*vectorTile.Tile_Feature, keyIndex map[s Values: values, Extent: &extent, } -} - -// Construct tile according to protobuf format -func BuildTile(layers ...*vectorTile.Tile_Layer) *vectorTile.Tile { - return &vectorTile.Tile{Layers: layers} -} - -// Main exported function that encodes a tile of a single layer. Assumes -// geometry and keyIndex have already been encoded. Returns marshalled byte -// string. -func BuildMVTTile(layerName string, keyIndex map[string]uint32, encFeatRows []EncodedFeatureRow, attributes InternalAttributeTable) ([]byte, error) { - valueIndex := BuildValueDictionary(attributes) - - features := make([]*vectorTile.Tile_Feature, 0, len(encFeatRows)) - for _, encodedFeature := range encFeatRows { - featureID := encodedFeature.FeatureID - attrs := attributesForFeature(attributes, featureID) - feature, err := BuildFeature(featureID, encodedFeature.Geom, attrs, keyIndex, valueIndex) - if err != nil { - return nil, err - } - features = append(features, feature) - } - - layer := BuildLayer(layerName, features, keyIndex, valueIndex) - - data, err := MarshalTile(BuildTile(layer)) - if err != nil { - return nil, fmt.Errorf("marshaling tile: %w", err) - } - return data, nil + return tile, nil } // attributesForFeature extracts a single feature's attributes from the map construct. diff --git a/tile/assemble_test.go b/tile/assemble_test.go index 79aecfb..d9536e9 100644 --- a/tile/assemble_test.go +++ b/tile/assemble_test.go @@ -5,7 +5,6 @@ import ( "testing" vectorTile "github.com/go-spatial/geom/encoding/mvt/vector_tile" - oldproto "github.com/golang/protobuf/proto" //nolint:staticcheck // matches MarshalTile's use of the reflection-based marshaler "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -155,82 +154,109 @@ func TestBuildTags(t *testing.T) { } func TestBuildLayer(t *testing.T) { - t.Run("happy path building layer", func(t *testing.T) { - keyIndex := map[string]uint32{"b": 1, "a": 0} - valueIndex := map[any]uint32{"second": 1, "first": 0} - features := []*vectorTile.Tile_Feature{{}, {}} - - layer := BuildLayer("mylayer", features, keyIndex, valueIndex) - - assert.Equal(t, "mylayer", layer.GetName()) - assert.Equal(t, uint32(mvtVersion), layer.GetVersion()) - assert.Equal(t, uint32(precision), layer.GetExtent()) - assert.Same(t, features[0], layer.GetFeatures()[0]) - assert.Same(t, features[1], layer.GetFeatures()[1]) - - // Keys/values must be ordered according to their assigned index, - // regardless of the (arbitrary) map iteration order used to build them. - assert.Equal(t, []string{"a", "b"}, layer.GetKeys()) - require.Len(t, layer.GetValues(), 2) - assert.Equal(t, "first", layer.GetValues()[0].GetStringValue()) - assert.Equal(t, "second", layer.GetValues()[1].GetStringValue()) - }) - t.Run("empty layer", func(t *testing.T) { - layer := BuildLayer("empty", nil, map[string]uint32{}, map[any]uint32{}) - - assert.Equal(t, "empty", layer.GetName()) - assert.Empty(t, layer.GetKeys()) - assert.Empty(t, layer.GetValues()) - assert.Empty(t, layer.GetFeatures()) - }) -} - -func TestBuildMVTTile(t *testing.T) { - t.Run("happy path roundtrips features, tags and attribute values", func(t *testing.T) { - keyIndex := BuildKeyDictionary([]string{"name"}) - attributes := InternalAttributeTable{ - 1: {"name": "foo"}, - 2: {"name": "bar"}, - } - encFeatRows := []EncodedFeatureRow{ - {FeatureID: 1, Geom: EncodedGeometry{GeometryType: int32(vectorTile.Tile_POINT)}}, - {FeatureID: 2, Geom: EncodedGeometry{GeometryType: int32(vectorTile.Tile_POINT)}}, - } - - data, err := BuildMVTTile("mylayer", keyIndex, encFeatRows, attributes) - require.NoError(t, err) - require.NotEmpty(t, data) - - got := &vectorTile.Tile{} - require.NoError(t, oldproto.Unmarshal(data, got)) - require.Len(t, got.GetLayers(), 1) - - layer := got.GetLayers()[0] - assert.Equal(t, "mylayer", layer.GetName()) - require.Len(t, layer.GetFeatures(), 2) - - // Collect the string values assigned to each feature via its tags, - // independent of feature/value ordering. - gotNames := make([]string, 0, 2) - for _, feature := range layer.GetFeatures() { - require.Len(t, feature.GetTags(), 2) - valueIdx := feature.GetTags()[1] - gotNames = append(gotNames, layer.GetValues()[valueIdx].GetStringValue()) - } - sort.Strings(gotNames) - assert.Equal(t, []string{"bar", "foo"}, gotNames) - }) + tests := []struct { + name string + layerName string + keyIndex map[string]uint32 + encFeatRows []EncodedFeatureRow + attributes InternalAttributeTable + wantErr bool + validate func(t *testing.T, layer *vectorTile.Tile_Layer) + }{ + { + name: "empty layer", + layerName: "empty", + keyIndex: map[string]uint32{}, + encFeatRows: nil, + attributes: InternalAttributeTable{}, + validate: func(t *testing.T, layer *vectorTile.Tile_Layer) { + t.Helper() + assert.Equal(t, "empty", layer.GetName()) + assert.Equal(t, uint32(mvtVersion), layer.GetVersion()) + assert.Equal(t, uint32(precision), layer.GetExtent()) + assert.Empty(t, layer.GetKeys()) + assert.Empty(t, layer.GetValues()) + assert.Empty(t, layer.GetFeatures()) + }, + }, + { + name: "keys are ordered according to key index", + layerName: "mylayer", + keyIndex: map[string]uint32{"b": 1, "a": 0}, + encFeatRows: []EncodedFeatureRow{ + {FeatureID: 1, Geom: EncodedGeometry{GeometryType: int32(vectorTile.Tile_POINT)}}, + }, + attributes: InternalAttributeTable{ + 1: {"a": "first", "b": "second"}, + }, + validate: func(t *testing.T, layer *vectorTile.Tile_Layer) { + t.Helper() + assert.Equal(t, "mylayer", layer.GetName()) + assert.Equal(t, uint32(mvtVersion), layer.GetVersion()) + assert.Equal(t, uint32(precision), layer.GetExtent()) - t.Run("error building a feature's tags propagates", func(t *testing.T) { - keyIndex := map[string]uint32{"name": 0} - attributes := InternalAttributeTable{ - 1: {"unknown-key": "foo"}, - } - encFeatRows := []EncodedFeatureRow{ - {FeatureID: 1, Geom: EncodedGeometry{}}, - } + // Keys must be ordered according to their assigned index, + // regardless of the (arbitrary) map iteration order. + assert.Equal(t, []string{"a", "b"}, layer.GetKeys()) + require.Len(t, layer.GetValues(), 2) + gotValues := make([]string, len(layer.GetValues())) + for i, v := range layer.GetValues() { + gotValues[i] = v.GetStringValue() + } + assert.ElementsMatch(t, []string{"first", "second"}, gotValues) + }, + }, + { + name: "happy path roundtrips features, tags and attribute values", + layerName: "mylayer", + keyIndex: BuildKeyDictionary([]string{"name"}), + encFeatRows: []EncodedFeatureRow{ + {FeatureID: 1, Geom: EncodedGeometry{GeometryType: int32(vectorTile.Tile_POINT)}}, + {FeatureID: 2, Geom: EncodedGeometry{GeometryType: int32(vectorTile.Tile_POINT)}}, + }, + attributes: InternalAttributeTable{ + 1: {"name": "foo"}, + 2: {"name": "bar"}, + }, + validate: func(t *testing.T, layer *vectorTile.Tile_Layer) { + t.Helper() + assert.Equal(t, "mylayer", layer.GetName()) + require.Len(t, layer.GetFeatures(), 2) - _, err := BuildMVTTile("mylayer", keyIndex, encFeatRows, attributes) - require.Error(t, err) - }) + // Collect the string values assigned to each feature via its + // tags, independent of feature/value ordering. + gotNames := make([]string, 0, 2) + for _, feature := range layer.GetFeatures() { + require.Len(t, feature.GetTags(), 2) + valueIdx := feature.GetTags()[1] + gotNames = append(gotNames, layer.GetValues()[valueIdx].GetStringValue()) + } + sort.Strings(gotNames) + assert.Equal(t, []string{"bar", "foo"}, gotNames) + }, + }, + { + name: "error building a feature's tags propagates", + layerName: "mylayer", + keyIndex: map[string]uint32{"name": 0}, + encFeatRows: []EncodedFeatureRow{ + {FeatureID: 1, Geom: EncodedGeometry{}}, + }, + attributes: InternalAttributeTable{ + 1: {"unknown-key": "foo"}, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + layer, err := BuildLayer(tt.layerName, tt.keyIndex, tt.encFeatRows, tt.attributes) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + tt.validate(t, layer) + }) + } }