diff --git a/CHANGELOG.md b/CHANGELOG.md index 690757b5..2a8637c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -141,6 +141,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - `flake8`, `black`, and `mypy` removed from dev dependencies. ### Fixed +- Recognize existing indexes for queryable property names that contain dots (e.g. `test:detail.value`), so `maintain_partitions()` no longer creates duplicate indexes on every queryables reload. +- Generate JSON property indexes against the split `properties` column instead of the removed `content` column. - Explicit search stats refresh now propagates through cached and uncached search paths when `updatestats` is requested, keeping `numberMatched`/context counts current. - `scripts/container-scripts/test` now refreshes collation metadata for the `postgres` database during setup to avoid noisy warning output. diff --git a/src/pgstac/pgstac.sql b/src/pgstac/pgstac.sql index 77894b32..395563b9 100644 --- a/src/pgstac/pgstac.sql +++ b/src/pgstac/pgstac.sql @@ -1548,7 +1548,7 @@ CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ q.property_path ); ELSE - out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, + out := format($q$CREATE INDEX ON %%I USING %s (%s((properties -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name @@ -1567,7 +1567,8 @@ SELECT regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( substring(indexdef FROM '\(([a-zA-Z0-9_]+)\)'), - substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'), + substring(indexdef FROM 'properties -> ''([^'']+)''::text'), + substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([^'']+)''::text'), CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, @@ -1585,7 +1586,8 @@ SELECT indexdef, COALESCE( substring(indexdef FROM '\(([a-zA-Z0-9_]+)\)'), - substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'), + substring(indexdef FROM 'properties -> ''([^'']+)''::text'), + substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([^'']+)''::text'), CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, @@ -1631,7 +1633,8 @@ WITH p AS ( regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( substring(indexdef FROM '\(([a-zA-Z0-9_]+)\)'), - substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'), + substring(indexdef FROM 'properties -> ''([^'']+)''::text'), + substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([^'']+)''::text'), CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM diff --git a/src/pgstac/sql/002a_queryables.sql b/src/pgstac/sql/002a_queryables.sql index 61c9c0db..cf9e047f 100644 --- a/src/pgstac/sql/002a_queryables.sql +++ b/src/pgstac/sql/002a_queryables.sql @@ -259,7 +259,7 @@ CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ q.property_path ); ELSE - out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, + out := format($q$CREATE INDEX ON %%I USING %s (%s((properties -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name @@ -278,7 +278,8 @@ SELECT regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( substring(indexdef FROM '\(([a-zA-Z0-9_]+)\)'), - substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'), + substring(indexdef FROM 'properties -> ''([^'']+)''::text'), + substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([^'']+)''::text'), CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, @@ -296,7 +297,8 @@ SELECT indexdef, COALESCE( substring(indexdef FROM '\(([a-zA-Z0-9_]+)\)'), - substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'), + substring(indexdef FROM 'properties -> ''([^'']+)''::text'), + substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([^'']+)''::text'), CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, @@ -342,7 +344,8 @@ WITH p AS ( regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( substring(indexdef FROM '\(([a-zA-Z0-9_]+)\)'), - substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'), + substring(indexdef FROM 'properties -> ''([^'']+)''::text'), + substring(indexdef FROM '\(content -> ''properties''::text\) -> ''([^'']+)''::text'), CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM diff --git a/src/pgstac/tests/pgtap.sql b/src/pgstac/tests/pgtap.sql index 4d75eb89..4d3210ec 100644 --- a/src/pgstac/tests/pgtap.sql +++ b/src/pgstac/tests/pgtap.sql @@ -17,7 +17,7 @@ CREATE EXTENSION IF NOT EXISTS pgtap; SET SEARCH_PATH TO pgstac, pgtap, public; -- Plan the tests. -SELECT plan(371); +SELECT plan(378); --SELECT * FROM no_plan(); -- Run the tests. diff --git a/src/pgstac/tests/pgtap/002a_queryables.sql b/src/pgstac/tests/pgtap/002a_queryables.sql index d95cf56b..60fe97f8 100644 --- a/src/pgstac/tests/pgtap/002a_queryables.sql +++ b/src/pgstac/tests/pgtap/002a_queryables.sql @@ -208,3 +208,58 @@ SELECT lives_ok( ); RESET pgstac.additional_properties; + +SELECT lives_ok( + $$ SELECT create_item('{"id":"pgstac-test-item-dotted","type":"Feature","collection":"pgstac-test-collection","geometry":{"type":"Point","coordinates":[0,0]},"bbox":[0,0,0,0],"properties":{"datetime":"2011-08-25T00:00:00Z","a.b":"x","test:a.b.c":"y","test:detail.value":1.5},"assets":{},"links":[],"stac_version":"1.0.0"}'); $$, + 'Create item for dotted queryable index tests.' +); + +SELECT lives_ok( + $$ INSERT INTO queryables (name, collection_ids, property_wrapper, property_index_type) VALUES + ('a.b', '{pgstac-test-collection}', 'to_text', 'BTREE'), + ('test:a.b.c', '{pgstac-test-collection}', 'to_text', 'BTREE'), + ('test:detail.value', '{pgstac-test-collection}', 'to_float', 'BTREE'), + ('test:prop', '{pgstac-test-collection}', 'to_text', 'BTREE'); $$, + 'Can index dotted and non-dotted queryables.' +); + +SELECT lives_ok( + $$ SELECT build_pending_indexes(); + SELECT maintain_partitions() FROM generate_series(1, 2); $$, + 'build_pending_indexes + repeated maintain_partitions succeeds for dotted queryables.' +); + +SELECT results_eq( + $$ SELECT field, count(*)::int + FROM pgstac_indexes + WHERE field IN ('a.b', 'test:a.b.c', 'test:detail.value', 'test:prop') + GROUP BY field + ORDER BY field; $$, + $$ VALUES ('a.b', 1), ('test:a.b.c', 1), ('test:detail.value', 1), ('test:prop', 1); $$, + 'Repeated maintain leaves exactly one index per dotted/non-dotted queryable.' +); + +SELECT is_empty( + $$ SELECT field FROM queryable_indexes('items', true) + WHERE field IN ('a.b', 'test:a.b.c', 'test:detail.value', 'test:prop'); $$, + 'queryable_indexes(changes:=true) finds no changes for matching defs.' +); + + +SELECT results_eq( + $q$ SELECT substring( + $i$CREATE INDEX ON pgstac._items_1 USING btree (to_float((properties -> 'test:detail.value'::text)))$i$, + 'properties -> ''([^'']+)''::text' + ); $q$, + $$ SELECT 'test:detail.value'; $$, + 'Extraction captures full dotted property names from split properties indexes.' +); + +SELECT results_eq( + $q$ SELECT substring( + $i$CREATE INDEX ON pgstac._items_1 USING btree (to_float(((content -> 'properties'::text) -> 'test:detail.value'::text)))$i$, + '\(content -> ''properties''::text\) -> ''([^'']+)''::text' + ); $q$, + $$ SELECT 'test:detail.value'; $$, + 'Legacy content->properties extraction captures dotted names.' +);