From d5c542ec5c71ab3996bbea4714f35ac9a3f60a06 Mon Sep 17 00:00:00 2001 From: rng Date: Fri, 4 Sep 2026 15:46:34 +1000 Subject: [PATCH] Implement IN CQL for dataset_group --- .../model/enumeration/CQLFeatureFields.java | 20 +- .../core/model/enumeration/CQLFields.java | 542 ++++++++++-------- .../model/enumeration/CQLFieldsInterface.java | 6 + .../server/core/parser/elastic/OrImpl.java | 42 +- .../parser/elastic/PropertyEqualToImpl.java | 11 +- .../ogcapi/server/common/RestApiTest.java | 29 +- .../CQLToElasticFilterFactoryTest.java | 105 ++++ 7 files changed, 508 insertions(+), 247 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java index b62b0a2f..90ca6fc8 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFeatureFields.java @@ -20,42 +20,49 @@ public enum CQLFeatureFields implements CQLFieldsInterface { StacBasicField.UUID.searchField, StacBasicField.UUID.displayField, null, + null, (order) -> new SortOptions.Builder().field(f -> f.field(StacBasicField.UUID.sortField).order(order)) ), collection( StacBasicField.Collection.searchField, StacBasicField.Collection.displayField, null, + null, (order) -> new SortOptions.Builder().field(f -> f.field(StacBasicField.Collection.sortField).order(order)) ), temporal( "properties.time", "properties.time", null, + null, null ), count( "properties.count", "properties.count", null, + null, null ), geometry( "geometry", "geometry", null, + null, (order) -> new SortOptions.Builder().field(f -> f.field("geometry.geometry.coordinates").order(order)) ), lat( "properties.lat", "properties.lat", null, + null, (order) -> new SortOptions.Builder().field(f -> f.field("properties.lat").order(order)) ), lng( "properties.lng", "properties.lng", null, + null, (order) -> new SortOptions.Builder().field(f -> f.field("properties.lng").order(order)) ); @@ -76,23 +83,30 @@ public enum CQLFeatureFields implements CQLFieldsInterface { @Getter private final Function overridePropertyEqualsToQuery; + // We provided a default match query but there are cases where it isn't enough and need more complex + // match, one example is multiple field. Move this logic out of the parser make it easier to read + @Getter + private final Function, Query> overridePropertyInQuery; CQLFeatureFields(String fields, String displayField, Function overridePropertyEqualsToQuery, + Function, Query> overridePropertyInQuery, Function> sortBuilder) { - this(fields, List.of(displayField), overridePropertyEqualsToQuery, sortBuilder); + this(fields, List.of(displayField), overridePropertyEqualsToQuery, overridePropertyInQuery, sortBuilder); } CQLFeatureFields(String fields, List displayField, Function overridePropertyEqualsToQuery, + Function, Query> overridePropertyInQuery, Function> sortBuilder) { this.searchField = fields; this.displayField = displayField; this.overridePropertyEqualsToQuery = overridePropertyEqualsToQuery; + this.overridePropertyInQuery = overridePropertyInQuery; this.sortBuilder = sortBuilder; } @@ -125,6 +139,10 @@ public Query getPropertyGreaterThanOrEqualsToQuery(String literal) { public Query getBoundingBoxQuery(TopLeftBottomRightGeoBounds tlbr) { return null; } + + @Override + public Query getPropertyInQuery(List literals) { return null; } + /** * Given param, find any of those is not a valid CQLCollectionsField * @param args - diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java index 4b9df62d..239a8381 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java @@ -26,179 +26,217 @@ */ public enum CQLFields implements CQLFieldsInterface { dataset_provider( - StacSummeries.DatasetProvider.searchField, - StacSummeries.DatasetProvider.displayField, - null, - null), + StacSummeries.DatasetProvider.searchField, + StacSummeries.DatasetProvider.displayField, + null, + null, + null), dataset_group( StacSummeries.DatasetGroup.searchField, StacSummeries.DatasetGroup.displayField, - literal -> datasetGroupTermsQuery(List.of( - normalizeDatasetGroupTerm(literal))), + literal -> { + // Exactly one item in the group and match exactly, ie. dataset_group = 'imos' match only + // if the group is exactly ['imos'] not ['imos', 'other'] + String field = StacSummeries.DatasetGroup.searchField; + Query term = TermQuery.of(query -> query + .field(field) + .value(normalizeDatasetGroupTerm(literal)) + )._toQuery(); + Query sizeOne = ScriptQuery.of(query -> query + .script(script -> script + .lang("painless") + .source("doc.containsKey('" + field + "') && !doc['" + field + + "'].empty && doc['" + field + "'].size() == 1"))) + ._toQuery(); + return BoolQuery.of(query -> query.filter(term, sizeOne))._toQuery(); + }, + literals -> datasetGroupTermsQuery(literals.stream() + .map(CQLFields::normalizeDatasetGroupTerm) + .toList()), null), update_frequency( - StacSummeries.UpdateFrequency.searchField, - StacSummeries.UpdateFrequency.displayField, - null, - null), + StacSummeries.UpdateFrequency.searchField, + StacSummeries.UpdateFrequency.displayField, + null, + null, + null), ai_update_frequency( - StacSummeries.AiUpdateFrequency.searchField, - StacSummeries.AiUpdateFrequency.displayField, - null, - null), + StacSummeries.AiUpdateFrequency.searchField, + StacSummeries.AiUpdateFrequency.displayField, + null, + null, + null), geometry( - StacSummeries.Geometry.searchField, - StacSummeries.Geometry.searchField, - null, - null), + StacSummeries.Geometry.searchField, + StacSummeries.Geometry.searchField, + null, + null, + null), bbox( - StacSummeries.Geometry.searchField, - StacSummeries.Geometry.displayField, - null, - null), + StacSummeries.Geometry.searchField, + StacSummeries.Geometry.displayField, + null, + null, + null), centroid( - StacSummeries.GeometryNoLand.searchField, - StacSummeries.GeometryNoLand.displayField, - null, - null), + StacSummeries.GeometryNoLand.searchField, + StacSummeries.GeometryNoLand.displayField, + null, + null, + null), centroid_nocache( - StacSummeries.GeometryNoLand.searchField, - StacSummeries.GeometryNoLand.displayField, - null, - null), + StacSummeries.GeometryNoLand.searchField, + StacSummeries.GeometryNoLand.displayField, + null, + null, + null), temporal( - StacSummeries.Temporal.searchField, - StacSummeries.Temporal.displayField, - null, - /* - * You need to test this in elastic console, basically if end is null aka not - * exist then set the value - * to max, else convert the time to epochMilli secs and get the largest if - * multiple exist, - * so it means, when null it is on going and always, then follow by some valid - * large end dates - * desc order always make on going on top. - * { - * "_script": { - * "type": "number", - * "nested": { - * "path": "summaries.temporal" - * }, - * "script": { - * "lang": "painless", - * "source": """ - * if (doc['summaries.temporal.end'].size() == 0) { - * return Double.MAX_VALUE; - * } - * else { - * return doc['summaries.temporal.end'].stream() - * .mapToLong(f -> f.toEpochMilli()) - * .max() - * .getAsLong() - * } - * """ - * }, - * "order": "desc" - * } - * } - */ - (order) -> new SortOptions.Builder().script(s -> s - .type(ScriptSortType.Number) - .nested(NestedSortValue.of(p -> p.path(StacSummeries.Temporal.sortField))) - .script(script -> script - .lang("painless") - .source("if (doc['" + StacSummeries.TemporalEnd.searchField - + "'].size() == 0) {" + - " return Long.MAX_VALUE; " + - " } " + - " else {" + - " return doc['" - + StacSummeries.TemporalEnd.searchField - + "'].stream()" + - " .mapToLong(f -> f.toEpochMilli())" + - " .max()" + - " .getAsLong()" + - " }")) - .order(order))), + StacSummeries.Temporal.searchField, + StacSummeries.Temporal.displayField, + null, + null, + /* + * You need to test this in elastic console, basically if end is null aka not + * exist then set the value + * to max, else convert the time to epochMilli secs and get the largest if + * multiple exist, + * so it means, when null it is on going and always, then follow by some valid + * large end dates + * desc order always make on going on top. + * { + * "_script": { + * "type": "number", + * "nested": { + * "path": "summaries.temporal" + * }, + * "script": { + * "lang": "painless", + * "source": """ + * if (doc['summaries.temporal.end'].size() == 0) { + * return Double.MAX_VALUE; + * } + * else { + * return doc['summaries.temporal.end'].stream() + * .mapToLong(f -> f.toEpochMilli()) + * .max() + * .getAsLong() + * } + * """ + * }, + * "order": "desc" + * } + * } + */ + (order) -> new SortOptions.Builder().script(s -> s + .type(ScriptSortType.Number) + .nested(NestedSortValue.of(p -> p.path(StacSummeries.Temporal.sortField))) + .script(script -> script + .lang("painless") + .source("if (doc['" + StacSummeries.TemporalEnd.searchField + + "'].size() == 0) {" + + " return Long.MAX_VALUE; " + + " } " + + " else {" + + " return doc['" + + StacSummeries.TemporalEnd.searchField + + "'].stream()" + + " .mapToLong(f -> f.toEpochMilli())" + + " .max()" + + " .getAsLong()" + + " }")) + .order(order))), title( - StacBasicField.Title.searchField, - StacBasicField.Title.displayField, - null, - (order) -> new SortOptions.Builder() - .field(f -> f.field(StacBasicField.Title.sortField).order(order))), + StacBasicField.Title.searchField, + StacBasicField.Title.displayField, + null, + null, + (order) -> new SortOptions.Builder() + .field(f -> f.field(StacBasicField.Title.sortField).order(order))), description( - StacBasicField.Description.searchField, - StacBasicField.Description.displayField, - null, - null), + StacBasicField.Description.searchField, + StacBasicField.Description.displayField, + null, + null, + null), providers( - StacBasicField.Providers.searchField, - StacBasicField.Providers.displayField, - null, - null), + StacBasicField.Providers.searchField, + StacBasicField.Providers.displayField, + null, + null, + null), parameter_vocabs( - StacBasicField.ParameterVocabs.searchField, - StacBasicField.ParameterVocabs.displayField, - null, - vocabPrioritySortBuilder(StacBasicField.ParameterVocabs.searchField)), + StacBasicField.ParameterVocabs.searchField, + StacBasicField.ParameterVocabs.displayField, + null, + null, + vocabPrioritySortBuilder(StacBasicField.ParameterVocabs.searchField)), platform_vocabs( - StacBasicField.PlatformVocabs.searchField, - StacBasicField.PlatformVocabs.displayField, - null, - vocabPrioritySortBuilder(StacBasicField.PlatformVocabs.searchField)), + StacBasicField.PlatformVocabs.searchField, + StacBasicField.PlatformVocabs.displayField, + null, + null, + vocabPrioritySortBuilder(StacBasicField.PlatformVocabs.searchField)), ai_parameter_vocabs( - StacSummeries.AiParameterVocabs.searchField, - StacSummeries.AiParameterVocabs.displayField, - null, - null), + StacSummeries.AiParameterVocabs.searchField, + StacSummeries.AiParameterVocabs.displayField, + null, + null, + null), ai_platform_vocabs( - StacSummeries.AiPlatformVocabs.searchField, - StacSummeries.AiPlatformVocabs.displayField, - null, - null), + StacSummeries.AiPlatformVocabs.searchField, + StacSummeries.AiPlatformVocabs.displayField, + null, + null, + null), organisation_vocabs( - StacBasicField.OrganisationVocabs.searchField, - StacBasicField.OrganisationVocabs.displayField, - null, - null), + StacBasicField.OrganisationVocabs.searchField, + StacBasicField.OrganisationVocabs.displayField, + null, + null, + null), id( - StacBasicField.UUID.searchField, - StacBasicField.UUID.displayField, - // Make sure if id match, it will show up as the first result - (literal) -> MatchPhraseQuery.of(builder -> builder - .field(StacBasicField.UUID.searchField) - .query(literal) - .boost(100.0F))._toQuery(), - (order) -> new SortOptions.Builder() - .field(f -> f.field(StacBasicField.UUID.sortField).order(order))), + StacBasicField.UUID.searchField, + StacBasicField.UUID.displayField, + // Make sure if id match, it will show up as the first result + (literal) -> MatchPhraseQuery.of(builder -> builder + .field(StacBasicField.UUID.searchField) + .query(literal) + .boost(100.0F))._toQuery(), + null, + (order) -> new SortOptions.Builder() + .field(f -> f.field(StacBasicField.UUID.sortField).order(order))), links( - StacBasicField.Links.searchField, - StacBasicField.Links.displayField, - null, - null), + StacBasicField.Links.searchField, + StacBasicField.Links.displayField, + null, + null, + null), links_airole_contains( - StacBasicField.LinksAiRole.searchField, - StacBasicField.LinksAiRole.displayField, - (literal) -> NestedQuery.of(m -> m - .path(StacBasicField.Links.searchField)// "links" - .query(q -> q - .term(t -> t - .field(StacBasicField.LinksAiRole.searchField)// "links.ai:role" - .value(literal)))) - ._toQuery(), - null), + StacBasicField.LinksAiRole.searchField, + StacBasicField.LinksAiRole.displayField, + (literal) -> NestedQuery.of(m -> m + .path(StacBasicField.Links.searchField)// "links" + .query(q -> q + .term(t -> t + .field(StacBasicField.LinksAiRole.searchField)// "links.ai:role" + .value(literal)))) + ._toQuery(), + null, + null), credit_contains( - StacSummeries.Credits.searchField, - StacSummeries.Credits.displayField, - (literal) -> MatchPhraseQuery.of(m -> m// We want the words exact so need to add space in front and end - .field(StacSummeries.Credits.searchField) - .query(literal))._toQuery(), - null), + StacSummeries.Credits.searchField, + StacSummeries.Credits.displayField, + (literal) -> MatchPhraseQuery.of(m -> m// We want the words exact so need to add space in front and end + .field(StacSummeries.Credits.searchField) + .query(literal))._toQuery(), + null, + null), status( - StacSummeries.Status.searchField, - StacSummeries.Status.displayField, - null, - null), + StacSummeries.Status.searchField, + StacSummeries.Status.displayField, + null, + null, + null), scope( StacSummeries.Scope.searchField, StacSummeries.Scope.displayField, @@ -209,101 +247,115 @@ public enum CQLFields implements CQLFieldsInterface { .field(StacSummeries.Scope.searchField) .value(literal)))) ._toQuery(), + null, null), score( - CQLElasticSetting.score.getSetting(), - CQLElasticSetting.score.getSetting(), - null, - (order) -> new SortOptions.Builder() - .field(f -> f.field(CQLElasticSetting.score.getSetting()).order(order))), + CQLElasticSetting.score.getSetting(), + CQLElasticSetting.score.getSetting(), + null, + null, + (order) -> new SortOptions.Builder() + .field(f -> f.field(CQLElasticSetting.score.getSetting()).order(order))), // Rank score is an internal calculated score, it is different from the one use // by ElasticSearch, // @see es-indexer RankingService rank( - StacSummeries.Score.searchField, - StacSummeries.Score.displayField, - null, - (order) -> new SortOptions.Builder() - .field(f -> f.field(StacSummeries.Score.sortField).order(order))), + StacSummeries.Score.searchField, + StacSummeries.Score.displayField, + null, + null, + (order) -> new SortOptions.Builder() + .field(f -> f.field(StacSummeries.Score.sortField).order(order))), fuzzy_title( - null, - StacBasicField.Title.displayField, - (literal) -> MatchQuery.of(m -> m - .fuzziness("AUTO") - .field(StacBasicField.Title.searchField) - .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records - // Increase the relevance of matches in title - .boost(2.0F) - .operator(Operator.And)// ensure all terms are matched with fuzziness - .query(literal))._toQuery(), - null), + null, + StacBasicField.Title.displayField, + (literal) -> MatchQuery.of(m -> m + .fuzziness("AUTO") + .field(StacBasicField.Title.searchField) + .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records + // Increase the relevance of matches in title + .boost(2.0F) + .operator(Operator.And)// ensure all terms are matched with fuzziness + .query(literal))._toQuery(), + null, + null), fuzzy_desc( - null, - StacBasicField.Description.displayField, - (literal) -> MatchQuery.of(m -> m - .fuzziness("AUTO") - .field(StacBasicField.Description.searchField) - .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records - .operator(Operator.And)// ensure all terms are matched with fuzziness - .query(literal))._toQuery(), - null), + null, + StacBasicField.Description.displayField, + (literal) -> MatchQuery.of(m -> m + .fuzziness("AUTO") + .field(StacBasicField.Description.searchField) + .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records + .operator(Operator.And)// ensure all terms are matched with fuzziness + .query(literal))._toQuery(), + null, + null), // Acronym match on the synonyms sub-fields (search-time expansion), e.g. "SOOP" -> "ships of opportunity". acronym_title( - StacBasicField.Title.searchField + ".synonyms", - StacBasicField.Title.displayField, - (literal) -> MatchQuery.of(m -> m - .field(StacBasicField.Title.searchField + ".synonyms") - .operator(Operator.And)// all expanded terms must match - .boost(2.0F)// align with fuzzy_title weighting - .query(literal))._toQuery(), - null), + StacBasicField.Title.searchField + ".synonyms", + StacBasicField.Title.displayField, + (literal) -> MatchQuery.of(m -> m + .field(StacBasicField.Title.searchField + ".synonyms") + .operator(Operator.And)// all expanded terms must match + .boost(2.0F)// align with fuzzy_title weighting + .query(literal))._toQuery(), + null, + null), acronym_desc( - StacBasicField.Description.searchField + ".synonyms", - StacBasicField.Description.displayField, - (literal) -> MatchQuery.of(m -> m - .field(StacBasicField.Description.searchField + ".synonyms") - .operator(Operator.And) - .query(literal))._toQuery(), - null), + StacBasicField.Description.searchField + ".synonyms", + StacBasicField.Description.displayField, + (literal) -> MatchQuery.of(m -> m + .field(StacBasicField.Description.searchField + ".synonyms") + .operator(Operator.And) + .query(literal))._toQuery(), + null, + null), // Contains cloud-optimized data assets_summary( - StacBasicField.AssetsSummary.searchField, - StacBasicField.AssetsSummary.displayField, - null, - null), + StacBasicField.AssetsSummary.searchField, + StacBasicField.AssetsSummary.displayField, + null, + null, + null), // Fields for training ML keyword classification model and delivery mode // classification model themes( - StacBasicField.Themes.searchField, - StacBasicField.Themes.searchField, - null, - null), + StacBasicField.Themes.searchField, + StacBasicField.Themes.searchField, + null, + null, + null), statement( - StacSummeries.Statement.searchField, - StacSummeries.Statement.displayField, - null, - null), + StacSummeries.Statement.searchField, + StacSummeries.Statement.displayField, + null, + null, + null), creation( - StacSummeries.Creation.searchField, - StacSummeries.Creation.displayField, - null, - null), + StacSummeries.Creation.searchField, + StacSummeries.Creation.displayField, + null, + null, + null), revision( - StacSummeries.Revision.searchField, - StacSummeries.Revision.displayField, - null, - null), + StacSummeries.Revision.searchField, + StacSummeries.Revision.displayField, + null, + null, + null), citation( - StacBasicField.Citation.searchField, - StacBasicField.Citation.displayField, - null, - null), + StacBasicField.Citation.searchField, + StacBasicField.Citation.displayField, + null, + null, + null), license( - StacBasicField.License.searchField, - StacBasicField.License.displayField, - null, - null), - ; + StacBasicField.License.searchField, + StacBasicField.License.displayField, + null, + null, + null), + ; private final String searchField; @@ -312,33 +364,39 @@ public enum CQLFields implements CQLFieldsInterface { // do search @Getter private final Function> sortBuilder; - - // We provided a default match query but there are cases where it isn't enough - // and need more complex - // match, one example is multiple field. Move this logic out of the parser make - // it easier to read + /* + * Override the default query for this field, e.g. dataset_group is a special case, it is a list of values, but it is not a property, so it cannot be used in getPropertyEqualToQuery() + */ @Getter private final Function overridePropertyEqualsToQuery; + /* + * Override the default query for this field, e.g. dataset_group is a special case, it is a list of values, but it is not a property, so it cannot be used in getPropertyInQuery() + */ + @Getter + private final Function, Query> overridePropertyInQuery; @Getter private final List displayField; CQLFields(String fields, - String displayField, - Function overridePropertyEqualsToQuery, - Function> sortBuilder) { + String displayField, + Function overridePropertyEqualsToQuery, + Function, Query> overridePropertyInQuery, + Function> sortBuilder) { - this(fields, List.of(displayField), overridePropertyEqualsToQuery, sortBuilder); + this(fields, List.of(displayField), overridePropertyEqualsToQuery, overridePropertyInQuery, sortBuilder); } CQLFields(String fields, - List displayField, - Function overridePropertyEqualsToQuery, - Function> sortBuilder) { + List displayField, + Function overridePropertyEqualsToQuery, + Function, Query> overridePropertyInQuery, + Function> sortBuilder) { this.searchField = fields; this.displayField = displayField; this.overridePropertyEqualsToQuery = overridePropertyEqualsToQuery; + this.overridePropertyInQuery = overridePropertyInQuery; this.sortBuilder = sortBuilder; } @@ -356,7 +414,7 @@ private static String normalizeDatasetGroupTerm(String value) { return value.toLowerCase(Locale.ROOT).trim(); } - // used by the dataset_group CQL filter only without boost + // used by dataset_group IN (...) without boost private static Query datasetGroupTermsQuery(List values) { List terms = values.stream() .filter(value -> !value.isBlank()) @@ -373,7 +431,7 @@ private static Query datasetGroupTermsQuery(List values) { /** * Expands a free-text term into dataset-group candidate values. For example, "csiro temperature" => ["csiro temperature", "csiro", "temperature"]. * Unquoted input includes both the complete normalized input and its individual words. Double-quoted free-text input is treated as one exact value. - * Dataset-group CQL filters continue to use getPropertyEqualToQuery(). + * Dataset-group equality uses getPropertyEqualToQuery(); membership uses getPropertyInQuery(). */ public static List getDatasetGroupCandidates( String literal, @@ -425,6 +483,16 @@ public Query getPropertyEqualToQuery(String literal) { } } + @Override + public Query getPropertyInQuery(List literals) { + if(getOverridePropertyInQuery() == null) { + return null; + } + else { + return getOverridePropertyInQuery().apply(literals); + } + } + @Override public Query getPropertyGreaterThanOrEqualsToQuery(String literal) { return RangeQuery.of(builder -> builder diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFieldsInterface.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFieldsInterface.java index 6a502cf4..cf7f2456 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFieldsInterface.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFieldsInterface.java @@ -19,4 +19,10 @@ public interface CQLFieldsInterface { Query getLikeQuery(String literal); Query getPropertyGreaterThanOrEqualsToQuery(String literal); Query getBoundingBoxQuery(TopLeftBottomRightGeoBounds tlbr); + + /** + * Membership query for {@code property IN (v1, v2, ...)}. Null means the caller should + * keep the default OR of equals clauses. + */ + Query getPropertyInQuery(List literals); } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java index 140786d0..08d74a9c 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/OrImpl.java @@ -1,5 +1,6 @@ package au.org.aodn.ogcapi.server.core.parser.elastic; +import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFieldsInterface; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.opengis.filter.Filter; @@ -28,6 +29,10 @@ private static boolean containsElasticSetting(Filter filter) { */ private static List collectQueries(Filter filter) { if (filter instanceof OrImpl orFilter) { + // Keep collapsed membership queries (e.g. dataset_group IN) intact; flatten bool/should only. + if (orFilter.getQuery() != null && !orFilter.getQuery().isBool()) { + return List.of(orFilter.getQuery()); + } return orFilter.getChildren().stream() .flatMap(child -> collectQueries(child).stream()) .toList(); @@ -40,10 +45,39 @@ private static List collectQueries(Filter filter) { return List.of(); } + /** + * GeoTools compiles {@code property IN (v1, v2)} as {@code or(equals...)} even for one value. + * If every child is equality on the same field that implements membership, emit that IN query. + */ + private static Query collapseToInQuery(List filters) { + if (filters.isEmpty()) { + return null; + } + + List> equals = new ArrayList<>(); + for (Filter filter : filters) { + if (filter instanceof PropertyEqualToImpl equal + && equal.getField() != null + && equal.getLiteralValue() != null) { + equals.add(equal); + } else { + return null; + } + } + + CQLFieldsInterface field = equals.get(0).getField(); + if (equals.stream().anyMatch(equal -> equal.getField() != field)) { + return null; + } + + return field.getPropertyInQuery(equals.stream() + .map(PropertyEqualToImpl::getLiteralValue) + .toList()); + } + /** * Builds the Elasticsearch representation of an OR expression. - * * A single query is returned directly. Multiple queries are combined into * one flat bool/should query to avoid deeply nested bool queries for large * vocabulary selections. @@ -53,6 +87,12 @@ private void buildQuery(List filters) { throw new IllegalArgumentException("Or combine with query setting do not make sense"); } + Query inQuery = collapseToInQuery(filters); + if (inQuery != null) { + this.query = inQuery; + return; + } + List queries = filters.stream() .flatMap(filter -> collectQueries(filter).stream()) .toList(); diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/PropertyEqualToImpl.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/PropertyEqualToImpl.java index 0234e85b..a92ce817 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/PropertyEqualToImpl.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/parser/elastic/PropertyEqualToImpl.java @@ -1,6 +1,7 @@ package au.org.aodn.ogcapi.server.core.parser.elastic; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFieldsInterface; +import lombok.Getter; import org.geotools.filter.AttributeExpressionImpl; import org.geotools.filter.LiteralExpressionImpl; import org.opengis.filter.FilterVisitor; @@ -15,6 +16,12 @@ public class PropertyEqualToImpl & CQLFieldsInterface> extends protected Boolean isMatchingCase; protected MultiValuedFilter.MatchAction matchAction; + @Getter + protected T field; + + @Getter + protected String literalValue; + public PropertyEqualToImpl(Expression expression1, Expression expression2, boolean isMatchingCase, MultiValuedFilter.MatchAction matchAction, Class enumType) { this.expression1 = expression1; this.expression2 = expression2; @@ -23,8 +30,10 @@ public PropertyEqualToImpl(Expression expression1, Expression expression2, boole if (expression1 instanceof AttributeExpressionImpl attribute && expression2 instanceof LiteralExpressionImpl literal) { T v = Enum.valueOf(enumType, attribute.toString().toLowerCase()); + this.field = v; + this.literalValue = literal.toString(); // It is not an Elastic setting, so normal route. - this.query = v.getPropertyEqualToQuery(literal.toString()); + this.query = v.getPropertyEqualToQuery(this.literalValue); } } diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java index cba6e690..371059da 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/common/RestApiTest.java @@ -485,8 +485,8 @@ public void verifyFunderCorrect() throws IOException { @Test public void verifyCQLPropertyDatasetGroup() throws IOException { super.insertJsonToElasticRecordIndex( - "5c418118-2581-4936-b6fd-d6bedfe74f62.json", // Provider null - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // Provider is IMOS + "5c418118-2581-4936-b6fd-d6bedfe74f62.json", // dataset_group is aodn + "7709f541-fc0c-4318-b5b9-9053aa474e0e.json" // dataset_group is IMOS ); ResponseEntity collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=dataset_group='aodn'", Collections.class); @@ -497,30 +497,45 @@ public void verifyCQLPropertyDatasetGroup() throws IOException { "UUID matches"); } /** - * Show that in case of dataset falls in multiple group, it works too + * Equals matches a single-group array only. IN matches any record that contains the group. * @throws IOException - Not expected to throw */ @Test public void verifyCQLPropertyMultiDatasetGroup() throws IOException { super.insertJsonToElasticRecordIndex( - "5c418118-2581-4936-b6fd-d6bedfe74f62.json", // dataset_group null - "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", // dataset_group is IMOS, - "b9bf6b57-54a0-44b3-bd17-30ccfb2b246f.json" // dataset_group aims, imas + "5c418118-2581-4936-b6fd-d6bedfe74f62.json", // dataset_group aodn + "7709f541-fc0c-4318-b5b9-9053aa474e0e.json", // dataset_group is IMOS + "b9bf6b57-54a0-44b3-bd17-30ccfb2b246f.json" // dataset_group aims, imas ); ResponseEntity collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=dataset_group='imas'", Collections.class); + assertEquals(0, Objects.requireNonNull(collections.getBody()).getCollections().size(), + "equals must not match a multi-group record"); + + collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=dataset_group='AIMS'", Collections.class); + assertEquals(0, Objects.requireNonNull(collections.getBody()).getCollections().size(), + "equals must not match a multi-group record"); + + collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=dataset_group IN ('imas')", Collections.class); assertEquals(1, Objects.requireNonNull(collections.getBody()).getCollections().size(), "hit 1, only one record"); assertEquals( "b9bf6b57-54a0-44b3-bd17-30ccfb2b246f", collections.getBody().getCollections().get(0).getId(), "UUID matches"); - collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=dataset_group='AIMS'", Collections.class); + collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=dataset_group IN ('AIMS')", Collections.class); assertEquals(1, Objects.requireNonNull(collections.getBody()).getCollections().size(), "hit 1, only one record"); assertEquals( "b9bf6b57-54a0-44b3-bd17-30ccfb2b246f", collections.getBody().getCollections().get(0).getId(), "UUID matches"); + + collections = testRestTemplate.getForEntity(getBasePath() + "/collections?filter=dataset_group='aodn'", Collections.class); + assertEquals(1, Objects.requireNonNull(collections.getBody()).getCollections().size(), "exact single-group still hits"); + assertEquals( + "5c418118-2581-4936-b6fd-d6bedfe74f62", + collections.getBody().getCollections().get(0).getId(), + "UUID matches"); } /** * You can use the score to tune the return result's relevancy, at this moment, only >= make sense other value diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java index 29738d70..8ccd385b 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/parser/elastic/CQLToElasticFilterFactoryTest.java @@ -3,7 +3,9 @@ import au.org.aodn.ogcapi.server.core.model.enumeration.CQLCrsType; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLElasticSetting; import au.org.aodn.ogcapi.server.core.model.enumeration.CQLFields; +import au.org.aodn.ogcapi.server.core.model.enumeration.StacBasicField; import au.org.aodn.ogcapi.server.core.model.enumeration.StacSummeries; +import co.elastic.clients.elasticsearch._types.FieldValue; import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode; import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.geotools.filter.text.commons.CompilerUtil; @@ -14,6 +16,8 @@ import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -123,6 +127,101 @@ public void temporalAfterUsesNestedRangeWithoutScoring() throws CQLException { assertTrue(Objects.requireNonNull(range.range().date().gte()).contains("1970-01-01")); } + @Test + public void datasetGroupEqualsRequiresSingleValueArray() throws CQLException { + Filter filter = CompilerUtil.parseFilter( + Language.ECQL, + "dataset_group='imas'", + newFactory()); + + PropertyEqualToImpl equalFilter = assertInstanceOf(PropertyEqualToImpl.class, filter); + Query query = equalFilter.getQuery(); + assertTrue(query.isBool()); + List filters = query.bool().filter(); + assertEquals(2, filters.size()); + + Query term = filters.stream().filter(Query::isTerm).findFirst().orElseThrow(); + assertEquals(StacSummeries.DatasetGroup.searchField, term.term().field()); + assertEquals("imas", term.term().value().stringValue()); + + Query script = filters.stream().filter(Query::isScript).findFirst().orElseThrow(); + assertTrue(Objects.requireNonNull(script.script().script().source()).contains("size() == 1")); + } + + @Test + public void datasetGroupInMatchesAnyArrayElement() throws CQLException { + Filter filter = CompilerUtil.parseFilter( + Language.ECQL, + "dataset_group IN ('imas')", + newFactory()); + + OrImpl inFilter = assertInstanceOf(OrImpl.class, filter); + Query query = inFilter.getQuery(); + assertTrue(query.isTerms(), "IN must be a terms membership query, not bool/should of exact equals"); + assertEquals(StacSummeries.DatasetGroup.searchField, query.terms().field()); + assertEquals(List.of("imas"), termValues(query)); + } + + @Test + public void datasetGroupInNormalizesAndCollectsAllLiterals() throws CQLException { + Filter filter = CompilerUtil.parseFilter( + Language.ECQL, + "dataset_group IN ('AIMS', 'imas')", + newFactory()); + + OrImpl inFilter = assertInstanceOf(OrImpl.class, filter); + Query query = inFilter.getQuery(); + assertTrue(query.isTerms()); + assertEquals(StacSummeries.DatasetGroup.searchField, query.terms().field()); + assertEquals(Set.of("aims", "imas"), Set.copyOf(termValues(query))); + } + + @Test + public void datasetGroupEqualsOrOtherFieldIsNotCollapsedToTerms() throws CQLException { + Filter filter = CompilerUtil.parseFilter( + Language.ECQL, + "dataset_group='imas' OR title='wave'", + newFactory()); + + OrImpl orFilter = assertInstanceOf(OrImpl.class, filter); + Query query = orFilter.getQuery(); + assertTrue(query.isBool()); + assertEquals(2, query.bool().should().size()); + assertTrue( + query.bool().should().stream().noneMatch(Query::isTerms), + "Mixed OR must not collapse to a dataset_group terms query"); + } + + @Test + public void datasetGroupInOrOtherFieldKeepsTermsMembership() throws CQLException { + Filter filter = CompilerUtil.parseFilter( + Language.ECQL, + "dataset_group IN ('imas') OR title='wave'", + newFactory()); + + OrImpl orFilter = assertInstanceOf(OrImpl.class, filter); + Query query = orFilter.getQuery(); + assertTrue(query.isBool()); + assertEquals(2, query.bool().should().size()); + assertTrue( + query.bool().should().stream().anyMatch(Query::isTerms), + "dataset_group IN must remain a terms query when OR'd with another field"); + } + + @Test + public void idInRemainsATermsQueryOnUuid() throws CQLException { + Filter filter = CompilerUtil.parseFilter( + Language.ECQL, + "id IN ('516811d7-cd1e-207a-e0440003ba8c79dd')", + newFactory()); + + IdImpl idFilter = assertInstanceOf(IdImpl.class, filter); + Query query = idFilter.getQuery(); + assertTrue(query.isTerms()); + assertEquals(StacBasicField.UUID.searchField, query.terms().field()); + assertEquals(List.of("516811d7-cd1e-207a-e0440003ba8c79dd"), termValues(query)); + } + @Test public void querySettingsCannotBeCombinedWithOr() { IllegalArgumentException settingFirst = assertThrows( @@ -159,4 +258,10 @@ private Query findDateRange(List queries, String field) { .findFirst() .orElseThrow(); } + + private List termValues(Query query) { + return query.terms().terms().value().stream() + .map(FieldValue::stringValue) + .collect(Collectors.toList()); + } }