From 845cc1bd4609959561d0b090935b54fea1446302 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 28 Jul 2026 14:43:05 +1000 Subject: [PATCH 1/8] add elasticsearch.semantic in application config --- server/src/main/resources/application-dev.yaml | 4 ++++ server/src/main/resources/application-edge.yaml | 4 ++++ server/src/main/resources/application-production.yaml | 4 ++++ server/src/main/resources/application-staging.yaml | 4 ++++ server/src/main/resources/application.yaml | 11 +++++++++++ server/src/test/resources/application-test.yaml | 4 ++++ 6 files changed, 31 insertions(+) diff --git a/server/src/main/resources/application-dev.yaml b/server/src/main/resources/application-dev.yaml index 63749f0e..28959e00 100644 --- a/server/src/main/resources/application-dev.yaml +++ b/server/src/main/resources/application-dev.yaml @@ -7,3 +7,7 @@ management: ogcapi: debug: elasticsearch-explain-enabled: true + +elasticsearch: + semantic: + enabled: false \ No newline at end of file diff --git a/server/src/main/resources/application-edge.yaml b/server/src/main/resources/application-edge.yaml index 3ffe7802..f7b49ac4 100644 --- a/server/src/main/resources/application-edge.yaml +++ b/server/src/main/resources/application-edge.yaml @@ -13,3 +13,7 @@ management: ogcapi: debug: elasticsearch-explain-enabled: true + +elasticsearch: + semantic: + enabled: true \ No newline at end of file diff --git a/server/src/main/resources/application-production.yaml b/server/src/main/resources/application-production.yaml index 5a5b12c2..da6aae2d 100644 --- a/server/src/main/resources/application-production.yaml +++ b/server/src/main/resources/application-production.yaml @@ -11,3 +11,7 @@ logging: ogcapi: debug: elasticsearch-explain-enabled: true + +elasticsearch: + semantic: + enabled: true \ No newline at end of file diff --git a/server/src/main/resources/application-staging.yaml b/server/src/main/resources/application-staging.yaml index 63749f0e..1a4a6a1b 100644 --- a/server/src/main/resources/application-staging.yaml +++ b/server/src/main/resources/application-staging.yaml @@ -7,3 +7,7 @@ management: ogcapi: debug: elasticsearch-explain-enabled: true + +elasticsearch: + semantic: + enabled: true \ No newline at end of file diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index 78a15d76..11f48a35 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -22,6 +22,17 @@ elasticsearch: search_suggestions: path: search_suggestions fields: abstract_phrases, parameter_vocabs_sayt, platform_vocabs_sayt, organisation_vocabs_sayt + # Semantic suggestions: the vocabs index is searched by meaning (ELSER via semantic_text), + # then filtered down to terms some record actually carries. + # Off by default - semantic_text needs the licensed `inference` feature. + semantic: + enabled: false + # field in vocab index for semantic text + concept_field: concept_semantic + # number of semantic search suggestion come back + size: 3 + # number of characters after user input to execute semantic query + min_input_length: 3 aws: region: ap-southeast-2 diff --git a/server/src/test/resources/application-test.yaml b/server/src/test/resources/application-test.yaml index f4713627..93ff6007 100644 --- a/server/src/test/resources/application-test.yaml +++ b/server/src/test/resources/application-test.yaml @@ -16,3 +16,7 @@ elasticsearch: search_suggestions: path: search_suggestions fields: abstract_phrases, parameter_vocabs_sayt, platform_vocabs_sayt, organisation_vocabs_sayt + + # no inference feature in test env so turn it off. + semantic: + enabled: false From ba6d9c914f31267e586c91eb1536797e140c44ee Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 28 Jul 2026 15:49:41 +1000 Subject: [PATCH 2/8] save point --- .../core/configuration/CacheConfig.java | 8 + .../configuration/ElasticSearchConfig.java | 5 +- .../server/core/service/ElasticSearch.java | 143 +++++++++++++++--- server/src/main/resources/application.yaml | 2 - .../server/service/ElasticSearchTest.java | 3 +- 5 files changed, 136 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java index a11a116d..25243dd5 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java @@ -38,6 +38,7 @@ public class CacheConfig { public static final String ALL_NO_LAND_GEOMETRY = "all-noland-geometry"; public static final String ALL_PARAM_VOCABS = "parameter-vocabs"; + public static final String USED_VOCAB_TERMS = "used-vocab-terms"; public static final String ELASTIC_SEARCH_UUID_ONLY = "elastic-search-uuid-only"; public static final String STRING_TO_GEOMETRY = "string-to-geometry"; public static final String STRING_TO_PREPARE_GEOMETRY = "string-to-prepared-geometry"; @@ -79,6 +80,13 @@ public JCacheCacheManager cacheManager() throws IOException { ResourcePoolsBuilder.heap(10) ).withExpiry(ExpiryPolicyBuilder.timeToLiveExpiration(Duration.ofHours(24))) ) + // One entry, rebuilt hourly so a reindex is picked up without a restart + .withCache(USED_VOCAB_TERMS, + CacheConfigurationBuilder.newCacheConfigurationBuilder( + Object.class, Object.class, + ResourcePoolsBuilder.heap(1) + ).withExpiry(ExpiryPolicyBuilder.timeToLiveExpiration(Duration.ofHours(1))) + ) .withCache(DOWNLOADABLE_FIELDS, CacheConfigurationBuilder.newCacheConfigurationBuilder( Object.class, Object.class, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java index 30f42b73..b14da720 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java @@ -3,6 +3,7 @@ import au.org.aodn.ogcapi.server.core.service.CacheNoLandGeometry; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import au.org.aodn.ogcapi.server.core.service.Search; +import au.org.aodn.ogcapi.server.core.service.VocabTermUsageService; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.json.jackson.JacksonJsonpMapper; import co.elastic.clients.transport.rest_client.RestClientTransport; @@ -62,10 +63,12 @@ public ElasticsearchClient geoNetworkElasticsearchClient(RestClientTransport tra public Search createElasticSearch(ElasticsearchClient client, CacheNoLandGeometry cacheNoLandGeometry, ObjectMapper mapper, + VocabTermUsageService vocabTermUsageService, @Value("${elasticsearch.index.name}") String indexName, @Value("${elasticsearch.index.pageSize:2200}") Integer pageSize, @Value("${elasticsearch.search_as_you_type.size:10}") Integer searchAsYouTypeSize) { - return new ElasticSearch(client, cacheNoLandGeometry, mapper, indexName, pageSize, searchAsYouTypeSize); + return new ElasticSearch(client, cacheNoLandGeometry, mapper, vocabTermUsageService, + indexName, pageSize, searchAsYouTypeSize); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index b73c2f47..8c5048c2 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -42,6 +42,9 @@ public class ElasticSearch extends ElasticSearchBase implements Search { protected Map defaultElasticSetting; + // the semantic_text field on the vocabs index + protected static final String SEMANTIC_CONCEPT_FIELD = "concept_semantic"; + @Value("${elasticsearch.search_as_you_type.search_suggestions.path}") protected String searchAsYouTypeFieldsPath; @@ -54,9 +57,24 @@ public class ElasticSearch extends ElasticSearchBase implements Search { @Value("${elasticsearch.search_after.split_regex:\\|\\|}") protected String searchAfterSplitRegex; + @Value("${elasticsearch.vocabs_index.name}") + protected String vocabsIndexName; + + @Value("${elasticsearch.semantic.enabled:false}") + protected Boolean semanticEnabled; + + @Value("${elasticsearch.semantic.size:3}") + protected Integer semanticSize; + + @Value("${elasticsearch.semantic.min_input_length:3}") + protected Integer semanticMinInputLength; + + protected final VocabTermUsageService vocabTermUsageService; + public ElasticSearch(ElasticsearchClient client, CacheNoLandGeometry cacheNoLandGeometry, ObjectMapper mapper, + VocabTermUsageService vocabTermUsageService, String indexName, Integer pageSize, Integer searchAsYouTypeSize) { @@ -67,6 +85,7 @@ public ElasticSearch(ElasticsearchClient client, this.setPageSize(pageSize); this.setSearchAsYouTypeSize(searchAsYouTypeSize); this.setCacheNoLandGeometry(cacheNoLandGeometry); + this.vocabTermUsageService = vocabTermUsageService; this.defaultElasticSetting = CQLToElasticFilterFactory.getDefaultSetting(); } /** @@ -117,27 +136,7 @@ protected List> getSuggestionsByField(String input, .query(bQ -> bQ.bool(b -> b.should(suggestFieldsQueries))) )); - /* - this is where the discovery parameter vocabs filter is applied - use term query for exact match of the parameter vocabs - (e.g you don't want "something", "something special" and "something secret" be returned when searching for "something") - see more: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-terms-query.html#query-dsl-terms-query - this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperature") - */ - List filters; - if (cql != null) { - CQLToElasticFilterFactory factory = new CQLToElasticFilterFactory<>(coor, CQLFields.class); - Filter filter = CompilerUtil.parseFilter(Language.ECQL, cql, factory); - if (filter instanceof QueryHandler elasticFilter) { - filters = List.of(elasticFilter.getQuery()); - } else { - // If no filter, then use the match_all{} to get all record - filters = List.of(MatchAllQuery.of(q -> q)._toQuery()); - } - } else { - // If no filter, then use the match_all{} to get all record - filters = List.of(MatchAllQuery.of(q -> q)._toQuery()); - } + List filters = buildSuggestionFilters(cql, coor); // create request SearchRequest searchRequest = this.buildSearchAsYouTypeRequest( @@ -155,6 +154,85 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu return response.hits().hits(); } + /* + this is where the discovery parameter vocabs filter is applied + use term query for exact match of the parameter vocabs + (e.g you don't want "something", "something special" and "something secret" be returned when searching for "something") + see more: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-terms-query.html#query-dsl-terms-query + this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperature") + */ + protected List buildSuggestionFilters(String cql, CQLCrsType coor) throws CQLException { + if (cql != null) { + CQLToElasticFilterFactory factory = new CQLToElasticFilterFactory<>(coor, CQLFields.class); + Filter filter = CompilerUtil.parseFilter(Language.ECQL, cql, factory); + if (filter instanceof QueryHandler elasticFilter) { + return List.of(elasticFilter.getQuery()); + } + } + // If no filter, then use the match_all{} to get all record + return List.of(MatchAllQuery.of(q -> q)._toQuery()); + } + + /** + * Only conduct semantic search if the length is long enough, + * the min_length is defined in application.yaml min_input_length: 3 + * */ + protected boolean isSemanticInputLongEnough(String input) { + return input != null && input.trim().length() >= semanticMinInputLength; + } + + /** + * Rank vocab terms by meaning. The vocabs index carries a `concept_semantic` semantic_text + * field built from each concept's label, alternative labels, definition and narrower terms + * (es-indexer VocabModel.toConceptText), so Elasticsearch scores the concepts themselves rather + * than the records that happen to mention them - which is how "underwater device" reaches + * "Glider" through its definition, something no lexical query can do. + *

+ * Deliberately over-fetches: the usage gate applied by the caller drops terms no record carries, + * and without headroom a page full of unused terms would leave nothing to suggest. + * + * @param input - The input text typed by the end user + */ + protected List> getSemanticTermHits(String input) throws IOException { + SearchRequest searchRequest = SearchRequest.of(s -> s + .index(vocabsIndexName) + .size(Math.max(semanticSize * 4, 10)) + .query(q -> q.semantic(sm -> sm + .field(SEMANTIC_CONCEPT_FIELD) + .query(input)))); + + log.info("getSemanticTermHits | Elastic search payload {}", searchRequest); + SearchResponse response = esClient.search(searchRequest, JsonNode.class); + log.info("getSemanticTermHits | Elastic search response {}", response); + + return response.hits().hits(); + } + + /** + * A vocabs doc holds exactly one of the three concept types (see es-indexer VocabDto), so the + * first one present is the one to label. `display_label` is the human-facing form and matches + * what a record's summaries.*_vocabs contain; `label` covers concepts that lack one. + */ + protected String extractLabel(JsonNode source) { + if (source == null) { + return null; + } + for (String type : List.of("parameter_vocab", "platform_vocab", "organisation_vocab")) { + JsonNode vocab = source.get(type); + if (vocab != null) { + JsonNode displayLabel = vocab.get("display_label"); + if (displayLabel != null && !displayLabel.asText().isBlank()) { + return displayLabel.asText(); + } + JsonNode label = vocab.get("label"); + if (label != null && !label.asText().isBlank()) { + return label.asText(); + } + } + } + return null; + } + public ResponseEntity> getAutocompleteSuggestions(String input, String cql, CQLCrsType coor) throws IOException, CQLException { Map> searchSuggestions = new HashMap<>(); List> suggestion = this.getSuggestionsByField(input, cql, coor); @@ -192,6 +270,29 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu .collect(Collectors.toSet()); searchSuggestions.put("suggested_phrases", abstractPhrases); + // Semantic suggestions - vocab terms ranked by meaning, then narrowed to terms in actual use. + if (Boolean.TRUE.equals(semanticEnabled) && isSemanticInputLongEnough(input)) { + try { + Set used = vocabTermUsageService.getUsedVocabTerms(); + + Set semanticSuggestions = this.getSemanticTermHits(input) + .stream() + .map(hit -> extractLabel(hit.source())) + .filter(Objects::nonNull) + .filter(term -> used.contains(term.toLowerCase())) + .distinct() + .limit(semanticSize) + // LinkedHashSet so the relevance order from Elastic survives into the response + .collect(Collectors.toCollection(LinkedHashSet::new)); + + searchSuggestions.put("suggested_semantic", semanticSuggestions); + } catch (Exception e) { + // Covers the case where the index was built without the semantic fields - the + // dropdown degrades to lexical suggestions rather than the request failing. + log.warn("Semantic suggestions unavailable, returning lexical suggestions only", e); + } + } + return new ResponseEntity<>(searchSuggestions, HttpStatus.OK); } diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index 11f48a35..d4c436ca 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -27,8 +27,6 @@ elasticsearch: # Off by default - semantic_text needs the licensed `inference` feature. semantic: enabled: false - # field in vocab index for semantic text - concept_field: concept_semantic # number of semantic search suggestion come back size: 3 # number of characters after user input to execute semantic query diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index 160672d2..bece5af8 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -46,6 +46,7 @@ public void setUp() { mockClient, null, // CacheNoLandGeometry new ObjectMapper(), + null, // VocabTermUsageService - the semantic path is not exercised here "test-index", 100, 10 @@ -289,7 +290,7 @@ private static class CapturingElasticSearch extends ElasticSearch { private SearchRequest explainRequest; private CapturingElasticSearch(ElasticsearchClient client) { - super(client, null, new ObjectMapper(), "test-index", 100, 10); + super(client, null, new ObjectMapper(), null, "test-index", 100, 10); this.searchAfterSplitRegex = "\\|\\|"; } From 5ea41d9971776ca709c59cfabbd74532762295c1 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Wed, 29 Jul 2026 13:45:04 +1000 Subject: [PATCH 3/8] add auto complete with vocab semantic --- .../server/core/configuration/CacheConfig.java | 8 -------- .../core/configuration/ElasticSearchConfig.java | 5 +---- .../server/core/service/ElasticSearch.java | 16 ++-------------- server/src/main/resources/application-dev.yaml | 2 +- server/src/main/resources/application-edge.yaml | 2 +- .../main/resources/application-production.yaml | 2 +- .../src/main/resources/application-staging.yaml | 2 +- .../ogcapi/server/service/ElasticSearchTest.java | 1 - 8 files changed, 7 insertions(+), 31 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java index 25243dd5..a11a116d 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/CacheConfig.java @@ -38,7 +38,6 @@ public class CacheConfig { public static final String ALL_NO_LAND_GEOMETRY = "all-noland-geometry"; public static final String ALL_PARAM_VOCABS = "parameter-vocabs"; - public static final String USED_VOCAB_TERMS = "used-vocab-terms"; public static final String ELASTIC_SEARCH_UUID_ONLY = "elastic-search-uuid-only"; public static final String STRING_TO_GEOMETRY = "string-to-geometry"; public static final String STRING_TO_PREPARE_GEOMETRY = "string-to-prepared-geometry"; @@ -80,13 +79,6 @@ public JCacheCacheManager cacheManager() throws IOException { ResourcePoolsBuilder.heap(10) ).withExpiry(ExpiryPolicyBuilder.timeToLiveExpiration(Duration.ofHours(24))) ) - // One entry, rebuilt hourly so a reindex is picked up without a restart - .withCache(USED_VOCAB_TERMS, - CacheConfigurationBuilder.newCacheConfigurationBuilder( - Object.class, Object.class, - ResourcePoolsBuilder.heap(1) - ).withExpiry(ExpiryPolicyBuilder.timeToLiveExpiration(Duration.ofHours(1))) - ) .withCache(DOWNLOADABLE_FIELDS, CacheConfigurationBuilder.newCacheConfigurationBuilder( Object.class, Object.class, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java index b14da720..30f42b73 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/ElasticSearchConfig.java @@ -3,7 +3,6 @@ import au.org.aodn.ogcapi.server.core.service.CacheNoLandGeometry; import au.org.aodn.ogcapi.server.core.service.ElasticSearch; import au.org.aodn.ogcapi.server.core.service.Search; -import au.org.aodn.ogcapi.server.core.service.VocabTermUsageService; import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.json.jackson.JacksonJsonpMapper; import co.elastic.clients.transport.rest_client.RestClientTransport; @@ -63,12 +62,10 @@ public ElasticsearchClient geoNetworkElasticsearchClient(RestClientTransport tra public Search createElasticSearch(ElasticsearchClient client, CacheNoLandGeometry cacheNoLandGeometry, ObjectMapper mapper, - VocabTermUsageService vocabTermUsageService, @Value("${elasticsearch.index.name}") String indexName, @Value("${elasticsearch.index.pageSize:2200}") Integer pageSize, @Value("${elasticsearch.search_as_you_type.size:10}") Integer searchAsYouTypeSize) { - return new ElasticSearch(client, cacheNoLandGeometry, mapper, vocabTermUsageService, - indexName, pageSize, searchAsYouTypeSize); + return new ElasticSearch(client, cacheNoLandGeometry, mapper, indexName, pageSize, searchAsYouTypeSize); } } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 1ff6c603..f1cc6c61 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -69,12 +69,9 @@ public class ElasticSearch extends ElasticSearchBase implements Search { @Value("${elasticsearch.semantic.min_input_length:3}") protected Integer semanticMinInputLength; - protected final VocabTermUsageService vocabTermUsageService; - public ElasticSearch(ElasticsearchClient client, CacheNoLandGeometry cacheNoLandGeometry, ObjectMapper mapper, - VocabTermUsageService vocabTermUsageService, String indexName, Integer pageSize, Integer searchAsYouTypeSize) { @@ -85,7 +82,6 @@ public ElasticSearch(ElasticsearchClient client, this.setPageSize(pageSize); this.setSearchAsYouTypeSize(searchAsYouTypeSize); this.setCacheNoLandGeometry(cacheNoLandGeometry); - this.vocabTermUsageService = vocabTermUsageService; this.defaultElasticSetting = CQLToElasticFilterFactory.getDefaultSetting(); } /** @@ -187,16 +183,13 @@ protected boolean isSemanticInputLongEnough(String input) { * (es-indexer VocabModel.toConceptText), so Elasticsearch scores the concepts themselves rather * than the records that happen to mention them - which is how "underwater device" reaches * "Glider" through its definition, something no lexical query can do. - *

- * Deliberately over-fetches: the usage gate applied by the caller drops terms no record carries, - * and without headroom a page full of unused terms would leave nothing to suggest. * * @param input - The input text typed by the end user */ protected List> getSemanticTermHits(String input) throws IOException { SearchRequest searchRequest = SearchRequest.of(s -> s .index(vocabsIndexName) - .size(Math.max(semanticSize * 4, 10)) + .size(semanticSize) .query(q -> q.semantic(sm -> sm .field(SEMANTIC_CONCEPT_FIELD) .query(input)))); @@ -270,18 +263,13 @@ protected String extractLabel(JsonNode source) { .collect(Collectors.toSet()); searchSuggestions.put("suggested_phrases", abstractPhrases); - // Semantic suggestions - vocab terms ranked by meaning, then narrowed to terms in actual use. + // Semantic suggestions - vocab terms ranked by meaning rather than by spelling. if (Boolean.TRUE.equals(semanticEnabled) && isSemanticInputLongEnough(input)) { try { - Set used = vocabTermUsageService.getUsedVocabTerms(); - Set semanticSuggestions = this.getSemanticTermHits(input) .stream() .map(hit -> extractLabel(hit.source())) .filter(Objects::nonNull) - .filter(term -> used.contains(term.toLowerCase())) - .distinct() - .limit(semanticSize) // LinkedHashSet so the relevance order from Elastic survives into the response .collect(Collectors.toCollection(LinkedHashSet::new)); diff --git a/server/src/main/resources/application-dev.yaml b/server/src/main/resources/application-dev.yaml index 28959e00..5eca292a 100644 --- a/server/src/main/resources/application-dev.yaml +++ b/server/src/main/resources/application-dev.yaml @@ -10,4 +10,4 @@ ogcapi: elasticsearch: semantic: - enabled: false \ No newline at end of file + enabled: false diff --git a/server/src/main/resources/application-edge.yaml b/server/src/main/resources/application-edge.yaml index f7b49ac4..c8f57fb1 100644 --- a/server/src/main/resources/application-edge.yaml +++ b/server/src/main/resources/application-edge.yaml @@ -16,4 +16,4 @@ ogcapi: elasticsearch: semantic: - enabled: true \ No newline at end of file + enabled: true diff --git a/server/src/main/resources/application-production.yaml b/server/src/main/resources/application-production.yaml index da6aae2d..44ad7293 100644 --- a/server/src/main/resources/application-production.yaml +++ b/server/src/main/resources/application-production.yaml @@ -14,4 +14,4 @@ ogcapi: elasticsearch: semantic: - enabled: true \ No newline at end of file + enabled: true diff --git a/server/src/main/resources/application-staging.yaml b/server/src/main/resources/application-staging.yaml index 1a4a6a1b..c27d2689 100644 --- a/server/src/main/resources/application-staging.yaml +++ b/server/src/main/resources/application-staging.yaml @@ -10,4 +10,4 @@ ogcapi: elasticsearch: semantic: - enabled: true \ No newline at end of file + enabled: true diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index 4012da3e..8e59069d 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -46,7 +46,6 @@ public void setUp() { mockClient, null, // CacheNoLandGeometry new ObjectMapper(), - null, // VocabTermUsageService - the semantic path is not exercised here "test-index", 100, 10 From b98600a0344f7558ae0e2cf69d557bb1dfdde148 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Fri, 31 Jul 2026 09:41:50 +1000 Subject: [PATCH 4/8] semantic search with fragments --- .../server/core/service/ElasticSearch.java | 67 ++++++++++++++++--- server/src/main/resources/application.yaml | 11 ++- .../server/service/ElasticSearchTest.java | 2 +- 3 files changed, 67 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index f1cc6c61..91e50b99 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -69,6 +69,12 @@ public class ElasticSearch extends ElasticSearchBase implements Search { @Value("${elasticsearch.semantic.min_input_length:3}") protected Integer semanticMinInputLength; + @Value("${elasticsearch.semantic.fragments:3}") + protected Integer semanticFragments; + + @Value("${elasticsearch.semantic.max_suggestions:5}") + protected Integer semanticMaxSuggestions; + public ElasticSearch(ElasticsearchClient client, CacheNoLandGeometry cacheNoLandGeometry, ObjectMapper mapper, @@ -178,12 +184,10 @@ protected boolean isSemanticInputLongEnough(String input) { } /** - * Rank vocab terms by meaning. The vocabs index carries a `concept_semantic` semantic_text - * field built from each concept's label, alternative labels, definition and narrower terms - * (es-indexer VocabModel.toConceptText), so Elasticsearch scores the concepts themselves rather - * than the records that happen to mention them - which is how "underwater device" reaches - * "Glider" through its definition, something no lexical query can do. - * + * Rank vocab terms by meaning similarity with query. Comparing with documents in vocabs index with the semantic_text + * field "concept_semantic", which is a list of combined text for per concepts (level-2 label) as "level-2 label's title. + * level-2 label's description. leaf labels' title". + * Using highlight option to get the real matched level-2 label. * @param input - The input text typed by the end user */ protected List> getSemanticTermHits(String input) throws IOException { @@ -192,7 +196,12 @@ protected List> getSemanticTermHits(String input) throws IOExcepti .size(semanticSize) .query(q -> q.semantic(sm -> sm .field(SEMANTIC_CONCEPT_FIELD) - .query(input)))); + .query(input))) + .highlight(h -> h + .fields(SEMANTIC_CONCEPT_FIELD, f -> f + .numberOfFragments(semanticFragments) + .preTags("") + .postTags("")))); log.info("getSemanticTermHits | Elastic search payload {}", searchRequest); SearchResponse response = esClient.search(searchRequest, JsonNode.class); @@ -226,6 +235,43 @@ protected String extractLabel(JsonNode source) { return null; } + /** + * concept_semantic holds one entry per narrower (level-2) concept, each starting with that + * concept's label (es-indexer VocabDto.getConceptSemantic). The semantic highlighter returns the + * matching entries ranked by score, so a fragment's leading segment names the concept that + * actually matched - not the broad level-1 category the document is keyed on. + */ + protected List extractSemanticLabels(Hit hit) { + List fragments = hit.highlight() == null + ? null + : hit.highlight().get(SEMANTIC_CONCEPT_FIELD); + + if (fragments == null || fragments.isEmpty()) { + // No highlight - e.g. an index still carrying the old single-valued concept_semantic. + String label = extractLabel(hit.source()); + return label == null ? List.of() : List.of(label); + } + return fragments.stream() + .map(this::toConceptLabel) + .filter(Objects::nonNull) + .toList(); + } + + /** + * Leading segment of a concept_semantic entry, which is the concept's label. Split on ". " + * rather than "." so labels that carry an internal period (e.g. "No.3 buoy") survive. + */ + protected String toConceptLabel(String fragment) { + if (fragment == null) { + return null; + } + // -1 means the fragment is a single segment and is the label; 0 means it opens with the + // separator, leaving no label at all - the two must not collapse into the same branch. + int end = fragment.indexOf(". "); + String label = (end >= 0 ? fragment.substring(0, end) : fragment).trim(); + return label.isBlank() ? null : label; + } + public ResponseEntity> getAutocompleteSuggestions(String input, String cql, CQLCrsType coor) throws IOException, CQLException { Map> searchSuggestions = new HashMap<>(); List> suggestion = this.getSuggestionsByField(input, cql, coor); @@ -268,8 +314,11 @@ protected String extractLabel(JsonNode source) { try { Set semanticSuggestions = this.getSemanticTermHits(input) .stream() - .map(hit -> extractLabel(hit.source())) - .filter(Objects::nonNull) + // Hit-major order: docs by score, then concepts within a doc by chunk score. + .flatMap(hit -> extractSemanticLabels(hit).stream()) + // distinct before limit so duplicates do not consume suggestion slots + .distinct() + .limit(semanticMaxSuggestions) // LinkedHashSet so the relevance order from Elastic survives into the response .collect(Collectors.toCollection(LinkedHashSet::new)); diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index c776a416..5cd9914b 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -22,15 +22,20 @@ elasticsearch: search_suggestions: path: search_suggestions fields: abstract_phrases, parameter_vocabs_sayt, platform_vocabs_sayt, organisation_vocabs_sayt - # Semantic suggestions: the vocabs index is searched by meaning (ELSER via semantic_text), - # then filtered down to terms some record actually carries. + # Semantic suggestions: the vocabs index is searched by meaning (ELSER via semantic_text). + # The suggested concepts are not filtered against records, so a suggestion may be a concept + # no record currently carries. # Off by default - semantic_text needs the licensed `inference` feature. semantic: enabled: false - # number of semantic search suggestion come back + # number of vocab docs (top-level categories) the semantic query returns size: 3 # number of characters after user input to execute semantic query min_input_length: 3 + # highlight fragments per doc - one per matching level-2 concept + fragments: 3 + # cap on the flattened suggestion list + max_suggestions: 5 aws: region: ap-southeast-2 diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index 8e59069d..c079a719 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -292,7 +292,7 @@ private static class CapturingElasticSearch extends ElasticSearch { private SearchRequest explainRequest; private CapturingElasticSearch(ElasticsearchClient client) { - super(client, null, new ObjectMapper(), null, "test-index", 100, 10); + super(client, null, new ObjectMapper(), "test-index", 100, 10); this.searchAfterSplitRegex = "\\|\\|"; } From 5617078128b81d14fa90c3b4d3d40de9cdbd323e Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Fri, 31 Jul 2026 14:51:15 +1000 Subject: [PATCH 5/8] return with fragments score --- .../server/core/service/ElasticSearch.java | 121 ++++++++++++++++-- server/src/main/resources/application.yaml | 4 +- 2 files changed, 115 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index a3c418de..43ada4e8 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -14,6 +14,7 @@ import co.elastic.clients.elasticsearch.core.SearchRequest; import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.HighlighterOrder; import co.elastic.clients.elasticsearch.core.search_mvt.GridType; import co.elastic.clients.transport.endpoints.BinaryResponse; import com.fasterxml.jackson.databind.JsonNode; @@ -45,6 +46,13 @@ public class ElasticSearch extends ElasticSearchBase implements Search { // the semantic_text field on the vocabs index protected static final String SEMANTIC_CONCEPT_FIELD = "concept_semantic"; + // marks a vocabs doc as an organisation one - see getSemanticTermHits for why they are skipped + protected static final String ORGANISATION_VOCAB_FIELD = "organisation_vocab"; + + // a vocabs doc holds exactly one of these (es-indexer VocabDto) + protected static final List VOCAB_TYPES = + List.of("parameter_vocab", "platform_vocab", ORGANISATION_VOCAB_FIELD); + @Value("${elasticsearch.search_as_you_type.search_suggestions.path}") protected String searchAsYouTypeFieldsPath; @@ -194,12 +202,18 @@ protected List> getSemanticTermHits(String input) throws IOExcepti SearchRequest searchRequest = SearchRequest.of(s -> s .index(vocabsIndexName) .size(semanticSize) - .query(q -> q.semantic(sm -> sm - .field(SEMANTIC_CONCEPT_FIELD) - .query(input))) + .query(q -> q.bool(b -> b + .must(m -> m.semantic(sm -> sm + .field(SEMANTIC_CONCEPT_FIELD) + .query(input))) + .mustNot(mn -> mn.exists(e -> e.field(ORGANISATION_VOCAB_FIELD))))) .highlight(h -> h .fields(SEMANTIC_CONCEPT_FIELD, f -> f .numberOfFragments(semanticFragments) + // The highlighter picks the top fragments by score but hands them + // back in field order unless asked otherwise, so without this the + // first fragment is merely the earliest concept, not the best match. + .order(HighlighterOrder.Score) .preTags("") .postTags("")))); @@ -219,7 +233,7 @@ protected String extractLabel(JsonNode source) { if (source == null) { return null; } - for (String type : List.of("parameter_vocab", "platform_vocab", "organisation_vocab")) { + for (String type : VOCAB_TYPES) { JsonNode vocab = source.get(type); if (vocab != null) { JsonNode displayLabel = vocab.get("display_label"); @@ -257,6 +271,80 @@ protected List extractSemanticLabels(Hit hit) { .toList(); } + /** + * Definitions of every concept in a vocabs doc, keyed by the name a suggestion can carry, so the + * portal can explain a semantic suggestion on hover. + *

+ * Read from the doc rather than parsed out of the highlight fragment. A fragment reads + * "label. definition. leaf labels", but definitions contain ". " of their own ("(e.g. Autosub + * Glider)"), so splitting one apart is guesswork - the doc has the fields verbatim. + */ + protected Map extractConceptDefinitions(JsonNode source) { + if (source == null) { + return Map.of(); + } + Map definitions = new HashMap<>(); + for (String type : VOCAB_TYPES) { + JsonNode vocab = source.get(type); + if (vocab == null) { + continue; + } + // The level-1 category, for the no-highlight path where extractLabel names the doc itself. + putDefinition(definitions, vocab); + + JsonNode narrower = vocab.get("narrower"); + if (narrower != null && narrower.isArray()) { + narrower.forEach(concept -> putDefinition(definitions, concept)); + } + } + return definitions; + } + + /** + * Index one concept's definition under both of its names: a highlight fragment opens with + * `label`, while {@link #extractLabel} emits `display_label`, and either can end up being the + * suggested string. + */ + protected void putDefinition(Map definitions, JsonNode concept) { + JsonNode definition = concept.get("definition"); + if (definition == null || definition.asText().isBlank()) { + return; + } + for (String key : List.of("label", "display_label")) { + JsonNode name = concept.get(key); + if (name != null && !name.asText().isBlank()) { + definitions.putIfAbsent(name.asText(), definition.asText()); + } + } + } + + /** + * Flatten the per-document concept labels round-robin: every document's best concept, then every + * document's second best, and so on. + *

+ * Concatenating the documents instead would let the third-best concept of the top document + * outrank the best concept of the second, and would let the first two documents consume every + * suggestion slot - the document-level version of that is the v1 bug this feature was rebuilt to + * fix. A true global sort is not available: highlight fragments carry no score in the response, + * so rank within a document is the only proxy there is. + * + * @param labelsPerDoc - concept labels per hit, hits in _score order, labels in fragment order + */ + protected List interleave(List> labelsPerDoc) { + int deepest = labelsPerDoc.stream().mapToInt(List::size).max().orElse(0); + + List flattened = new ArrayList<>(); + for (int rank = 0; rank < deepest; rank++) { + for (List labels : labelsPerDoc) { + // A document that ran out of concepts simply stops contributing at this rank. + if (rank < labels.size()) { + flattened.add(labels.get(rank)); + } + } + } + return flattened; + } + /** * Leading segment of a concept_semantic entry, which is the concept's label. Split on ". " * rather than "." so labels that carry an internal period (e.g. "No.3 buoy") survive. @@ -273,7 +361,9 @@ protected String toConceptLabel(String fragment) { } public ResponseEntity> getAutocompleteSuggestions(String input, String cql, CQLCrsType coor) throws IOException, CQLException { - Map> searchSuggestions = new HashMap<>(); + // Object rather than Set: every key is a set of suggestions except + // semantic_definitions, which is a label -> definition map. + Map searchSuggestions = new HashMap<>(); List> suggestion = this.getSuggestionsByField(input, cql, coor); // extract parameter vocab suggestions Set parameterVocabSuggestions = suggestion @@ -312,10 +402,15 @@ protected String toConceptLabel(String fragment) { // Semantic suggestions - vocab terms ranked by meaning rather than by spelling. if (Boolean.TRUE.equals(semanticEnabled) && isSemanticInputLongEnough(input)) { try { - Set semanticSuggestions = this.getSemanticTermHits(input) + List> semanticHits = this.getSemanticTermHits(input); + + List> labelsPerDoc = semanticHits + .stream() + .map(this::extractSemanticLabels) + .toList(); + + Set semanticSuggestions = interleave(labelsPerDoc) .stream() - // Hit-major order: docs by score, then concepts within a doc by chunk score. - .flatMap(hit -> extractSemanticLabels(hit).stream()) // distinct before limit so duplicates do not consume suggestion slots .distinct() .limit(semanticMaxSuggestions) @@ -323,6 +418,16 @@ protected String toConceptLabel(String fragment) { .collect(Collectors.toCollection(LinkedHashSet::new)); searchSuggestions.put("suggested_semantic", semanticSuggestions); + + // Definitions of the terms actually suggested, so the portal can explain one on + // hover. Only those - the docs carry definitions for concepts that never made the + // cut, and shipping them would dwarf the suggestions themselves. + Map definitions = new HashMap<>(); + semanticHits.forEach(hit -> + extractConceptDefinitions(hit.source()).forEach(definitions::putIfAbsent)); + definitions.keySet().retainAll(semanticSuggestions); + + searchSuggestions.put("semantic_definitions", definitions); } catch (Exception e) { // Covers the case where the index was built without the semantic fields - the // dropdown degrades to lexical suggestions rather than the request failing. diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index d58b44d5..3831f695 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -33,9 +33,9 @@ elasticsearch: # number of characters after user input to execute semantic query min_input_length: 3 # highlight fragments per doc - one per matching level-2 concept - fragments: 3 + fragments: 2 # cap on the flattened suggestion list - max_suggestions: 5 + max_suggestions: 3 aws: region: ap-southeast-2 From 06bd79bc3636041c8acb2ce8faa73d8377aee2e4 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 4 Aug 2026 15:01:52 +1000 Subject: [PATCH 6/8] better comment and remove unused code --- .../server/core/service/ElasticSearch.java | 64 +------------------ server/src/main/resources/application.yaml | 2 +- 2 files changed, 3 insertions(+), 63 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 43ada4e8..33f24f4c 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -184,7 +184,7 @@ protected List buildSuggestionFilters(String cql, CQLCrsType coor) throws } /** - * Only conduct semantic search if the length is long enough, + * Only conduct semantic search if the input is long enough, * the min_length is defined in application.yaml min_input_length: 3 * */ protected boolean isSemanticInputLongEnough(String input) { @@ -252,8 +252,7 @@ protected String extractLabel(JsonNode source) { /** * concept_semantic holds one entry per narrower (level-2) concept, each starting with that * concept's label (es-indexer VocabDto.getConceptSemantic). The semantic highlighter returns the - * matching entries ranked by score, so a fragment's leading segment names the concept that - * actually matched - not the broad level-1 category the document is keyed on. + * matching entries ranked by score, so a fragment's leading segment names the concept that actually matched. */ protected List extractSemanticLabels(Hit hit) { List fragments = hit.highlight() == null @@ -271,53 +270,6 @@ protected List extractSemanticLabels(Hit hit) { .toList(); } - /** - * Definitions of every concept in a vocabs doc, keyed by the name a suggestion can carry, so the - * portal can explain a semantic suggestion on hover. - *

- * Read from the doc rather than parsed out of the highlight fragment. A fragment reads - * "label. definition. leaf labels", but definitions contain ". " of their own ("(e.g. Autosub - * Glider)"), so splitting one apart is guesswork - the doc has the fields verbatim. - */ - protected Map extractConceptDefinitions(JsonNode source) { - if (source == null) { - return Map.of(); - } - Map definitions = new HashMap<>(); - for (String type : VOCAB_TYPES) { - JsonNode vocab = source.get(type); - if (vocab == null) { - continue; - } - // The level-1 category, for the no-highlight path where extractLabel names the doc itself. - putDefinition(definitions, vocab); - - JsonNode narrower = vocab.get("narrower"); - if (narrower != null && narrower.isArray()) { - narrower.forEach(concept -> putDefinition(definitions, concept)); - } - } - return definitions; - } - - /** - * Index one concept's definition under both of its names: a highlight fragment opens with - * `label`, while {@link #extractLabel} emits `display_label`, and either can end up being the - * suggested string. - */ - protected void putDefinition(Map definitions, JsonNode concept) { - JsonNode definition = concept.get("definition"); - if (definition == null || definition.asText().isBlank()) { - return; - } - for (String key : List.of("label", "display_label")) { - JsonNode name = concept.get(key); - if (name != null && !name.asText().isBlank()) { - definitions.putIfAbsent(name.asText(), definition.asText()); - } - } - } - /** * Flatten the per-document concept labels round-robin: every document's best concept, then every * document's second best, and so on. @@ -361,8 +313,6 @@ protected String toConceptLabel(String fragment) { } public ResponseEntity> getAutocompleteSuggestions(String input, String cql, CQLCrsType coor) throws IOException, CQLException { - // Object rather than Set: every key is a set of suggestions except - // semantic_definitions, which is a label -> definition map. Map searchSuggestions = new HashMap<>(); List> suggestion = this.getSuggestionsByField(input, cql, coor); // extract parameter vocab suggestions @@ -418,16 +368,6 @@ protected String toConceptLabel(String fragment) { .collect(Collectors.toCollection(LinkedHashSet::new)); searchSuggestions.put("suggested_semantic", semanticSuggestions); - - // Definitions of the terms actually suggested, so the portal can explain one on - // hover. Only those - the docs carry definitions for concepts that never made the - // cut, and shipping them would dwarf the suggestions themselves. - Map definitions = new HashMap<>(); - semanticHits.forEach(hit -> - extractConceptDefinitions(hit.source()).forEach(definitions::putIfAbsent)); - definitions.keySet().retainAll(semanticSuggestions); - - searchSuggestions.put("semantic_definitions", definitions); } catch (Exception e) { // Covers the case where the index was built without the semantic fields - the // dropdown degrades to lexical suggestions rather than the request failing. diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index 3831f695..c11293f3 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -34,7 +34,7 @@ elasticsearch: min_input_length: 3 # highlight fragments per doc - one per matching level-2 concept fragments: 2 - # cap on the flattened suggestion list + # the max number of semantic suggestion returned from the flattened size * fragments max_suggestions: 3 aws: From 2006faee5eb9a86bbe5e616373ac091958c86fb7 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 4 Aug 2026 15:21:32 +1000 Subject: [PATCH 7/8] better comment --- .../server/core/service/ElasticSearch.java | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 33f24f4c..fa80e631 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -46,10 +46,10 @@ public class ElasticSearch extends ElasticSearchBase implements Search { // the semantic_text field on the vocabs index protected static final String SEMANTIC_CONCEPT_FIELD = "concept_semantic"; - // marks a vocabs doc as an organisation one - see getSemanticTermHits for why they are skipped + // organisation vocabs are skipped from sementic query protected static final String ORGANISATION_VOCAB_FIELD = "organisation_vocab"; - // a vocabs doc holds exactly one of these (es-indexer VocabDto) + // a vocabs doc holds exactly one of these protected static final List VOCAB_TYPES = List.of("parameter_vocab", "platform_vocab", ORGANISATION_VOCAB_FIELD); @@ -184,17 +184,15 @@ protected List buildSuggestionFilters(String cql, CQLCrsType coor) throws } /** - * Only conduct semantic search if the input is long enough, - * the min_length is defined in application.yaml min_input_length: 3 + * Only conduct semantic search if the input is long enough * */ protected boolean isSemanticInputLongEnough(String input) { return input != null && input.trim().length() >= semanticMinInputLength; } /** - * Rank vocab terms by meaning similarity with query. Comparing with documents in vocabs index with the semantic_text - * field "concept_semantic", which is a list of combined text for per concepts (level-2 label) as "level-2 label's title. - * level-2 label's description. leaf labels' title". + * Rank vocab terms by meaning similarity with query. Comparing with documents in vocabs index with the semantic_text field "concept_semantic", + * which is a list of combined text for per concepts (level-2 label) as "level-2 label's title. level-2 label's description. leaf labels' title". * Using highlight option to get the real matched level-2 label. * @param input - The input text typed by the end user */ @@ -210,9 +208,8 @@ protected List> getSemanticTermHits(String input) throws IOExcepti .highlight(h -> h .fields(SEMANTIC_CONCEPT_FIELD, f -> f .numberOfFragments(semanticFragments) - // The highlighter picks the top fragments by score but hands them - // back in field order unless asked otherwise, so without this the - // first fragment is merely the earliest concept, not the best match. + // The highlighter picks the top fragments by score but hands them back in field order unless asked otherwise, + // so add highligherorder to makesure the fragment is ordered by score. .order(HighlighterOrder.Score) .preTags("") .postTags("")))); @@ -225,9 +222,8 @@ protected List> getSemanticTermHits(String input) throws IOExcepti } /** - * A vocabs doc holds exactly one of the three concept types (see es-indexer VocabDto), so the - * first one present is the one to label. `display_label` is the human-facing form and matches - * what a record's summaries.*_vocabs contain; `label` covers concepts that lack one. + * A vocabs doc holds exactly one of the three concept types (see es-indexer VocabDto), so the first one present is the one to label. + * `display_label` is the human-facing form and matches what a record's summaries.*_vocabs contain; If it's empty return `label`. */ protected String extractLabel(JsonNode source) { if (source == null) { @@ -250,9 +246,8 @@ protected String extractLabel(JsonNode source) { } /** - * concept_semantic holds one entry per narrower (level-2) concept, each starting with that - * concept's label (es-indexer VocabDto.getConceptSemantic). The semantic highlighter returns the - * matching entries ranked by score, so a fragment's leading segment names the concept that actually matched. + * concept_semantic holds one entry per narrower (level-2) concept, each starting with that concept's label (es-indexer VocabDto.getConceptSemantic). + * The semantic highlighter returns the matching entries ranked by score, so a fragment's leading segment names the concept that actually matched. */ protected List extractSemanticLabels(Hit hit) { List fragments = hit.highlight() == null @@ -271,14 +266,8 @@ protected List extractSemanticLabels(Hit hit) { } /** - * Flatten the per-document concept labels round-robin: every document's best concept, then every - * document's second best, and so on. - *

- * Concatenating the documents instead would let the third-best concept of the top document - * outrank the best concept of the second, and would let the first two documents consume every - * suggestion slot - the document-level version of that is the v1 bug this feature was rebuilt to - * fix. A true global sort is not available: highlight fragments carry no score in the response, - * so rank within a document is the only proxy there is. + * Flatten the per-document concept labels round-robin: every document's best concept, then every document's second best, and so on. + * Determined by semantic.size (number of documents should return) and semantic.fragments (number of best concpets should return for each documents) * * @param labelsPerDoc - concept labels per hit, hits in _score order, labels in fragment order */ From d27c04bd704a3add58efe5bb550071fcdc88ac51 Mon Sep 17 00:00:00 2001 From: Yuxuan HU Date: Tue, 4 Aug 2026 15:47:40 +1000 Subject: [PATCH 8/8] update application-dev --- server/src/main/resources/application-dev.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/src/main/resources/application-dev.yaml b/server/src/main/resources/application-dev.yaml index 5eca292a..f0891264 100644 --- a/server/src/main/resources/application-dev.yaml +++ b/server/src/main/resources/application-dev.yaml @@ -8,6 +8,15 @@ ogcapi: debug: elasticsearch-explain-enabled: true +data-discovery-ai: + host: https://data-discovery-ai.edge.aodn.org.au + +es-indexer: + host: https://es-indexer.edge.aodn.org.au + +geonetwork4: + host: https://geonetwork.edge.aodn.org.au + elasticsearch: semantic: enabled: false