diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 4e3e8392..0b7ceaff 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -12,6 +12,7 @@ import co.elastic.clients.elasticsearch.core.SearchRequest; import co.elastic.clients.elasticsearch.core.SearchResponse; import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.HighlighterOrder; import co.elastic.clients.elasticsearch.core.search_mvt.GridType; import co.elastic.clients.transport.endpoints.BinaryResponse; import com.fasterxml.jackson.databind.JsonNode; @@ -39,6 +40,16 @@ public class ElasticSearch extends ElasticSearchBase implements Search { protected Map defaultElasticSetting; + // the semantic_text field on the vocabs index + protected static final String SEMANTIC_CONCEPT_FIELD = "concept_semantic"; + + // organisation vocabs are skipped from sementic query + protected static final String ORGANISATION_VOCAB_FIELD = "organisation_vocab"; + + // a vocabs doc holds exactly one of these + protected static final List VOCAB_TYPES = + List.of("parameter_vocab", "platform_vocab", ORGANISATION_VOCAB_FIELD); + @Value("${elasticsearch.search_as_you_type.search_suggestions.path}") protected String searchAsYouTypeFieldsPath; @@ -51,6 +62,24 @@ public class ElasticSearch extends ElasticSearchBase implements Search { @Value("${elasticsearch.search_after.split_regex:\\|\\|}") protected String searchAfterSplitRegex; + @Value("${elasticsearch.vocabs_index.name}") + protected String vocabsIndexName; + + @Value("${elasticsearch.semantic.enabled:false}") + protected Boolean semanticEnabled; + + @Value("${elasticsearch.semantic.size:3}") + protected Integer semanticSize; + + @Value("${elasticsearch.semantic.min_input_length:3}") + protected Integer semanticMinInputLength; + + @Value("${elasticsearch.semantic.fragments:3}") + protected Integer semanticFragments; + + @Value("${elasticsearch.semantic.max_suggestions:5}") + protected Integer semanticMaxSuggestions; + public ElasticSearch(ElasticsearchClient client, CacheNoLandGeometry cacheNoLandGeometry, ObjectMapper mapper, @@ -114,27 +143,7 @@ protected List> getSuggestionsByField(String input, .query(bQ -> bQ.bool(b -> b.should(suggestFieldsQueries))) )); - /* - this is where the discovery parameter vocabs filter is applied - use term query for exact match of the parameter vocabs - (e.g you don't want "something", "something special" and "something secret" be returned when searching for "something") - see more: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-terms-query.html#query-dsl-terms-query - this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperature") - */ - List filters; - if (cql != null) { - CQLToElasticFilterFactory factory = new CQLToElasticFilterFactory<>(coor, CQLFields.class); - Filter filter = CompilerUtil.parseFilter(Language.ECQL, cql, factory); - if (filter instanceof QueryHandler elasticFilter) { - filters = List.of(elasticFilter.getQuery()); - } else { - // If no filter, then use the match_all{} to get all record - filters = List.of(MatchAllQuery.of(q -> q)._toQuery()); - } - } else { - // If no filter, then use the match_all{} to get all record - filters = List.of(MatchAllQuery.of(q -> q)._toQuery()); - } + List filters = buildSuggestionFilters(cql, coor); // create request SearchRequest searchRequest = this.buildSearchAsYouTypeRequest( @@ -152,8 +161,145 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu return response.hits().hits(); } + /* + this is where the discovery parameter vocabs filter is applied + use term query for exact match of the parameter vocabs + (e.g you don't want "something", "something special" and "something secret" be returned when searching for "something") + see more: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-terms-query.html#query-dsl-terms-query + this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperature") + */ + protected List buildSuggestionFilters(String cql, CQLCrsType coor) throws CQLException { + if (cql != null) { + CQLToElasticFilterFactory factory = new CQLToElasticFilterFactory<>(coor, CQLFields.class); + Filter filter = CompilerUtil.parseFilter(Language.ECQL, cql, factory); + if (filter instanceof QueryHandler elasticFilter) { + return List.of(elasticFilter.getQuery()); + } + } + // If no filter, then use the match_all{} to get all record + return List.of(MatchAllQuery.of(q -> q)._toQuery()); + } + + /** + * Only conduct semantic search if the input is long enough + * */ + protected boolean isSemanticInputLongEnough(String input) { + return input != null && input.trim().length() >= semanticMinInputLength; + } + + /** + * Rank vocab terms by meaning similarity with query. Comparing with documents in vocabs index with the semantic_text field "concept_semantic", + * which is a list of combined text for per concepts (level-2 label) as "level-2 label's title. level-2 label's description. leaf labels' title". + * Using highlight option to get the real matched level-2 label. + * @param input - The input text typed by the end user + */ + protected List> getSemanticTermHits(String input) throws IOException { + SearchRequest searchRequest = SearchRequest.of(s -> s + .index(vocabsIndexName) + .size(semanticSize) + .query(q -> q.bool(b -> b + .must(m -> m.semantic(sm -> sm + .field(SEMANTIC_CONCEPT_FIELD) + .query(input))) + .mustNot(mn -> mn.exists(e -> e.field(ORGANISATION_VOCAB_FIELD))))) + .highlight(h -> h + .fields(SEMANTIC_CONCEPT_FIELD, f -> f + .numberOfFragments(semanticFragments) + // The highlighter picks the top fragments by score but hands them back in field order unless asked otherwise, + // so add highligherorder to makesure the fragment is ordered by score. + .order(HighlighterOrder.Score) + .preTags("") + .postTags("")))); + + log.info("getSemanticTermHits | Elastic search payload {}", searchRequest); + SearchResponse response = esClient.search(searchRequest, JsonNode.class); + log.info("getSemanticTermHits | Elastic search response {}", response); + + return response.hits().hits(); + } + + /** + * A vocabs doc holds exactly one of the three concept types (see es-indexer VocabDto), so the first one present is the one to label. + * `display_label` is the human-facing form and matches what a record's summaries.*_vocabs contain; If it's empty return `label`. + */ + protected String extractLabel(JsonNode source) { + if (source == null) { + return null; + } + for (String type : VOCAB_TYPES) { + JsonNode vocab = source.get(type); + if (vocab != null) { + JsonNode displayLabel = vocab.get("display_label"); + if (displayLabel != null && !displayLabel.asText().isBlank()) { + return displayLabel.asText(); + } + JsonNode label = vocab.get("label"); + if (label != null && !label.asText().isBlank()) { + return label.asText(); + } + } + } + return null; + } + + /** + * concept_semantic holds one entry per narrower (level-2) concept, each starting with that concept's label (es-indexer VocabDto.getConceptSemantic). + * The semantic highlighter returns the matching entries ranked by score, so a fragment's leading segment names the concept that actually matched. + */ + protected List extractSemanticLabels(Hit hit) { + List fragments = hit.highlight() == null + ? null + : hit.highlight().get(SEMANTIC_CONCEPT_FIELD); + + if (fragments == null || fragments.isEmpty()) { + // No highlight - e.g. an index still carrying the old single-valued concept_semantic. + String label = extractLabel(hit.source()); + return label == null ? List.of() : List.of(label); + } + return fragments.stream() + .map(this::toConceptLabel) + .filter(Objects::nonNull) + .toList(); + } + + /** + * Flatten the per-document concept labels round-robin: every document's best concept, then every document's second best, and so on. + * Determined by semantic.size (number of documents should return) and semantic.fragments (number of best concpets should return for each documents) + * + * @param labelsPerDoc - concept labels per hit, hits in _score order, labels in fragment order + */ + protected List interleave(List> labelsPerDoc) { + int deepest = labelsPerDoc.stream().mapToInt(List::size).max().orElse(0); + + List flattened = new ArrayList<>(); + for (int rank = 0; rank < deepest; rank++) { + for (List labels : labelsPerDoc) { + // A document that ran out of concepts simply stops contributing at this rank. + if (rank < labels.size()) { + flattened.add(labels.get(rank)); + } + } + } + return flattened; + } + + /** + * Leading segment of a concept_semantic entry, which is the concept's label. Split on ". " + * rather than "." so labels that carry an internal period (e.g. "No.3 buoy") survive. + */ + protected String toConceptLabel(String fragment) { + if (fragment == null) { + return null; + } + // -1 means the fragment is a single segment and is the label; 0 means it opens with the + // separator, leaving no label at all - the two must not collapse into the same branch. + int end = fragment.indexOf(". "); + String label = (end >= 0 ? fragment.substring(0, end) : fragment).trim(); + return label.isBlank() ? null : label; + } + public ResponseEntity> getAutocompleteSuggestions(String input, String cql, CQLCrsType coor) throws IOException, CQLException { - Map> searchSuggestions = new HashMap<>(); + Map searchSuggestions = new HashMap<>(); List> suggestion = this.getSuggestionsByField(input, cql, coor); // extract parameter vocab suggestions Set parameterVocabSuggestions = suggestion @@ -189,6 +335,32 @@ this query uses AND operator for the parameter vocabs (e.g "wave" AND "temperatu .collect(Collectors.toSet()); searchSuggestions.put("suggested_phrases", abstractPhrases); + // Semantic suggestions - vocab terms ranked by meaning rather than by spelling. + if (Boolean.TRUE.equals(semanticEnabled) && isSemanticInputLongEnough(input)) { + try { + List> semanticHits = this.getSemanticTermHits(input); + + List> labelsPerDoc = semanticHits + .stream() + .map(this::extractSemanticLabels) + .toList(); + + Set semanticSuggestions = interleave(labelsPerDoc) + .stream() + // distinct before limit so duplicates do not consume suggestion slots + .distinct() + .limit(semanticMaxSuggestions) + // LinkedHashSet so the relevance order from Elastic survives into the response + .collect(Collectors.toCollection(LinkedHashSet::new)); + + searchSuggestions.put("suggested_semantic", semanticSuggestions); + } catch (Exception e) { + // Covers the case where the index was built without the semantic fields - the + // dropdown degrades to lexical suggestions rather than the request failing. + log.warn("Semantic suggestions unavailable, returning lexical suggestions only", e); + } + } + return new ResponseEntity<>(searchSuggestions, HttpStatus.OK); } diff --git a/server/src/main/resources/application-dev.yaml b/server/src/main/resources/application-dev.yaml index cffa88d1..f0891264 100644 --- a/server/src/main/resources/application-dev.yaml +++ b/server/src/main/resources/application-dev.yaml @@ -16,3 +16,7 @@ es-indexer: geonetwork4: host: https://geonetwork.edge.aodn.org.au + +elasticsearch: + semantic: + enabled: false diff --git a/server/src/main/resources/application-edge.yaml b/server/src/main/resources/application-edge.yaml index 3ffe7802..c8f57fb1 100644 --- a/server/src/main/resources/application-edge.yaml +++ b/server/src/main/resources/application-edge.yaml @@ -13,3 +13,7 @@ management: ogcapi: debug: elasticsearch-explain-enabled: true + +elasticsearch: + semantic: + enabled: true diff --git a/server/src/main/resources/application-production.yaml b/server/src/main/resources/application-production.yaml index 5a5b12c2..44ad7293 100644 --- a/server/src/main/resources/application-production.yaml +++ b/server/src/main/resources/application-production.yaml @@ -11,3 +11,7 @@ logging: ogcapi: debug: elasticsearch-explain-enabled: true + +elasticsearch: + semantic: + enabled: true diff --git a/server/src/main/resources/application-staging.yaml b/server/src/main/resources/application-staging.yaml index 63749f0e..c27d2689 100644 --- a/server/src/main/resources/application-staging.yaml +++ b/server/src/main/resources/application-staging.yaml @@ -7,3 +7,7 @@ management: ogcapi: debug: elasticsearch-explain-enabled: true + +elasticsearch: + semantic: + enabled: true diff --git a/server/src/main/resources/application.yaml b/server/src/main/resources/application.yaml index d63fd46c..c11293f3 100644 --- a/server/src/main/resources/application.yaml +++ b/server/src/main/resources/application.yaml @@ -22,6 +22,20 @@ elasticsearch: search_suggestions: path: search_suggestions fields: abstract_phrases, parameter_vocabs_sayt, platform_vocabs_sayt, organisation_vocabs_sayt + # Semantic suggestions: the vocabs index is searched by meaning (ELSER via semantic_text). + # The suggested concepts are not filtered against records, so a suggestion may be a concept + # no record currently carries. + # Off by default - semantic_text needs the licensed `inference` feature. + semantic: + enabled: false + # number of vocab docs (top-level categories) the semantic query returns + size: 3 + # number of characters after user input to execute semantic query + min_input_length: 3 + # highlight fragments per doc - one per matching level-2 concept + fragments: 2 + # the max number of semantic suggestion returned from the flattened size * fragments + max_suggestions: 3 aws: region: ap-southeast-2 diff --git a/server/src/test/resources/application-test.yaml b/server/src/test/resources/application-test.yaml index f4713627..93ff6007 100644 --- a/server/src/test/resources/application-test.yaml +++ b/server/src/test/resources/application-test.yaml @@ -16,3 +16,7 @@ elasticsearch: search_suggestions: path: search_suggestions fields: abstract_phrases, parameter_vocabs_sayt, platform_vocabs_sayt, organisation_vocabs_sayt + + # no inference feature in test env so turn it off. + semantic: + enabled: false