diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt index 997aece5..5bb3bbb9 100644 --- a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngine.kt @@ -196,22 +196,36 @@ class LuceneSearchEngine( } override fun buildSnippet(rawText: String, query: String, near: Int): String { - val norm = HebrewTextUtils.normalizeHebrew(query) - if (norm.isBlank()) return Jsoup.clean(rawText, Safelist.none()) + val parsed = SearchQueryParser.parse(query) + val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText) + val exactPhrasesNorm = parsed.exactPhrases + .map { HebrewTextUtils.normalizeHebrew(it) } + .filter { it.isNotBlank() } + if (norm.isBlank() && exactPhrasesNorm.isEmpty()) return Jsoup.clean(rawText, Safelist.none()) val rawClean = Jsoup.clean(rawText, Safelist.none()) val analyzedStd = (analyzeToTerms(stdAnalyzer, norm) ?: emptyList()) val hasHashem = query.contains("ה׳") || query.contains("ה'") val hashemTerms = if (hasHashem) loadHashemHighlightTerms() else emptyList() + // Verbatim tokens from the quoted phrases must be highlighted too (no expansion). + val exactPhraseTokens = exactPhrasesNorm.flatMap { analyzeToTerms(stdAnalyzer, it) ?: emptyList() } val highlightTerms = filterTermsForHighlight( - analyzedStd + buildNgramTerms(analyzedStd, gram = 4) + hashemTerms + analyzedStd + buildNgramTerms(analyzedStd, gram = 4) + hashemTerms + exactPhraseTokens ) - val anchorTerms = buildAnchorTerms(norm, highlightTerms) + val anchorBasis = (sequenceOf(norm) + exactPhrasesNorm.asSequence()) + .filter { it.isNotBlank() } + .joinToString(" ") + val anchorTerms = buildAnchorTerms(anchorBasis, highlightTerms) return buildSnippetInternal(rawClean, anchorTerms, highlightTerms) } override fun buildHighlightTerms(query: String): List { - val norm = HebrewTextUtils.normalizeHebrew(query) - if (norm.isBlank()) return emptyList() + val parsed = SearchQueryParser.parse(query) + val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText) + // Verbatim tokens from the quoted phrases, highlighted without dictionary expansion. + val exactPhraseTokens = parsed.exactPhrases + .flatMap { analyzeToTerms(stdAnalyzer, HebrewTextUtils.normalizeHebrew(it)) ?: emptyList() } + + if (norm.isBlank()) return filterTermsForHighlight(exactPhraseTokens) val analyzedRaw = analyzeToTerms(stdAnalyzer, norm) ?: emptyList() val hasHashem = query.contains("ה׳") || query.contains("ה'") @@ -248,7 +262,7 @@ class LuceneSearchEngine( val ngramTerms = buildNgramTerms(analyzedStd, gram = 4) val hashemTerms = if (hasHashem) loadHashemHighlightTerms() else emptyList() - return filterTermsForHighlight(analyzedStd + expandedTerms + ngramTerms + hashemTerms) + return filterTermsForHighlight(analyzedStd + expandedTerms + ngramTerms + hashemTerms + exactPhraseTokens) } override fun close() { @@ -386,8 +400,15 @@ class LuceneSearchEngine( lineIds: Collection?, baseBookOnly: Boolean = false ): SearchContext? { - val norm = HebrewTextUtils.normalizeHebrew(rawQuery) - if (norm.isBlank()) return null + // Split the raw query into exact phrases (wrapped in ASCII double quotes) and free text. + // Quoted segments are matched verbatim WITHOUT dictionary expansion (Google-style). + val parsed = SearchQueryParser.parse(rawQuery) + val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText) + val exactPhrasesNorm = parsed.exactPhrases + .map { HebrewTextUtils.normalizeHebrew(it) } + .filter { it.isNotBlank() } + + if (norm.isBlank() && exactPhrasesNorm.isEmpty()) return null val analyzedRaw = analyzeToTerms(stdAnalyzer, norm) ?: emptyList() @@ -452,12 +473,15 @@ class LuceneSearchEngine( // mentions Hashem explicitly, also include dictionary-based variants of the // divine name from the lexical DB val hashemTerms = if (hasHashem) loadHashemHighlightTerms() else emptyList() - val highlightTerms = filterTermsForHighlight(analyzedStd + expandedTerms + ngramTerms + hashemTerms) - val anchorTerms = buildAnchorTerms(norm, highlightTerms) - - val rankedQuery = buildExpandedQuery(norm, near, analyzedStd, tokenExpansions) - val mustAllTokensQuery: Query? = buildPresenceFilterForTokens(analyzedStd, near, tokenExpansions) - val phraseQuery: Query? = buildSynonymPhraseQuery(analyzedStd, tokenExpansions, near) + // Verbatim tokens from the quoted phrases must be highlighted too (no expansion). + val exactPhraseTokens = exactPhrasesNorm.flatMap { analyzeToTerms(stdAnalyzer, it) ?: emptyList() } + val highlightTerms = filterTermsForHighlight( + analyzedStd + expandedTerms + ngramTerms + hashemTerms + exactPhraseTokens + ) + val anchorBasis = (sequenceOf(norm) + exactPhrasesNorm.asSequence()) + .filter { it.isNotBlank() } + .joinToString(" ") + val anchorTerms = buildAnchorTerms(anchorBasis, highlightTerms) val builder = BooleanQuery.Builder() builder.add(TermQuery(Term("type", "line")), BooleanClause.Occur.FILTER) @@ -473,18 +497,36 @@ class LuceneSearchEngine( if (lineIdsArray != null && lineIdsArray.isNotEmpty()) { builder.add(IntPoint.newSetQuery("line_id", *lineIdsArray), BooleanClause.Occur.FILTER) } - if (mustAllTokensQuery != null) { - builder.add(mustAllTokensQuery, BooleanClause.Occur.FILTER) - logger.d { "[DEBUG] Added mustAllTokensQuery as FILTER" } + // Quoted phrases: each must appear verbatim as an exact, in-order, adjacent phrase. + var exactClausesAdded = 0 + for (phrase in exactPhrasesNorm) { + val exactQuery = buildExactPhraseQuery(phrase) ?: continue + builder.add(exactQuery, BooleanClause.Occur.MUST) + exactClausesAdded++ + logger.d { "[DEBUG] Added exact-phrase MUST for: \"$phrase\"" } } - val analyzedCount = analyzedStd.size - if (phraseQuery != null && analyzedCount >= 2) { - val occur = if (near == 0) BooleanClause.Occur.MUST else BooleanClause.Occur.SHOULD - builder.add(phraseQuery, occur) - logger.d { "[DEBUG] Added phraseQuery with occur=$occur, near=$near" } + + // A fully-quoted query that produced no usable phrase clause (e.g. just punctuation) + // must not fall through to a match-all query. + if (norm.isBlank() && exactClausesAdded == 0) return null + + // Free (unquoted) text: dictionary-aware ranking + presence filtering. + if (norm.isNotBlank()) { + val rankedQuery = buildExpandedQuery(norm, near, analyzedStd, tokenExpansions) + val mustAllTokensQuery: Query? = buildPresenceFilterForTokens(analyzedStd, near, tokenExpansions) + val phraseQuery: Query? = buildSynonymPhraseQuery(analyzedStd, tokenExpansions, near) + if (mustAllTokensQuery != null) { + builder.add(mustAllTokensQuery, BooleanClause.Occur.FILTER) + logger.d { "[DEBUG] Added mustAllTokensQuery as FILTER" } + } + if (phraseQuery != null && analyzedStd.size >= 2) { + val occur = if (near == 0) BooleanClause.Occur.MUST else BooleanClause.Occur.SHOULD + builder.add(phraseQuery, occur) + logger.d { "[DEBUG] Added phraseQuery with occur=$occur, near=$near" } + } + builder.add(rankedQuery, BooleanClause.Occur.SHOULD) + logger.d { "[DEBUG] Added rankedQuery as SHOULD" } } - builder.add(rankedQuery, BooleanClause.Occur.SHOULD) - logger.d { "[DEBUG] Added rankedQuery as SHOULD" } val finalQuery = builder.build() logger.d { "[DEBUG] Final query: $finalQuery" } @@ -671,6 +713,21 @@ class LuceneSearchEngine( return bool ?: BooleanQuery.Builder().build() } + /** + * Builds a strict, verbatim phrase query for a quoted segment: terms must appear in the + * given order and be adjacent (slop = 0), with NO dictionary expansion, fuzzy matching or + * n-gram substring matching. A single-token segment becomes an exact [TermQuery]. + * + * Nikud and final letters are still normalised (the segment is normalised by the caller and + * tokenised through [stdAnalyzer]) so that the phrase matches the indexed form. + */ + private fun buildExactPhraseQuery(phraseNorm: String): Query? { + if (phraseNorm.isBlank()) return null + val qb = QueryBuilder(stdAnalyzer) + qb.createPhraseQuery("text", phraseNorm, 0)?.let { return it } + return qb.createBooleanQuery("text", phraseNorm, BooleanClause.Occur.MUST) + } + private fun buildMagicBoostQuery(expansions: List): Query? { if (expansions.isEmpty()) return null val surfaceTerms = LinkedHashSet() diff --git a/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt new file mode 100644 index 00000000..2f2fb60a --- /dev/null +++ b/search/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParser.kt @@ -0,0 +1,88 @@ +package io.github.kdroidfilter.seforimlibrary.search + +/** + * Parses a raw search query into "exact" phrases (delimited by double quotes) and the + * remaining free text. + * + * Google-style behaviour: text wrapped in double quotes is searched verbatim, as an exact + * ordered phrase, WITHOUT the magic-dictionary / synonym expansion, fuzzy matching or n-gram + * substring matching. + * + * Two delimiter characters are recognised, so it works whatever the keyboard layout produces: + * - the ASCII straight quote `"` (U+0022) + * - the Hebrew gershayim `״` (U+05F4) + * They are interchangeable (a phrase may be opened with one and closed with the other). + * + * Hebrew acronyms (rashei tevot) are very commonly written with a double quote between the last + * two letters (e.g. רש״י / רש"י, רמב״ם, שו״ע, תנ״ך, תשפ״א). Such a quote is NOT a phrase + * delimiter: a double quote acts as a delimiter only when it sits at a word boundary, i.e. it is + * NOT flanked by Hebrew letters on both sides. This keeps acronym searches working as before. + */ +object SearchQueryParser { + + private const val ASCII_QUOTE = '"' + private const val GERSHAYIM = '״' + private val WHITESPACE = "\\s+".toRegex() + + data class ParsedQuery( + /** Substrings that were enclosed in double quotes, each to be matched as an exact phrase. */ + val exactPhrases: List, + /** The remaining (unquoted) text, processed by the normal dictionary-aware pipeline. */ + val freeText: String, + ) { + /** True when the query contains at least one real phrase delimiter. */ + val hasExactPhrases: Boolean get() = exactPhrases.isNotEmpty() + } + + private fun isQuoteDelimiter(c: Char): Boolean = c == ASCII_QUOTE || c == GERSHAYIM + + /** Hebrew letters block (alef..tav), including final forms which are all within this range. */ + private fun isHebrewLetter(c: Char): Boolean = c.code in 0x05D0..0x05EA + + /** + * A double quote at [index] is an acronym marker (NOT a phrase delimiter) when it is + * immediately surrounded by Hebrew letters on both sides, e.g. the ״ in רש״י. + */ + private fun isAcronymQuote(raw: String, index: Int): Boolean { + if (index <= 0 || index >= raw.length - 1) return false + return isHebrewLetter(raw[index - 1]) && isHebrewLetter(raw[index + 1]) + } + + fun parse(rawQuery: String): ParsedQuery { + if (rawQuery.none { isQuoteDelimiter(it) }) { + return ParsedQuery(emptyList(), rawQuery) + } + + val exactPhrases = mutableListOf() + val free = StringBuilder() + val phrase = StringBuilder() + var inQuote = false + + for (i in rawQuery.indices) { + val c = rawQuery[i] + if (isQuoteDelimiter(c) && !isAcronymQuote(rawQuery, i)) { + if (inQuote) { + addPhrase(exactPhrases, phrase) + inQuote = false + } else { + inQuote = true + } + } else { + (if (inQuote) phrase else free).append(c) + } + } + // Unclosed quote: treat the trailing buffered content as an exact phrase. + if (inQuote) addPhrase(exactPhrases, phrase) + + return ParsedQuery( + exactPhrases = exactPhrases, + freeText = free.toString().trim().replace(WHITESPACE, " "), + ) + } + + private fun addPhrase(into: MutableList, buffer: StringBuilder) { + val phrase = buffer.toString().trim().replace(WHITESPACE, " ") + if (phrase.isNotEmpty()) into.add(phrase) + buffer.setLength(0) + } +} diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngineTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngineTest.kt index ec2b6739..f75ce602 100644 --- a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngineTest.kt +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/LuceneSearchEngineTest.kt @@ -407,8 +407,125 @@ class LuceneSearchEngineTest { } } + // --- Exact phrase (quoted) tests --- + + @Test + fun `quoted query matches only the exact ordered adjacent phrase`() { + val tempDir = createTempIndexDir() + try { + createIndexWithPhrases(tempDir) + val engine = LuceneSearchEngine(tempDir) + + val session = engine.openSession("\"מלך דוד\"", 5) + assertNotNull(session) + // line 1 ("מלך דוד") and line 5 ("ירושלים מלך דוד עיר") contain the exact phrase. + // line 2 ("דוד מלך", reversed) and line 3 ("מלך גדול דוד", gap) must be excluded. + assertEquals(setOf(1L, 5L), collectLineIds(session)) + + engine.close() + } finally { + deleteDirectory(tempDir) + } + } + + @Test + fun `unquoted query matches every doc containing the words in any order`() { + val tempDir = createTempIndexDir() + try { + createIndexWithPhrases(tempDir) + val engine = LuceneSearchEngine(tempDir) + + val session = engine.openSession("מלך דוד", 5) + assertNotNull(session) + // Without quotes the order/adjacency is relaxed: lines 1, 2, 3 and 5 all qualify. + assertEquals(setOf(1L, 2L, 3L, 5L), collectLineIds(session)) + + engine.close() + } finally { + deleteDirectory(tempDir) + } + } + + @Test + fun `mixed free word plus quoted phrase requires both`() { + val tempDir = createTempIndexDir() + try { + createIndexWithPhrases(tempDir) + val engine = LuceneSearchEngine(tempDir) + + val session = engine.openSession("ירושלים \"מלך דוד\"", 5) + assertNotNull(session) + // Only line 5 has both the free word "ירושלים" and the exact phrase "מלך דוד". + assertEquals(setOf(5L), collectLineIds(session)) + + engine.close() + } finally { + deleteDirectory(tempDir) + } + } + + @Test + fun `gershayim-quoted query behaves like an exact phrase`() { + val tempDir = createTempIndexDir() + try { + createIndexWithPhrases(tempDir) + val engine = LuceneSearchEngine(tempDir) + + // Same phrase as the ASCII test but delimited with Hebrew gershayim (U+05F4). + val session = engine.openSession("״מלך דוד״", 5) + assertNotNull(session) + assertEquals(setOf(1L, 5L), collectLineIds(session)) + + engine.close() + } finally { + deleteDirectory(tempDir) + } + } + // --- Helper methods --- + private fun collectLineIds(session: SearchSession): Set = runBlocking { + val ids = mutableSetOf() + while (true) { + val page = session.nextPage(50) ?: break + page.hits.forEach { ids.add(it.lineId) } + if (page.isLastPage) break + } + session.close() + ids + } + + private fun createIndexWithPhrases(indexDir: Path) { + FSDirectory.open(indexDir).use { dir -> + val config = IndexWriterConfig(StandardAnalyzer()) + IndexWriter(dir, config).use { writer -> + val rows = listOf( + 1L to "מלך דוד", + 2L to "דוד מלך", + 3L to "מלך גדול דוד", + 4L to "שלום עולם", + 5L to "ירושלים מלך דוד עיר", + ) + rows.forEach { (lineId, text) -> + val doc = Document().apply { + add(StringField("type", "line", Field.Store.YES)) + add(StoredField("book_id", 1)) + add(IntPoint("book_id", 1)) + add(StoredField("book_title", "ספר בדיקה")) + add(StoredField("line_id", lineId)) + add(IntPoint("line_id", lineId.toInt())) + add(StoredField("line_index", (lineId - 1).toInt())) + add(TextField("text", HebrewTextUtils.normalizeHebrew(text), Field.Store.NO)) + add(StoredField("text_raw", text)) + add(StoredField("is_base_book", 1)) + add(StoredField("order_index", 1)) + } + writer.addDocument(doc) + } + } + } + } + private fun createTempIndexDir(): Path { return Files.createTempDirectory("lucene_test_index") } diff --git a/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParserTest.kt b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParserTest.kt new file mode 100644 index 00000000..9844518f --- /dev/null +++ b/search/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/search/SearchQueryParserTest.kt @@ -0,0 +1,130 @@ +package io.github.kdroidfilter.seforimlibrary.search + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class SearchQueryParserTest { + + @Test + fun `query without quotes is all free text`() { + val parsed = SearchQueryParser.parse("מלך דוד") + assertEquals("מלך דוד", parsed.freeText) + assertTrue(parsed.exactPhrases.isEmpty()) + assertFalse(parsed.hasExactPhrases) + } + + @Test + fun `single quoted phrase`() { + val parsed = SearchQueryParser.parse("\"מלך דוד\"") + assertEquals(listOf("מלך דוד"), parsed.exactPhrases) + assertEquals("", parsed.freeText) + assertTrue(parsed.hasExactPhrases) + } + + @Test + fun `mixed quoted phrase and free text`() { + val parsed = SearchQueryParser.parse("ירושלים \"מלך דוד\" עיר") + assertEquals(listOf("מלך דוד"), parsed.exactPhrases) + assertEquals("ירושלים עיר", parsed.freeText) + } + + @Test + fun `multiple quoted phrases`() { + val parsed = SearchQueryParser.parse("\"בראשית ברא\" \"אלהים את\"") + assertEquals(listOf("בראשית ברא", "אלהים את"), parsed.exactPhrases) + assertEquals("", parsed.freeText) + } + + @Test + fun `acronym with ascii quote is not a delimiter`() { + // רש"י (Rashi) is written with an ASCII double quote between the last two letters. + val parsed = SearchQueryParser.parse("רש\"י") + assertTrue(parsed.exactPhrases.isEmpty()) + assertEquals("רש\"י", parsed.freeText) + } + + @Test + fun `acronym is preserved alongside a real quoted phrase`() { + val parsed = SearchQueryParser.parse("רש\"י \"בראשית ברא\"") + assertEquals(listOf("בראשית ברא"), parsed.exactPhrases) + assertEquals("רש\"י", parsed.freeText) + } + + @Test + fun `common hebrew acronyms are never treated as delimiters`() { + for (acronym in listOf("רמב\"ם", "שו\"ע", "תנ\"ך", "תשפ\"א", "י\"ד")) { + val parsed = SearchQueryParser.parse(acronym) + assertTrue(parsed.exactPhrases.isEmpty(), "Acronym should not produce phrases: $acronym") + assertEquals(acronym, parsed.freeText) + } + } + + @Test + fun `single gershayim-quoted phrase`() { + val parsed = SearchQueryParser.parse("״מלך דוד״") + assertEquals(listOf("מלך דוד"), parsed.exactPhrases) + assertEquals("", parsed.freeText) + assertTrue(parsed.hasExactPhrases) + } + + @Test + fun `mixed gershayim phrase and free text`() { + val parsed = SearchQueryParser.parse("ירושלים ״מלך דוד״ עיר") + assertEquals(listOf("מלך דוד"), parsed.exactPhrases) + assertEquals("ירושלים עיר", parsed.freeText) + } + + @Test + fun `gershayim acronym is not a delimiter`() { + // רש״י written with the Hebrew gershayim (U+05F4) between the last two letters. + val parsed = SearchQueryParser.parse("רש״י") + assertTrue(parsed.exactPhrases.isEmpty()) + assertEquals("רש״י", parsed.freeText) + } + + @Test + fun `common hebrew acronyms with gershayim are not delimiters`() { + for (acronym in listOf("רמב״ם", "שו״ע", "תנ״ך", "תשפ״א", "י״ד")) { + val parsed = SearchQueryParser.parse(acronym) + assertTrue(parsed.exactPhrases.isEmpty(), "Acronym should not produce phrases: $acronym") + assertEquals(acronym, parsed.freeText) + } + } + + @Test + fun `ascii and gershayim quotes are interchangeable delimiters`() { + val parsed = SearchQueryParser.parse("\"מלך דוד״") + assertEquals(listOf("מלך דוד"), parsed.exactPhrases) + assertEquals("", parsed.freeText) + } + + @Test + fun `unclosed quote treats remainder as a phrase`() { + val parsed = SearchQueryParser.parse("\"מלך דוד") + assertEquals(listOf("מלך דוד"), parsed.exactPhrases) + assertEquals("", parsed.freeText) + } + + @Test + fun `empty quotes produce no phrase`() { + val parsed = SearchQueryParser.parse("מלך \"\" דוד") + assertTrue(parsed.exactPhrases.isEmpty()) + assertEquals("מלך דוד", parsed.freeText) + } + + @Test + fun `whitespace inside and around phrases is collapsed`() { + val parsed = SearchQueryParser.parse(" ירושלים \"מלך דוד\" ") + assertEquals(listOf("מלך דוד"), parsed.exactPhrases) + assertEquals("ירושלים", parsed.freeText) + } + + @Test + fun `blank input yields empty free text and no phrases`() { + val parsed = SearchQueryParser.parse("") + assertEquals("", parsed.freeText) + assertTrue(parsed.exactPhrases.isEmpty()) + } +}