Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -196,22 +196,36 @@ class LuceneSearchEngine(
}

override fun buildSnippet(rawText: String, query: String, near: Int): String {
val norm = HebrewTextUtils.normalizeHebrew(query)
if (norm.isBlank()) return Jsoup.clean(rawText, Safelist.none())
val parsed = SearchQueryParser.parse(query)
val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText)
val exactPhrasesNorm = parsed.exactPhrases
.map { HebrewTextUtils.normalizeHebrew(it) }
.filter { it.isNotBlank() }
if (norm.isBlank() && exactPhrasesNorm.isEmpty()) return Jsoup.clean(rawText, Safelist.none())
val rawClean = Jsoup.clean(rawText, Safelist.none())
val analyzedStd = (analyzeToTerms(stdAnalyzer, norm) ?: emptyList())
val hasHashem = query.contains("ה׳") || query.contains("ה'")
val hashemTerms = if (hasHashem) loadHashemHighlightTerms() else emptyList()
// Verbatim tokens from the quoted phrases must be highlighted too (no expansion).
val exactPhraseTokens = exactPhrasesNorm.flatMap { analyzeToTerms(stdAnalyzer, it) ?: emptyList() }
val highlightTerms = filterTermsForHighlight(
analyzedStd + buildNgramTerms(analyzedStd, gram = 4) + hashemTerms
analyzedStd + buildNgramTerms(analyzedStd, gram = 4) + hashemTerms + exactPhraseTokens
)
val anchorTerms = buildAnchorTerms(norm, highlightTerms)
val anchorBasis = (sequenceOf(norm) + exactPhrasesNorm.asSequence())
.filter { it.isNotBlank() }
.joinToString(" ")
val anchorTerms = buildAnchorTerms(anchorBasis, highlightTerms)
return buildSnippetInternal(rawClean, anchorTerms, highlightTerms)
}

override fun buildHighlightTerms(query: String): List<String> {
val norm = HebrewTextUtils.normalizeHebrew(query)
if (norm.isBlank()) return emptyList()
val parsed = SearchQueryParser.parse(query)
val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText)
// Verbatim tokens from the quoted phrases, highlighted without dictionary expansion.
val exactPhraseTokens = parsed.exactPhrases
.flatMap { analyzeToTerms(stdAnalyzer, HebrewTextUtils.normalizeHebrew(it)) ?: emptyList() }

if (norm.isBlank()) return filterTermsForHighlight(exactPhraseTokens)

val analyzedRaw = analyzeToTerms(stdAnalyzer, norm) ?: emptyList()
val hasHashem = query.contains("ה׳") || query.contains("ה'")
Expand Down Expand Up @@ -248,7 +262,7 @@ class LuceneSearchEngine(
val ngramTerms = buildNgramTerms(analyzedStd, gram = 4)
val hashemTerms = if (hasHashem) loadHashemHighlightTerms() else emptyList()

return filterTermsForHighlight(analyzedStd + expandedTerms + ngramTerms + hashemTerms)
return filterTermsForHighlight(analyzedStd + expandedTerms + ngramTerms + hashemTerms + exactPhraseTokens)
}

override fun close() {
Expand Down Expand Up @@ -386,8 +400,15 @@ class LuceneSearchEngine(
lineIds: Collection<Long>?,
baseBookOnly: Boolean = false
): SearchContext? {
val norm = HebrewTextUtils.normalizeHebrew(rawQuery)
if (norm.isBlank()) return null
// Split the raw query into exact phrases (wrapped in ASCII double quotes) and free text.
// Quoted segments are matched verbatim WITHOUT dictionary expansion (Google-style).
val parsed = SearchQueryParser.parse(rawQuery)
val norm = HebrewTextUtils.normalizeHebrew(parsed.freeText)
val exactPhrasesNorm = parsed.exactPhrases
.map { HebrewTextUtils.normalizeHebrew(it) }
.filter { it.isNotBlank() }

if (norm.isBlank() && exactPhrasesNorm.isEmpty()) return null

val analyzedRaw = analyzeToTerms(stdAnalyzer, norm) ?: emptyList()

Expand Down Expand Up @@ -452,12 +473,15 @@ class LuceneSearchEngine(
// mentions Hashem explicitly, also include dictionary-based variants of the
// divine name from the lexical DB
val hashemTerms = if (hasHashem) loadHashemHighlightTerms() else emptyList()
val highlightTerms = filterTermsForHighlight(analyzedStd + expandedTerms + ngramTerms + hashemTerms)
val anchorTerms = buildAnchorTerms(norm, highlightTerms)

val rankedQuery = buildExpandedQuery(norm, near, analyzedStd, tokenExpansions)
val mustAllTokensQuery: Query? = buildPresenceFilterForTokens(analyzedStd, near, tokenExpansions)
val phraseQuery: Query? = buildSynonymPhraseQuery(analyzedStd, tokenExpansions, near)
// Verbatim tokens from the quoted phrases must be highlighted too (no expansion).
val exactPhraseTokens = exactPhrasesNorm.flatMap { analyzeToTerms(stdAnalyzer, it) ?: emptyList() }
val highlightTerms = filterTermsForHighlight(
analyzedStd + expandedTerms + ngramTerms + hashemTerms + exactPhraseTokens
)
val anchorBasis = (sequenceOf(norm) + exactPhrasesNorm.asSequence())
.filter { it.isNotBlank() }
.joinToString(" ")
val anchorTerms = buildAnchorTerms(anchorBasis, highlightTerms)

val builder = BooleanQuery.Builder()
builder.add(TermQuery(Term("type", "line")), BooleanClause.Occur.FILTER)
Expand All @@ -473,18 +497,36 @@ class LuceneSearchEngine(
if (lineIdsArray != null && lineIdsArray.isNotEmpty()) {
builder.add(IntPoint.newSetQuery("line_id", *lineIdsArray), BooleanClause.Occur.FILTER)
}
if (mustAllTokensQuery != null) {
builder.add(mustAllTokensQuery, BooleanClause.Occur.FILTER)
logger.d { "[DEBUG] Added mustAllTokensQuery as FILTER" }
// Quoted phrases: each must appear verbatim as an exact, in-order, adjacent phrase.
var exactClausesAdded = 0
for (phrase in exactPhrasesNorm) {
val exactQuery = buildExactPhraseQuery(phrase) ?: continue
builder.add(exactQuery, BooleanClause.Occur.MUST)
exactClausesAdded++
logger.d { "[DEBUG] Added exact-phrase MUST for: \"$phrase\"" }
}
val analyzedCount = analyzedStd.size
if (phraseQuery != null && analyzedCount >= 2) {
val occur = if (near == 0) BooleanClause.Occur.MUST else BooleanClause.Occur.SHOULD
builder.add(phraseQuery, occur)
logger.d { "[DEBUG] Added phraseQuery with occur=$occur, near=$near" }

// A fully-quoted query that produced no usable phrase clause (e.g. just punctuation)
// must not fall through to a match-all query.
if (norm.isBlank() && exactClausesAdded == 0) return null

// Free (unquoted) text: dictionary-aware ranking + presence filtering.
if (norm.isNotBlank()) {
val rankedQuery = buildExpandedQuery(norm, near, analyzedStd, tokenExpansions)
val mustAllTokensQuery: Query? = buildPresenceFilterForTokens(analyzedStd, near, tokenExpansions)
val phraseQuery: Query? = buildSynonymPhraseQuery(analyzedStd, tokenExpansions, near)
if (mustAllTokensQuery != null) {
builder.add(mustAllTokensQuery, BooleanClause.Occur.FILTER)
logger.d { "[DEBUG] Added mustAllTokensQuery as FILTER" }
}
if (phraseQuery != null && analyzedStd.size >= 2) {
val occur = if (near == 0) BooleanClause.Occur.MUST else BooleanClause.Occur.SHOULD
builder.add(phraseQuery, occur)
logger.d { "[DEBUG] Added phraseQuery with occur=$occur, near=$near" }
}
builder.add(rankedQuery, BooleanClause.Occur.SHOULD)
logger.d { "[DEBUG] Added rankedQuery as SHOULD" }
}
builder.add(rankedQuery, BooleanClause.Occur.SHOULD)
logger.d { "[DEBUG] Added rankedQuery as SHOULD" }

val finalQuery = builder.build()
logger.d { "[DEBUG] Final query: $finalQuery" }
Expand Down Expand Up @@ -671,6 +713,21 @@ class LuceneSearchEngine(
return bool ?: BooleanQuery.Builder().build()
}

/**
* Builds a strict, verbatim phrase query for a quoted segment: terms must appear in the
* given order and be adjacent (slop = 0), with NO dictionary expansion, fuzzy matching or
* n-gram substring matching. A single-token segment becomes an exact [TermQuery].
*
* Nikud and final letters are still normalised (the segment is normalised by the caller and
* tokenised through [stdAnalyzer]) so that the phrase matches the indexed form.
*/
private fun buildExactPhraseQuery(phraseNorm: String): Query? {
if (phraseNorm.isBlank()) return null
val qb = QueryBuilder(stdAnalyzer)
qb.createPhraseQuery("text", phraseNorm, 0)?.let { return it }
return qb.createBooleanQuery("text", phraseNorm, BooleanClause.Occur.MUST)
}

private fun buildMagicBoostQuery(expansions: List<MagicDictionaryIndex.Expansion>): Query? {
if (expansions.isEmpty()) return null
val surfaceTerms = LinkedHashSet<String>()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package io.github.kdroidfilter.seforimlibrary.search

/**
* Parses a raw search query into "exact" phrases (delimited by double quotes) and the
* remaining free text.
*
* Google-style behaviour: text wrapped in double quotes is searched verbatim, as an exact
* ordered phrase, WITHOUT the magic-dictionary / synonym expansion, fuzzy matching or n-gram
* substring matching.
*
* Two delimiter characters are recognised, so it works whatever the keyboard layout produces:
* - the ASCII straight quote `"` (U+0022)
* - the Hebrew gershayim `״` (U+05F4)
* They are interchangeable (a phrase may be opened with one and closed with the other).
*
* Hebrew acronyms (rashei tevot) are very commonly written with a double quote between the last
* two letters (e.g. רש״י / רש"י, רמב״ם, שו״ע, תנ״ך, תשפ״א). Such a quote is NOT a phrase
* delimiter: a double quote acts as a delimiter only when it sits at a word boundary, i.e. it is
* NOT flanked by Hebrew letters on both sides. This keeps acronym searches working as before.
*/
object SearchQueryParser {

private const val ASCII_QUOTE = '"'
private const val GERSHAYIM = '״'
private val WHITESPACE = "\\s+".toRegex()

data class ParsedQuery(
/** Substrings that were enclosed in double quotes, each to be matched as an exact phrase. */
val exactPhrases: List<String>,
/** The remaining (unquoted) text, processed by the normal dictionary-aware pipeline. */
val freeText: String,
) {
/** True when the query contains at least one real phrase delimiter. */
val hasExactPhrases: Boolean get() = exactPhrases.isNotEmpty()
}

private fun isQuoteDelimiter(c: Char): Boolean = c == ASCII_QUOTE || c == GERSHAYIM

/** Hebrew letters block (alef..tav), including final forms which are all within this range. */
private fun isHebrewLetter(c: Char): Boolean = c.code in 0x05D0..0x05EA

/**
* A double quote at [index] is an acronym marker (NOT a phrase delimiter) when it is
* immediately surrounded by Hebrew letters on both sides, e.g. the ״ in רש״י.
*/
private fun isAcronymQuote(raw: String, index: Int): Boolean {
if (index <= 0 || index >= raw.length - 1) return false
return isHebrewLetter(raw[index - 1]) && isHebrewLetter(raw[index + 1])
}

fun parse(rawQuery: String): ParsedQuery {
if (rawQuery.none { isQuoteDelimiter(it) }) {
return ParsedQuery(emptyList(), rawQuery)
}

val exactPhrases = mutableListOf<String>()
val free = StringBuilder()
val phrase = StringBuilder()
var inQuote = false

for (i in rawQuery.indices) {
val c = rawQuery[i]
if (isQuoteDelimiter(c) && !isAcronymQuote(rawQuery, i)) {
if (inQuote) {
addPhrase(exactPhrases, phrase)
inQuote = false
} else {
inQuote = true
}
} else {
(if (inQuote) phrase else free).append(c)
}
}
// Unclosed quote: treat the trailing buffered content as an exact phrase.
if (inQuote) addPhrase(exactPhrases, phrase)

return ParsedQuery(
exactPhrases = exactPhrases,
freeText = free.toString().trim().replace(WHITESPACE, " "),
)
}

private fun addPhrase(into: MutableList<String>, buffer: StringBuilder) {
val phrase = buffer.toString().trim().replace(WHITESPACE, " ")
if (phrase.isNotEmpty()) into.add(phrase)
buffer.setLength(0)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -407,8 +407,125 @@ class LuceneSearchEngineTest {
}
}

// --- Exact phrase (quoted) tests ---

@Test
fun `quoted query matches only the exact ordered adjacent phrase`() {
val tempDir = createTempIndexDir()
try {
createIndexWithPhrases(tempDir)
val engine = LuceneSearchEngine(tempDir)

val session = engine.openSession("\"מלך דוד\"", 5)
assertNotNull(session)
// line 1 ("מלך דוד") and line 5 ("ירושלים מלך דוד עיר") contain the exact phrase.
// line 2 ("דוד מלך", reversed) and line 3 ("מלך גדול דוד", gap) must be excluded.
assertEquals(setOf(1L, 5L), collectLineIds(session))

engine.close()
} finally {
deleteDirectory(tempDir)
}
}

@Test
fun `unquoted query matches every doc containing the words in any order`() {
val tempDir = createTempIndexDir()
try {
createIndexWithPhrases(tempDir)
val engine = LuceneSearchEngine(tempDir)

val session = engine.openSession("מלך דוד", 5)
assertNotNull(session)
// Without quotes the order/adjacency is relaxed: lines 1, 2, 3 and 5 all qualify.
assertEquals(setOf(1L, 2L, 3L, 5L), collectLineIds(session))

engine.close()
} finally {
deleteDirectory(tempDir)
}
}

@Test
fun `mixed free word plus quoted phrase requires both`() {
val tempDir = createTempIndexDir()
try {
createIndexWithPhrases(tempDir)
val engine = LuceneSearchEngine(tempDir)

val session = engine.openSession("ירושלים \"מלך דוד\"", 5)
assertNotNull(session)
// Only line 5 has both the free word "ירושלים" and the exact phrase "מלך דוד".
assertEquals(setOf(5L), collectLineIds(session))

engine.close()
} finally {
deleteDirectory(tempDir)
}
}

@Test
fun `gershayim-quoted query behaves like an exact phrase`() {
val tempDir = createTempIndexDir()
try {
createIndexWithPhrases(tempDir)
val engine = LuceneSearchEngine(tempDir)

// Same phrase as the ASCII test but delimited with Hebrew gershayim (U+05F4).
val session = engine.openSession("״מלך דוד״", 5)
assertNotNull(session)
assertEquals(setOf(1L, 5L), collectLineIds(session))

engine.close()
} finally {
deleteDirectory(tempDir)
}
}

// --- Helper methods ---

private fun collectLineIds(session: SearchSession): Set<Long> = runBlocking {
val ids = mutableSetOf<Long>()
while (true) {
val page = session.nextPage(50) ?: break
page.hits.forEach { ids.add(it.lineId) }
if (page.isLastPage) break
}
session.close()
ids
}

private fun createIndexWithPhrases(indexDir: Path) {
FSDirectory.open(indexDir).use { dir ->
val config = IndexWriterConfig(StandardAnalyzer())
IndexWriter(dir, config).use { writer ->
val rows = listOf(
1L to "מלך דוד",
2L to "דוד מלך",
3L to "מלך גדול דוד",
4L to "שלום עולם",
5L to "ירושלים מלך דוד עיר",
)
rows.forEach { (lineId, text) ->
val doc = Document().apply {
add(StringField("type", "line", Field.Store.YES))
add(StoredField("book_id", 1))
add(IntPoint("book_id", 1))
add(StoredField("book_title", "ספר בדיקה"))
add(StoredField("line_id", lineId))
add(IntPoint("line_id", lineId.toInt()))
add(StoredField("line_index", (lineId - 1).toInt()))
add(TextField("text", HebrewTextUtils.normalizeHebrew(text), Field.Store.NO))
add(StoredField("text_raw", text))
add(StoredField("is_base_book", 1))
add(StoredField("order_index", 1))
}
writer.addDocument(doc)
}
}
}
}

private fun createTempIndexDir(): Path {
return Files.createTempDirectory("lucene_test_index")
}
Expand Down
Loading
Loading