diff --git a/internal/index/hybrid.go b/internal/index/hybrid.go index 8280fa2..88ff4cd 100644 --- a/internal/index/hybrid.go +++ b/internal/index/hybrid.go @@ -3,6 +3,7 @@ package index import ( "fmt" "strings" + "unicode" "github.com/Coder8124/logos/internal/memory" "github.com/Coder8124/logos/internal/provider" @@ -116,11 +117,15 @@ func (ix *Index) memoryHits(p *provider.Provider, model, query string, k int) [] // The tokenizer indexed WG-4471 as wg then 4471, so gluing the query into // "WG4471" matched nothing; the phrase "WG 4471" matches the identifier as a // unit, and not a note that only has wg and 4471 apart. +// +// A letter is any letter, not just ASCII: unicode61 indexes café and naïve as +// whole words, and an ASCII-only split turned the query into "caf" and "na ve", +// which never matched them — and did match a note with those fragments. func ftsQuery(q string) string { var tokens []string for _, f := range strings.Fields(q) { pieces := strings.FieldsFunc(f, func(r rune) bool { - return !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9') + return !(unicode.IsLetter(r) || unicode.IsDigit(r)) }) if len(strings.Join(pieces, "")) > 1 { tokens = append(tokens, `"`+strings.Join(pieces, " ")+`"`) diff --git a/internal/index/hybrid_test.go b/internal/index/hybrid_test.go index b70448f..2e465f0 100644 --- a/internal/index/hybrid_test.go +++ b/internal/index/hybrid_test.go @@ -61,6 +61,25 @@ func TestLexicalFindsExactToken(t *testing.T) { } } +func TestAWordWithANonASCIILetterIsFoundAsWritten(t *testing.T) { + ix := newTestIndex(t) + seed(t, ix, "ideas/naive", "Naïve Bayes", "a naïve baseline before the café survey") + seed(t, ix, "trips/greece", "Trip", "two weeks in Ελλάδα") + // The ASCII remnants of each word, so a query that drops the accented + // letter matches this note instead and the failure is a wrong answer. + seed(t, ix, "ideas/other", "Other", "na ve caf") + + for q, want := range map[string]string{"naïve": "ideas/naive", "café": "ideas/naive", "Ελλάδα": "trips/greece"} { + got, err := ix.lexical(q, 5) + if err != nil { + t.Fatal(err) + } + if len(got) == 0 || got[0] != want { + t.Errorf("searching %q should surface %s first, got %v", q, want, got) + } + } +} + func TestFtsQueryStripsPunctuation(t *testing.T) { // A query full of FTS syntax characters must not error — it should reduce to // a safe OR of the real tokens. diff --git a/internal/textmatch/textmatch.go b/internal/textmatch/textmatch.go index 623bc6e..f299e82 100644 --- a/internal/textmatch/textmatch.go +++ b/internal/textmatch/textmatch.go @@ -76,10 +76,12 @@ func numeric(s string) bool { // Akin reports whether two words are the same term in different clothes. // -// A shared prefix of five characters catches the inflections that matter — -// manufactures/manufacturer, proposal/proposals, quote/quoted — without pulling -// in a stemmer. Exact matching missed all of them, and a question asked in the -// user's words rarely uses the same inflection as the note that answers it. +// One word being a prefix of the other, at least five characters long, catches +// the inflections that add a suffix — proposal/proposals, quote/quoted, +// manufacture/manufacturer — without pulling in a stemmer. Exact matching +// missed all of them, and a question asked in the user's words rarely uses the +// same inflection as the note that answers it. Two words that both change the +// ending, manufactures/manufacturer, are not akin. func Akin(a, b string) bool { if a == b { return true diff --git a/internal/textmatch/textmatch_test.go b/internal/textmatch/textmatch_test.go index a6d0e0c..1e3b5c8 100644 --- a/internal/textmatch/textmatch_test.go +++ b/internal/textmatch/textmatch_test.go @@ -31,6 +31,10 @@ func TestAkinMatchesInflectionsButNotShortWords(t *testing.T) { if Akin("cart", "carton") { t.Error(`Akin("cart", "carton") = true, want false`) } + // The comment once promised this pair; the prefix rule never matched it. + if Akin("manufactures", "manufacturer") { + t.Error(`Akin("manufactures", "manufacturer") = true, want false — update the comment if this changes`) + } } // Jaccard scored this pair at 0.29, so a superseded price was handed over as