From f08781e3f91df3451404087452b0cb0d00c04f83 Mon Sep 17 00:00:00 2001 From: jus1d Date: Sun, 26 Apr 2026 22:27:59 +0400 Subject: [PATCH] feat: Make forms accordingly to string case --- analyzer.go | 111 +++++++++++++++++++++++++++++++++++------------ analyzer_test.go | 73 ++++++++++++++++++++++++++++--- 2 files changed, 150 insertions(+), 34 deletions(-) diff --git a/analyzer.go b/analyzer.go index 3644f74..6e31eef 100644 --- a/analyzer.go +++ b/analyzer.go @@ -21,6 +21,8 @@ import ( "encoding/json" "strings" "sync" + "unicode" + "unicode/utf8" ) //go:embed data/words.dawg data/paradigms.array data/suffixes.json data/gramtab-opencorpora-int.json data/meta.json @@ -53,12 +55,14 @@ func Default() (*Analyzer, error) { // The word may be supplied in any grammatical form // Returns nil if the word is not found in the dictionary func (a *Analyzer) WordForms(word string) []string { - word = strings.ToLower(strings.TrimSpace(word)) - if word == "" { + trimmed := strings.TrimSpace(word) + style := detectCase(trimmed) + lower := strings.ToLower(trimmed) + if lower == "" { return nil } - entries := a.words.get(word) + entries := a.words.get(lower) if len(entries) == 0 { return nil } @@ -72,7 +76,7 @@ func (a *Analyzer) WordForms(word string) []string { return nil } - stem, ok := a.extractStem(word, para, n, int(e.formIdx)) + stem, ok := a.extractStem(lower, para, n, int(e.formIdx)) if !ok { return nil } @@ -80,7 +84,7 @@ func (a *Analyzer) WordForms(word string) []string { seen := make(map[string]struct{}, n) forms := make([]string, 0, n) for i := 0; i < n; i++ { - f := paradigmPrefixes[para[2*n+i]] + stem + a.suffixes[para[i]] + f := applyCase(paradigmPrefixes[para[2*n+i]]+stem+a.suffixes[para[i]], style) if _, dup := seen[f]; !dup { seen[f] = struct{}{} forms = append(forms, f) @@ -147,21 +151,19 @@ func (a *Analyzer) bestTag(entries []wordEntry) string { // are always preserved verbatim and never declined. // The original phrase is always the first element of the returned slice func (a *Analyzer) PhraseFormsConcordant(phrase string) []string { - phrase = strings.ToLower(strings.TrimSpace(phrase)) - tokens := tokenizePhrase(phrase) + origPhrase := strings.TrimSpace(phrase) + lowerPhrase := strings.ToLower(origPhrase) + tokens := tokenizePhrase(lowerPhrase) if len(tokens) == 0 { return nil } - // Collect plain (non-quoted) words for single-word fast path - plainTokens := 0 - firstPlain := "" - for _, t := range tokens { - if !t.quoted { - plainTokens++ - if firstPlain == "" { - firstPlain = t.text - } + // Detect case styles from original phrase tokens + origTokens := tokenizePhrase(origPhrase) + styles := make([]caseStyle, len(tokens)) + for i := range tokens { + if i < len(origTokens) && !tokens[i].quoted { + styles[i] = detectCase(origTokens[i].text) } } @@ -169,12 +171,17 @@ func (a *Analyzer) PhraseFormsConcordant(phrase string) []string { w := tokens[0].text forms := a.WordForms(w) if forms == nil { - return []string{w} + return []string{origPhrase} + } + // Apply original case to all forms + for i := range forms { + forms[i] = applyCase(forms[i], styles[0]) } - // Ensure the input form (which may be non-nominative) is first. - if forms[0] != w { + // Ensure the input form is first. + target := applyCase(w, styles[0]) + if forms[0] != target { for i, f := range forms { - if f == w { + if f == target { forms = append([]string{f}, append(forms[:i:i], forms[i+1:]...)...) break } @@ -210,11 +217,10 @@ func (a *Analyzer) PhraseFormsConcordant(phrase string) []string { } } - seen := map[string]struct{}{phrase: {}} - result := []string{phrase} + seen := map[string]struct{}{origPhrase: {}} + result := []string{origPhrase} if headIdx == -1 { - // No noun found -- cannot decline the phrase, return as-is return result } @@ -230,18 +236,20 @@ func (a *Analyzer) PhraseFormsConcordant(phrase string) []string { declined[i] = t.text continue } + var raw string switch infos[i].pos { case "NOUN", "NPRO": if i == headIdx { - declined[i] = a.inflect(t.text, cas, number, "", "") + raw = a.inflect(t.text, cas, number, "", "") } else { - declined[i] = t.text // genitive dependent -- leave unchanged + raw = t.text } case "ADJF", "PRTF": - declined[i] = a.inflectAdj(t.text, cas, number, head.gender, head.animacy) + raw = a.inflectAdj(t.text, cas, number, head.gender, head.animacy) default: - declined[i] = t.text + raw = t.text } + declined[i] = applyCase(raw, styles[i]) } form := strings.Join(declined, " ") if _, ok := seen[form]; !ok { @@ -253,6 +261,55 @@ func (a *Analyzer) PhraseFormsConcordant(phrase string) []string { return result } +// caseStyle represents the casing pattern of a word. +type caseStyle int + +const ( + caseLower caseStyle = iota // all lowercase + caseUpper // ALL UPPERCASE + caseTitle // Title case (first letter upper, rest lower) +) + +// detectCase returns the casing style of a word. +func detectCase(word string) caseStyle { + if word == "" { + return caseLower + } + firstRune, _ := utf8.DecodeRuneInString(word) + if !unicode.IsUpper(firstRune) { + return caseLower + } + // First letter is upper — check if all are upper + allUpper := true + for _, r := range word { + if unicode.IsLetter(r) && !unicode.IsUpper(r) { + allUpper = false + break + } + } + if allUpper { + return caseUpper + } + return caseTitle +} + +// applyCase transforms a lowercase word to match the given casing style. +func applyCase(word string, style caseStyle) string { + switch style { + case caseUpper: + return strings.ToUpper(word) + case caseTitle: + runes := []rune(word) + if len(runes) == 0 { + return word + } + runes[0] = unicode.ToUpper(runes[0]) + return string(runes) + default: + return word + } +} + // phraseToken is a single element of a tokenized phrase. // quoted tokens are quoted segments that must not be declined. type phraseToken struct { diff --git a/analyzer_test.go b/analyzer_test.go index 0b2c9b3..0f895ba 100644 --- a/analyzer_test.go +++ b/analyzer_test.go @@ -4,6 +4,7 @@ import ( "slices" "strings" "testing" + "unicode" ) // shared analyzer instance reused across all tests @@ -99,18 +100,29 @@ func TestWordForms_EdgeCases(t *testing.T) { } }) - t.Run("case insensitive", func(t *testing.T) { + t.Run("case preservation", func(t *testing.T) { lower := a.WordForms("кошка") upper := a.WordForms("КОШКА") - mixed := a.WordForms("Кошка") - if lower == nil || upper == nil || mixed == nil { + title := a.WordForms("Кошка") + if lower == nil || upper == nil || title == nil { t.Fatal("WordForms returned nil for some casing variant") } - if strings.Join(lower, ",") != strings.Join(upper, ",") { - t.Error("WordForms differs for lower vs upper case") + // Lowercase input returns lowercase forms + for _, f := range lower { + if f != strings.ToLower(f) { + t.Errorf("lowercase input produced non-lowercase form: %q", f) + } + } + // Uppercase input returns uppercase forms + for _, f := range upper { + if f != strings.ToUpper(f) { + t.Errorf("uppercase input produced non-uppercase form: %q", f) + } } - if strings.Join(lower, ",") != strings.Join(mixed, ",") { - t.Error("WordForms differs for lower vs mixed case") + // Title case input returns title case forms + runes := []rune(title[0]) + if !unicode.IsUpper(runes[0]) { + t.Errorf("title case input first form should start with upper: %q", title[0]) } }) @@ -338,6 +350,53 @@ func TestPhraseFormsConcordant_QuotedSegments(t *testing.T) { }) } +func TestPhraseFormsConcordant_CasePreservation(t *testing.T) { + a := testAnalyzer + + t.Run("title case phrase", func(t *testing.T) { + forms := a.PhraseFormsConcordant("Красивая Кошка") + if len(forms) == 0 { + t.Fatal("empty result") + } + if forms[0] != "Красивая Кошка" { + t.Errorf("first form = %q, want %q", forms[0], "Красивая Кошка") + } + // All forms should have title case words + for _, f := range forms { + words := strings.Fields(f) + for _, w := range words { + runes := []rune(w) + if len(runes) > 0 && unicode.IsLetter(runes[0]) && !unicode.IsUpper(runes[0]) { + t.Errorf("form %q has lowercase word %q, expected title case", f, w) + } + } + } + }) + + t.Run("uppercase phrase", func(t *testing.T) { + forms := a.PhraseFormsConcordant("КРАСИВАЯ КОШКА") + if len(forms) == 0 { + t.Fatal("empty result") + } + if forms[0] != "КРАСИВАЯ КОШКА" { + t.Errorf("first form = %q, want %q", forms[0], "КРАСИВАЯ КОШКА") + } + }) + + t.Run("single word title case", func(t *testing.T) { + forms := a.PhraseFormsConcordant("Кошка") + if len(forms) == 0 { + t.Fatal("empty result") + } + if forms[0] != "Кошка" { + t.Errorf("first form = %q, want %q", forms[0], "Кошка") + } + if !slices.Contains(forms, "Кошки") { + t.Errorf("expected Кошки in forms: %v", forms) + } + }) +} + func TestTagPOS(t *testing.T) { tests := []struct { tag string