Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 84 additions & 27 deletions analyzer.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (
"encoding/json"
"strings"
"sync"
"unicode"
"unicode/utf8"
)

//go:embed data/words.dawg data/paradigms.array data/suffixes.json data/gramtab-opencorpora-int.json data/meta.json
Expand Down Expand Up @@ -53,12 +55,14 @@ func Default() (*Analyzer, error) {
// The word may be supplied in any grammatical form
// Returns nil if the word is not found in the dictionary
func (a *Analyzer) WordForms(word string) []string {
word = strings.ToLower(strings.TrimSpace(word))
if word == "" {
trimmed := strings.TrimSpace(word)
style := detectCase(trimmed)
lower := strings.ToLower(trimmed)
if lower == "" {
return nil
}

entries := a.words.get(word)
entries := a.words.get(lower)
if len(entries) == 0 {
return nil
}
Expand All @@ -72,15 +76,15 @@ func (a *Analyzer) WordForms(word string) []string {
return nil
}

stem, ok := a.extractStem(word, para, n, int(e.formIdx))
stem, ok := a.extractStem(lower, para, n, int(e.formIdx))
if !ok {
return nil
}

seen := make(map[string]struct{}, n)
forms := make([]string, 0, n)
for i := 0; i < n; i++ {
f := paradigmPrefixes[para[2*n+i]] + stem + a.suffixes[para[i]]
f := applyCase(paradigmPrefixes[para[2*n+i]]+stem+a.suffixes[para[i]], style)
if _, dup := seen[f]; !dup {
seen[f] = struct{}{}
forms = append(forms, f)
Expand Down Expand Up @@ -147,34 +151,37 @@ func (a *Analyzer) bestTag(entries []wordEntry) string {
// are always preserved verbatim and never declined.
// The original phrase is always the first element of the returned slice
func (a *Analyzer) PhraseFormsConcordant(phrase string) []string {
phrase = strings.ToLower(strings.TrimSpace(phrase))
tokens := tokenizePhrase(phrase)
origPhrase := strings.TrimSpace(phrase)
lowerPhrase := strings.ToLower(origPhrase)
tokens := tokenizePhrase(lowerPhrase)
if len(tokens) == 0 {
return nil
}

// Collect plain (non-quoted) words for single-word fast path
plainTokens := 0
firstPlain := ""
for _, t := range tokens {
if !t.quoted {
plainTokens++
if firstPlain == "" {
firstPlain = t.text
}
// Detect case styles from original phrase tokens
origTokens := tokenizePhrase(origPhrase)
styles := make([]caseStyle, len(tokens))
for i := range tokens {
if i < len(origTokens) && !tokens[i].quoted {
styles[i] = detectCase(origTokens[i].text)
}
}

if len(tokens) == 1 && !tokens[0].quoted {
w := tokens[0].text
forms := a.WordForms(w)
if forms == nil {
return []string{w}
return []string{origPhrase}
}
// Apply original case to all forms
for i := range forms {
forms[i] = applyCase(forms[i], styles[0])
}
// Ensure the input form (which may be non-nominative) is first.
if forms[0] != w {
// Ensure the input form is first.
target := applyCase(w, styles[0])
if forms[0] != target {
for i, f := range forms {
if f == w {
if f == target {
forms = append([]string{f}, append(forms[:i:i], forms[i+1:]...)...)
break
}
Expand Down Expand Up @@ -210,11 +217,10 @@ func (a *Analyzer) PhraseFormsConcordant(phrase string) []string {
}
}

seen := map[string]struct{}{phrase: {}}
result := []string{phrase}
seen := map[string]struct{}{origPhrase: {}}
result := []string{origPhrase}

if headIdx == -1 {
// No noun found -- cannot decline the phrase, return as-is
return result
}

Expand All @@ -230,18 +236,20 @@ func (a *Analyzer) PhraseFormsConcordant(phrase string) []string {
declined[i] = t.text
continue
}
var raw string
switch infos[i].pos {
case "NOUN", "NPRO":
if i == headIdx {
declined[i] = a.inflect(t.text, cas, number, "", "")
raw = a.inflect(t.text, cas, number, "", "")
} else {
declined[i] = t.text // genitive dependent -- leave unchanged
raw = t.text
}
case "ADJF", "PRTF":
declined[i] = a.inflectAdj(t.text, cas, number, head.gender, head.animacy)
raw = a.inflectAdj(t.text, cas, number, head.gender, head.animacy)
default:
declined[i] = t.text
raw = t.text
}
declined[i] = applyCase(raw, styles[i])
}
form := strings.Join(declined, " ")
if _, ok := seen[form]; !ok {
Expand All @@ -253,6 +261,55 @@ func (a *Analyzer) PhraseFormsConcordant(phrase string) []string {
return result
}

// caseStyle represents the casing pattern of a word.
type caseStyle int

const (
caseLower caseStyle = iota // all lowercase
caseUpper // ALL UPPERCASE
caseTitle // Title case (first letter upper, rest lower)
)

// detectCase returns the casing style of a word.
func detectCase(word string) caseStyle {
if word == "" {
return caseLower
}
firstRune, _ := utf8.DecodeRuneInString(word)
if !unicode.IsUpper(firstRune) {
return caseLower
}
// First letter is upper — check if all are upper
allUpper := true
for _, r := range word {
if unicode.IsLetter(r) && !unicode.IsUpper(r) {
allUpper = false
break
}
}
if allUpper {
return caseUpper
}
return caseTitle
}

// applyCase transforms a lowercase word to match the given casing style.
func applyCase(word string, style caseStyle) string {
switch style {
case caseUpper:
return strings.ToUpper(word)
case caseTitle:
runes := []rune(word)
if len(runes) == 0 {
return word
}
runes[0] = unicode.ToUpper(runes[0])
return string(runes)
default:
return word
}
}

// phraseToken is a single element of a tokenized phrase.
// quoted tokens are quoted segments that must not be declined.
type phraseToken struct {
Expand Down
73 changes: 66 additions & 7 deletions analyzer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"slices"
"strings"
"testing"
"unicode"
)

// shared analyzer instance reused across all tests
Expand Down Expand Up @@ -99,18 +100,29 @@ func TestWordForms_EdgeCases(t *testing.T) {
}
})

t.Run("case insensitive", func(t *testing.T) {
t.Run("case preservation", func(t *testing.T) {
lower := a.WordForms("кошка")
upper := a.WordForms("КОШКА")
mixed := a.WordForms("Кошка")
if lower == nil || upper == nil || mixed == nil {
title := a.WordForms("Кошка")
if lower == nil || upper == nil || title == nil {
t.Fatal("WordForms returned nil for some casing variant")
}
if strings.Join(lower, ",") != strings.Join(upper, ",") {
t.Error("WordForms differs for lower vs upper case")
// Lowercase input returns lowercase forms
for _, f := range lower {
if f != strings.ToLower(f) {
t.Errorf("lowercase input produced non-lowercase form: %q", f)
}
}
// Uppercase input returns uppercase forms
for _, f := range upper {
if f != strings.ToUpper(f) {
t.Errorf("uppercase input produced non-uppercase form: %q", f)
}
}
if strings.Join(lower, ",") != strings.Join(mixed, ",") {
t.Error("WordForms differs for lower vs mixed case")
// Title case input returns title case forms
runes := []rune(title[0])
if !unicode.IsUpper(runes[0]) {
t.Errorf("title case input first form should start with upper: %q", title[0])
}
})

Expand Down Expand Up @@ -338,6 +350,53 @@ func TestPhraseFormsConcordant_QuotedSegments(t *testing.T) {
})
}

func TestPhraseFormsConcordant_CasePreservation(t *testing.T) {
a := testAnalyzer

t.Run("title case phrase", func(t *testing.T) {
forms := a.PhraseFormsConcordant("Красивая Кошка")
if len(forms) == 0 {
t.Fatal("empty result")
}
if forms[0] != "Красивая Кошка" {
t.Errorf("first form = %q, want %q", forms[0], "Красивая Кошка")
}
// All forms should have title case words
for _, f := range forms {
words := strings.Fields(f)
for _, w := range words {
runes := []rune(w)
if len(runes) > 0 && unicode.IsLetter(runes[0]) && !unicode.IsUpper(runes[0]) {
t.Errorf("form %q has lowercase word %q, expected title case", f, w)
}
}
}
})

t.Run("uppercase phrase", func(t *testing.T) {
forms := a.PhraseFormsConcordant("КРАСИВАЯ КОШКА")
if len(forms) == 0 {
t.Fatal("empty result")
}
if forms[0] != "КРАСИВАЯ КОШКА" {
t.Errorf("first form = %q, want %q", forms[0], "КРАСИВАЯ КОШКА")
}
})

t.Run("single word title case", func(t *testing.T) {
forms := a.PhraseFormsConcordant("Кошка")
if len(forms) == 0 {
t.Fatal("empty result")
}
if forms[0] != "Кошка" {
t.Errorf("first form = %q, want %q", forms[0], "Кошка")
}
if !slices.Contains(forms, "Кошки") {
t.Errorf("expected Кошки in forms: %v", forms)
}
})
}

func TestTagPOS(t *testing.T) {
tests := []struct {
tag string
Expand Down
Loading