diff --git a/docs/architecture.md b/docs/architecture.md index 147be23..4733326 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -91,6 +91,15 @@ Analyzer の前分割や文ごとの API でも元の入力を使って判定す `word_cost` は元トークンのコストを広い整数型で足し、最後に一度だけ i16 の範囲に飽和させる。最良パス決定後の情報であり、 経路の再探索には使わない。もともと数詞1トークンなら既存の素性を維持し、非空の既知の読みも尊重する。 `2026年度`・`2.5倍` のように数と単位を含む既知語も、分割や素性を変更しない。 + +暦月は別に、半角空白・タブを挟む `9 月`・`9 月`・`九 月` をク / ガツに補正する。 +1〜12の半角数字・全角数字・漢数字の通常表記を対象に、四・七・九はシ・シチ・ク、十の発音はジューにする。 +数詞が一語なら読み・発音以外を保つ。`1 / 2 / 月` のような数詞列は数の部分だけを数量と同じ形の一語にまとめ、 +`12` と `月` の表層形・バイト範囲には空白を含めない。「月」の品詞・原形なども保つ。 +数詞・月の品詞と既存の標準読みを確認し、未知のASCII数詞だけは空の読みも補う。 +一般の月、`3か月`、`月間`・`月曜`、小数・識別子・先頭ゼロ・混在表記・13以上、改行・全角空白を挟む形は対象にしない。 +空白のない既知の月名は辞書のままにする。人名が数詞と普通名詞に分かれる場合など、同じ表記の意味を判別するものではない。 + 対応表は [Open JTalk 1.11 の数詞規則](https://github.com/r9y9/open_jtalk/blob/1.11/src/njd_set_digit/njd_set_digit_rule_utf_8.h) を参考に、トークンの読みを補正する独立した処理として実装している。 diff --git a/src/lattice.rs b/src/lattice.rs index 0e810aa..0b91987 100644 --- a/src/lattice.rs +++ b/src/lattice.rs @@ -1276,6 +1276,7 @@ impl LatticeWorkspace { |surface| unit_reading(surface).is_some_and(is_katakana_str), ); apply_contextual_readings_with_protection(&mut out[first..], &protected); + crate::numeric::apply_spaced_months(input, offset, out, first, plan); Ok(()) } } diff --git a/src/numeric.rs b/src/numeric.rs index 09b30c0..5b3cf58 100644 --- a/src/numeric.rs +++ b/src/numeric.rs @@ -8,7 +8,9 @@ use std::ops::Range; use std::sync::Arc; mod ascii; +mod month; pub(crate) use ascii::{AsciiPlan, apply as apply_ascii_quantities}; +pub(crate) use month::apply as apply_spaced_months; #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Ending { diff --git a/src/numeric/ascii.rs b/src/numeric/ascii.rs index d544790..5a467fa 100644 --- a/src/numeric/ascii.rs +++ b/src/numeric/ascii.rs @@ -150,6 +150,7 @@ struct Candidate { #[derive(Default, Debug)] pub(crate) struct AsciiPlan { candidates: Vec, + month_starts: Vec, } fn envelope_byte(b: u8) -> bool { @@ -157,14 +158,36 @@ fn envelope_byte(b: u8) -> bool { } impl AsciiPlan { + pub(super) fn has_month_in(&self, start: usize, end: usize) -> bool { + let i = self.month_starts.partition_point(|&s| s < start); + self.month_starts.get(i).is_some_and(|&s| s < end) + } + + pub(super) fn month_number_start(&self, start: usize) -> bool { + self.month_starts.binary_search(&start).is_ok() + } + + /// 識別子・小数・単位を含まない、独立した整数候補との完全一致。 + pub(super) fn bare_integer(&self, start: usize, end: usize) -> bool { + let i = self.candidates.partition_point(|c| c.number.start < start); + self.candidates.get(i).is_some_and(|c| { + c.number == (start..end) + && c.envelope == c.number + && c.parsed.is_some_and(|n| n.dot.is_none()) + }) + } + /// 位置は offset を加えた入力全体の位置。空白越しの英字・裸の数字も識別子の前置とみなす。 pub(crate) fn new(input: &str, offset: usize) -> Self { + let mut plan = Self { + month_starts: super::month::number_starts(input, offset), + ..Self::default() + }; if !input.bytes().any(|b| b.is_ascii_digit()) { - return Self::default(); + return plan; } let bytes = input.as_bytes(); let (mut pos, mut prefix) = (0, false); - let mut plan = Self::default(); while pos < bytes.len() { if envelope_byte(bytes[pos]) { let start = pos; diff --git a/src/numeric/month.rs b/src/numeric/month.rs new file mode 100644 index 0000000..d1dd59b --- /dev/null +++ b/src/numeric/month.rs @@ -0,0 +1,274 @@ +//! 水平空白を挟む暦月。数詞と「月」の範囲を保ち、数詞が複数トークンなら数量と同じ形に合成する。 + +use super::{AsciiPlan, digit, has_reading, is_numeral, place, pos_has, set_readings}; +use crate::lattice::Token; +use std::sync::Arc; + +const MONTHS: [([&str; 3], &str, &str); 12] = [ + (["1", "1", "一"], "イチ", "イチ"), + (["2", "2", "二"], "ニ", "ニ"), + (["3", "3", "三"], "サン", "サン"), + (["4", "4", "四"], "シ", "シ"), + (["5", "5", "五"], "ゴ", "ゴ"), + (["6", "6", "六"], "ロク", "ロク"), + (["7", "7", "七"], "シチ", "シチ"), + (["8", "8", "八"], "ハチ", "ハチ"), + (["9", "9", "九"], "ク", "ク"), + (["10", "10", "十"], "ジュウ", "ジュー"), + (["11", "11", "十一"], "ジュウイチ", "ジューイチ"), + (["12", "12", "十二"], "ジュウニ", "ジューニ"), +]; + +fn horizontal(c: char) -> bool { + matches!(c, ' ' | '\t') +} + +/// 前分割で小数点などの前半を失わないよう、入力全体で月名の数詞の開始位置を確定する。 +pub(super) fn number_starts(input: &str, offset: usize) -> Vec { + let mut starts = Vec::new(); + for (i, _) in input.match_indices('月') { + let left = input[..i].trim_end_matches(horizontal); + if left.len() == i { + continue; + } + let prefix = left.trim_end_matches(|c| digit(c).is_some() || place(c).is_some()); + let raw = &left[prefix.len()..]; + if !MONTHS.iter().any(|(forms, _, _)| forms.contains(&raw)) { + continue; + } + // 小数・番号・英字の末尾だけを拾わない。括弧を挟む識別子の前置も守る。 + let previous = prefix + .trim_end_matches(|c| horizontal(c) || matches!(c, '(' | '[' | '(' | '[')) + .chars() + .next_back(); + if previous.is_some_and(|c| { + c.is_ascii_alphanumeric() + || matches!(c, 'A'..='Z' | 'a'..='z' | '_' | '.' | '.' | '点' | ',' | ',' | ':' | '/' | '/' | '∕' | '÷' | '×' | '+' | '+' | '-' | '-' | '−' | '‐' | '‑' | '‒' | '–' | '—' | '―' | '=' | '*' | '^' | '%') + || digit(c).is_some() + || place(c).is_some() + }) { + continue; + } + starts.push(offset + prefix.len()); + } + starts +} + +fn standard_readings(token: &Token) -> bool { + // 数詞として確定した未知語だけは、欠けた読みを救済できる。 + if !token.is_known + && token.surface.is_ascii() + && token.reading.is_empty() + && token.pronunciation.is_empty() + { + return true; + } + let spelling: String = token + .surface + .chars() + .map(|c| { + digit(c).map_or(c, |n| { + ['〇', '一', '二', '三', '四', '五', '六', '七', '八', '九'][n as usize] + }) + }) + .collect(); + // 既に一語の 10〜12 は位取りで、分かれた各桁は元の桁読みで照合する。 + let allowed = match spelling.as_str() { + "一〇" => &["ジュウ", "ジュー"][..], + "一一" | "十一" => &["ジュウイチ", "ジューイチ"], + "一二" | "十二" => &["ジュウニ", "ジューニ"], + _ => { + return super::standard_kanji_readings(&spelling, &token.reading, &token.pronunciation); + } + }; + has_reading(token, allowed) +} + +fn candidate<'a>( + input: &'a str, + offset: usize, + tokens: &[Token], + start: usize, + end: usize, + plan: &AsciiPlan, +) -> Option<(&'a str, &'static str, &'static str)> { + let month = tokens.get(end)?; + if month.surface.as_ref() != "月" + || !(pos_has(&month.pos, "名詞,一般") || pos_has(&month.pos, "名詞,副詞可能")) + || !has_reading(month, &["ツキ", "ガツ"]) + { + return None; + } + let from = tokens[start].start.checked_sub(offset)?; + let to = tokens[end - 1].end.checked_sub(offset)?; + let month_start = month.start.checked_sub(offset)?; + let gap = input.get(to..month_start)?; + if gap.is_empty() || !gap.chars().all(horizontal) { + return None; + } + let raw = input.get(from..to)?; + let (_, reading, pronunciation) = MONTHS.iter().find(|(forms, _, _)| forms.contains(&raw))?; + if !plan.month_number_start(tokens[start].start) { + return None; + } + if raw.is_ascii() && !plan.bare_integer(tokens[start].start, tokens[end - 1].end) { + return None; + } + // 「月間」「月曜」が別トークンでも期間・曜日のままにする。 + if input + .get(month.end.checked_sub(offset)?..)? + .starts_with(['間', '曜']) + || !tokens[start..end].iter().all(standard_readings) + { + return None; + } + Some((raw, reading, pronunciation)) +} + +/// 今回のチャンクで追加した範囲だけを線形に圧縮する。前の出力・チャンクには触れない。 +pub(crate) fn apply( + input: &str, + offset: usize, + out: &mut Vec, + first: usize, + plan: &AsciiPlan, +) { + if !plan.has_month_in(offset, offset + input.len()) { + return; + } + let (mut read, mut write) = (first, first); + while read < out.len() { + let start = read; + read += 1; + if is_numeral(&out[start]) { + while read < out.len() && out[read - 1].end == out[read].start && is_numeral(&out[read]) + { + read += 1; + } + if let Some((raw, reading, pronunciation)) = + candidate(input, offset, out, start, read, plan) + { + if read - start == 1 { + set_readings(&mut out[start], reading, pronunciation); + out.swap(write, start); + } else { + let cost: i64 = out[start..read] + .iter() + .map(|t| i64::from(t.word_cost)) + .sum(); + let surface: Arc = Arc::from(raw); + out[write] = Token { + surface: Arc::clone(&surface), + base_form: surface, + start: out[start].start, + end: out[read - 1].end, + pos: Arc::from("名詞,数,*,*"), + conj_type: Arc::from(""), + conj_form: Arc::from(""), + reading: Arc::from(reading), + pronunciation: Arc::from(pronunciation), + word_cost: cost.clamp(i64::from(i16::MIN), i64::from(i16::MAX)) as i16, + is_known: false, + }; + } + write += 1; + set_readings(&mut out[read], "ガツ", "ガツ"); + out.swap(write, read); + write += 1; + read += 1; + continue; + } + } + for i in start..read { + out.swap(write, i); + write += 1; + } + } + out.truncate(write); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tokens(number: &str, reading: &str, pronunciation: &str) -> Vec { + [ + (number, "名詞,数,*,*", reading, pronunciation), + ("月", "名詞,一般,*,*", "ツキ", "ツキ"), + ] + .into_iter() + .enumerate() + .map(|(i, (surface, pos, reading, pronunciation))| Token { + surface: surface.into(), + start: if i == 0 { 37 } else { 38 + number.len() }, + end: if i == 0 { + 37 + number.len() + } else { + 41 + number.len() + }, + pos: pos.into(), + base_form: format!("base:{surface}").into(), + conj_type: "fixture-type".into(), + conj_form: "fixture-form".into(), + reading: reading.into(), + pronunciation: pronunciation.into(), + word_cost: 321, + is_known: true, + }) + .collect() + } + + #[test] + fn single_number_and_month_keep_every_non_reading_field_and_are_idempotent() { + for (number, reading, pronunciation, expected_r, expected_p) in [ + ("9", "キュウ", "キュー", "ク", "ク"), + ("9", "キュウ", "キュー", "ク", "ク"), + ("九", "キュウ", "キュー", "ク", "ク"), + ("4", "ヨン", "ヨン", "シ", "シ"), + ("7", "ナナ", "ナナ", "シチ", "シチ"), + ("12", "ジュウニ", "ジューニ", "ジュウニ", "ジューニ"), + ] { + let text = format!("{number} 月"); + let mut out = tokens(number, reading, pronunciation); + let before = out.clone(); + let plan = AsciiPlan::new(&text, 37); + apply(&text, 37, &mut out, 0, &plan); + assert_eq!(out[0].reading.as_ref(), expected_r); + assert_eq!(out[0].pronunciation.as_ref(), expected_p); + assert_eq!(out[1].reading.as_ref(), "ガツ"); + let once = format!("{out:?}"); + apply(&text, 37, &mut out, 0, &plan); + assert_eq!(format!("{out:?}"), once); + for (token, original) in out.iter_mut().zip(before) { + token.reading = Arc::clone(&original.reading); + token.pronunciation = Arc::clone(&original.pronunciation); + assert_eq!(format!("{token:?}"), format!("{original:?}")); + } + } + } + + #[test] + fn unknown_ascii_numbers_can_gain_readings_but_known_unusual_readings_are_kept() { + let text = "9 月"; + let plan = AsciiPlan::new(text, 37); + let mut out = tokens("9", "", ""); + out[0].is_known = false; + apply(text, 37, &mut out, 0, &plan); + assert_eq!(out[0].reading.as_ref(), "ク"); + assert!(!out[0].is_known); + for (reading, pronunciation) in [("", ""), ("キュウ", "ベツ"), ("ナイン", "ナイン")] + { + let mut out = tokens("9", reading, pronunciation); + let before = format!("{out:?}"); + apply(text, 37, &mut out, 0, &plan); + assert_eq!(format!("{out:?}"), before); + } + for index in [0, 1] { + let mut out = tokens("9", "キュウ", "キュー"); + out[index].pos = "名詞,固有名詞,人名,名".into(); + let before = format!("{out:?}"); + apply(text, 37, &mut out, 0, &plan); + assert_eq!(format!("{out:?}"), before); + } + } +} diff --git a/tests/month_readings.rs b/tests/month_readings.rs new file mode 100644 index 0000000..567e712 --- /dev/null +++ b/tests/month_readings.rs @@ -0,0 +1,214 @@ +//! 空白を挟む暦月の読みと、原入力のバイト範囲。 +#![cfg(feature = "build")] + +use hasami::dict::{DictBuilder, DictEntry}; +use hasami::{Analyzer, Dictionary, Token}; + +fn dictionary() -> Dictionary { + let mut builder = DictBuilder::new(); + for (glyphs, reading, pronunciation) in [ + ("00〇", "ゼロ", "ゼロ"), + ("11一", "イチ", "イチ"), + ("22二", "ニ", "ニ"), + ("33三", "サン", "サン"), + ("44四", "ヨン", "ヨン"), + ("55五", "ゴ", "ゴ"), + ("66六", "ロク", "ロク"), + ("77七", "ナナ", "ナナ"), + ("88八", "ハチ", "ハチ"), + ("99九", "キュウ", "キュー"), + ("十", "ジュウ", "ジュー"), + ] { + for glyph in glyphs.chars() { + builder.add_entry(DictEntry { + surface: glyph.to_string().into(), + pos: "名詞,数,*,*".into(), + reading: reading.into(), + pronunciation: pronunciation.into(), + cost: -100, + ..Default::default() + }); + } + } + for (surface, pos, reading) in [ + ("月", "名詞,一般,*,*", "ツキ"), + ("か月", "名詞,接尾,助数詞,*", "カゲツ"), + ("月曜日", "名詞,副詞可能,*,*", "ゲツヨウビ"), + ("曜", "名詞,一般,*,*", "ヨウ"), + ("間", "名詞,接尾,一般,*", "カン"), + ("本", "名詞,接尾,助数詞,*", "ホン"), + ("前", "名詞,副詞可能,*,*", "マエ"), + ] { + builder.add_entry(DictEntry { + surface: surface.into(), + pos: pos.into(), + reading: reading.into(), + pronunciation: reading.into(), + cost: -100, + ..Default::default() + }); + } + builder.build().unwrap() +} + +fn readings(tokens: &[Token]) -> (String, String) { + ( + tokens.iter().map(|t| t.reading.as_ref()).collect(), + tokens.iter().map(|t| t.pronunciation.as_ref()).collect(), + ) +} + +const MONTHS: [(&str, &str, &str, &str, &str); 12] = [ + ("1", "1", "一", "イチ", "イチ"), + ("2", "2", "二", "ニ", "ニ"), + ("3", "3", "三", "サン", "サン"), + ("4", "4", "四", "シ", "シ"), + ("5", "5", "五", "ゴ", "ゴ"), + ("6", "6", "六", "ロク", "ロク"), + ("7", "7", "七", "シチ", "シチ"), + ("8", "8", "八", "ハチ", "ハチ"), + ("9", "9", "九", "ク", "ク"), + ("10", "10", "十", "ジュウ", "ジュー"), + ("11", "11", "十一", "ジュウイチ", "ジューイチ"), + ("12", "12", "十二", "ジュウニ", "ジューニ"), +]; + +#[test] +fn spaced_months_cover_all_months_and_notations() { + let mut analyzer = Analyzer::from_dict(dictionary()); + for (ascii, wide, kanji, reading, pronunciation) in MONTHS { + for number in [ascii, wide, kanji] { + for gap in [" ", "\t", " \t "] { + let text = format!("{number}{gap}月"); + let out = analyzer.try_tokenize(&text).unwrap(); + assert_eq!( + readings(&out), + (format!("{reading}ガツ"), format!("{pronunciation}ガツ")), + "{text:?}: {out:?}" + ); + for token in &out { + assert_eq!(&text[token.start..token.end], token.surface.as_ref()); + } + assert_eq!(out.last().unwrap().start, number.len() + gap.len()); + } + } + } +} + +#[test] +fn non_calendar_contexts_keep_the_moon_reading() { + let mut analyzer = Analyzer::from_dict(dictionary()); + for text in [ + "月がきれい", + "3か月", + "3 か月", + "9 月曜", + "9 月曜日", + "9 月間", + "四 月間", + "9\n月", + "9\r\n月", + "9 月", + "0 月", + "13 月", + "109 月", + "十九 月", + "09 月", + "09 月", + "一二 月", + "十 二 月", + "19 月", + "v9 月", + "v 9 月", + "v (9 月", + "1.9 月", + "1/9 月", + "1.9 月", + "九.九 月", + "1e9 月", + "9x 月", + "99九 月", + "3 本", + ] { + let out = analyzer.try_tokenize(text).unwrap(); + assert!( + out.iter().filter(|t| t.surface.as_ref() == "月").all(|t| { + t.reading.as_ref() == "ツキ" && t.pronunciation.as_ref() == "ツキ" + }), + "{text:?}: {out:?}" + ); + } +} + +#[test] +fn month_readings_preserve_offsets_previous_output_and_the_dictionary_cache() { + let dict = dictionary(); + let mut analyzer = Analyzer::from_shared(std::sync::Arc::new(dict)); + for _ in 0..2 { + for text in [ + "9 月の利用実績", + "(9 月)", + "2026年9 月1日", + "前。9 月", + "9 月、7 月", + ] { + let out = analyzer.try_tokenize(text).unwrap(); + assert!( + out.iter() + .any(|t| t.surface.as_ref() == "月" && t.reading.as_ref() == "ガツ") + ); + for token in &out { + assert_eq!(&text[token.start..token.end], token.surface.as_ref()); + } + } + assert_eq!( + readings(&analyzer.tokenize("月")), + ("ツキ".into(), "ツキ".into()) + ); + assert_eq!( + readings(&analyzer.tokenize("九本")), + ("キュウホン".into(), "キューホン".into()) + ); + } + let mut workspace = hasami::lattice::LatticeWorkspace::new(); + let mut out = workspace.tokenize("九", analyzer.dictionary()).unwrap(); + let previous = format!("{:?}", out[0]); + workspace + .tokenize_into("12 月", analyzer.dictionary(), 91, &mut out) + .unwrap(); + assert_eq!(format!("{:?}", out[0]), previous); + assert_eq!( + readings(&out[1..]), + ("ジュウニガツ".into(), "ジューニガツ".into()) + ); + for t in &out[1..] { + assert_eq!(&"12 月"[t.start - 91..t.end - 91], t.surface.as_ref()); + } + let mut out = workspace.tokenize("九", analyzer.dictionary()).unwrap(); + workspace + .tokenize_into(" 月", analyzer.dictionary(), 3, &mut out) + .unwrap(); + assert_eq!(readings(&out), ("キュウツキ".into(), "キューツキ".into())); +} + +#[test] +#[ignore = "配布辞書が必要"] +fn distributed_spaced_month_readings() { + for name in ["ipadic", "ipadic-neologd", "ipadic-neologd-sudachi"] { + let mut analyzer = Analyzer::load(format!("dict/{name}.hsd")).unwrap(); + for (ascii, wide, kanji, reading, pronunciation) in MONTHS { + for number in [ascii, wide, kanji] { + let text = format!("{number} 月"); + let out = analyzer.try_tokenize(&text).unwrap(); + assert_eq!( + readings(&out), + (format!("{reading}ガツ"), format!("{pronunciation}ガツ")), + "{name}: {text}: {out:?}" + ); + for token in &out { + assert_eq!(&text[token.start..token.end], token.surface.as_ref()); + } + } + } + } +}