diff --git a/CHANGELOG.md b/CHANGELOG.md index 643a76cc..92c4a41b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,37 @@ GitHub Release 的发布说明；版本与发布流程见 [docs/RELEASE.md](docs ## [Unreleased] +## [0.0.3-beta.1] - 2026-10-04 + +第三个公开测试版：空格滑动移光标、密码管理器的自动填充直接出现在键盘上、选词后的表情联想、26 键字母键上的数字和符号（上滑输入），以及语音输入的去语气词和“标点用空格代替”。仍是测试版，欢迎在 Issues 反馈问题。从 0.0.2-beta.1 覆盖安装即可，用户数据保留；升级后首次启动会重新部署词典，这段时间候选由备用词库提供。 + +### 亮点 +- 空格左右滑动移动光标；滑动期间底行其他按键被锁定（变灰、不响应），拇指漂移或第二根手指不会误触。正在输入拼音时，滑动移动的是拼音预编辑里的光标。 +- 自动填充（Android 11 及以上）：密码管理器和系统自动填充的账号、验证码直接显示在工具栏位置，点一下就填入。键盘只负责展示，拿不到填入的内容。 +- 选词后，联想栏最前面先给出相关表情（开心 → 😊），词表内置在应用里，不联网。 +- 语音输入：去掉“嗯”“呃”等口头停顿；可选把语音里的标点写成空格。 +- 26 键字母键右上角印着数字和符号，上滑（或长按）直接输入，不用切换键盘，也不多占一行。 + +### 新增 +- 空格键横向滑动 18dp 进入光标模式，之后每 12dp 移动一格，松手结束；长按空格说话和单击空格不受影响。九键、数字键盘的空格同样支持。 +- 自动填充：工具栏位置出现账号 / 验证码条带，最左边的“‹”回到工具栏；填入或离开输入框后自动收起。系统自动填充服务需要在“设置 → 密码和账号 → 自动填充服务”里选择。 +- 表情联想：提交一个词（或以该词结尾的短语）后，联想栏最前面最多给出 3 个相关表情，覆盖约 500 个常用词；单字词只在整词相同时匹配。可在“设置 → 智能输入 → 表情联想”关闭。 +- 字母键上的数字和符号：中文、英文 26 键的每个字母键右上角都有一个提示，上滑或长按输入它。第一排是 1–0；其余按键是标点：`~ ! @ # $ % & * ?` 在 a–l 行，`( ) - _ : ; /` 在 z–m 行。中文模式输入全角形式（！ ￥ ？ （ ） ： ；），英文模式输入半角。按键布局和键盘高度不变。可在“设置 → 按键与输入 → 数字和符号提示”关闭提示；“上滑输入数字”开关同时控制九键和 26 键的上滑。 +- 语音输入设置： + - “语音去语气词”（默认开启）：去掉“嗯”“呃”和英文 um / uh / er / hmm；“额”只在被单独识别为一个词时去掉，“额度”“金额”不受影响；整句只有一个语气词时原样保留。 + - “标点用空格代替”（默认关闭）：语音里的逗号、句号、问号等写成一个空格，结尾的标点直接去掉；括号和 3.5、a.b 这类内容不变。 +- 上述四个开关随用户数据一起导出和导入。 + +### 变更 +- 26 键中文模式原来只有第一排有数字提示（长按输入）；现在整个键盘都有数字和符号提示，上滑也能输入，英文 26 键同样有。 +- 工具栏位置现在有四种互斥状态：图标、自动填充、联想、撤销提示；联想和撤销提示优先于自动填充。 + +### 已知限制 +- 自动填充条带的外观由系统渲染，使用提供者的默认样式，深色主题下是浅色底的圆角条。 +- 自动填充需要提供者支持内嵌建议（Android 11+）；不支持时系统仍然使用原来的下拉菜单。 +- 首次安装后第一次启动输入法进程时，系统发出的第一个自动填充请求可能没有带窗口信息，条带要到切换一次输入框才出现。 +- 空格滑动光标在密码框里不移动光标（密码框不允许输入法模拟方向键）。 + ## [0.0.2-beta.1] - 2026-10-03 第二个公开测试版，重点是游戏场景、语音词表和九键手势。仍是测试版，请不要把它作为唯一的输入法；欢迎在 Issues 反馈问题。从 0.0.1-beta.1 覆盖安装即可，用户数据保留；升级后首次启动会重新部署词典，这段时间候选由备用词库提供。 diff --git a/README.md b/README.md index 1fe56671..06233c5f 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,12 @@ openIME 是一款独立的 Android 系统输入法。拼音候选、用户词库 - **拼音**：全拼、简拼、手动分词、候选展开、用户词库学习、简繁转换。 引擎为 librime，内置约 90 万条 Rime Ice 词典记录；首次部署完整词典期间，高频快速词库先提供候选。 - **九键**：输入时左栏列出下一个字的拼音，一个字选一个音节，选定后自动移到下一个字。 -- **语音输入**：长按空格说话，松手后识别并上屏；使用内置的中英双语模型，不联网。 +- **语音输入**：长按空格说话，松手后识别并上屏；使用内置的中英双语模型，不联网；可去掉“嗯”“呃”等语气词，也可把标点写成空格。 - **语音词表**：内置科技、应用、游戏词表，可导入自己的文本词表；识别后把同音词改成词表写法，不联网。 -- **手势**：删除键上滑清空，5 秒内下滑可撤回。 +- **手势**：删除键上滑清空，5 秒内下滑可撤回；空格左右滑动移动光标，滑动时底行其他按键锁定。 +- **表情联想**：选词后联想栏先给出相关表情（开心 → 😊），词表内置，不联网。 +- **自动填充**：Android 11+ 上，密码管理器的账号、验证码直接显示在键盘工具栏位置。 +- **数字和符号**：26 键字母键右上角印着数字和符号，上滑或长按直接输入。 - **适配**：横竖屏、平板与折叠屏、深色模式、大字号；终端、远程桌面、游戏等原始按键输入框； 外接键盘可直接打拼音。详见 [兼容性说明](docs/COMPATIBILITY.md)。 - **自我保护**：连续崩溃或卡死后自动进入安全模式，保证仍然可以打字； diff --git a/VERSION b/VERSION index d20da904..4c98a1d8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.0.2-beta.1 +0.0.3-beta.1 diff --git a/app/src/androidTest/java/llc/slacker/openime/SpaceCursorAndLetterHintsInstrumentedTest.kt b/app/src/androidTest/java/llc/slacker/openime/SpaceCursorAndLetterHintsInstrumentedTest.kt new file mode 100644 index 00000000..fac0c870 --- /dev/null +++ b/app/src/androidTest/java/llc/slacker/openime/SpaceCursorAndLetterHintsInstrumentedTest.kt @@ -0,0 +1,462 @@ +package llc.slacker.openime + +import android.os.SystemClock +import android.view.MotionEvent +import android.view.View +import android.view.ViewGroup +import android.widget.EditText +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import java.lang.reflect.Proxy +import llc.slacker.openime.core.KeyboardMode +import llc.slacker.openime.core.Panel +import llc.slacker.openime.data.ImeSettingsRepository +import llc.slacker.openime.keyboard.ImeKeyboardView +import llc.slacker.openime.voice.VoiceModelLifecycleState +import llc.slacker.openime.widget.ImeKeyView +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +/** + * The space-bar cursor drag (with the bottom row locked while it runs), the + * digits and symbols printed on the letter keys (typed by swipe up or long + * press), and the settings toggles added in 0.0.3-beta.1. + */ +@RunWith(AndroidJUnit4::class) +class SpaceCursorAndLetterHintsInstrumentedTest { + private val context = InstrumentationRegistry.getInstrumentation().targetContext + + private class Recorder { + val characters = mutableListOf() + val textEdits = mutableListOf() + var spaces = 0 + val listener = Proxy.newProxyInstance( + ImeKeyboardView.Listener::class.java.classLoader, + arrayOf(ImeKeyboardView.Listener::class.java), + ) { _, method, args -> + when (method.name) { + "voiceModelState" -> VoiceModelLifecycleState.COLD + "onCharacter" -> { characters += args!![0] as String; null } + "onTextEdit" -> { textEdits += args!![0] as String; null } + "onSpace" -> { spaces++; null } + else -> if (method.returnType == java.lang.Boolean.TYPE) false else null + } + } as ImeKeyboardView.Listener + } + + @After + fun restoreSettings() { + ImeSettingsRepository.saveLetterHints(context, true) + ImeSettingsRepository.saveSwipeUpDigits(context, true) + ImeSettingsRepository.saveEmojiAssociation(context, true) + ImeSettingsRepository.saveVoiceStripFillers(context, true) + ImeSettingsRepository.saveVoicePunctuationAsSpace(context, false) + } + + private fun withKeyboard( + test: (DirectActivityHarness, Recorder, ImeKeyboardView) -> Any?, + ) { + DirectActivityHarness(DebugKeyboardActivity::class.java).use { harness -> + harness.launch() + val recorder = Recorder() + val keyboard = harness.awaitMain { activity -> + ImeKeyboardView(activity, recorder.listener).also { + activity.findViewById(android.R.id.content).addView(it) + } + } + try { + harness.awaitMain { if (keyboard.width > 0) true else null } + test(harness, recorder, keyboard) + } finally { + harness.awaitMain { + keyboard.shutdown() + (keyboard.parent as? ViewGroup)?.removeView(keyboard) + true + } + } + } + } + + private fun settled( + harness: DirectActivityHarness, + keyboard: ImeKeyboardView, + tag: String, + ): View = harness.awaitMain { keyboard.findViewWithTag(tag)?.takeIf { it.width > 0 } } + + private fun send(view: View, action: Int, downTime: Long, x: Float, y: Float) { + val event = MotionEvent.obtain(downTime, SystemClock.uptimeMillis(), action, x, y, 0) + try { + view.dispatchTouchEvent(event) + } finally { + event.recycle() + } + } + + private fun tap(harness: DirectActivityHarness, view: View) { + val downTime = SystemClock.uptimeMillis() + val sent = harness.awaitMain { + send(view, MotionEvent.ACTION_DOWN, downTime, view.width / 2f, view.height / 2f) + send(view, MotionEvent.ACTION_UP, downTime, view.width / 2f, view.height / 2f) + true + } + check(sent) + // View posts the click after ACTION_UP; let it run before the caller looks. + SystemClock.sleep(150) + check(harness.awaitMain { true }) + } + + private class Drag(val view: View, val downTime: Long, val y: Float, var x: Float) + + private fun dragStart(harness: DirectActivityHarness, view: View): Drag { + val drag = Drag(view, SystemClock.uptimeMillis(), view.height / 2f, view.width / 2f) + harness.awaitMain { + send(view, MotionEvent.ACTION_DOWN, drag.downTime, drag.x, drag.y) + true + } + return drag + } + + /** Moves the pointer [dxDp] to the right (negative: left), in small steps like a finger. */ + private fun dragBy(harness: DirectActivityHarness, drag: Drag, dxDp: Int) { + val density = context.resources.displayMetrics.density + val steps = 20 + val stepPx = dxDp * density / steps + repeat(steps) { + drag.x += stepPx + val moved = harness.awaitMain { + send(drag.view, MotionEvent.ACTION_MOVE, drag.downTime, drag.x, drag.y) + true + } + check(moved) + } + } + + private fun dragEnd(harness: DirectActivityHarness, drag: Drag) { + harness.awaitMain { + send(drag.view, MotionEvent.ACTION_UP, drag.downTime, drag.x, drag.y) + true + } + } + + private fun neighbours(space: View): List { + val row = space.parent as ViewGroup + return (0 until row.childCount).map(row::getChildAt).filter { it !== space }.filterIsInstance() + } + + // ----- space-bar cursor drag ------------------------------------------------- + + @Test + fun draggingTheSpaceBarMovesTheCursorOneStepPerTwelveDp() = withKeyboard { harness, recorder, keyboard -> + harness.awaitMain { keyboard.setMode(KeyboardMode.PINYIN_26); true } + val space = settled(harness, keyboard, "key-space") + val drag = dragStart(harness, space) + dragBy(harness, drag, 18 + 12 * 4) + dragEnd(harness, drag) + assertTrue("expected right steps, got ${recorder.textEdits}", recorder.textEdits.size in 3..5) + assertTrue(recorder.textEdits.all { it == "right" }) + assertEquals("a drag is not a space", 0, recorder.spaces) + + val back = dragStart(harness, space) + dragBy(harness, back, -(18 + 12 * 3 + 6)) + dragEnd(harness, back) + assertTrue(recorder.textEdits.takeLast(2).all { it == "left" }) + assertEquals(0, recorder.spaces) + } + + @Test + fun theRestOfTheBottomRowIsLockedWhileDraggingAndFreeAfterwards() = withKeyboard { harness, recorder, keyboard -> + harness.awaitMain { keyboard.setMode(KeyboardMode.PINYIN_26); true } + val space = settled(harness, keyboard, "key-space") + val others = harness.awaitMain { neighbours(space).takeIf { it.size >= 3 } } + harness.awaitMain { assertTrue(others.none { it.touchLocked }); true } + + val drag = dragStart(harness, space) + dragBy(harness, drag, 60) + harness.awaitMain { + assertTrue("every neighbour must be locked", others.all { it.touchLocked }) + assertTrue("locked keys are drawn dimmed", others.all { it.alpha < 0.5f }) + true + } + + // A tap on a locked key (a second finger) does nothing. + val punctuation = settled(harness, keyboard, "key-punctuation") + tap(harness, punctuation) + assertTrue("locked key must not type: ${recorder.characters}", recorder.characters.isEmpty()) + + dragEnd(harness, drag) + harness.awaitMain { + assertTrue(others.none { it.touchLocked }) + assertTrue(others.all { it.alpha == 1f }) + true + } + tap(harness, punctuation) + assertEquals("the key works again after the drag", 1, recorder.characters.size) + } + + @Test + fun aPlainTapOnSpaceStaysASpaceAndLocksNothing() = withKeyboard { harness, recorder, keyboard -> + harness.awaitMain { keyboard.setMode(KeyboardMode.PINYIN_26); true } + val space = settled(harness, keyboard, "key-space") + val others = harness.awaitMain { neighbours(space) } + tap(harness, space) + assertEquals(1, recorder.spaces) + assertTrue(recorder.textEdits.isEmpty()) + assertTrue(others.none { it.touchLocked }) + } + + @Test + fun aSmallWobbleIsStillATapNotACursorDrag() = withKeyboard { harness, recorder, keyboard -> + harness.awaitMain { keyboard.setMode(KeyboardMode.PINYIN_26); true } + val space = settled(harness, keyboard, "key-space") + val others = harness.awaitMain { neighbours(space) } + val drag = dragStart(harness, space) + dragBy(harness, drag, 8) + harness.awaitMain { assertTrue(others.none { it.touchLocked }); true } + dragEnd(harness, drag) + assertTrue(recorder.textEdits.isEmpty()) + } + + @Test + fun theLockAlsoWorksOnTheNineKeyAndDigitKeyboards() = withKeyboard { harness, recorder, keyboard -> + listOf(KeyboardMode.PINYIN_9, KeyboardMode.DIGITS).forEach { mode -> + harness.awaitMain { keyboard.setMode(mode); true } + val space = settled(harness, keyboard, "key-space") + val others = harness.awaitMain { neighbours(space).takeIf { it.isNotEmpty() } } + recorder.textEdits.clear() + val drag = dragStart(harness, space) + dragBy(harness, drag, 50) + harness.awaitMain { assertTrue("$mode", others.all { it.touchLocked }); true } + dragEnd(harness, drag) + harness.awaitMain { assertTrue("$mode", others.none { it.touchLocked }); true } + assertTrue("$mode: ${recorder.textEdits}", recorder.textEdits.isNotEmpty()) + } + } + + @Test + fun whileComposingTheDragMovesThePinyinCursorNotTheTargetText() = withKeyboard { harness, recorder, keyboard -> + harness.awaitMain { keyboard.setMode(KeyboardMode.PINYIN_26); true } + tap(harness, settled(harness, keyboard, "key:n")) + tap(harness, settled(harness, keyboard, "key:i")) + tap(harness, settled(harness, keyboard, "key:h")) + val editor = harness.awaitMain { + keyboard.findViewWithTag("pinyin-composition-editor")?.takeIf { it.text.length == 3 } + } + val before = harness.awaitMain { editor.selectionStart } + val space = settled(harness, keyboard, "key-space") + val drag = dragStart(harness, space) + dragBy(harness, drag, -(18 + 12 * 2)) + dragEnd(harness, drag) + val after = harness.awaitMain { editor.selectionStart } + assertTrue("pinyin cursor $before -> $after", after < before) + assertTrue("the target text must not move: ${recorder.textEdits}", recorder.textEdits.isEmpty()) + assertEquals("the pinyin itself is untouched", "nih", editor.text.toString()) + } + + @Test + fun theLockIsReleasedIfTheKeyboardIsRebuiltMidDrag() = withKeyboard { harness, _, keyboard -> + harness.awaitMain { keyboard.setMode(KeyboardMode.PINYIN_26); true } + val space = settled(harness, keyboard, "key-space") + val drag = dragStart(harness, space) + dragBy(harness, drag, 60) + // Android cancels the gesture (for example the window is hidden). + harness.awaitMain { + send(space, MotionEvent.ACTION_CANCEL, drag.downTime, drag.x, drag.y) + true + } + harness.awaitMain { + assertTrue(neighbours(space).none { it.touchLocked }) + true + } + } + + // ----- digits and symbols on the letter keys ------------------------------------ + + private fun texts(view: View): List { + val found = mutableListOf() + fun walk(v: View) { + if (v is android.widget.TextView && v.text.isNotEmpty()) found += v.text.toString() + if (v is ViewGroup) for (i in 0 until v.childCount) walk(v.getChildAt(i)) + } + walk(view) + return found + } + + private fun swipeUp(harness: DirectActivityHarness, view: View) { + val downTime = SystemClock.uptimeMillis() + val x = view.width / 2f + val y = view.height / 2f + val distance = view.height * 1.2f + val ok = harness.awaitMain { + send(view, MotionEvent.ACTION_DOWN, downTime, x, y) + send(view, MotionEvent.ACTION_MOVE, downTime, x, y - distance / 2) + send(view, MotionEvent.ACTION_MOVE, downTime, x, y - distance) + send(view, MotionEvent.ACTION_UP, downTime, x, y - distance) + true + } + check(ok) + SystemClock.sleep(150) + check(harness.awaitMain { true }) + } + + private val chineseHints = mapOf( + 'q' to "1", 'w' to "2", 'e' to "3", 'r' to "4", 't' to "5", 'y' to "6", 'u' to "7", 'i' to "8", + 'o' to "9", 'p' to "0", 'a' to "~", 's' to "！", 'd' to "@", 'f' to "#", 'g' to "￥", 'h' to "%", + 'j' to "&", 'k' to "*", 'l' to "？", 'z' to "（", 'x' to "）", 'c' to "-", 'v' to "_", + 'b' to "：", 'n' to "；", 'm' to "/", + ) + private val englishHints = chineseHints + mapOf( + 's' to "!", 'g' to "$", 'l' to "?", 'z' to "(", 'x' to ")", 'b' to ":", 'n' to ";", + ) + + @Test + fun noExtraRowIsAddedAndTheKeyboardKeepsItsFourRows() = withKeyboard { harness, _, keyboard -> + harness.awaitMain { keyboard.setMode(KeyboardMode.PINYIN_26); true } + settled(harness, keyboard, "key-space") + harness.awaitMain { + assertNull(keyboard.findViewWithTag("key-row-numbers")) + val body = (keyboard.findViewWithTag("key-space").parent as View).parent as ViewGroup + assertEquals("four key rows", 4, body.childCount) + true + } + } + + @Test + fun everyLetterKeyShowsItsHintInBothLanguages() = withKeyboard { harness, _, keyboard -> + mapOf(KeyboardMode.PINYIN_26 to chineseHints, KeyboardMode.ENGLISH_26 to englishHints) + .forEach { (mode, expected) -> + harness.awaitMain { keyboard.setMode(KeyboardMode.DIGITS); keyboard.setMode(mode); true } + settled(harness, keyboard, "key:q") + val checked = harness.awaitMain { + expected.forEach { (letter, hint) -> + val key = keyboard.findViewWithTag("key:$letter") + assertNotNull("$mode $letter", key) + assertTrue("$mode $letter shows $hint in ${texts(key)}", hint in texts(key)) + } + true + } + check(checked) + } + } + + @Test + fun swipeUpTypesTheHintOnceAndStartsNoComposition() = withKeyboard { harness, recorder, keyboard -> + listOf(KeyboardMode.PINYIN_26 to chineseHints, KeyboardMode.ENGLISH_26 to englishHints) + .forEach { (mode, hints) -> + harness.awaitMain { keyboard.setMode(KeyboardMode.DIGITS); keyboard.setMode(mode); true } + recorder.characters.clear() + val letters = listOf('q', 'p', 'a', 's', 'g', 'l', 'z', 'm') + letters.forEach { swipeUp(harness, settled(harness, keyboard, "key:$it"), ) } + assertEquals(mode.name, letters.map { hints.getValue(it) }, recorder.characters.toList()) + val composing = harness.awaitMain { + keyboard.findViewWithTag("pinyin-composition-editor")?.text?.toString().orEmpty() + } + assertEquals("a swipe must not also start a composition", "", composing) + } + } + + @Test + fun longPressTypesTheSameHint() = withKeyboard { harness, recorder, keyboard -> + harness.awaitMain { keyboard.setMode(KeyboardMode.PINYIN_26); true } + listOf('w', 'd', 'k', 'v').forEach { letter -> + val key = settled(harness, keyboard, "key:$letter") + assertTrue(harness.awaitMain { key.performLongClick() }) + } + assertEquals(listOf("2", "@", "*", "_"), recorder.characters.toList()) + } + + @Test + fun aPlainTapStillTypesTheLetter() = withKeyboard { harness, recorder, keyboard -> + harness.awaitMain { keyboard.setMode(KeyboardMode.ENGLISH_26); true } + tap(harness, settled(harness, keyboard, "key:q")) + assertTrue("a tap types the letter, not the hint: ${recorder.characters}", "1" !in recorder.characters) + } + + @Test + fun theSwipeSwitchTurnsOffSwipeButNotTheHints() = withKeyboard { harness, recorder, keyboard -> + ImeSettingsRepository.saveSwipeUpDigits(context, false) + harness.awaitMain { keyboard.setMode(KeyboardMode.PINYIN_26); true } + val q = settled(harness, keyboard, "key:q") + swipeUp(harness, q) + assertTrue("swipe is off: ${recorder.characters}", recorder.characters.isEmpty()) + harness.awaitMain { assertTrue("1" in texts(q)); true } + assertTrue(harness.awaitMain { q.performLongClick() }) + assertEquals("long press still works", listOf("1"), recorder.characters.toList()) + } + + @Test + fun turningTheHintsOffRemovesThemAndTheirGestures() = withKeyboard { harness, recorder, keyboard -> + ImeSettingsRepository.saveLetterHints(context, false) + harness.awaitMain { keyboard.setMode(KeyboardMode.DIGITS); keyboard.setMode(KeyboardMode.PINYIN_26); true } + val q = settled(harness, keyboard, "key:q") + harness.awaitMain { + assertEquals(listOf("Q"), texts(q)) + assertEquals(listOf("S"), texts(keyboard.findViewWithTag("key:s"))) + true + } + swipeUp(harness, q) + harness.awaitMain { q.performLongClick(); true } + assertTrue("no hint, no digit: ${recorder.characters}", recorder.characters.isEmpty()) + } + + @Test + fun turningTheHintsOffFromSettingsRebuildsTheKeyboard() = withKeyboard { harness, _, keyboard -> + harness.awaitMain { keyboard.setMode(KeyboardMode.PINYIN_26); true } + settled(harness, keyboard, "key-space") + harness.awaitMain { keyboard.showPanel(Panel.SETTINGS); true } + val flipped = harness.awaitMain { + val toggle = keyboard.findToggle("数字和符号提示") + assertEquals("数字和符号提示，已开启", toggle.contentDescription.toString()) + assertTrue(toggle.performClick()) + true + } + check(flipped) + assertFalse(ImeSettingsRepository.loadLetterHints(context)) + harness.awaitMain { keyboard.closePanelToKeyboard(); true } + val q = settled(harness, keyboard, "key:q") + harness.awaitMain { assertEquals(listOf("Q"), texts(q)); true } + } + + // ----- new settings toggles ---------------------------------------------------- + + @Test + fun theNewTogglesAreListedAndPersist() = withKeyboard { harness, _, keyboard -> + harness.awaitMain { keyboard.showPanel(Panel.SETTINGS); true } + val expected = mapOf( + "数字和符号提示" to ("开启" to { ImeSettingsRepository.loadLetterHints(context) }), + "表情联想" to ("开启" to { ImeSettingsRepository.loadEmojiAssociation(context) }), + "语音去语气词" to ("开启" to { ImeSettingsRepository.loadVoiceStripFillers(context) }), + "标点用空格代替" to ("关闭" to { ImeSettingsRepository.loadVoicePunctuationAsSpace(context) }), + ) + expected.forEach { (label, state) -> + val (initial, read) = state + val flipped = harness.awaitMain { + val toggle = keyboard.findToggle(label) + assertEquals("$label，已$initial", toggle.contentDescription.toString()) + val before = read() + assertTrue(toggle.performClick()) + assertEquals("$label must flip", !before, read()) + assertTrue(toggle.performClick()) + assertEquals("$label must flip back", before, read()) + true + } + check(flipped) + } + } + + private fun View.findToggle(label: String): View { + var found: View? = null + fun walk(view: View) { + if (found != null) return + if (view.tag == "toggle" && view.contentDescription?.startsWith("$label，") == true) found = view + if (view is ViewGroup) for (i in 0 until view.childCount) walk(view.getChildAt(i)) + } + walk(this) + return found ?: error("no toggle for $label") + } +} diff --git a/app/src/debug/AndroidManifest.xml b/app/src/debug/AndroidManifest.xml index 007264e4..a7207417 100644 --- a/app/src/debug/AndroidManifest.xml +++ b/app/src/debug/AndroidManifest.xml @@ -23,6 +23,22 @@ android:name=".LifecycleTestActivity" android:exported="true" android:permission="android.permission.DUMP" /> + + + + + + + password.requestFocus() + "plain" -> plain.requestFocus() + else -> username.requestFocus() + } + } +} diff --git a/app/src/debug/java/llc/slacker/openime/TestAutofillService.kt b/app/src/debug/java/llc/slacker/openime/TestAutofillService.kt new file mode 100644 index 00000000..2b837734 --- /dev/null +++ b/app/src/debug/java/llc/slacker/openime/TestAutofillService.kt @@ -0,0 +1,97 @@ +package llc.slacker.openime + +import android.annotation.TargetApi +import android.app.PendingIntent +import android.app.assist.AssistStructure +import android.app.slice.Slice +import android.app.slice.SliceSpec +import android.content.Intent +import android.net.Uri +import android.os.CancellationSignal +import android.service.autofill.AutofillService +import android.service.autofill.Dataset +import android.service.autofill.FillCallback +import android.service.autofill.FillContext +import android.service.autofill.FillRequest +import android.service.autofill.FillResponse +import android.service.autofill.InlinePresentation +import android.service.autofill.SaveCallback +import android.service.autofill.SaveRequest +import android.view.autofill.AutofillId +import android.view.autofill.AutofillValue +import android.widget.RemoteViews +import android.widget.inline.InlinePresentationSpec + +/** + * Debug-only autofill provider for the end-to-end check of the keyboard's + * inline suggestion strip. It offers two canned accounts for any field whose + * autofill hint is `username`, and a canned password for `password`. It builds + * the chip slice by hand in the androidx.autofill v1 layout so the app needs no + * extra dependency. + */ +@TargetApi(30) +class TestAutofillService : AutofillService() { + override fun onFillRequest( + request: FillRequest, + cancellationSignal: CancellationSignal, + callback: FillCallback, + ) { + val fields = mutableListOf>() + request.fillContexts.lastOrNull()?.let { collect(it, fields) } + val specs = request.inlineSuggestionsRequest?.inlinePresentationSpecs.orEmpty() + val response = FillResponse.Builder() + var datasets = 0 + fields.forEach { (id, hint) -> + val values = if (hint == "password") PASSWORDS else USERNAMES + values.forEachIndexed { index, (label, value) -> + val spec = specs.getOrNull(index) ?: specs.lastOrNull() + val menu = RemoteViews("android", android.R.layout.simple_list_item_1).apply { + setTextViewText(android.R.id.text1, label) + } + val dataset = Dataset.Builder(menu).setValue(id, AutofillValue.forText(value), menu) + if (spec != null) dataset.setInlinePresentation(chip(label, spec)) + response.addDataset(dataset.build()) + datasets++ + } + } + if (datasets == 0) callback.onSuccess(null) else callback.onSuccess(response.build()) + } + + override fun onSaveRequest(request: SaveRequest, callback: SaveCallback) = callback.onSuccess() + + private fun collect(context: FillContext, out: MutableList>) { + val structure = context.structure + for (i in 0 until structure.windowNodeCount) walk(structure.getWindowNodeAt(i).rootViewNode, out) + } + + private fun walk(node: AssistStructure.ViewNode, out: MutableList>) { + val hint = node.autofillHints?.firstOrNull { it == "username" || it == "password" } + val id = node.autofillId + if (hint != null && id != null) out += id to hint + for (i in 0 until node.childCount) walk(node.getChildAt(i), out) + } + + private fun chip(title: String, spec: InlinePresentationSpec): InlinePresentation { + val attribution = PendingIntent.getService( + this, + 0, + Intent(this, TestAutofillService::class.java), + PendingIntent.FLAG_IMMUTABLE, + ) + val sliceBuilder = Slice.Builder(Uri.parse("inline.slice"), SliceSpec(V1, 1)) + sliceBuilder.addText(title, null, listOf("inline_title")) + sliceBuilder.addAction( + attribution, + Slice.Builder(sliceBuilder).addHints(listOf("inline_attribution")).build(), + null, + ) + sliceBuilder.addText(title, null, listOf("inline_content_description")) + return InlinePresentation(sliceBuilder.build(), spec, false) + } + + private companion object { + const val V1 = "androidx.autofill.inline.ui.version:v1" + val USERNAMES = listOf("工作账号" to "work@openime.dev", "个人账号" to "me@openime.dev") + val PASSWORDS = listOf("测试密码" to "openime-test-secret") + } +} diff --git a/app/src/debug/res/xml/test_autofill_service.xml b/app/src/debug/res/xml/test_autofill_service.xml new file mode 100644 index 00000000..43747ded --- /dev/null +++ b/app/src/debug/res/xml/test_autofill_service.xml @@ -0,0 +1,3 @@ + + diff --git a/app/src/main/assets/emoji/keywords.tsv b/app/src/main/assets/emoji/keywords.tsv new file mode 100644 index 00000000..b95b4709 --- /dev/null +++ b/app/src/main/assets/emoji/keywords.tsv @@ -0,0 +1,505 @@ +# 表情联想词表：每行 “词表情 表情 …”，# 开头为注释。 +# 提交一个词（或以该词结尾的短语）后，联想栏最前面会出现对应表情，最多 3 个。 +# 单字词只在整词完全相同时匹配；两字及以上的词也匹配短语结尾。 +开心 😊 😄 😁 +高兴 😄 😊 🥳 +快乐 😄 🥳 😊 +哈哈 😄 😂 🤣 +哈哈哈 🤣 😂 😆 +嘿嘿 😁 😏 😄 +笑 😄 😂 🤣 +笑死 🤣 😂 💀 +好笑 😂 🤣 😆 +搞笑 🤣 😂 🤪 +微笑 🙂 😊 😄 +大笑 😆 😂 🤣 +偷笑 🤭 😏 🙊 +憨笑 😁 🤪 😄 +哭 😭 😢 🥲 +哭了 😭 😢 😿 +大哭 😭 😢 😿 +难过 😢 😞 😔 +伤心 😢 😭 💔 +悲伤 😢 😔 🥺 +失望 😞 😔 😟 +委屈 🥺 😢 😔 +心碎 💔 😭 😢 +可怜 🥺 😿 😢 +感动 🥹 😭 🥺 +想哭 🥺 😢 😭 +生气 😡 😠 🤬 +愤怒 😡 🤬 😤 +气死 😤 😡 🤬 +烦 😤 😩 😫 +烦死 😫 😩 😤 +无语 😑 🙄 😶 +无奈 🤷 😮‍💨 😔 +尴尬 😅 😬 🫠 +害羞 😳 🙈 😊 +脸红 😳 🥰 😊 +害怕 😨 😱 😰 +恐怖 😱 👻 😨 +吓死 😱 😨 🫣 +惊讶 😲 😮 🤯 +震惊 😱 🤯 😲 +惊喜 🎉 😲 🥳 +哇 😲 😮 🤩 +厉害 👍 💪 🔥 +牛 🐮 👍 🔥 +牛逼 🔥 👍 💪 +棒 👍 💯 👏 +真棒 👍 👏 💯 +很棒 👍 👏 🌟 +太棒了 👏 🎉 💯 +优秀 💯 👏 🌟 +完美 💯 ✨ 👌 +赞 👍 👏 💯 +点赞 👍 ❤️ 👏 +加油 💪 🔥 👊 +努力 💪 🔥 🏃 +奋斗 💪 🔥 ✊ +坚持 💪 ✊ 🔥 +胜利 ✌️ 🏆 🎉 +成功 🎉 🏆 💪 +恭喜 🎉 🥳 👏 +祝贺 🎉 🥳 🍾 +庆祝 🎉 🥳 🍾 +生日 🎂 🎉 🎁 +生日快乐 🎂 🎉 🎁 +节日 🎉 🎊 🎈 +新年 🎆 🧧 🎊 +春节 🧧 🏮 🎆 +过年 🧧 🎆 🏮 +新年快乐 🎆 🧧 🎉 +圣诞 🎄 🎅 🎁 +圣诞节 🎄 🎅 🎁 +礼物 🎁 🎀 🎉 +红包 🧧 💰 🎁 +中秋 🥮 🌕 🏮 +月亮 🌙 🌕 ✨ +太阳 ☀️ 🌞 🌅 +晚安 😴 🌙 💤 +早安 ☀️ 🌅 😊 +早上好 ☀️ 🌅 😊 +午安 ☀️ 😊 🍱 +你好 👋 😊 🙂 +您好 👋 🙏 😊 +嗨 👋 😄 🙂 +再见 👋 🥲 🫡 +拜拜 👋 😘 🥲 +谢谢 🙏 😊 ❤️ +感谢 🙏 ❤️ 😊 +多谢 🙏 😊 🤝 +抱歉 🙏 😔 🙇 +对不起 🙏 🥺 🙇 +不好意思 🙏 😅 🙇 +请 🙏 🙇 🤲 +拜托 🙏 🥺 🤲 +麻烦 🙏 😅 🙇 +辛苦 🙏 💪 🫡 +辛苦了 🙏 💪 🫶 +欢迎 🤗 🎉 👋 +抱抱 🤗 🫂 💕 +拥抱 🤗 🫂 💞 +爱 ❤️ 💕 😍 +爱你 ❤️ 😘 😍 +喜欢 😍 ❤️ 💕 +爱情 💕 ❤️ 💑 +想你 🥺 💕 😘 +想念 🥺 💭 💕 +亲亲 😘 💋 😗 +么么哒 😘 💋 😚 +飞吻 😘 💋 😗 +比心 🫶 ❤️ 💕 +心 ❤️ 💖 💗 +红心 ❤️ 💖 💕 +可爱 🥰 😍 🐰 +萌 🥰 🐰 😻 +帅 😎 🤩 👍 +帅气 😎 🤩 🔥 +酷 😎 🆒 🤙 +漂亮 💄 ✨ 😍 +美女 👩 💄 😍 +美丽 🌸 ✨ 😍 +好看 😍 ✨ 👍 +晕 😵 😵‍💫 🥴 +头晕 😵‍💫 😵 🤢 +累 😫 😩 🥱 +好累 😫 😩 😮‍💨 +疲惫 😩 😫 🥱 +困 😴 🥱 💤 +好困 😴 🥱 😪 +睡觉 😴 🛌 💤 +睡了 😴 💤 🌙 +失眠 😵‍💫 😩 🌙 +熬夜 🥱 🌙 😵‍💫 +忙 🏃 💼 😵‍💫 +加班 💼 😭 🌙 +上班 💼 🏢 😩 +下班 🎉 🏃 🚗 +放假 🏖️ 🎉 😎 +休息 😌 🛌 ☕ +旅行 ✈️ 🧳 🌍 +旅游 ✈️ 🧳 📸 +出发 🚀 🚗 ✈️ +飞机 ✈️ 🛫 🛬 +火车 🚄 🚆 🚂 +高铁 🚄 🚅 🚆 +汽车 🚗 🚙 🚘 +开车 🚗 🛣️ 🚙 +地铁 🚇 🚈 🚉 +公交 🚌 🚏 🚍 +自行车 🚲 🚴 🛴 +海边 🏖️ 🌊 🏝️ +海 🌊 🏖️ 🐳 +山 ⛰️ 🏔️ 🗻 +下雨 🌧️ ☔ 🌂 +雨 🌧️ ☔ ☂️ +下雪 ❄️ ⛄ 🌨️ +雪 ❄️ ⛄ ☃️ +刮风 💨 🌬️ 🍃 +风 💨 🌬️ 🍃 +天气 ⛅ 🌤️ ☀️ +晴天 ☀️ 🌞 😎 +阴天 ☁️ 🌥️ 😶 +打雷 ⚡ 🌩️ ⛈️ +热 🥵 🔥 ☀️ +好热 🥵 🔥 😓 +冷 🥶 ❄️ 🧊 +好冷 🥶 ❄️ 😨 +冻 🥶 ❄️ 🧊 +吃饭 🍚 🍜 🍽️ +吃 😋 🍽️ 🍴 +好吃 😋 🤤 👍 +美味 😋 🤤 🍽️ +饿 🤤 🍽️ 😋 +好饿 🤤 🍽️ 😭 +馋 🤤 😋 🍖 +早餐 🥐 🍳 ☕ +早饭 🥐 🍳 🥛 +午饭 🍱 🍜 🍚 +午餐 🍱 🍜 🍚 +晚饭 🍽️ 🍲 🍚 +晚餐 🍽️ 🍷 🍲 +夜宵 🍢 🍻 🌙 +火锅 🍲 🌶️ 🥘 +烧烤 🍢 🍖 🍻 +面条 🍜 🍝 🥢 +拉面 🍜 🍥 🥢 +米饭 🍚 🍙 🥢 +饺子 🥟 🥢 😋 +包子 🥟 🍞 😋 +汉堡 🍔 🍟 😋 +披萨 🍕 😋 🧀 +炸鸡 🍗 🍟 😋 +薯条 🍟 🍔 😋 +寿司 🍣 🍱 🥢 +蛋糕 🍰 🎂 🧁 +甜品 🍰 🍮 🧁 +冰淇淋 🍦 🍨 🍧 +零食 🍿 🍪 🍫 +巧克力 🍫 😋 ❤️ +糖 🍬 🍭 🍫 +水果 🍎 🍇 🍓 +苹果 🍎 🍏 📱 +香蕉 🍌 😋 🐒 +西瓜 🍉 😋 ☀️ +草莓 🍓 😋 🍰 +葡萄 🍇 😋 🍷 +橙子 🍊 😋 🧃 +柠檬 🍋 😖 🍹 +桃子 🍑 😋 🍃 +荔枝 😋 🍒 🍑 +樱桃 🍒 😋 🍰 +蔬菜 🥬 🥦 🥕 +土豆 🥔 🍟 😋 +番茄 🍅 😋 🥗 +玉米 🌽 😋 🍿 +辣 🌶️ 🔥 🥵 +好辣 🥵 🌶️ 🔥 +水 💧 🚰 🥤 +喝水 💧 🥤 🚰 +喝 🥤 🍹 🍺 +茶 🍵 🫖 ☕ +奶茶 🧋 🥤 😋 +咖啡 ☕ 😌 🥱 +果汁 🧃 🍹 🍊 +可乐 🥤 🍔 😋 +啤酒 🍺 🍻 😋 +干杯 🍻 🥂 🍾 +酒 🍷 🍺 🥂 +红酒 🍷 🥂 😌 +香槟 🍾 🥂 🎉 +烟花 🎆 🎇 ✨ +猫 🐱 😺 🐈 +小猫 🐱 😺 🐾 +狗 🐶 🐕 🐾 +小狗 🐶 🐕 🐾 +兔子 🐰 🐇 🥕 +熊猫 🐼 🎋 😍 +老虎 🐯 🐅 🔥 +老鼠 🐭 🐁 🧀 +猪 🐷 🐖 🐽 +小猪 🐷 🐽 😄 +牛奶 🥛 🐮 ☕ +马 🐴 🐎 🏇 +龙 🐲 🐉 🔥 +蛇 🐍 🧪 😱 +鸟 🐦 🕊️ 🪶 +鸡 🐔 🐓 🍗 +鱼 🐟 🐠 🎣 +鲸鱼 🐳 🐋 🌊 +企鹅 🐧 ❄️ 🐾 +猴 🐵 🙈 🐒 +猴子 🐵 🙈 🐒 +蝴蝶 🦋 🌸 ✨ +蜜蜂 🐝 🍯 🌼 +花 🌸 🌹 💐 +鲜花 💐 🌸 🌹 +玫瑰 🌹 ❤️ 💐 +樱花 🌸 🌺 🌷 +树 🌳 🌲 🌴 +绿色 💚 🟢 🌿 +红色 ❤️ 🔴 🌹 +蓝色 💙 🔵 🌊 +黄色 💛 🟡 🌻 +粉色 💗 🌸 🩷 +黑色 🖤 ⚫ 🌑 +白色 🤍 ⚪ ☁️ +钱 💰 💵 🤑 +发财 💰 🤑 🧧 +赚钱 💰 🤑 💸 +花钱 💸 💳 🛍️ +没钱 💸 😭 🥲 +穷 💸 🥲 😭 +富 💰 🤑 💎 +钻石 💎 ✨ 💍 +购物 🛍️ 🛒 💳 +买买买 🛍️ 🛒 💸 +手机 📱 📲 🔋 +电脑 💻 🖥️ ⌨️ +笔记本 💻 📓 ✏️ +游戏 🎮 🕹️ 👾 +打游戏 🎮 🕹️ 👾 +电影 🎬 🍿 🎥 +看电影 🎬 🍿 🎥 +音乐 🎵 🎶 🎧 +歌 🎤 🎵 🎶 +唱歌 🎤 🎶 🎵 +跳舞 💃 🕺 🎶 +拍照 📷 📸 🤳 +照片 📷 🖼️ 📸 +自拍 🤳 📸 😎 +视频 🎥 📹 🎬 +直播 📹 🎥 📡 +书 📚 📖 📕 +读书 📖 📚 🤓 +看书 📖 📚 👓 +学习 📚 ✏️ 🤓 +学校 🏫 🎒 📚 +考试 📝 😰 💯 +作业 📝 ✏️ 😭 +老师 👩‍🏫 👨‍🏫 📚 +毕业 🎓 🎉 👨‍🎓 +工作 💼 🧑‍💻 💪 +会议 👥 📊 💼 +开会 👥 💼 📊 +老板 👔 💼 🤑 +报告 📊 📝 📈 +文件 📄 📁 📎 +邮件 📧 ✉️ 📨 +电话 📞 ☎️ 📱 +微信 💬 📱 🟢 +消息 💬 📩 🔔 +生病 🤒 🤧 😷 +感冒 🤧 🤒 😷 +发烧 🤒 🌡️ 🥵 +咳嗽 😷 🤧 🤒 +头疼 🤕 😣 😖 +肚子疼 🤕 😣 😖 +医院 🏥 🚑 💊 +药 💊 💉 🏥 +口罩 😷 🙂 🏥 +健康 💪 🍎 🧘 +运动 🏃 🏋️ ⚽ +跑步 🏃 👟 💨 +健身 🏋️ 💪 🔥 +游泳 🏊 🌊 💦 +篮球 🏀 ⛹️ 🔥 +足球 ⚽ 🥅 🏟️ +乒乓球 🏓 😄 🔥 +羽毛球 🏸 😄 🔥 +网球 🎾 😄 🔥 +瑜伽 🧘 🌿 ✨ +冠军 🏆 🥇 🎉 +第一 🥇 🏆 💯 +奖杯 🏆 🥇 🎉 +奖 🏆 🏅 🎖️ +好运 🍀 🤞 ✨ +幸运 🍀 🤞 🌟 +祝福 🙏 🍀 ❤️ +平安 🙏 🕊️ ✨ +许愿 🌠 🙏 ✨ +星星 ⭐ 🌟 ✨ +彩虹 🌈 ☀️ 🦄 +闪 ✨ ⚡ 🌟 +火 🔥 🧯 🚒 +火爆 🔥 💥 🚀 +爆炸 💥 🤯 🔥 +炸 💥 💣 🤯 +炸弹 💣 💥 😱 +危险 ⚠️ 🚨 ☠️ +警告 ⚠️ 🚨 🔔 +注意 ⚠️ ❗ 👀 +重要 ❗ ⚠️ ⭐ +问题 ❓ 🤔 🧐 +疑问 🤔 ❓ 🧐 +思考 🤔 💭 🧠 +想想 🤔 💭 🧐 +思索 🤔 💭 🧠 +聪明 🧠 💡 🤓 +智慧 🧠 💡 ✨ +想法 💡 🤔 💭 +主意 💡 🤔 ✨ +灵感 💡 ✨ 🎨 +创意 💡 🎨 ✨ +好主意 💡 👍 😄 +对 ✅ 👌 👍 +没错 ✅ 👌 💯 +正确 ✅ 👍 💯 +是的 ✅ 👌 👍 +好的 👌 👍 ✅ +好 👌 👍 😊 +行 👌 👍 ✅ +可以 👌 👍 ✅ +同意 👍 ✅ 🤝 +不 🙅 ❌ 🚫 +不行 🙅 ❌ 🚫 +不要 🙅 🚫 ❌ +不可以 🙅 🚫 ❌ +错 ❌ 🙅 😬 +错误 ❌ ⚠️ 🐛 +拒绝 🙅 🚫 ❌ +禁止 🚫 ⛔ 🙅 +取消 ❌ 🚫 ↩️ +完成 ✅ 🎉 💯 +搞定 ✅ 💪 😎 +完毕 ✅ 👌 🎉 +结束 🔚 🏁 ✅ +开始 🚀 ▶️ 🏁 +等等 ⏳ ✋ 🕐 +等待 ⏳ ⌛ 🕐 +时间 ⏰ ⌛ 🕐 +快 💨 ⚡ 🏃 +快点 ⏩ 🏃 💨 +慢 🐢 🐌 ⏳ +慢慢来 🐢 🍵 😌 +再来一次 🔁 🔄 😄 +重复 🔁 🔄 ♻️ +收到 👌 ✅ 📩 +知道了 👌 ✅ 🫡 +明白 💡 👌 ✅ +了解 👌 🫡 ✅ +遵命 🫡 👌 ✅ +报到 🙋 🫡 👋 +举手 🙋 ✋ 🙌 +鼓掌 👏 🎉 🙌 +拍手 👏 🙌 😄 +握手 🤝 🙌 😊 +合作 🤝 💪 🙌 +朋友 🫂 🤝 👭 +好朋友 🫂 💞 🤝 +家 🏠 🏡 ❤️ +回家 🏠 🏃 🚗 +到家 🏠 ✅ 😌 +家人 👨‍👩‍👧‍👦 🏠 ❤️ +爸爸 👨 👨‍👧 ❤️ +妈妈 👩 👩‍👧 ❤️ +宝宝 👶 🍼 🥰 +孩子 👶 🧒 👧 +结婚 💒 💍 💑 +婚礼 💒 💍 🤵 +情侣 💑 💏 ❤️ +约会 💑 🌹 🍷 +单身 🙋 😎 🕺 +脱单 💑 🎉 ❤️ +鬼 👻 😱 💀 +外星人 👽 🛸 🌌 +机器人 🤖 ⚙️ 🦾 +死 💀 ☠️ 😵 +死了 💀 ☠️ 😵 +完蛋 💀 😱 🫠 +糟糕 😱 😰 🫠 +崩溃 🤯 😭 🫠 +裂开 🫠 💔 😵 +凉了 🥶 💀 😭 +摸鱼 🐟 😎 🤫 +躺平 🛌 😌 🫠 +佛系 🧘 🙏 😌 +吃瓜 🍉 👀 🤭 +围观 👀 🍿 🤭 +看 👀 👁️ 🔍 +看看 👀 🔍 🧐 +偷看 👀 🫣 🙈 +搜索 🔍 🔎 🌐 +找 🔍 🔎 👀 +秘密 🤫 🔒 🙊 +嘘 🤫 🤐 🙊 +闭嘴 🤐 🤫 🙊 +保密 🤫 🔒 🤐 +安静 🤫 🙊 🌙 +吵 🔊 😣 🙉 +声音 🔊 🎙️ 🎧 +耳机 🎧 🎵 🎶 +音量 🔊 🔉 🎚️ +电 ⚡ 🔌 🔋 +没电 🪫 🔋 😭 +充电 🔋 🔌 ⚡ +网络 🌐 📶 💻 +没网 📶 😭 🛜 +信号 📶 📡 📱 +代码 💻 🧑‍💻 ⌨️ +程序员 🧑‍💻 💻 ☕ +程序 💻 ⌨️ 🧑‍💻 +调试 🐛 🔧 💻 +漏洞 🐛 🔓 ⚠️ +修复 🔧 ✅ 🛠️ +发布 🚀 📢 🎉 +上线 🚀 ✅ 🎉 +测试 🧪 ✅ 🐛 +设计 🎨 ✏️ 📐 +画画 🎨 🖌️ 🖼️ +画 🎨 🖼️ 🖌️ +写字 ✍️ 📝 ✏️ +写 ✍️ 📝 ✏️ +记录 📝 📒 ✍️ +笔记 📝 📒 📓 +日历 📅 🗓️ 📆 +日期 📅 🗓️ 📆 +地图 🗺️ 📍 🧭 +位置 📍 🗺️ 🧭 +地址 📍 🏠 🗺️ +方向 🧭 ➡️ 🗺️ +钥匙 🔑 🗝️ 🔐 +锁 🔒 🔐 🗝️ +门 🚪 🔑 🏠 +灯 💡 🔦 🏮 +伞 ☂️ 🌂 ☔ +包 👜 🎒 💼 +衣服 👕 👗 🧥 +裙子 👗 👚 💃 +鞋 👟 👠 🥾 +帽子 🧢 🎩 👒 +眼镜 👓 🕶️ 🤓 +戒指 💍 💎 ❤️ +手表 ⌚ ⏰ 🕐 +太空 🚀 🌌 🪐 +火箭 🚀 🌌 🔥 +地球 🌍 🌎 🌏 +世界 🌍 🌐 ✨ +宇宙 🌌 🪐 ✨ +和平 ☮️ 🕊️ 🙏 +中国 🇨🇳 🧧 🐼 +加油中国 🇨🇳 💪 🔥 +国旗 🇨🇳 🚩 🎌 +旗 🚩 🏁 🎌 diff --git a/app/src/main/java/llc/slacker/openime/LocalVoiceImeService.kt b/app/src/main/java/llc/slacker/openime/LocalVoiceImeService.kt index 5a2b3d2c..6ac5072c 100644 --- a/app/src/main/java/llc/slacker/openime/LocalVoiceImeService.kt +++ b/app/src/main/java/llc/slacker/openime/LocalVoiceImeService.kt @@ -2,6 +2,8 @@ package llc.slacker.openime import android.content.Intent import android.inputmethodservice.InputMethodService +import android.os.Build +import android.os.Bundle import android.os.Handler import android.os.Looper import android.text.InputType @@ -10,6 +12,8 @@ import android.util.Log import android.view.KeyEvent import android.view.View import android.view.inputmethod.EditorInfo +import android.view.inputmethod.InlineSuggestionsRequest +import android.view.inputmethod.InlineSuggestionsResponse import android.view.inputmethod.InputMethodManager import android.view.inputmethod.InputMethodSubtype import llc.slacker.openime.candidate.CandidateEngine @@ -18,6 +22,7 @@ import llc.slacker.openime.candidate.CandidateQueryCoordinator import llc.slacker.openime.candidate.CandidateResolver import llc.slacker.openime.candidate.CandidateSnapshot import llc.slacker.openime.candidate.CandidateSnapshotEntry +import llc.slacker.openime.candidate.EmojiAssociationIndex import llc.slacker.openime.candidate.NineKeyReading import llc.slacker.openime.candidate.PinyinLexicon import llc.slacker.openime.candidate.personalizedLearningAllowed @@ -48,6 +53,7 @@ import llc.slacker.openime.keyboard.HardwareKey import llc.slacker.openime.keyboard.HardwareKeyAction import llc.slacker.openime.keyboard.HardwareKeyPolicy import llc.slacker.openime.keyboard.ImeKeyboardView +import llc.slacker.openime.keyboard.InlineAutofill import llc.slacker.openime.rime.NativeCandidateReference import llc.slacker.openime.rime.RimeEngine import llc.slacker.openime.rime.RimeInputNormalizer @@ -62,6 +68,7 @@ import llc.slacker.openime.voice.VoiceModelLifecycleManager import llc.slacker.openime.voice.VoiceModelLifecycleState import llc.slacker.openime.voice.VoicePerformanceTrace import llc.slacker.openime.voice.VoiceRecognitionEvents +import llc.slacker.openime.voice.VoiceTextProcessor import java.io.File /** @@ -180,6 +187,9 @@ class LocalVoiceImeService : InputMethodService(), ImeKeyboardView.Listener, Can VoiceEditorContext.allowNaturalPunctuation = { EditorInfoAdapter.allowNaturalLanguageVoicePunctuation(currentInputEditorInfo) } + VoiceEditorContext.stripFillers = { ImeSettingsRepository.loadVoiceStripFillers(this) } + VoiceEditorContext.punctuationAsSpace = + { ImeSettingsRepository.loadVoicePunctuationAsSpace(this) } UserPhraseRepository.configure(this) VoiceCorrectionRepository.configure(this) HotwordRuntime.configure(this) @@ -256,6 +266,9 @@ class LocalVoiceImeService : InputMethodService(), ImeKeyboardView.Listener, Can private fun createKeyboardView(): ImeKeyboardView { return ImeKeyboardView(this, this).also { view -> + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R && inlineSuggestions.isNotEmpty()) { + view.post { view.showInlineSuggestions(inlineSuggestions) } + } view.setMode(state.keyboardMode, notifyListener = false) view.setTheme(state.theme) view.setAppearance(state.appearance) @@ -269,6 +282,26 @@ class LocalVoiceImeService : InputMethodService(), ImeKeyboardView.Listener, Can } } + /** Autofill chips in the toolbar row (Android 11+); see [InlineAutofill]. */ + @android.annotation.TargetApi(Build.VERSION_CODES.R) + override fun onCreateInlineSuggestionsRequest(uiExtras: Bundle): InlineSuggestionsRequest? = + InlineAutofill.request(this) + + /** + * The latest response, kept until the field goes away. The response can + * arrive while the keyboard view is still being (re)created for the new + * field; it is then shown as soon as the view exists instead of being + * refused, which would send the system back to its dropdown. + */ + private var inlineSuggestions: List = emptyList() + + @android.annotation.TargetApi(Build.VERSION_CODES.R) + override fun onInlineSuggestionsResponse(response: InlineSuggestionsResponse): Boolean { + inlineSuggestions = response.inlineSuggestions + val view = keyboardView ?: return inlineSuggestions.isNotEmpty() + return view.showInlineSuggestions(inlineSuggestions) + } + override fun onCreateInputView(): View { keyboardView = createKeyboardView() return keyboardView!! @@ -486,6 +519,7 @@ class LocalVoiceImeService : InputMethodService(), ImeKeyboardView.Listener, Can } override fun onFinishInput() { + inlineSuggestions = emptyList() invalidateCandidateQueries() voiceCorrectionTracker.finalizeIfNeeded() voiceMediaMute.restore() @@ -562,6 +596,8 @@ class LocalVoiceImeService : InputMethodService(), ImeKeyboardView.Listener, Can if (::voiceLifecycle.isInitialized) voiceLifecycle.destroy() activeInstance = null VoiceEditorContext.allowNaturalPunctuation = { true } + VoiceEditorContext.stripFillers = { true } + VoiceEditorContext.punctuationAsSpace = { false } super.onDestroy() } @@ -639,6 +675,24 @@ class LocalVoiceImeService : InputMethodService(), ImeKeyboardView.Listener, Can true } }.getOrDefault(false) + // The recogniser's final text goes through the same post-processing as + // a real session (spoken punctuation, fillers, punctuation as spaces, + // word lists) before it reaches the editor. + command.startsWith("voice-processed64:") -> runCatching { + val raw = String( + java.util.Base64.getDecoder().decode(command.substringAfter("voice-processed64:")), + Charsets.UTF_8, + ) + val processed = VoiceTextProcessor.process(raw, "zh-CN") + if (processed.isBlank()) { + false + } else { + VoicePerformanceTrace.abandon() + onVoiceSessionStarted(autoCommitOnFinal = true) + onVoiceFinal(HotwordRuntime.apply(VoiceCorrectionRepository.apply(processed))) + true + } + }.getOrDefault(false) command.startsWith("quick-phrase-edit64:") -> runCatching { val text = String( java.util.Base64.getDecoder().decode(command.substringAfter("quick-phrase-edit64:")), @@ -1207,7 +1261,18 @@ class LocalVoiceImeService : InputMethodService(), ImeKeyboardView.Listener, Can gateway.commitText(text) gateway.finishComposing() keyboardView?.clearAssociationCandidates() - keyboardView?.setAssociationCandidates(candidatePipeline.associationsFor(text)) + keyboardView?.setAssociationCandidates(associationsAfterCommit(text)) + } + + private val emojiAssociations: EmojiAssociationIndex by lazy { + EmojiAssociationIndex.load(this) + } + + /** Next-word suggestions, with the emoji that go with the committed text first. */ + private fun associationsAfterCommit(committed: String): List { + val words = candidatePipeline.associationsFor(committed) + if (state.passwordField || !ImeSettingsRepository.loadEmojiAssociation(this)) return words + return emojiAssociations.emojiFor(committed) + words } override fun onCompositionBackspace() { @@ -1717,7 +1782,7 @@ class LocalVoiceImeService : InputMethodService(), ImeKeyboardView.Listener, Can lastComposition = "" state = state.copy(composition = "", candidates = emptyList()) keyboardView?.renderState(state) - keyboardView?.setAssociationCandidates(candidatePipeline.associationsFor(committed)) + keyboardView?.setAssociationCandidates(associationsAfterCommit(committed)) } private fun allowsPersonalizedLearning(): Boolean = diff --git a/app/src/main/java/llc/slacker/openime/candidate/EmojiAssociationIndex.kt b/app/src/main/java/llc/slacker/openime/candidate/EmojiAssociationIndex.kt new file mode 100644 index 00000000..fe5dda63 --- /dev/null +++ b/app/src/main/java/llc/slacker/openime/candidate/EmojiAssociationIndex.kt @@ -0,0 +1,65 @@ +package llc.slacker.openime.candidate + +import android.content.Context + +/** + * Local keyword to emoji lookup behind the association row ("联想"). + * + * After a word is committed, the emoji that go with it are offered first: + * 开心 -> 😊 😄 😁. A keyword matches when the committed text equals it, or, + * for keywords of two characters or more, when the committed phrase ends with + * it (今天好开心 -> 开心). One-character keywords such as 笑 or 猫 only match + * exactly so that ordinary words never sprout unrelated emoji. + * + * The table is a bundled text asset; nothing here touches the network. + */ +class EmojiAssociationIndex internal constructor( + private val byKeyword: Map>, +) { + private val longestKeyword = byKeyword.keys.maxOfOrNull { it.length } ?: 0 + + /** Up to [limit] emoji for [committed]; empty when nothing matches. */ + fun emojiFor(committed: String, limit: Int = MAX_EMOJI): List { + val text = committed.trim() + if (text.isEmpty() || byKeyword.isEmpty()) return emptyList() + byKeyword[text]?.let { return it.take(limit) } + val longest = minOf(longestKeyword, text.length - 1) + for (length in longest downTo MIN_SUFFIX_KEYWORD_LENGTH) { + byKeyword[text.takeLast(length)]?.let { return it.take(limit) } + } + return emptyList() + } + + companion object { + const val MAX_EMOJI = 3 + private const val MIN_SUFFIX_KEYWORD_LENGTH = 2 + private const val ASSET = "emoji/keywords.tsv" + + val EMPTY = EmojiAssociationIndex(emptyMap()) + + /** Lines are `keywordemoji emoji ...`; `#` starts a comment. */ + fun parse(lines: Sequence): EmojiAssociationIndex { + val table = LinkedHashMap>() + lines.forEach { raw -> + val line = raw.trim() + if (line.isEmpty() || line.startsWith("#")) return@forEach + val tab = line.indexOf('\t') + if (tab <= 0) return@forEach + val keyword = line.substring(0, tab).trim() + val emoji = line.substring(tab + 1).trim() + .split(Regex("\\s+")) + .filter { it.isNotEmpty() } + .distinct() + if (keyword.isNotEmpty() && emoji.isNotEmpty()) table.putIfAbsent(keyword, emoji) + } + return EmojiAssociationIndex(table) + } + + fun load(context: Context): EmojiAssociationIndex = + runCatching { + context.assets.open(ASSET).bufferedReader(Charsets.UTF_8).use { reader -> + parse(reader.lineSequence().toList().asSequence()) + } + }.getOrDefault(EMPTY) + } +} diff --git a/app/src/main/java/llc/slacker/openime/data/ImeSettingsRepository.kt b/app/src/main/java/llc/slacker/openime/data/ImeSettingsRepository.kt index 3a91e5fd..0d615f58 100644 --- a/app/src/main/java/llc/slacker/openime/data/ImeSettingsRepository.kt +++ b/app/src/main/java/llc/slacker/openime/data/ImeSettingsRepository.kt @@ -23,6 +23,10 @@ object ImeSettingsRepository { private const val KEY_POPUP = "popup" private const val KEY_FUZZY = "fuzzy" private const val KEY_SWIPE_UP_DIGITS = "swipe_up_digits" + private const val KEY_LETTER_HINTS = "letter_hints" + private const val KEY_EMOJI_ASSOCIATION = "emoji_association" + private const val KEY_VOICE_STRIP_FILLERS = "voice_strip_fillers" + private const val KEY_VOICE_PUNCTUATION_AS_SPACE = "voice_punctuation_as_space" private const val KEY_SKIN_OPACITY = "skin_opacity" private const val KEY_SKIN_RADIUS = "skin_radius" private const val KEY_SKIN_FONT = "skin_font" @@ -157,6 +161,46 @@ object ImeSettingsRepository { .edit().putBoolean(KEY_SWIPE_UP_DIGITS, enabled).apply() } + /** Digits and symbols printed on the letter keys, typed by swipe up or long press. */ + fun loadLetterHints(context: Context): Boolean = + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + .getBoolean(KEY_LETTER_HINTS, true) + + fun saveLetterHints(context: Context, enabled: Boolean) { + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + .edit().putBoolean(KEY_LETTER_HINTS, enabled).apply() + } + + /** Emoji suggestions in the association row after a word is committed. */ + fun loadEmojiAssociation(context: Context): Boolean = + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + .getBoolean(KEY_EMOJI_ASSOCIATION, true) + + fun saveEmojiAssociation(context: Context, enabled: Boolean) { + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + .edit().putBoolean(KEY_EMOJI_ASSOCIATION, enabled).apply() + } + + /** Drop hesitation sounds such as 嗯 / 呃 from voice input. */ + fun loadVoiceStripFillers(context: Context): Boolean = + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + .getBoolean(KEY_VOICE_STRIP_FILLERS, true) + + fun saveVoiceStripFillers(context: Context, enabled: Boolean) { + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + .edit().putBoolean(KEY_VOICE_STRIP_FILLERS, enabled).apply() + } + + /** Write a space instead of a punctuation mark between voice clauses. */ + fun loadVoicePunctuationAsSpace(context: Context): Boolean = + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + .getBoolean(KEY_VOICE_PUNCTUATION_AS_SPACE, false) + + fun saveVoicePunctuationAsSpace(context: Context, enabled: Boolean) { + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + .edit().putBoolean(KEY_VOICE_PUNCTUATION_AS_SPACE, enabled).apply() + } + fun loadPopup(context: Context): Boolean = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) .getBoolean(KEY_POPUP, true) diff --git a/app/src/main/java/llc/slacker/openime/data/UserDataArchive.kt b/app/src/main/java/llc/slacker/openime/data/UserDataArchive.kt index e7231fd6..5272a6e5 100644 --- a/app/src/main/java/llc/slacker/openime/data/UserDataArchive.kt +++ b/app/src/main/java/llc/slacker/openime/data/UserDataArchive.kt @@ -44,6 +44,10 @@ internal data class ArchiveSettings( val keyboardHeightPercent: Int, val floatingWidthPercent: Int, val floatingOpacityPercent: Int, + val letterHints: Boolean = true, + val emojiAssociation: Boolean = true, + val voiceStripFillers: Boolean = true, + val voicePunctuationAsSpace: Boolean = false, ) internal data class RimeUserDictionaryArchive( @@ -188,6 +192,10 @@ internal object UserDataArchiveCodec { .put("keyboard_height_percent", value.keyboardHeightPercent) .put("floating_width_percent", value.floatingWidthPercent) .put("floating_opacity_percent", value.floatingOpacityPercent) + .put("letter_hints", value.letterHints) + .put("emoji_association", value.emojiAssociation) + .put("voice_strip_fillers", value.voiceStripFillers) + .put("voice_punctuation_as_space", value.voicePunctuationAsSpace) private fun settingsFromJson(value: JSONObject): ArchiveSettings = ArchiveSettings( @@ -213,6 +221,11 @@ internal object UserDataArchiveCodec { value.optInt("floating_width_percent", 88).coerceIn(72, 96), floatingOpacityPercent = value.optInt("floating_opacity_percent", 100).coerceIn(82, 100), + letterHints = value.optBoolean("letter_hints", true), + emojiAssociation = value.optBoolean("emoji_association", true), + voiceStripFillers = value.optBoolean("voice_strip_fillers", true), + voicePunctuationAsSpace = + value.optBoolean("voice_punctuation_as_space", false), ) private inline fun JSONArray?.toObjects(block: (JSONObject) -> T): List { @@ -340,6 +353,11 @@ internal object UserDataRepository { ImeSettingsRepository.loadFloatingWidthPercent(context), floatingOpacityPercent = ImeSettingsRepository.loadFloatingOpacityPercent(context), + letterHints = ImeSettingsRepository.loadLetterHints(context), + emojiAssociation = ImeSettingsRepository.loadEmojiAssociation(context), + voiceStripFillers = ImeSettingsRepository.loadVoiceStripFillers(context), + voicePunctuationAsSpace = + ImeSettingsRepository.loadVoicePunctuationAsSpace(context), ), rimeUserDictionaries = rimeUserDictionaries, ) @@ -438,5 +456,12 @@ internal object UserDataRepository { context, value.floatingOpacityPercent, ) + ImeSettingsRepository.saveLetterHints(context, value.letterHints) + ImeSettingsRepository.saveEmojiAssociation(context, value.emojiAssociation) + ImeSettingsRepository.saveVoiceStripFillers(context, value.voiceStripFillers) + ImeSettingsRepository.saveVoicePunctuationAsSpace( + context, + value.voicePunctuationAsSpace, + ) } } diff --git a/app/src/main/java/llc/slacker/openime/keyboard/ImeKeyboardView.kt b/app/src/main/java/llc/slacker/openime/keyboard/ImeKeyboardView.kt index e019c582..dc12f19a 100644 --- a/app/src/main/java/llc/slacker/openime/keyboard/ImeKeyboardView.kt +++ b/app/src/main/java/llc/slacker/openime/keyboard/ImeKeyboardView.kt @@ -194,10 +194,37 @@ open class ImeKeyboardView( onCancelPreviewChanged = { cancelling -> voicePanelController.setCancelPreview(cancelling) }, - onCursorStep = { direction -> - listener.onTextEdit(if (direction < 0) "left" else "right") - }, + onCursorStep = ::moveCursorFromSpace, + onCursorModeChanged = ::lockBottomRowForCursorDrag, ) + + /** + * One horizontal step of a space-bar drag. While pinyin is being composed + * the drag moves the pre-edit cursor (the same one a tap on the pre-edit + * moves); otherwise it moves the cursor in the target text. + */ + private fun moveCursorFromSpace(direction: Int) { + val preedit = composition.text + if (preedit != null && preedit.isNotEmpty()) { + val next = (composition.selectionStart + direction).coerceIn(0, preedit.length) + composition.setSelection(next) + } else { + listener.onTextEdit(if (direction < 0) "left" else "right") + } + } + + /** + * The drag keeps the pointer on the space key, but a second finger or a + * drifting thumb must not press the neighbouring keys of the bottom row. + */ + private fun lockBottomRowForCursorDrag(spaceKey: View, locked: Boolean) { + val row = spaceKey.parent as? ViewGroup ?: return + for (index in 0 until row.childCount) { + val sibling = row.getChildAt(index) + if (sibling !== spaceKey && sibling is ImeKeyView) sibling.touchLocked = locked + } + } + private val spaceVoiceKeyFactory: SpaceVoiceKeyFactory by lazy { SpaceVoiceKeyFactory( gestureController = spaceVoiceGestureController, @@ -622,6 +649,7 @@ open class ImeKeyboardView( currentHaptic = { hapticEnabled }, currentPopup = { popupEnabled }, currentSwipeUpDigits = { ImeSettingsRepository.loadSwipeUpDigits(context) }, + currentExtraToggle = ::onState, currentFuzzy = { fuzzyEnabled }, currentSkinOpacity = { skinOpacity }, currentSkinRadius = { skinRadius }, @@ -737,6 +765,8 @@ open class ImeKeyboardView( onPinyinSegment = ::onPinyinSegment, onShowChoicePopup = ::showChoicePopup, onCommitCharacter = ::commitKeyboardCharacter, + hintsEnabled = { ImeSettingsRepository.loadLetterHints(context) }, + swipeUpEnabled = { ImeSettingsRepository.loadSwipeUpDigits(context) }, onShift = ::cycleShift, onDigits = { setMode(KeyboardMode.DIGITS) }, onModeSwitch = ::cycleMode, @@ -1505,6 +1535,36 @@ open class ImeKeyboardView( enter.contentDescription = label } + private var inlineAutofillController: InlineAutofillController? = null + + /** The autofill host, created on first use (Android 11+ only). */ + @android.annotation.TargetApi(Build.VERSION_CODES.R) + private fun inlineAutofill(): InlineAutofillController? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) return null + inlineAutofillController?.let { return it } + lateinit var controller: InlineAutofillController + controller = InlineAutofillController(context, context.mainExecutor) { chips -> + if (::topZone.isInitialized) { + topZone.setAutofillChips(chips, controller.chipSize.width, controller.chipSize.height) + } + } + inlineAutofillController = controller + return controller + } + + /** + * Hosts the autofill chips of one response (Android 11+). Returns false when + * nothing can be shown, in which case the system falls back to its dropdown. + */ + @android.annotation.TargetApi(Build.VERSION_CODES.R) + fun showInlineSuggestions(suggestions: List): Boolean = + inlineAutofill()?.show(suggestions) ?: false + + fun clearInlineSuggestions() { + // Nothing to clear until a response has been hosted. + inlineAutofillController?.clear() + } + fun setAssociationCandidates(candidates: List) { associationRow.removeAllViews() candidates.distinct().take(8).forEach { candidate -> @@ -1720,6 +1780,8 @@ open class ImeKeyboardView( } open fun shutdown() { + // The field these chips belong to is going away. + clearInlineSuggestions() if (panel == Panel.CLIPBOARD) clipboardPanelController.invalidatePendingLoad() stopVoiceIfActive() // View.removeCallbacks(null) is a no-op; cancel the root-owned @@ -2211,6 +2273,10 @@ open class ImeKeyboardView( "模糊音纠错", "启用模糊音" -> fuzzyEnabled "按键气泡" -> popupEnabled "上滑输入数字" -> ImeSettingsRepository.loadSwipeUpDigits(context) + "数字和符号提示" -> ImeSettingsRepository.loadLetterHints(context) + "表情联想" -> ImeSettingsRepository.loadEmojiAssociation(context) + "语音去语气词" -> ImeSettingsRepository.loadVoiceStripFillers(context) + "标点用空格代替" -> ImeSettingsRepository.loadVoicePunctuationAsSpace(context) else -> true } @@ -2234,6 +2300,15 @@ open class ImeKeyboardView( } // Read at gesture time, so it needs no listener round trip. "上滑输入数字" -> ImeSettingsRepository.saveSwipeUpDigits(context, enabled) + "数字和符号提示" -> { + ImeSettingsRepository.saveLetterHints(context, enabled) + // Rebuilt when the keyboard is next shown (right away if it is). + renderedMode = null + if (panel == Panel.NONE) renderModeBody() + } + "表情联想" -> ImeSettingsRepository.saveEmojiAssociation(context, enabled) + "语音去语气词" -> ImeSettingsRepository.saveVoiceStripFillers(context, enabled) + "标点用空格代替" -> ImeSettingsRepository.saveVoicePunctuationAsSpace(context, enabled) } } diff --git a/app/src/main/java/llc/slacker/openime/keyboard/ImeTopZone.kt b/app/src/main/java/llc/slacker/openime/keyboard/ImeTopZone.kt index aab803e5..f0a7e723 100644 --- a/app/src/main/java/llc/slacker/openime/keyboard/ImeTopZone.kt +++ b/app/src/main/java/llc/slacker/openime/keyboard/ImeTopZone.kt @@ -83,9 +83,23 @@ internal class ImeTopZone( private lateinit var associationScroll: HorizontalScrollView private var toolbarMode = ToolbarMode.NORMAL private var associationsShown = false + private var autofillShown = false - /** What the toolbar row shows; icons, associations and the undo banner are exclusive. */ - private enum class ToolbarMode { NORMAL, ASSOCIATION, UNDO } + /** Chips from the system autofill service, hosted in a strip of their own. */ + private lateinit var autofillScroll: HorizontalScrollView + private val autofillRow = LinearLayout(context) + + /** + * What the toolbar row shows; icons, autofill chips, associations and the + * undo banner are exclusive. + */ + private enum class ToolbarMode { NORMAL, AUTOFILL, ASSOCIATION, UNDO } + + private fun idleToolbarMode(): ToolbarMode = when { + associationsShown -> ToolbarMode.ASSOCIATION + autofillShown -> ToolbarMode.AUTOFILL + else -> ToolbarMode.NORMAL + } private val hideUndoClearRunnable = Runnable { if (toolbarRow.findViewWithTag("undo-toolbar")?.isActivated == true) { toolbarRow.findViewWithTag("undo-toolbar")?.isActivated = false @@ -133,7 +147,8 @@ internal class ImeTopZone( isFocusable = true setOnClickListener { onFeedback() - onAssociationDismiss() + // The same back control closes whichever strip is open. + if (toolbarMode == ToolbarMode.AUTOFILL) dismissAutofillChips() else onAssociationDismiss() } } toolbarRow.addView( @@ -164,6 +179,29 @@ internal class ImeTopZone( LinearLayout.LayoutParams(0, toPx(ImeGeometryTokens.TOUCH_TARGET_DP), 1f), ) + autofillRow.apply { + orientation = LinearLayout.HORIZONTAL + gravity = Gravity.CENTER_VERTICAL + tag = "autofill-row" + } + autofillScroll = HorizontalScrollView(context).apply { + tag = "autofill-scroll" + visibility = View.GONE + isHorizontalScrollBarEnabled = false + overScrollMode = View.OVER_SCROLL_IF_CONTENT_SCROLLS + addView( + autofillRow, + ViewGroup.LayoutParams( + ViewGroup.LayoutParams.WRAP_CONTENT, + toPx(ImeGeometryTokens.TOUCH_TARGET_DP), + ), + ) + } + toolbarRow.addView( + autofillScroll, + LinearLayout.LayoutParams(0, toPx(ImeGeometryTokens.TOUCH_TARGET_DP), 1f), + ) + keyboardHide = toolbarIcon(R.drawable.ic_chevron_down, "收起键盘", "keyboard-hide", onHideKeyboard) toolbarRow.addView(keyboardHide, LinearLayout.LayoutParams(0, toPx(48), 1f)) @@ -410,9 +448,41 @@ internal class ImeTopZone( fun showAssociations(show: Boolean) { associationsShown = show - if (toolbarMode != ToolbarMode.UNDO) { - toolbarMode = if (show) ToolbarMode.ASSOCIATION else ToolbarMode.NORMAL + if (toolbarMode != ToolbarMode.UNDO) toolbarMode = idleToolbarMode() + refreshToolbar() + } + + /** + * Shows the autofill [chips] (already inflated by the system) in the toolbar + * row, or the normal toolbar again when the list is empty. Associations and + * the undo banner keep priority for as long as they are showing. + */ + fun setAutofillChips(chips: List, chipWidthPx: Int = 0, chipHeightPx: Int = 0) { + autofillRow.removeAllViews() + chips.forEach { chip -> + (chip.parent as? ViewGroup)?.removeView(chip) + autofillRow.addView( + chip, + LinearLayout.LayoutParams( + if (chipWidthPx > 0) chipWidthPx else LinearLayout.LayoutParams.WRAP_CONTENT, + if (chipHeightPx > 0) chipHeightPx else LinearLayout.LayoutParams.WRAP_CONTENT, + ).apply { + gravity = Gravity.CENTER_VERTICAL + marginEnd = toPx(8) + }, + ) } + autofillShown = chips.isNotEmpty() + autofillScroll.scrollTo(0, 0) + if (toolbarMode != ToolbarMode.UNDO) toolbarMode = idleToolbarMode() + refreshToolbar() + } + + /** Back control of the autofill strip: hide the chips until the next response. */ + private fun dismissAutofillChips() { + autofillRow.removeAllViews() + autofillShown = false + if (toolbarMode != ToolbarMode.UNDO) toolbarMode = idleToolbarMode() refreshToolbar() } @@ -429,7 +499,7 @@ internal class ImeTopZone( val wasVisible = toolbarRow.findViewWithTag("undo-toolbar")?.isActivated == true toolbarRow.findViewWithTag("undo-toolbar")?.isActivated = false if (toolbarMode == ToolbarMode.UNDO) { - toolbarMode = if (associationsShown) ToolbarMode.ASSOCIATION else ToolbarMode.NORMAL + toolbarMode = idleToolbarMode() refreshToolbar() } if (discardSnapshot && wasVisible) onUndoClearExpired() @@ -516,16 +586,18 @@ internal class ImeTopZone( private fun refreshToolbar() { val normal = toolbarMode == ToolbarMode.NORMAL val association = toolbarMode == ToolbarMode.ASSOCIATION + val autofill = toolbarMode == ToolbarMode.AUTOFILL val shown = if (compact) compactToolbarIcons else toolbarIcons (toolbarIcons + compactToolbarIcons).distinct().forEach { it.visibility = if (normal && it in shown) View.VISIBLE else View.GONE } - associationBack.visibility = if (association) View.VISIBLE else View.GONE + associationBack.visibility = if (association || autofill) View.VISIBLE else View.GONE associationScroll.visibility = if (association) View.VISIBLE else View.GONE + autofillScroll.visibility = if (autofill) View.VISIBLE else View.GONE undoBanner.visibility = if (toolbarMode == ToolbarMode.UNDO) View.VISIBLE else View.GONE keyboardHide.visibility = if (toolbarMode == ToolbarMode.UNDO) View.GONE else View.VISIBLE (keyboardHide.layoutParams as? LinearLayout.LayoutParams)?.let { params -> - if (association) { + if (association || autofill) { params.width = toPx(ImeGeometryTokens.TOUCH_TARGET_DP) params.weight = 0f } else { diff --git a/app/src/main/java/llc/slacker/openime/keyboard/InlineAutofill.kt b/app/src/main/java/llc/slacker/openime/keyboard/InlineAutofill.kt new file mode 100644 index 00000000..18b5ec6d --- /dev/null +++ b/app/src/main/java/llc/slacker/openime/keyboard/InlineAutofill.kt @@ -0,0 +1,116 @@ +package llc.slacker.openime.keyboard + +import android.annotation.TargetApi +import android.content.Context +import android.os.Build +import android.os.Bundle +import android.util.Size +import android.view.inputmethod.InlineSuggestion +import android.view.inputmethod.InlineSuggestionsRequest +import android.widget.inline.InlineContentView +import android.widget.inline.InlinePresentationSpec +import java.util.concurrent.Executor +import llc.slacker.openime.theme.ImeGeometryTokens + +/** + * Autofill suggestions inside the keyboard (Android 11+). + * + * Password managers and the platform autofill service hand the keyboard small + * remote chips ("work account", "123456") to show in the suggestion strip. + * The keyboard only hosts the chips: it never receives the underlying values, + * the system fills the field itself when a chip is tapped, and nothing here + * reads the field's content or touches the network. + */ +@TargetApi(Build.VERSION_CODES.R) +internal object InlineAutofill { + const val MAX_SUGGESTIONS = 5 + + private const val STYLE_VERSIONS_KEY = "androidx.autofill.inline.ui.version:key" + private const val STYLE_VERSION_V1 = "androidx.autofill.inline.ui.version:v1" + private const val STYLE_V1_MARKER = "style_v1" + + /** + * The request the platform sends providers: up to [MAX_SUGGESTIONS] chips + * that fit the 48dp toolbar row. The empty v1 style tells providers to draw + * their default chip, which every provider understands. + */ + fun request(context: Context): InlineSuggestionsRequest { + val density = context.resources.displayMetrics.density + val screenWidth = context.resources.displayMetrics.widthPixels + val minSize = Size((MIN_CHIP_WIDTH_DP * density).toInt(), (MIN_CHIP_HEIGHT_DP * density).toInt()) + val maxSize = Size( + (MAX_CHIP_WIDTH_DP * density).toInt().coerceAtMost(screenWidth).coerceAtLeast(minSize.width), + ((ImeGeometryTokens.TOUCH_TARGET_DP - 4) * density).toInt().coerceAtLeast(minSize.height), + ) + val spec = InlinePresentationSpec.Builder(minSize, maxSize) + .setStyle(defaultStyles()) + .build() + return InlineSuggestionsRequest.Builder(listOf(spec)) + .setMaxSuggestionCount(MAX_SUGGESTIONS) + .build() + } + + /** + * Size every chip is rendered at. A chip is a remote surface, which has no + * size of its own, so it is given a fixed width that fits two accounts on a + * phone and scrolls beyond that; height leaves a margin in the toolbar row. + */ + fun chipSize(context: Context): Size { + val density = context.resources.displayMetrics.density + val request = request(context).inlinePresentationSpecs.first() + val width = (CHIP_WIDTH_DP * density).toInt() + .coerceIn(request.minSize.width, request.maxSize.width) + val height = ((ImeGeometryTokens.TOUCH_TARGET_DP - 8) * density).toInt() + .coerceIn(request.minSize.height, request.maxSize.height) + return Size(width, height) + } + + /** Version table + one empty v1 style, as the androidx.autofill library writes it. */ + fun defaultStyles(): Bundle = Bundle().apply { + putStringArrayList(STYLE_VERSIONS_KEY, arrayListOf(STYLE_VERSION_V1)) + putBundle(STYLE_VERSION_V1, Bundle().apply { putBoolean(STYLE_V1_MARKER, true) }) + } + + private const val CHIP_WIDTH_DP = 150 + private const val MIN_CHIP_WIDTH_DP = 52 + private const val MIN_CHIP_HEIGHT_DP = 32 + private const val MAX_CHIP_WIDTH_DP = 280 +} + +/** + * Inflates the suggestions of one response and hands the ready chips to + * [onChips]. Callbacks run on the main thread, which is also where the chips + * are added to the toolbar. + */ +@TargetApi(Build.VERSION_CODES.R) +internal class InlineAutofillController( + private val context: Context, + private val mainExecutor: Executor, + onChips: (List) -> Unit, +) { + private val tracker = InlineChipTracker(InlineAutofill.MAX_SUGGESTIONS, onChips) + + /** The pixel size chips are inflated at; the strip lays them out at the same size. */ + val chipSize: Size = InlineAutofill.chipSize(context) + + /** Returns false (the system then uses its own dropdown) when there is nothing to host. */ + fun show(suggestions: List): Boolean { + if (!tracker.accepts(suggestions.size)) { + tracker.clear() + return false + } + val token = tracker.begin(suggestions.size) + suggestions.take(InlineAutofill.MAX_SUGGESTIONS).forEachIndexed { index, suggestion -> + runCatching { + suggestion.inflate( + context, + chipSize, + mainExecutor, + ) { chip -> tracker.deliver(token, index, chip) } + }.onFailure { tracker.deliver(token, index, null) } + } + return true + } + + fun clear() = tracker.clear() +} diff --git a/app/src/main/java/llc/slacker/openime/keyboard/InlineChipTracker.kt b/app/src/main/java/llc/slacker/openime/keyboard/InlineChipTracker.kt new file mode 100644 index 00000000..b98ac192 --- /dev/null +++ b/app/src/main/java/llc/slacker/openime/keyboard/InlineChipTracker.kt @@ -0,0 +1,42 @@ +package llc.slacker.openime.keyboard + +/** + * Keeps the autofill chips of the current response in the order the provider + * sent them, even though each chip is inflated asynchronously. + * + * A new response (or [clear]) starts a new generation; a chip that arrives for + * an older one is dropped, so a slow inflate can never put last field's + * suggestion into this field's strip. [publish] always receives the chips that + * are ready, in provider order, and an empty list when nothing is left. + */ +internal class InlineChipTracker( + private val maxChips: Int, + private val publish: (List) -> Unit, +) { + private var generation = 0 + private var slots: MutableList = mutableListOf() + + /** Starts a response of [count] suggestions; returns the token for its chips. */ + fun begin(count: Int): Int { + generation++ + slots = MutableList(count.coerceIn(0, maxChips)) { null } + if (slots.isEmpty()) publish(emptyList()) + return generation + } + + /** Whether a response of [count] suggestions would be shown at all. */ + fun accepts(count: Int): Boolean = count > 0 && maxChips > 0 + + /** A chip finished inflating; [chip] is null when the system could not build it. */ + fun deliver(token: Int, index: Int, chip: V?) { + if (token != generation || index !in slots.indices || chip == null) return + slots[index] = chip + publish(slots.filterNotNull()) + } + + fun clear() { + generation++ + slots = mutableListOf() + publish(emptyList()) + } +} diff --git a/app/src/main/java/llc/slacker/openime/keyboard/KeyboardGeometry.kt b/app/src/main/java/llc/slacker/openime/keyboard/KeyboardGeometry.kt index b0ec4206..8ccfb490 100644 --- a/app/src/main/java/llc/slacker/openime/keyboard/KeyboardGeometry.kt +++ b/app/src/main/java/llc/slacker/openime/keyboard/KeyboardGeometry.kt @@ -95,5 +95,3 @@ internal data class KeyboardLayoutMetrics( val panelBodyHeightDp: Int = (imeHeightDp - ImeGeometryTokens.TOUCH_TARGET_DP).coerceAtLeast(0) } - - diff --git a/app/src/main/java/llc/slacker/openime/keyboard/LetterHintPolicy.kt b/app/src/main/java/llc/slacker/openime/keyboard/LetterHintPolicy.kt new file mode 100644 index 00000000..c73c1e99 --- /dev/null +++ b/app/src/main/java/llc/slacker/openime/keyboard/LetterHintPolicy.kt @@ -0,0 +1,33 @@ +package llc.slacker.openime.keyboard + +/** + * The digit or symbol printed in the top-right corner of each 26-key letter key + * (the scheme Sogou, iFlytek and WeChat Keyboard use): the top row carries + * 1–0, the other rows carry punctuation. Swipe up on the key, or long press it, + * commits the hint instead of the letter. + * + * Chinese mode commits the full-width marks a Chinese IME types; English mode + * the ASCII ones. The two layouts differ only where a full-width mark exists. + */ +internal object LetterHintPolicy { + private val DIGITS = mapOf( + 'q' to "1", 'w' to "2", 'e' to "3", 'r' to "4", 't' to "5", + 'y' to "6", 'u' to "7", 'i' to "8", 'o' to "9", 'p' to "0", + ) + + private val ENGLISH_SYMBOLS = mapOf( + 'a' to "~", 's' to "!", 'd' to "@", 'f' to "#", 'g' to "$", + 'h' to "%", 'j' to "&", 'k' to "*", 'l' to "?", + 'z' to "(", 'x' to ")", 'c' to "-", 'v' to "_", + 'b' to ":", 'n' to ";", 'm' to "/", + ) + + private val CHINESE_SYMBOLS = ENGLISH_SYMBOLS + mapOf( + 's' to "！", 'g' to "￥", 'l' to "？", + 'z' to "（", 'x' to "）", 'b' to "：", 'n' to "；", + ) + + /** The hint of [letter] (lower case), or null for none. */ + fun hint(letter: Char, english: Boolean): String? = + DIGITS[letter] ?: (if (english) ENGLISH_SYMBOLS else CHINESE_SYMBOLS)[letter] +} diff --git a/app/src/main/java/llc/slacker/openime/keyboard/Pinyin26KeyboardRenderer.kt b/app/src/main/java/llc/slacker/openime/keyboard/Pinyin26KeyboardRenderer.kt index 5ac0a363..38b2d376 100644 --- a/app/src/main/java/llc/slacker/openime/keyboard/Pinyin26KeyboardRenderer.kt +++ b/app/src/main/java/llc/slacker/openime/keyboard/Pinyin26KeyboardRenderer.kt @@ -34,6 +34,8 @@ internal class Pinyin26KeyboardRenderer( private val onPinyinSegment: () -> Unit, private val onShowChoicePopup: (View, List) -> Unit, private val onCommitCharacter: (String) -> Unit, + private val hintsEnabled: () -> Boolean, + private val swipeUpEnabled: () -> Boolean, private val onShift: () -> Unit, private val onDigits: () -> Unit, private val onModeSwitch: () -> Unit, @@ -155,7 +157,8 @@ internal class Pinyin26KeyboardRenderer( } else { base } - val secondary = if (english) null else DIGIT_HINTS[character] + // The corner hint is a digit or symbol; swipe up or long press types it. + val secondary = if (hintsEnabled()) LetterHintPolicy.hint(character, english) else null return createKey(main, false, secondary, ImeTypographyTokens.KEY_LETTER_COMPACT_SP, 0) { onLetter(base) }.apply { @@ -167,6 +170,8 @@ internal class Pinyin26KeyboardRenderer( onCommitCharacter(secondary) true } + onSwipeUp = { onCommitCharacter(secondary) } + swipeUpEnabled = this@Pinyin26KeyboardRenderer.swipeUpEnabled } } } @@ -192,17 +197,5 @@ internal class Pinyin26KeyboardRenderer( private companion object { val ROWS = listOf("qwertyuiop", "asdfghjkl", "zxcvbnm") - val DIGIT_HINTS = mapOf( - 'q' to "1", - 'w' to "2", - 'e' to "3", - 'r' to "4", - 't' to "5", - 'y' to "6", - 'u' to "7", - 'i' to "8", - 'o' to "9", - 'p' to "0", - ) } } diff --git a/app/src/main/java/llc/slacker/openime/keyboard/SpaceVoiceGestureController.kt b/app/src/main/java/llc/slacker/openime/keyboard/SpaceVoiceGestureController.kt index ab06c9f5..dc7686f4 100644 --- a/app/src/main/java/llc/slacker/openime/keyboard/SpaceVoiceGestureController.kt +++ b/app/src/main/java/llc/slacker/openime/keyboard/SpaceVoiceGestureController.kt @@ -23,6 +23,8 @@ internal class SpaceVoiceGestureController( private val onVoiceCancel: () -> Unit, private val onCancelPreviewChanged: (Boolean) -> Unit, private val onCursorStep: (Int) -> Unit, + /** Called with the space key when a cursor drag starts and when it ends. */ + private val onCursorModeChanged: (anchor: View, active: Boolean) -> Unit = { _, _ -> }, ) { private val handler = Handler(Looper.getMainLooper()) private val longPressTimeoutMs = ViewConfiguration.getLongPressTimeout().toLong() @@ -92,7 +94,10 @@ internal class SpaceVoiceGestureController( cursorTravel = 0f lastX = rawX handler.removeCallbacks(armVoice) - anchor?.isPressed = false + anchor?.let { + it.isPressed = false + onCursorModeChanged(it, true) + } } if (cursorMode) { @@ -131,9 +136,9 @@ internal class SpaceVoiceGestureController( handler.removeCallbacks(armVoice) if (cursorMode) { + endCursorMode() trackingTouch = false active = false - cursorMode = false pointerId = -1 downX = 0f downY = 0f @@ -171,6 +176,7 @@ internal class SpaceVoiceGestureController( */ fun reset() { handler.removeCallbacks(armVoice) + endCursorMode() if (active) anchor?.isPressed = false trackingTouch = false active = false @@ -184,6 +190,13 @@ internal class SpaceVoiceGestureController( anchor = null } + /** Leaves cursor mode, telling the host to unlock whatever it locked. */ + private fun endCursorMode() { + if (!cursorMode) return + cursorMode = false + anchor?.let { onCursorModeChanged(it, false) } + } + fun shutdown() { reset() handler.removeCallbacksAndMessages(null) diff --git a/app/src/main/java/llc/slacker/openime/panel/SettingsPanelController.kt b/app/src/main/java/llc/slacker/openime/panel/SettingsPanelController.kt index eeb9fc5a..f19d355d 100644 --- a/app/src/main/java/llc/slacker/openime/panel/SettingsPanelController.kt +++ b/app/src/main/java/llc/slacker/openime/panel/SettingsPanelController.kt @@ -58,6 +58,8 @@ internal class SettingsPanelController( private val currentHaptic: () -> Boolean, private val currentPopup: () -> Boolean, private val currentSwipeUpDigits: () -> Boolean, + /** State of the toggles added after the first four, looked up by their label. */ + private val currentExtraToggle: (String) -> Boolean, private val currentFuzzy: () -> Boolean, private val currentSkinOpacity: () -> Int, private val currentSkinRadius: () -> Int, @@ -173,7 +175,8 @@ internal class SettingsPanelController( settingToggleRow("按键音效", "机械轴敲击反馈"), settingToggleRow("触感震动", "轻微触感反馈"), settingToggleRow("按键气泡", "按下时显示字母预览"), - settingToggleRow("上滑输入数字", "九键按键上滑直接输入数字"), + settingToggleRow("数字和符号提示", "字母键右上角显示数字和符号"), + settingToggleRow("上滑输入数字", "按键上滑输入右上角的数字或符号"), ), groupParams(), ) @@ -186,6 +189,16 @@ internal class SettingsPanelController( "进入后配置 z/zh、c/ch、s/sh 等规则", onShowFuzzySettings, ), + settingToggleRow("表情联想", "选词后在联想栏先给出相关表情"), + ), + groupParams(), + ) + + content.addView(createSectionTitle("语音输入"), wrapParams()) + content.addView( + settingGroup( + settingToggleRow("语音去语气词", "去掉“嗯”“呃”等口头停顿"), + settingToggleRow("标点用空格代替", "语音里的逗号、句号等写成空格"), ), groupParams(), ) @@ -579,7 +592,7 @@ internal class SettingsPanelController( "模糊音纠错", "启用模糊音" -> currentFuzzy() "按键气泡" -> currentPopup() "上滑输入数字" -> currentSwipeUpDigits() - else -> true + else -> currentExtraToggle(seed) } private fun accentColorRow(): LinearLayout { diff --git a/app/src/main/java/llc/slacker/openime/voice/VoiceEditorContext.kt b/app/src/main/java/llc/slacker/openime/voice/VoiceEditorContext.kt index c62bf365..db626b46 100644 --- a/app/src/main/java/llc/slacker/openime/voice/VoiceEditorContext.kt +++ b/app/src/main/java/llc/slacker/openime/voice/VoiceEditorContext.kt @@ -9,4 +9,12 @@ internal object VoiceEditorContext { /** Whether voice text may get sentence punctuation added in the current editor. */ @Volatile var allowNaturalPunctuation: () -> Boolean = { true } + + /** Whether hesitation sounds (嗯, 呃, um) are dropped from the recognised text. */ + @Volatile + var stripFillers: () -> Boolean = { true } + + /** Whether clause punctuation is written as a plain space instead. */ + @Volatile + var punctuationAsSpace: () -> Boolean = { false } } diff --git a/app/src/main/java/llc/slacker/openime/voice/VoiceTextProcessor.kt b/app/src/main/java/llc/slacker/openime/voice/VoiceTextProcessor.kt index 2f78c3e4..c7cd1e45 100644 --- a/app/src/main/java/llc/slacker/openime/voice/VoiceTextProcessor.kt +++ b/app/src/main/java/llc/slacker/openime/voice/VoiceTextProcessor.kt @@ -2,6 +2,8 @@ package llc.slacker.openime.voice internal data class VoiceTextProcessingPolicy( val autoTerminalPunctuation: Boolean, + val stripFillers: Boolean = false, + val punctuationAsSpace: Boolean = false, ) /** @@ -42,6 +44,8 @@ object VoiceTextProcessor { fun process(raw: String, languageTag: String): String { val policy = VoiceTextProcessingPolicy( autoTerminalPunctuation = VoiceEditorContext.allowNaturalPunctuation(), + stripFillers = VoiceEditorContext.stripFillers(), + punctuationAsSpace = VoiceEditorContext.punctuationAsSpace(), ) return process(raw, languageTag, policy) } @@ -57,6 +61,14 @@ object VoiceTextProcessor { .trim() if (text.isEmpty()) return "" + val original = text + if (policy.stripFillers) { + text = stripFillers(text) + // An utterance that is nothing but a hesitation ("嗯") is a real + // answer, not noise; keep it as spoken. + if (text.isBlank()) text = original + } + if (languageTag.startsWith("zh", ignoreCase = true)) { text = replaceChineseSpokenPunctuation(text) // Chinese characters do not need an ASR-inserted space between @@ -86,6 +98,38 @@ object VoiceTextProcessor { text = text.replace(Regex("\\s+([,.!?;:)])"), "$1") text = collapsePunctuation(text) } + if (policy.punctuationAsSpace) text = punctuationToSpaces(text) + return text + } + + /** + * Hesitation sounds carry no content. 嗯 and 呃 are removed wherever they + * occur (the one real word containing 呃, 呃逆, is kept); 额 is also a + * component of 额度 / 金额, so it only goes when the recogniser set it off + * as a word of its own. English um / uh / er / hmm go only as whole words. + */ + private fun stripFillers(value: String): String { + var text = value + text = text.replace(Regex("[嗯呃]+(?!逆)"), "") + text = text.replace(Regex("(? { @@ -357,5 +377,6 @@ class ImeKeyView( private companion object { const val MAX_LABEL_FONT_SCALE = 1.3f const val SWIPE_UP_THRESHOLD_DP = 22 + const val LOCKED_ALPHA = 0.38f } } diff --git a/app/src/main/res/xml/method.xml b/app/src/main/res/xml/method.xml index 8b97e17d..485a9257 100644 --- a/app/src/main/res/xml/method.xml +++ b/app/src/main/res/xml/method.xml @@ -1,6 +1,7 @@ = 400) + + val keywords = mutableSetOf() + rows.forEach { row -> + val parts = row.split('\t') + assertEquals("one tab per row: $row", 2, parts.size) + val keyword = parts[0] + assertTrue("duplicate keyword $keyword", keywords.add(keyword)) + val emoji = parts[1].split(' ').filter { it.isNotEmpty() } + assertTrue("$keyword needs 1..3 emoji", emoji.size in 1..3) + assertEquals("$keyword has repeated emoji", emoji.size, emoji.toSet().size) + emoji.forEach { + assertTrue( + "$keyword -> $it is not an emoji", + it.codePoints().anyMatch { cp -> cp >= 0x2190 }, + ) + } + } + + val real = EmojiAssociationIndex.parse(lines.asSequence()) + assertEquals("😊", real.emojiFor("开心").first()) + assertEquals("😊", real.emojiFor("今天真的好开心").first()) + assertTrue(real.emojiFor("我们").isEmpty()) + } +} diff --git a/app/src/test/java/llc/slacker/openime/InlineChipTrackerTest.kt b/app/src/test/java/llc/slacker/openime/InlineChipTrackerTest.kt new file mode 100644 index 00000000..3810f9bf --- /dev/null +++ b/app/src/test/java/llc/slacker/openime/InlineChipTrackerTest.kt @@ -0,0 +1,78 @@ +package llc.slacker.openime + +import llc.slacker.openime.keyboard.InlineChipTracker +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class InlineChipTrackerTest { + private val published = mutableListOf>() + private val tracker = InlineChipTracker(maxChips = 3) { published += it } + + private fun last() = published.lastOrNull().orEmpty() + + @Test + fun chipsKeepProviderOrderWhateverOrderTheyInflateIn() { + val token = tracker.begin(3) + tracker.deliver(token, 2, "third") + assertEquals(listOf("third"), last()) + tracker.deliver(token, 0, "first") + assertEquals(listOf("first", "third"), last()) + tracker.deliver(token, 1, "second") + assertEquals(listOf("first", "second", "third"), last()) + } + + @Test + fun aChipFromAnOlderResponseIsDropped() { + val old = tracker.begin(2) + val current = tracker.begin(1) + tracker.deliver(old, 0, "stale") + assertTrue(last().isEmpty()) + tracker.deliver(current, 0, "fresh") + assertEquals(listOf("fresh"), last()) + } + + @Test + fun clearDropsPendingChipsAndPublishesNothing() { + val token = tracker.begin(2) + tracker.deliver(token, 0, "a") + tracker.clear() + assertTrue(last().isEmpty()) + tracker.deliver(token, 1, "late") + assertTrue(last().isEmpty()) + } + + @Test + fun anEmptyResponseClearsTheStrip() { + val token = tracker.begin(2) + tracker.deliver(token, 0, "a") + tracker.begin(0) + assertTrue(last().isEmpty()) + } + + @Test + fun suggestionsBeyondTheLimitAreIgnored() { + val token = tracker.begin(10) + tracker.deliver(token, 5, "ignored") + assertTrue(last().isEmpty()) + (0..2).forEach { tracker.deliver(token, it, "chip$it") } + assertEquals(listOf("chip0", "chip1", "chip2"), last()) + } + + @Test + fun aChipThatCouldNotBeBuiltLeavesAGapNotACrash() { + val token = tracker.begin(3) + tracker.deliver(token, 0, "a") + tracker.deliver(token, 1, null) + tracker.deliver(token, 2, "c") + assertEquals(listOf("a", "c"), last()) + } + + @Test + fun onlyNonEmptyResponsesAreAccepted() { + assertTrue(tracker.accepts(1)) + assertFalse(tracker.accepts(0)) + assertFalse(InlineChipTracker(maxChips = 0) {}.accepts(1)) + } +} diff --git a/app/src/test/java/llc/slacker/openime/LetterHintPolicyTest.kt b/app/src/test/java/llc/slacker/openime/LetterHintPolicyTest.kt new file mode 100644 index 00000000..eec8f259 --- /dev/null +++ b/app/src/test/java/llc/slacker/openime/LetterHintPolicyTest.kt @@ -0,0 +1,71 @@ +package llc.slacker.openime + +import llc.slacker.openime.keyboard.LetterHintPolicy +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class LetterHintPolicyTest { + private val letters = ('a'..'z').toList() + + @Test + fun theTopRowCarriesTheDigitsInKeyboardOrder() { + listOf(false, true).forEach { english -> + assertEquals( + listOf("1", "2", "3", "4", "5", "6", "7", "8", "9", "0"), + "qwertyuiop".map { LetterHintPolicy.hint(it, english) }, + ) + } + } + + @Test + fun everyLetterHasAHintInBothLanguages() { + listOf(false, true).forEach { english -> + letters.forEach { assertNotNull("$it english=$english", LetterHintPolicy.hint(it, english)) } + } + } + + @Test + fun noHintIsRepeatedWithinALayout() { + listOf(false, true).forEach { english -> + val hints = letters.map { LetterHintPolicy.hint(it, english) } + assertEquals("english=$english", hints.size, hints.toSet().size) + } + } + + @Test + fun englishUsesAsciiMarks() { + assertEquals("!", LetterHintPolicy.hint('s', english = true)) + assertEquals("$", LetterHintPolicy.hint('g', english = true)) + assertEquals("?", LetterHintPolicy.hint('l', english = true)) + assertEquals("(", LetterHintPolicy.hint('z', english = true)) + assertEquals(")", LetterHintPolicy.hint('x', english = true)) + assertEquals(":", LetterHintPolicy.hint('b', english = true)) + assertEquals(";", LetterHintPolicy.hint('n', english = true)) + } + + @Test + fun chineseUsesFullWidthMarksWhereOneExists() { + assertEquals("！", LetterHintPolicy.hint('s', english = false)) + assertEquals("￥", LetterHintPolicy.hint('g', english = false)) + assertEquals("？", LetterHintPolicy.hint('l', english = false)) + assertEquals("（", LetterHintPolicy.hint('z', english = false)) + assertEquals("）", LetterHintPolicy.hint('x', english = false)) + assertEquals("：", LetterHintPolicy.hint('b', english = false)) + assertEquals("；", LetterHintPolicy.hint('n', english = false)) + } + + @Test + fun marksWithoutAFullWidthFormAreTheSameInBothLanguages() { + "adfhjkcvm".forEach { + assertEquals("$it", LetterHintPolicy.hint(it, true), LetterHintPolicy.hint(it, false)) + } + } + + @Test + fun capitalLettersAndNonLettersHaveNoHint() { + assertTrue(LetterHintPolicy.hint('Q', false) == null) + assertTrue(LetterHintPolicy.hint('1', false) == null) + } +} diff --git a/app/src/test/java/llc/slacker/openime/UserDataArchiveTest.kt b/app/src/test/java/llc/slacker/openime/UserDataArchiveTest.kt index 421b1501..f1ebf8b9 100644 --- a/app/src/test/java/llc/slacker/openime/UserDataArchiveTest.kt +++ b/app/src/test/java/llc/slacker/openime/UserDataArchiveTest.kt @@ -35,6 +35,10 @@ class UserDataArchiveTest { keyboardHeightPercent = 104, floatingWidthPercent = 84, floatingOpacityPercent = 96, + letterHints = false, + emojiAssociation = false, + voiceStripFillers = false, + voicePunctuationAsSpace = true, ) @Test @@ -60,6 +64,31 @@ class UserDataArchiveTest { assertEquals(archive, decoded) } + @Test + fun archivesWrittenBeforeTheNewTogglesDecodeToTheirDefaults() { + val archive = UserDataArchive( + quickPhrases = emptyList(), + customSymbols = emptyList(), + userPhrases = emptyList(), + settings = settings, + ) + val json = org.json.JSONObject(UserDataArchiveCodec.encode(archive)) + val legacy = json.getJSONObject("settings") + listOf( + "letter_hints", + "emoji_association", + "voice_strip_fillers", + "voice_punctuation_as_space", + ).forEach { legacy.remove(it) } + + val decoded = UserDataArchiveCodec.decode(json.toString()).settings + + assertTrue(decoded.letterHints) + assertTrue(decoded.emojiAssociation) + assertTrue(decoded.voiceStripFillers) + assertFalse(decoded.voicePunctuationAsSpace) + } + @Test fun decodeDropsRimeDictionaryNamesThatCanEscapeTheUserDataDirectory() { val archive = UserDataArchive( diff --git a/app/src/test/java/llc/slacker/openime/VoiceTextProcessorFillerTest.kt b/app/src/test/java/llc/slacker/openime/VoiceTextProcessorFillerTest.kt new file mode 100644 index 00000000..abee2c01 --- /dev/null +++ b/app/src/test/java/llc/slacker/openime/VoiceTextProcessorFillerTest.kt @@ -0,0 +1,118 @@ +package llc.slacker.openime + +import llc.slacker.openime.voice.VoiceTextProcessingPolicy +import llc.slacker.openime.voice.VoiceTextProcessor +import org.junit.Assert.assertEquals +import org.junit.Test + +class VoiceTextProcessorFillerTest { + private val strip = VoiceTextProcessingPolicy( + autoTerminalPunctuation = false, + stripFillers = true, + ) + private val stripProse = VoiceTextProcessingPolicy( + autoTerminalPunctuation = true, + stripFillers = true, + ) + private val asSpace = VoiceTextProcessingPolicy( + autoTerminalPunctuation = true, + punctuationAsSpace = true, + ) + private val off = VoiceTextProcessingPolicy(autoTerminalPunctuation = false) + + private fun zh(raw: String, policy: VoiceTextProcessingPolicy) = + VoiceTextProcessor.process(raw, "zh-CN", policy) + + private fun en(raw: String, policy: VoiceTextProcessingPolicy) = + VoiceTextProcessor.process(raw, "en-US", policy) + + @Test + fun hesitationsAreRemovedFromTheStartMiddleAndEndOfChineseText() { + assertEquals("我觉得这个不错", zh("嗯我觉得这个不错", strip)) + assertEquals("我觉得这个不错", zh("我觉得呃这个不错", strip)) + assertEquals("我觉得这个不错", zh("我觉得 呃 这个不错", strip)) + assertEquals("我觉得这个不错", zh("我觉得这个不错嗯", strip)) + assertEquals("我觉得这个不错", zh("嗯嗯呃我觉得这个不错", strip)) + } + + @Test + fun fillersStayWhenTheSwitchIsOff() { + assertEquals("嗯我觉得这个不错", zh("嗯我觉得这个不错", off)) + assertEquals("um I think so", en("um I think so", off)) + } + + @Test + fun anUtteranceThatIsOnlyAHesitationIsKept() { + assertEquals("嗯", zh("嗯", strip)) + assertEquals("嗯嗯", zh("嗯嗯", strip)) + assertEquals("um", en("um", strip)) + } + + @Test + fun realWordsContainingFillerCharactersSurvive() { + assertEquals("他一直在打呃逆", zh("他一直在打呃逆", strip)) + assertEquals("额度不够", zh("额度不够", strip)) + assertEquals("总金额是五百", zh("总金额是五百", strip)) + // 额 only goes when the recogniser set it off as a word of its own. + assertEquals("这个不错", zh("额 这个不错", strip)) + } + + @Test + fun englishFillersAreRemovedOnlyAsWholeWords() { + assertEquals("I think so", en("um I think so", strip)) + assertEquals("I think so", en("I uh think so", strip)) + assertEquals("I think so", en("I think so er", strip)) + assertEquals("I think so", en("Um, I think so", strip)) + assertEquals("humble summer", en("humble summer", strip)) + assertEquals("to err is human", en("to err is human", strip)) + assertEquals("the herd", en("the herd", strip)) + } + + @Test + fun removingAFillerNextToPunctuationDoesNotLeaveStrayMarks() { + assertEquals("你好，世界", zh("嗯 你好 逗号 世界", strip)) + assertEquals("你好，世界", zh("嗯，你好 逗号 世界", strip)) + } + + @Test + fun automaticTerminalPunctuationStillAppliesAfterStripping() { + assertEquals("我觉得这个不错。", zh("嗯我觉得这个不错", stripProse)) + } + + @Test + fun punctuationBecomesSpacesBetweenClauses() { + assertEquals("你好 世界", zh("你好 逗号 世界", asSpace)) + assertEquals("你好 世界 再见", zh("你好 逗号 世界 句号 再见", asSpace)) + assertEquals("你准备好了吗", zh("你准备好了吗", asSpace)) + assertEquals("今天天气很好", zh("今天天气很好", asSpace)) + } + + @Test + fun trailingPunctuationIsDroppedNotReplacedByATrailingSpace() { + assertEquals("你好", zh("你好句号", asSpace)) + assertEquals("你好", zh("你好 感叹号", asSpace)) + } + + @Test + fun bracketsAndLatinNumbersAreLeftAlone() { + assertEquals("价格 3.5 元", zh("价格 3.5 元", asSpace)) + assertEquals("见 a.b 网站", zh("见 a.b 网站", asSpace)) + assertEquals("（备注）", zh("左括号 备注 右括号", asSpace).replace(" ", "")) + } + + @Test + fun englishPunctuationBecomesSpacesToo() { + assertEquals("hello world", en("hello comma world period", asSpace)) + assertEquals("hello world", en("hello world", asSpace)) + } + + @Test + fun bothSwitchesCompose() { + val both = VoiceTextProcessingPolicy( + autoTerminalPunctuation = true, + stripFillers = true, + punctuationAsSpace = true, + ) + assertEquals("你好 世界", zh("嗯 你好 逗号 呃 世界", both)) + } +} diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index 94b2179a..9f049c70 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -15,6 +15,14 @@ | 无个性化学习标志（隐身模式） | 不学习、不记录剪贴板 | `PersonalizedLearningPolicy` | | 一次提交几十万字（大段粘贴、长语音） | 分块提交，每块不超过 32000 个字符且不拆代理对，避免超过 Binder 单次事务上限 | `CrashResilienceTest` | +## 自动填充（Android 11+） + +`method.xml` 声明 `supportsInlineSuggestions`，键盘请求最多 5 个 48dp 高的条目并把系统渲染的条目放进工具栏位置。键盘只托管条目： +填入的内容由系统直接写入输入框，键盘既拿不到也不读取，所以密码框里同样可以显示条目。条目响应可能先于键盘视图到达（新输入框刚获得焦点）， +服务会暂存最近一次响应，视图建好后再显示，而不是拒绝（拒绝会让系统退回下拉菜单）。离开输入框时清除。 +Android 11 以下和不支持内嵌建议的提供者仍使用系统的下拉菜单。 +验证：`InlineChipTrackerTest`；`scripts/beta3_e2e.py autofill`（debug 构建自带测试提供者 `TestAutofillService` 和 `AutofillTestActivity`）。 + ## 物理键盘（平板、折叠屏键盘套、Chromebook、桌面模式、模拟器） 中文 26 键模式下：字母组成拼音，空格选首选，1–9 选候选，回车保留已输入拼音，Esc 取消，`'` 分词，退格删拼音； @@ -26,7 +34,7 @@ 横屏（不进入全屏提取模式，键盘是底部面板）、字体 130% / 200%（按键标签最多放大到 1.3 倍，功能键标签自动缩小）、 深色、小屏、窄屏、平板竖 / 横、折叠屏内屏。验证：`scripts/display_matrix_regression.py`（断言底部面板且每个键都在窗口内）、 -`DisplayEnvironmentInstrumentedTest`。 +`DisplayEnvironmentInstrumentedTest`、`scripts/beta3_e2e.py`（横屏下空格滑动光标、字母键数字和符号、表情联想、语音处理、自动填充）。 ## Android 版本 diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 78ce811b..8b645dd4 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -11,3 +11,7 @@ - 2026-09-29｜流式语音模型｜默认模型切换为 `sherpa-onnx-streaming-paraformer-bilingual-zh-en` 的 INT8 encoder/decoder；运行时使用 `OnlineParaformerModelConfig` + `greedy_search`，结束时补 300ms 静音；不再向 Paraformer stream 传 transducer-only 动态 hotwords｜优先降低模型体积和保持中英流式识别，同时遵循 sherpa-onnx v1.13.6 官方 Paraformer 配置。 - 2026-10-03｜语音词表｜内置词表随版本发布（`assets/hotwords/`），用户可导入文本词表；二者都不联网，不新增 `INTERNET` 权限，不做在线定期更新｜Paraformer 不能把热词传进解码器，所以词表走识别后的“同音替换”：读音（取自 `8105.dict.yaml`，含多音字）相同而字不同的片段改成词表写法。游戏词表默认关闭，避免日常聊天被误改。 - 2026-10-03｜整体分包｜99 个平铺文件按功能分进 theme/core/editor/data/setup/widget/floating/handwriting/rime/candidate/hotword/voice/panel/keyboard，根包只留 Service、Activity 和 JNI 类；依赖方向由 `ArchitectureLayeringTest` 固定｜包名只影响组织和可见性，不改 Manifest、native 符号和测试脚本引用的名字；Gradle 多模块暂不做，因为 `keyboard` 与 `panel`、`voice` 仍通过大接口耦合，需要先拆 `ImeKeyboardView.Listener`。 +- 2026-10-04｜自动填充条带｜只声明 `supportsInlineSuggestions`、自己拼装 androidx.autofill v1 的样式 Bundle（版本表 + 一个空的 v1 样式），不引入 androidx.autofill 依赖；条目按固定像素尺寸（150dp × 40dp）渲染并横向滚动｜`InlineContentView` 是远程 Surface，没有固有尺寸，`WRAP_CONTENT` 会得到 0×0；样式 Bundle 只有十几行，为它引入第一个 androidx 运行时依赖不值得。响应早于键盘视图到达时先暂存，不能返回 false（系统会退回下拉菜单）。 +- 2026-10-04｜字母键上的数字和符号｜不加一行数字，而是像搜狗、讯飞、微信键盘那样把数字和符号印在字母键右上角，上滑（复用九键的“上滑输入数字”开关）或长按输入；第一排 q–p 是 1–0，其余是标点，中文模式用全角形式（！ ￥ ？ （ ） ： ；），英文模式半角；提示可单独关闭（“数字和符号提示”，默认开启）｜多一行数字要么挤压每一行（竖屏约 43dp，横屏约 34dp），要么让键盘变高；输入法窗口高度一变，面板、浮动键盘、九键都要跟着变。提示 + 上滑不改布局，也是国内主流输入法的做法。 +- 2026-10-04｜空格滑动光标锁定底行｜进入光标模式（横向 18dp、横向分量大于纵向 1.25 倍、早于长按语音超时）后，同一行的其他按键 `touchLocked`（变灰、不响应）；拼音预编辑存在时移动的是预编辑光标｜手指一直在空格上，但第二根手指或漂移的拇指可能按到邻键；预编辑期间把方向键事件发给应用会打断组合。 +- 2026-10-04｜“标点用空格代替”｜只作用于语音识别结果：逗号、句号、问号等写成一个空格，结尾标点直接去掉，括号和 3.5、a.b 不变｜把这项需求理解为语音文本后处理的开关（与去语气词同属“语音输入”设置组），默认关闭。 diff --git a/docs/TEST_SOP_CHECKLIST.md b/docs/TEST_SOP_CHECKLIST.md index a4cced3f..b182efa6 100644 --- a/docs/TEST_SOP_CHECKLIST.md +++ b/docs/TEST_SOP_CHECKLIST.md @@ -40,6 +40,7 @@ - [ ] 长按空格约 150ms 启动语音，松手上屏，上滑取消无残留。 - [ ] 左右边缘和底行按键气泡锚点正确、无越界。 - [ ] 默认震动轻、按下即反馈、快速输入不堆积。 +- [ ] 空格横向滑动移动光标；滑动期间底行其他按键变灰且不响应，松手恢复；单击空格仍是空格；输入拼音时滑动移动拼音光标。 ## 语音语料 @@ -49,6 +50,8 @@ - [ ] 中英混合句、连续 30 秒中文、连续 30 秒中英混合。 - [ ] 环境噪音、静音、口述逗号/句号/问号/换行。 - [ ] 断网可用、权限拒绝可恢复、密码框保护正确。 +- [ ] “嗯我觉得呃这个不错”去掉语气词；“额度”“金额”“呃逆”不受影响；只说“嗯”时原样保留。 +- [ ] 打开“标点用空格代替”：口述逗号/句号变成空格，结尾标点消失，3.5、a.b 不变。 ## 面板与浮动键盘 @@ -59,6 +62,9 @@ - [ ] 常用语分类及模板增删改、插入、重启和升级保留通过。 - [ ] 工具与全部设置页面可滚动、无死按钮、返回正确、无旧主题/圆角选项。 - [ ] 浮动键盘可拖到五个位置和边缘，横竖屏后重新约束且可恢复停靠。 +- [ ] 中文/英文 26 键每个字母键右上角有数字或符号提示；上滑和长按都输入提示、点按仍输入字母；中文模式输入全角（！ ￥ ？ （ ） ： ；）、英文模式半角；横屏同样可用；关闭“数字和符号提示”后提示和手势一起消失，布局不变。 +- [ ] 选“开心”后联想栏最前面是 😊 😄 😁；关闭“表情联想”后不再出现；密码框不出现。 +- [ ] 自动填充（Android 11+）：登录框里条带显示账号，点一下填入并回到工具栏；“‹”收起条带；普通输入框不出现；横屏同样可用。 ## 发布稳定性 diff --git a/scripts/beta3_e2e.py b/scripts/beta3_e2e.py new file mode 100644 index 00000000..8ba29042 --- /dev/null +++ b/scripts/beta3_e2e.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 +"""Real-IME end-to-end check of the 0.0.3-beta.1 features. + +Drives the live openIME service on one device or emulator with real touch +events (`input swipe`, `input motionevent`), the debug E2E receiver for the +keyboard's own click targets, and `uiautomator dump` to read the screen. Needs +a debug build of openIME; it selects openIME as the default input method. + +Covered, each in portrait and (where it matters) landscape: + + * space-bar cursor drag: the cursor moves, the bottom row is locked while the + finger is down and free afterwards, a plain tap is still a space + * digits and symbols on the letter keys: printed in the corner, typed by swipe + up or long press, no extra row, the layout does not change + * emoji association after committing a word + * voice: filler removal and punctuation-as-space through the real + post-processing chain + * autofill strip (needs the debug test autofill service, installed with the + debug build): chips show in the keyboard and tapping one fills the field + +Screenshots go to --out so a person can look at them. Settings and rotation are +restored afterwards, also on error. + + python3 scripts/beta3_e2e.py [--serial SERIAL] [--out DIR] [case ...] +""" + +from __future__ import annotations + +import argparse +import base64 +import re +import subprocess +import sys +import time +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path + +PKG = "llc.slacker.openime" +IME = f"{PKG}/.LocalVoiceImeService" +LAB = f"{PKG}/.ImeTestLabActivity" +AUTOFILL_LAB = f"{PKG}/.AutofillTestActivity" +AUTOFILL_SERVICE = f"{PKG}/.TestAutofillService" +RECEIVER = f"{PKG}/.E2ETestReceiver" +ACTION = f"{PKG}.TEST_COMMAND" + +PREFS_HEAD = "" +PREFS_TAIL = "" + + +@dataclass +class Node: + cls: str + text: str + desc: str + x0: int + y0: int + x1: int + y1: int + + @property + def cx(self) -> int: + return (self.x0 + self.x1) // 2 + + @property + def cy(self) -> int: + return (self.y0 + self.y1) // 2 + + @property + def w(self) -> int: + return self.x1 - self.x0 + + @property + def h(self) -> int: + return self.y1 - self.y0 + + +class Device: + def __init__(self, serial: str | None, out: Path): + self.base = ["adb"] + (["-s", serial] if serial else []) + self.out = out + out.mkdir(parents=True, exist_ok=True) + self.shots = 0 + + def run(self, *args: str, binary: bool = False): + result = subprocess.run(self.base + list(args), capture_output=True) + return result.stdout if binary else result.stdout.decode("utf-8", "replace") + + def shell(self, *args: str) -> str: + return self.run("shell", *args) + + def density(self) -> float: + match = re.search(r"(\d+)", self.shell("wm", "density")) + return int(match.group(1)) / 160.0 if match else 2.625 + + def size(self) -> tuple[int, int]: + match = re.search(r"(\d+)x(\d+)", self.shell("wm", "size")) + return (int(match.group(1)), int(match.group(2))) if match else (1080, 2400) + + def shot(self, name: str) -> Path: + self.shots += 1 + path = self.out / f"{self.shots:02d}-{name}.png" + path.write_bytes(self.run("exec-out", "screencap", "-p", binary=True)) + return path + + def dump(self) -> list[Node]: + for _ in range(3): + self.shell("uiautomator", "dump", "--windows", "/sdcard/beta3.xml") + xml = self.shell("cat", "/sdcard/beta3.xml") + start = xml.find(" bool: + self.run("logcat", "-c") + self.shell("am", "broadcast", "-n", RECEIVER, "-a", ACTION, "--es", "cmd", cmd) + time.sleep(0.6) + log = self.run("logcat", "-d", "-s", "OpenImeE2E:I") + return "ok=true" in log.split("cmd=")[-1] if "cmd=" in log else False + + def tap(self, x: int, y: int) -> None: + self.shell("input", "tap", str(x), str(y)) + + +def b64(text: str) -> str: + return base64.b64encode(text.encode("utf-8")).decode("ascii") + + +def find(nodes: list[Node], desc_prefix: str) -> Node | None: + for node in nodes: + if node.desc.startswith(desc_prefix): + return node + return None + + +# uiautomator reports an empty field's hint as its text. +LAB_HINTS = {"单行普通文本", "用户名"} + + +def field_text(dev: Device) -> str: + for node in dev.dump(): + if node.cls.endswith("EditText"): + return "" if node.text in LAB_HINTS else node.text + return "" + + +def write_prefs(dev: Device, booleans: dict[str, bool]) -> None: + body = "".join(f'' for k, v in booleans.items()) + dev.shell("am", "force-stop", PKG) + time.sleep(1) + encoded = base64.b64encode(f"{PREFS_HEAD}{body}{PREFS_TAIL}".encode()).decode() + # One quoted string for the device shell: the payload is plain base64. + dev.run( + "shell", + f"run-as {PKG} sh -c 'mkdir -p shared_prefs && echo {encoded} | base64 -d > shared_prefs/ime_settings.xml'", + ) + + +def ensure_ime(dev: Device) -> None: + if dev.shell("settings", "get", "secure", "default_input_method").strip() != IME: + dev.shell("ime", "enable", "--user", "0", IME) + dev.shell("ime", "set", "--user", "0", IME) + time.sleep(1) + dev.shell("pm", "grant", PKG, "android.permission.RECORD_AUDIO") + + +def show_keyboard(dev: Device, focus: str = "lab_single") -> bool: + ensure_ime(dev) + dev.shell("am", "start", "-n", LAB, "--es", "focus_id", focus, "-f", "0x10008000") + time.sleep(3) + for _ in range(5): + if "mInputShown=true" in dev.shell("dumpsys", "input_method"): + break + edits = [n for n in dev.dump() if n.cls.endswith("EditText")] + if len(edits) >= 2: + dev.tap(edits[1].cx, edits[1].cy) + time.sleep(1) + dev.tap(edits[0].cx, edits[0].cy) + elif edits: + dev.tap(edits[0].cx, edits[0].cy) + time.sleep(2) + time.sleep(1.5) + return "mInputShown=true" in dev.shell("dumpsys", "input_method") + + +def set_rotation(dev: Device, rotation: int) -> None: + dev.shell("settings", "put", "system", "accelerometer_rotation", "0") + dev.shell("settings", "put", "system", "user_rotation", str(rotation)) + time.sleep(2.5) + + +def rime_ready(dev: Device) -> bool: + dev.run("logcat", "-c") + dev.command("state") + return "rimeReady=true" in dev.run("logcat", "-d", "-s", "OpenImeE2E:I") + + +# ----------------------------------------------------------------------------- cases + + +def case_space_cursor(dev: Device, landscape: bool) -> list[str]: + problems: list[str] = [] + write_prefs(dev, {"sound": False, "haptic": False}) + if not show_keyboard(dev): + return ["the keyboard did not show"] + dev.command("mode:ENGLISH_26") + time.sleep(1) + original = "hello world" + dev.command(f"type64:{b64(original)}") + time.sleep(1) + before = field_text(dev) + if before != original: + problems.append(f"typed text is {before!r}, expected {original!r}") + nodes = dev.dump() + space = find(nodes, "空格") + if not space: + return problems + ["space key not found"] + density = dev.density() + start_x = space.cx + y = space.cy + # DOWN, then horizontal moves in ~3dp steps, like a finger. + dev.shell("input", "motionevent", "DOWN", str(start_x), str(y)) + steps = 20 + total_dp = 18 + 12 * 4 + for i in range(1, steps + 1): + dev.shell("input", "motionevent", "MOVE", str(int(start_x - total_dp * density * i / steps)), str(y)) + time.sleep(0.4) + dev.shot(f"space-drag-held{'-landscape' if landscape else ''}") + # While the finger is down the bottom row is dimmed, so the other keys have + # no usable touch. A tap on the punctuation key must not type. + punct = [n for n in dev.dump() if n.desc.startswith(",")] + if punct: + # A second finger would hit it; `input tap` is a separate gesture. + dev.tap(punct[0].cx, punct[0].cy) + time.sleep(0.5) + dev.shell("input", "motionevent", "UP", str(int(start_x - total_dp * density)), str(y)) + time.sleep(0.8) + after_drag = field_text(dev) + if after_drag != original: + problems.append(f"the drag changed the text to {after_drag!r}") + dev.command("type:X") + time.sleep(1) + after = field_text(dev) + if after.replace("X", "") != original or "X" not in after: + problems.append(f"after the drag the text is {after!r}") + else: + index = after.index("X") + moved = len(original) - index + if not 2 <= moved <= 5: + problems.append(f"cursor moved {moved} characters (expected about 4)") + dev.shot(f"space-drag-after{'-landscape' if landscape else ''}") + # A plain tap on space is still a space. + nodes = dev.dump() + space = find(nodes, "空格") + if space: + dev.tap(space.cx, space.cy) + time.sleep(0.8) + if field_text(dev) == after: + problems.append("a tap on the space bar no longer types a space") + return problems + + +CHINESE_HINTS = dict( + zip("qwertyuiop", "1234567890"), + a="~", s="！", d="@", f="#", g="￥", h="%", j="&", k="*", l="？", + z="（", x="）", c="-", v="_", b="：", n="；", m="/", +) +ENGLISH_HINTS = {**CHINESE_HINTS, "s": "!", "g": "$", "l": "?", "z": "(", "x": ")", "b": ":", "n": ";"} + + +def key_node(nodes: list[Node], letter: str) -> Node | None: + return next((n for n in nodes if n.desc in (letter, letter.upper()) and n.w > 0), None) + + +def hint_inside(nodes: list[Node], key: Node, hint: str) -> bool: + return any(n.text == hint and n.x0 >= key.x0 and n.x1 <= key.x1 and n.y0 >= key.y0 and n.y1 <= key.y1 for n in nodes) + + +def case_letter_hints(dev: Device, landscape: bool) -> list[str]: + problems: list[str] = [] + space_tops: dict[bool, int] = {} + for enabled in (True, False): + write_prefs(dev, {"letter_hints": enabled, "sound": False, "haptic": False}) + if not show_keyboard(dev): + return ["the keyboard did not show"] + for mode, hints in (("PINYIN_26", CHINESE_HINTS), ("ENGLISH_26", ENGLISH_HINTS)): + dev.command(f"mode:{mode}") + time.sleep(1.2) + nodes = dev.dump() + label = f"{mode}-hints-{'on' if enabled else 'off'}{'-landscape' if landscape else ''}" + dev.shot(label) + space = find(nodes, "空格") + if not space: + problems.append(f"{mode}: space key missing") + continue + if mode == "PINYIN_26": + space_tops[enabled] = space.y0 + width, height = dev.size() + if landscape: + width, height = height, width + if space.y1 > height: + problems.append(f"{mode}: bottom row runs off the screen ({space.y1} > {height})") + shown = 0 + for letter, hint in hints.items(): + key = key_node(nodes, letter) + if not key: + problems.append(f"{mode}: key {letter} missing") + continue + if hint_inside(nodes, key, hint): + shown += 1 + if enabled and shown != 26: + problems.append(f"{mode}: only {shown} of 26 keys show their hint") + if not enabled and shown: + problems.append(f"{mode}: {shown} hints shown while the setting is off") + if not enabled: + continue + # Real gestures: swipe up and long press type the hint, a tap types the letter. + for letter in ("q", "s", "g", "z"): + key = key_node(nodes, letter) + if not key: + continue + hint = hints[letter] + before = field_text(dev) + dev.shell("input", "swipe", str(key.cx), str(key.cy), str(key.cx), str(key.cy - int(key.h * 1.3)), "140") + time.sleep(0.9) + typed = field_text(dev) + if typed != before + hint: + problems.append(f"{mode}: swipe up on {letter} typed {typed[len(before):]!r}, expected {hint!r}") + dev.shell("input", "swipe", str(key.cx), str(key.cy), str(key.cx), str(key.cy), "900") + time.sleep(1) + held = field_text(dev) + if held != typed + hint: + problems.append(f"{mode}: long press on {letter} typed {held[len(typed):]!r}, expected {hint!r}") + dev.shot(f"{mode}-hints-typed{'-landscape' if landscape else ''}") + if len(space_tops) == 2 and abs(space_tops[True] - space_tops[False]) > 2: + problems.append(f"the hints changed the keyboard layout (space at {space_tops[True]} vs {space_tops[False]})") + return problems + + +def case_emoji(dev: Device, landscape: bool) -> list[str]: + problems: list[str] = [] + for enabled in (True, False): + write_prefs(dev, {"emoji_association": enabled, "sound": False, "haptic": False}) + if not show_keyboard(dev): + return ["the keyboard did not show"] + dev.command("mode:PINYIN_26") + time.sleep(1) + for letter in "kaixin": + dev.command(f"tap:{letter}") + time.sleep(1.5) + nodes = dev.dump() + candidate = next((n for n in nodes if n.text == "开心" or n.desc.startswith("开心")), None) + if not candidate: + problems.append("candidate 开心 not offered for 'kaixin'") + continue + dev.tap(candidate.cx, candidate.cy) + time.sleep(1.2) + nodes = dev.dump() + emoji = [n for n in nodes if n.desc.startswith("联想:") and any(ord(c) > 0x2000 for c in n.desc)] + dev.shot(f"emoji-{'on' if enabled else 'off'}{'-landscape' if landscape else ''}") + if enabled and not emoji: + problems.append("no emoji association after committing 开心") + if enabled and emoji and not emoji[0].desc.startswith("联想:😊"): + problems.append(f"first association is {emoji[0].desc!r}, expected 😊") + if not enabled and emoji: + problems.append("emoji association shown while the setting is off") + if enabled and emoji: + dev.tap(emoji[0].cx, emoji[0].cy) + time.sleep(1) + if "😊" not in field_text(dev): + problems.append(f"tapping the emoji typed {field_text(dev)!r}") + return problems + + +def case_voice(dev: Device, landscape: bool) -> list[str]: + problems: list[str] = [] + cases = [ + ({"voice_strip_fillers": True, "voice_punctuation_as_space": False}, "嗯我觉得呃这个不错", "我觉得这个不错"), + ({"voice_strip_fillers": False, "voice_punctuation_as_space": False}, "嗯我觉得呃这个不错", "嗯我觉得呃这个不错"), + ({"voice_strip_fillers": True, "voice_punctuation_as_space": True}, "嗯 你好 逗号 世界 句号", "你好 世界"), + ({"voice_strip_fillers": True, "voice_punctuation_as_space": False}, "你好 逗号 世界", "你好，世界"), + ] + for prefs, spoken, expected in cases: + write_prefs(dev, {**prefs, "sound": False, "haptic": False}) + if not show_keyboard(dev): + return ["the keyboard did not show"] + dev.command("mode:PINYIN_26") + if not dev.command(f"voice-processed64:{b64(spoken)}"): + problems.append(f"voice-processed command refused for {spoken!r}") + continue + time.sleep(2.5) + got = field_text(dev).strip() + # Natural terminal punctuation may follow in prose fields. + if got.rstrip("。？！") != expected: + problems.append(f"{prefs}: {spoken!r} -> {got!r}, expected {expected!r}") + dev.shot(f"voice{'-landscape' if landscape else ''}") + return problems + + +def case_autofill(dev: Device, landscape: bool) -> list[str]: + problems: list[str] = [] + write_prefs(dev, {"sound": False, "haptic": False}) + ensure_ime(dev) + dev.shell("settings", "put", "secure", "autofill_service", AUTOFILL_SERVICE) + try: + # The first request after a process start has no window token yet, so + # warm the service up before the real attempt. + dev.shell("am", "start", "-n", AUTOFILL_LAB, "-f", "0x10008000") + time.sleep(4) + dev.shell("am", "start", "-n", AUTOFILL_LAB, "-f", "0x10008000") + time.sleep(4) + edits = [n for n in dev.dump() if n.cls.endswith("EditText")] + if len(edits) < 2: + return ["autofill lab not on screen"] + dev.tap(edits[1].cx, edits[1].cy) + time.sleep(2) + dev.tap(edits[0].cx, edits[0].cy) + time.sleep(4) + nodes = dev.dump() + back = find(nodes, "返回工具栏") + dev.shot(f"autofill-strip{'-landscape' if landscape else ''}") + if not back: + return problems + ["the autofill strip did not appear in the keyboard"] + # Chips are remote surfaces; the first one starts right after the back control. + chip_x = back.x1 + int(80 * dev.density()) + dev.tap(chip_x, back.cy) + time.sleep(3) + text = next((n.text for n in dev.dump() if n.cls.endswith("EditText") and "@" in n.text), "") + dev.shot(f"autofill-filled{'-landscape' if landscape else ''}") + if text != "work@openime.dev": + problems.append(f"tapping the chip filled {text!r}") + # After filling, the toolbar comes back. + if not find(dev.dump(), "切换键盘"): + problems.append("the toolbar did not come back after filling") + finally: + dev.shell("settings", "delete", "secure", "autofill_service") + return problems + + +CASES = { + "space_cursor": case_space_cursor, + "letter_hints": case_letter_hints, + "emoji": case_emoji, + "voice": case_voice, + "autofill": case_autofill, +} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--serial") + parser.add_argument("--out", type=Path, default=Path(".local/test-runs/beta3-e2e")) + parser.add_argument("--no-landscape", action="store_true") + parser.add_argument("cases", nargs="*", help=f"subset of: {', '.join(CASES)}") + args = parser.parse_args() + unknown = [name for name in args.cases if name not in CASES] + if unknown: + parser.error(f"unknown case(s): {', '.join(unknown)}") + + dev = Device(args.serial, args.out) + results: dict[str, bool] = {} + try: + for orientation in ([False] if args.no_landscape else [False, True]): + set_rotation(dev, 1 if orientation else 0) + for name, case in CASES.items(): + if args.cases and name not in args.cases: + continue + label = f"{name}{' (landscape)' if orientation else ''}" + try: + problems = case(dev, orientation) + except Exception as error: # a crashed case must not hide the others + problems = [f"error: {error}"] + results[label] = not problems + print(f"{'PASS' if not problems else 'FAIL'} {label}" + (f" :: {'; '.join(problems)}" if problems else "")) + finally: + set_rotation(dev, 0) + dev.shell("settings", "delete", "secure", "autofill_service") + write_prefs(dev, {}) + failed = [name for name, ok in results.items() if not ok] + print(f"SUMMARY {len(results) - len(failed)} passed, {len(failed)} failed" + (f": {failed}" if failed else "")) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main())