diff --git a/app/src/androidTest/java/io/livekit/android/example/voiceassistant/HermesApprovalUiTest.kt b/app/src/androidTest/java/io/livekit/android/example/voiceassistant/HermesApprovalUiTest.kt new file mode 100644 index 0000000..e1c266e --- /dev/null +++ b/app/src/androidTest/java/io/livekit/android/example/voiceassistant/HermesApprovalUiTest.kt @@ -0,0 +1,42 @@ +package io.livekit.android.example.voiceassistant + +import androidx.activity.ComponentActivity +import androidx.compose.ui.test.junit4.createAndroidComposeRule +import androidx.compose.ui.test.onNodeWithText +import androidx.compose.ui.test.performClick +import io.livekit.android.example.voiceassistant.screen.HermesApprovalDialog +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test + +class HermesApprovalUiTest { + @get:Rule + val composeRule = createAndroidComposeRule() + + @Test + fun destructiveDialogHasOnlyPhysicalConfirmAndCancel() { + var choice: String? = null + composeRule.setContent { + HermesApprovalDialog( + request = ApprovalRequest( + runId = "run-1", + target = "fixture", + action = "Delete", + reason = "Requested for a no-op test", + agent = "Computer Operator" + ), + onConfirm = { choice = "once" }, + onCancel = { choice = "deny" } + ) + } + + composeRule.onNodeWithText("⚠ DESTRUCTIVE ACTION").assertExists() + composeRule.onNodeWithText("Agent: Computer Operator").assertExists() + composeRule.onNodeWithText("Action: Delete").assertExists() + composeRule.onNodeWithText("Target: fixture").assertExists() + composeRule.onNodeWithText("CANCEL").assertExists() + composeRule.onNodeWithText("CONFIRM").assertExists().performClick() + composeRule.onNodeWithText("APPROVE").assertDoesNotExist() + assertEquals("once", choice) + } +} diff --git a/app/src/androidTest/java/io/livekit/android/example/voiceassistant/HermesChatUiTest.kt b/app/src/androidTest/java/io/livekit/android/example/voiceassistant/HermesChatUiTest.kt new file mode 100644 index 0000000..fb34cce --- /dev/null +++ b/app/src/androidTest/java/io/livekit/android/example/voiceassistant/HermesChatUiTest.kt @@ -0,0 +1,105 @@ +package io.livekit.android.example.voiceassistant + +import androidx.activity.ComponentActivity +import androidx.compose.ui.test.junit4.createAndroidComposeRule +import androidx.compose.ui.test.onNodeWithTag +import androidx.compose.ui.test.onNodeWithText +import androidx.compose.ui.test.performClick +import androidx.compose.ui.test.performTextClearance +import androidx.compose.ui.test.performTextInput +import io.livekit.android.example.voiceassistant.realtime.ControlPacket +import io.livekit.android.example.voiceassistant.screen.HermesChatTransport +import io.livekit.android.example.voiceassistant.screen.HermesScreen +import io.livekit.android.example.voiceassistant.screen.HermesSessionController +import kotlinx.coroutines.CompletableDeferred +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Rule +import org.junit.Test + +class HermesChatUiTest { + @get:Rule + val composeRule = createAndroidComposeRule() + + @Test + fun sendAddsBubbleBeforeTransportCompletes() { + val transport = FakeChatTransport(blockSend = true) + composeRule.setContent { + HermesScreen(FakeVoiceController(), chatTransport = transport) + } + + submit("hello") + + composeRule.onNodeWithText("hello").assertExists() + assertFalse(transport.completion.isCompleted) + assertEquals(1, transport.chatSends) + } + + @Test + fun slashAndMentionSuggestionsAppearImmediately() { + composeRule.setContent { + HermesScreen(FakeVoiceController(), chatTransport = FakeChatTransport()) + } + + composeRule.onNodeWithTag("message_input").performTextInput("@c") + composeRule.onNodeWithText("@coder").assertExists() + composeRule.onNodeWithText("@computer-operator").assertExists() + composeRule.onNodeWithTag("message_input").performTextClearance() + composeRule.onNodeWithTag("message_input").performTextInput("/m") + composeRule.onNodeWithText("/mute").assertExists() + composeRule.onNodeWithText("/memory").assertExists() + } + + @Test + fun muteAndEndCallDoNotSendChat() { + val transport = FakeChatTransport() + val voice = FakeVoiceController() + composeRule.setContent { HermesScreen(voice, chatTransport = transport) } + + submit("/mute") + submit("/endcall") + composeRule.waitForIdle() + + assertEquals(0, transport.chatSends) + assertEquals(listOf(false, false), voice.micStates) + } + + private fun submit(text: String) { + composeRule.onNodeWithTag("message_input").performTextInput(text) + composeRule.onNodeWithTag("send_button").performClick() + composeRule.waitForIdle() + } + + private class FakeChatTransport( + private val blockSend: Boolean = false + ) : HermesChatTransport { + var chatSends = 0 + val completion = CompletableDeferred() + + override suspend fun sendMessage( + text: String, + localId: String, + operationId: String + ): String { + chatSends += 1 + if (blockSend) completion.await() + return "stream-$chatSends" + } + + override suspend fun sendControl(packet: ControlPacket): Boolean = true + } + + private class FakeVoiceController : HermesSessionController { + val micStates = mutableListOf() + override val isConnected = true + override val isReconnecting = false + + override suspend fun start(microphoneEnabled: Boolean) = Unit + + override suspend fun setMicrophoneEnabled(enabled: Boolean) { + micStates += enabled + } + + override suspend fun setAgentVolume(volume: Double) = Unit + } +} diff --git a/app/src/androidTest/java/io/livekit/android/example/voiceassistant/HermesLifecycleTest.kt b/app/src/androidTest/java/io/livekit/android/example/voiceassistant/HermesLifecycleTest.kt new file mode 100644 index 0000000..cf24584 --- /dev/null +++ b/app/src/androidTest/java/io/livekit/android/example/voiceassistant/HermesLifecycleTest.kt @@ -0,0 +1,51 @@ +package io.livekit.android.example.voiceassistant + +import androidx.activity.ComponentActivity +import androidx.compose.ui.test.junit4.createAndroidComposeRule +import androidx.compose.ui.test.onNodeWithTag +import androidx.compose.ui.test.onNodeWithText +import io.livekit.android.example.voiceassistant.screen.HermesScreen +import io.livekit.android.example.voiceassistant.screen.HermesSessionController +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test + +class HermesLifecycleTest { + @get:Rule + val composeRule = createAndroidComposeRule() + + @Test + fun launchShowsChatAndCallWithMicOff() { + val controller = FakeHermesSessionController() + + composeRule.setContent { HermesScreen(controller) } + composeRule.waitForIdle() + + composeRule.onNodeWithTag("conversation_timeline").assertExists() + composeRule.onNodeWithText("CALL HERMES").assertExists() + composeRule.onNodeWithText("মাইক্রোফোন বন্ধ").assertExists() + assertEquals(1, controller.startCalls) + assertEquals(false, controller.startMicrophoneEnabled) + assertEquals(0, controller.enableMicCalls) + } + + private class FakeHermesSessionController : HermesSessionController { + var startCalls = 0 + var startMicrophoneEnabled: Boolean? = null + var enableMicCalls = 0 + + override val isConnected = false + override val isReconnecting = false + + override suspend fun start(microphoneEnabled: Boolean) { + startCalls += 1 + startMicrophoneEnabled = microphoneEnabled + } + + override suspend fun setMicrophoneEnabled(enabled: Boolean) { + if (enabled) enableMicCalls += 1 + } + + override suspend fun setAgentVolume(volume: Double) = Unit + } +} diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/ApprovalProtocol.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/ApprovalProtocol.kt new file mode 100644 index 0000000..d070c61 --- /dev/null +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/ApprovalProtocol.kt @@ -0,0 +1,46 @@ +package io.livekit.android.example.voiceassistant + +import com.google.gson.Gson + +const val APPROVAL_REQUEST_TOPIC = "hermes.approval.request" +const val APPROVAL_RESPONSE_TOPIC = "hermes.approval.response" + +data class ApprovalRequest( + val runId: String, + val target: String, + val action: String, + val reason: String, + val agent: String? = null +) { + val displayAgent: String + get() = agent?.takeIf(String::isNotBlank) ?: "Hermes Main" +} + +private data class ApprovalResponse(val runId: String, val choice: String) + +private val approvalGson = Gson() + +fun parseApprovalRequest(data: ByteArray): ApprovalRequest? { + if (data.isEmpty() || data.size > 4096) return null + val request = runCatching { + approvalGson.fromJson(data.decodeToString(), ApprovalRequest::class.java) + }.getOrNull() ?: return null + + return runCatching { + request.takeIf { + it.runId.isNotBlank() && + it.target.isNotBlank() && + it.action.isNotBlank() && + it.reason.isNotBlank() + } + }.getOrNull()?.copy( + agent = request.agent + ?.trim() + ?.takeIf { it.length <= 80 && it.none(Char::isISOControl) } + ) +} + +fun approvalResponseJson(runId: String, choice: String): String? { + if (runId.isBlank() || choice !in setOf("once", "deny")) return null + return approvalGson.toJson(ApprovalResponse(runId, choice)) +} diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/MainActivity.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/MainActivity.kt index d4d2e31..1b0e076 100644 --- a/app/src/main/java/io/livekit/android/example/voiceassistant/MainActivity.kt +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/MainActivity.kt @@ -8,13 +8,7 @@ import androidx.compose.foundation.layout.padding import androidx.compose.material3.Scaffold import androidx.compose.ui.Modifier import androidx.lifecycle.viewmodel.compose.viewModel -import androidx.navigation.compose.NavHost -import androidx.navigation.compose.composable -import androidx.navigation.compose.rememberNavController import io.livekit.android.LiveKit -import io.livekit.android.example.voiceassistant.screen.ConnectRoute -import io.livekit.android.example.voiceassistant.screen.ConnectScreen -import io.livekit.android.example.voiceassistant.screen.VoiceAssistantRoute import io.livekit.android.example.voiceassistant.screen.VoiceAssistantScreen import io.livekit.android.example.voiceassistant.ui.theme.LiveKitVoiceAssistantExampleTheme import io.livekit.android.example.voiceassistant.viewmodel.VoiceAssistantViewModel @@ -24,34 +18,14 @@ class MainActivity : ComponentActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - LiveKit.loggingLevel = LoggingLevel.DEBUG + LiveKit.loggingLevel = LoggingLevel.INFO setContent { - val navController = rememberNavController() LiveKitVoiceAssistantExampleTheme(dynamicColor = false) { Scaffold { innerPadding -> Box(modifier = Modifier.padding(innerPadding)) { - - // Set up NavHost for the app - NavHost(navController, startDestination = ConnectRoute) { - composable { - ConnectScreen(navigateToVoiceAssistant = { voiceAssistantRoute -> - runOnUiThread { - navController.navigate(voiceAssistantRoute) - } - }) - } - - composable { - val viewModel = viewModel() - VoiceAssistantScreen( - viewModel = viewModel, - onEndCall = { - runOnUiThread { navController.navigateUp() } - } - ) - } - } + val viewModel = viewModel() + VoiceAssistantScreen(viewModel = viewModel) } } } diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/HermesInput.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/HermesInput.kt new file mode 100644 index 0000000..01f7ca0 --- /dev/null +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/HermesInput.kt @@ -0,0 +1,86 @@ +package io.livekit.android.example.voiceassistant.realtime + +enum class HermesCommand(val wireValue: String) { + NEW("new"), + STATUS("status"), + AGENTS("agents"), + TASKS("tasks"), + STOP("stop"), + VOICE("voice"), + CALL("call"), + ENDCALL("endcall"), + MUTE("mute"), + UNMUTE("unmute"), + MEMORY("memory"), + HELP("help") +} + +sealed interface InputIntent { + data class Local(val command: HermesCommand) : InputIntent + + data class Control(val command: HermesCommand) : InputIntent + + data class Message(val text: String) : InputIntent +} + +val SUPPORTED_MENTIONS = listOf( + "@main", + "@architect", + "@researcher", + "@coder", + "@browser", + "@computer-operator", + "@qa", + "@reviewer", + "@security", + "@ops" +) + +val SUPPORTED_SLASH_COMMANDS = listOf( + "/new", + "/status", + "/agents", + "/tasks", + "/stop", + "/voice", + "/call", + "/endcall", + "/mute", + "/unmute", + "/memory", + "/help" +) + +private val LOCAL_COMMANDS = mapOf( + "/mute" to HermesCommand.MUTE, + "/unmute" to HermesCommand.UNMUTE, + "/voice" to HermesCommand.VOICE, + "/call" to HermesCommand.CALL, + "/endcall" to HermesCommand.ENDCALL, + "/help" to HermesCommand.HELP +) + +private val CONTROL_COMMANDS = mapOf( + "/new" to HermesCommand.NEW, + "/stop" to HermesCommand.STOP, + "/status" to HermesCommand.STATUS +) + +fun parseInput(text: String): InputIntent { + val normalized = text.trim().lowercase() + LOCAL_COMMANDS[normalized]?.let { return InputIntent.Local(it) } + CONTROL_COMMANDS[normalized]?.let { return InputIntent.Control(it) } + return InputIntent.Message(text) +} + +fun suggestInputs(text: String): List { + val prefix = text.trimStart().lowercase() + if (prefix.isEmpty() || prefix.any(Char::isWhitespace)) return emptyList() + return when { + prefix.startsWith("@") -> SUPPORTED_MENTIONS.filter { it.startsWith(prefix) } + prefix.startsWith("/") -> SUPPORTED_SLASH_COMMANDS.filter { + it.startsWith(prefix) + } + else -> emptyList() + } +} diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/HistoryRepository.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/HistoryRepository.kt new file mode 100644 index 0000000..fb5a39b --- /dev/null +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/HistoryRepository.kt @@ -0,0 +1,115 @@ +package io.livekit.android.example.voiceassistant.realtime + +import android.content.SharedPreferences +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken + +interface HistoryStorage { + fun read(key: String): String? + + fun write(key: String, value: String) +} + +class SharedPreferencesHistoryStorage( + private val preferences: SharedPreferences +) : HistoryStorage { + override fun read(key: String): String? = preferences.getString(key, null) + + override fun write(key: String, value: String) { + preferences.edit().putString(key, value).apply() + } +} + +private data class HistoryItem( + val id: String, + val text: String, + val role: MessageRole, + val source: MessageSource, + val timestampMs: Long +) + +class HistoryRepository( + private val storage: HistoryStorage, + private val gson: Gson = Gson() +) { + fun save(conversationId: String, messages: List) { + val items = messages + .asSequence() + .filter(TimelineMessage::persistable) + .takeLast(MAX_HISTORY_MESSAGES) + .map { + HistoryItem( + id = it.id, + text = redactForHistory(it.text), + role = it.role, + source = it.source, + timestampMs = it.timestampMs + ) + } + .toList() + storage.write(historyKey(conversationId), gson.toJson(items)) + } + + fun load(conversationId: String): List { + val value = storage.read(historyKey(conversationId)) ?: return emptyList() + val type = object : TypeToken>() {}.type + val items: List = runCatching { + gson.fromJson>(value, type) + }.getOrNull() ?: return emptyList() + + return items + .takeLast(MAX_HISTORY_MESSAGES) + .filter { it.role in setOf(MessageRole.USER, MessageRole.HERMES) } + .filter { it.source != MessageSource.STATUS } + .map { + TimelineMessage( + id = it.id, + text = redactForHistory(it.text), + role = it.role, + source = it.source, + timestampMs = it.timestampMs, + isFinal = true, + delivery = DeliveryState.SENT + ) + } + } + + private fun historyKey(conversationId: String): String { + require(identifierPattern.matches(conversationId)) { + "conversation identifier is invalid" + } + return "history:$conversationId" + } + + private companion object { + const val MAX_HISTORY_MESSAGES = 200 + val identifierPattern = Regex("[A-Za-z0-9_.:-]{1,128}") + } +} + +private val bearerPattern = Regex( + """(?i)\bBearer\s+[A-Za-z0-9._~+/=-]{8,}""" +) +private val assignmentPattern = Regex( + """(?i)\b(api[-_ ]?key|secret|token|password)\b\s*[:=]\s*[^\s,;]+""" +) +private val jwtPattern = Regex( + """\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b""" +) +private val longHexPattern = Regex("""\b[0-9a-fA-F]{32,}\b""") +private val longBase64Pattern = Regex( + """\b(?=[A-Za-z0-9+/=_-]{40,}\b)(?=[A-Za-z0-9+/=_-]*[A-Z])(?=[A-Za-z0-9+/=_-]*[a-z])[A-Za-z0-9+/=_-]+""" +) + +fun redactForHistory(text: String): String = text + .replace(bearerPattern, "Bearer [REDACTED]") + .replace(assignmentPattern) { match -> + "${match.groupValues[1]}=[REDACTED]" + } + .replace(jwtPattern, "[REDACTED]") + .replace(longHexPattern, "[REDACTED]") + .replace(longBase64Pattern, "[REDACTED]") + +private fun Sequence.takeLast(count: Int): Sequence = toList() + .takeLast(count) + .asSequence() diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/LatencyTracker.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/LatencyTracker.kt new file mode 100644 index 0000000..afe6e29 --- /dev/null +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/LatencyTracker.kt @@ -0,0 +1,45 @@ +package io.livekit.android.example.voiceassistant.realtime + +data class LatencyPayload( + val opId: String, + val durationsMs: Map +) + +class LatencyTracker(val opId: String) { + private val marks = linkedMapOf() + + init { + require(identifierPattern.matches(opId)) { "latency operation identifier is invalid" } + } + + fun mark(name: String, timestampNanos: Long = System.nanoTime()) { + require(markNamePattern.matches(name)) { "latency mark name is invalid" } + marks.putIfAbsent(name, timestampNanos) + } + + fun durationMs(start: String, end: String): Int? { + val startNanos = marks[start] ?: return null + val endNanos = marks[end] ?: return null + return millisecondsBetween(startNanos, endNanos) + } + + fun payload(): LatencyPayload { + val durations = linkedMapOf() + marks.entries.zipWithNext().forEach { (start, end) -> + durations["${start.key}_to_${end.key}"] = + millisecondsBetween(start.value, end.value) + } + return LatencyPayload(opId, durations) + } + + private fun millisecondsBetween(startNanos: Long, endNanos: Long): Int = + ((endNanos - startNanos) / NANOS_PER_MILLISECOND) + .coerceIn(0, Int.MAX_VALUE.toLong()) + .toInt() + + private companion object { + const val NANOS_PER_MILLISECOND = 1_000_000L + val identifierPattern = Regex("[A-Za-z0-9_.:-]{1,128}") + val markNamePattern = Regex("[A-Za-z0-9_.:-]{1,96}") + } +} diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/RealtimeProtocol.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/RealtimeProtocol.kt new file mode 100644 index 0000000..71cd62f --- /dev/null +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/RealtimeProtocol.kt @@ -0,0 +1,203 @@ +package io.livekit.android.example.voiceassistant.realtime + +import com.google.gson.Gson +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import com.google.gson.annotations.SerializedName + +const val CONTROL_TOPIC = "hermes.control" +const val STATUS_TOPIC = "hermes.status" +const val REALTIME_PROTOCOL_VERSION = 1 +private const val MAX_PACKET_BYTES = 4096 + +private val identifierPattern = Regex("[A-Za-z0-9_.:-]{1,128}") +private val safeNamePattern = Regex("[A-Za-z0-9_.:-]{1,64}") +private val fingerprintPattern = Regex("[0-9a-f]{12}") +private val durationNamePattern = Regex("[A-Za-z0-9_.:-]{1,96}") +private val controlCommands = setOf("new", "stop", "status") + +data class ControlPacket( + val version: Int = REALTIME_PROTOCOL_VERSION, + @SerializedName("op_id") val opId: String, + val command: String, + @SerializedName("conversation_id") val conversationId: String? = null +) + +data class StatusPacket( + val version: Int, + val type: String, + @SerializedName("conversation_fingerprint") + val conversationFingerprint: String? = null, + val tool: String? = null, + val status: String? = null, + val mention: String? = null, + val state: String? = null, + @SerializedName("op_id") val opId: String? = null, + @SerializedName("duration_ms") val durationMs: Int? = null, + @SerializedName("durations_ms") val durationsMs: Map? = null, + val duration: Double? = null, + @SerializedName("duration_seconds") val durationSeconds: Double? = null, + val error: Boolean? = null, + @SerializedName("is_final") val isFinal: Boolean? = null, + @SerializedName("is_interruption") val isInterruption: Boolean? = null, + @SerializedName("streamed") val streamed: Boolean? = null, + @SerializedName("connection_reused") val connectionReused: Boolean? = null, + @SerializedName("ttfb_ms") val ttfbMs: Int? = null, + @SerializedName("audio_duration_ms") val audioDurationMs: Int? = null, + @SerializedName("end_of_utterance_delay_ms") val endOfUtteranceDelayMs: Int? = null, + @SerializedName("transcription_delay_ms") val transcriptionDelayMs: Int? = null, + @SerializedName("on_user_turn_completed_delay_ms") + val onUserTurnCompletedDelayMs: Int? = null, + @SerializedName("total_duration_ms") val totalDurationMs: Int? = null, + @SerializedName("prediction_duration_ms") val predictionDurationMs: Int? = null, + @SerializedName("detection_delay_ms") val detectionDelayMs: Int? = null +) + +private val realtimeGson = Gson() + +fun controlPacketJson(packet: ControlPacket): String? { + if (packet.version != REALTIME_PROTOCOL_VERSION) return null + if (!identifierPattern.matches(packet.opId)) return null + if (packet.command !in controlCommands) return null + if (packet.command == "new") { + if (packet.conversationId == null || + !identifierPattern.matches(packet.conversationId) + ) { + return null + } + } else if (packet.conversationId != null) { + return null + } + return realtimeGson.toJson(packet) +} + +private val statusFields = mapOf( + "session.ready" to setOf("conversation_fingerprint"), + "tool.started" to setOf("tool"), + "tool.completed" to setOf("tool", "duration", "error"), + "subagent.start" to setOf("status"), + "subagent.complete" to setOf("status", "duration_seconds"), + "approval.request" to emptySet(), + "run.completed" to emptySet(), + "run.failed" to emptySet(), + "run.cancelled" to emptySet(), + "delegation.requested" to setOf("mention", "status"), + "first_hermes_delta" to setOf("duration_ms"), + "agent.state" to setOf("state"), + "user.state" to setOf("state"), + "user.transcription" to setOf("is_final"), + "speech.overlap" to setOf( + "is_interruption", + "detection_delay_ms", + "prediction_duration_ms" + ), + "metrics.eou" to setOf( + "end_of_utterance_delay_ms", + "transcription_delay_ms", + "on_user_turn_completed_delay_ms" + ), + "metrics.tts" to setOf("ttfb_ms", "streamed", "connection_reused"), + "metrics.stt" to setOf( + "duration_ms", + "audio_duration_ms", + "streamed", + "connection_reused" + ), + "metrics.interruption" to setOf( + "total_duration_ms", + "prediction_duration_ms", + "detection_delay_ms" + ), + "latency" to setOf("op_id", "durations_ms") +) + +private val stringFields = setOf( + "conversation_fingerprint", + "tool", + "status", + "mention", + "state", + "op_id" +) +private val booleanFields = setOf( + "error", + "is_final", + "is_interruption", + "streamed", + "connection_reused" +) +private val numberFields = statusFields.values.flatten().toSet() - + stringFields - booleanFields - setOf("durations_ms") + +fun parseStatusPacket(data: ByteArray): StatusPacket? { + if (data.isEmpty() || data.size > MAX_PACKET_BYTES) return null + val json = runCatching { + JsonParser.parseString(data.decodeToString()).asJsonObject + }.getOrNull() ?: return null + + if (json.keySet().let { "version" !in it || "type" !in it }) return null + val version = json.intValue("version") ?: return null + if (version != REALTIME_PROTOCOL_VERSION) return null + val type = json.stringValue("type") ?: return null + val allowed = statusFields[type] ?: return null + if ((json.keySet() - (allowed + setOf("version", "type"))).isNotEmpty()) { + return null + } + if (!validateStatusValues(json, allowed)) return null + + return runCatching { + realtimeGson.fromJson(json, StatusPacket::class.java) + }.getOrNull() +} + +private fun validateStatusValues(json: JsonObject, fields: Set): Boolean { + for (field in fields.intersect(json.keySet())) { + val value = json[field] + when (field) { + in stringFields -> { + val text = json.stringValue(field) ?: return false + val valid = when (field) { + "conversation_fingerprint" -> fingerprintPattern.matches(text) + "tool" -> safeNamePattern.matches(text) + "mention" -> "@$text" in SUPPORTED_MENTIONS + "op_id" -> identifierPattern.matches(text) + else -> text.length in 1..128 && text.none(Char::isISOControl) + } + if (!valid) return false + } + in booleanFields -> if (!value.isJsonPrimitive || + !value.asJsonPrimitive.isBoolean + ) { + return false + } + in numberFields -> if (json.nonNegativeNumber(field) == null) return false + "durations_ms" -> { + if (!value.isJsonObject) return false + for ((name, duration) in value.asJsonObject.entrySet()) { + if (!durationNamePattern.matches(name) || + !duration.isJsonPrimitive || + !duration.asJsonPrimitive.isNumber || + runCatching { duration.asInt }.getOrNull()?.let { it >= 0 } != true + ) { + return false + } + } + } + } + } + return true +} + +private fun JsonObject.stringValue(name: String): String? = runCatching { + get(name).takeIf { it.isJsonPrimitive && it.asJsonPrimitive.isString }?.asString +}.getOrNull() + +private fun JsonObject.intValue(name: String): Int? = runCatching { + get(name).takeIf { it.isJsonPrimitive && it.asJsonPrimitive.isNumber }?.asInt +}.getOrNull() + +private fun JsonObject.nonNegativeNumber(name: String): Double? = runCatching { + get(name).takeIf { it.isJsonPrimitive && it.asJsonPrimitive.isNumber } + ?.asDouble + ?.takeIf { it.isFinite() && it >= 0 } +}.getOrNull() diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/SessionIdentity.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/SessionIdentity.kt new file mode 100644 index 0000000..962434d --- /dev/null +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/SessionIdentity.kt @@ -0,0 +1,71 @@ +package io.livekit.android.example.voiceassistant.realtime + +import android.content.SharedPreferences +import java.util.UUID + +interface IdentityStorage { + fun read(key: String): String? + + fun write(key: String, value: String) +} + +class SharedPreferencesIdentityStorage( + private val preferences: SharedPreferences +) : IdentityStorage { + override fun read(key: String): String? = preferences.getString(key, null) + + override fun write(key: String, value: String) { + preferences.edit().putString(key, value).apply() + } +} + +data class SessionIdentity( + val installationId: String, + val conversationId: String +) + +class SessionIdentityStore( + private val storage: IdentityStorage, + private val generateId: () -> String = { UUID.randomUUID().toString() } +) { + @Synchronized + fun current(): SessionIdentity { + val installation = validStored(INSTALLATION_KEY) ?: newIdentifier().also { + storage.write(INSTALLATION_KEY, it) + } + val conversation = validStored(CONVERSATION_KEY) ?: newIdentifier().also { + storage.write(CONVERSATION_KEY, it) + } + return SessionIdentity(installation, conversation) + } + + @Synchronized + fun rotateConversation(): SessionIdentity { + val before = current() + var next = newIdentifier() + if (next == before.conversationId) { + next = UUID.randomUUID().toString() + } + storage.write(CONVERSATION_KEY, next) + return SessionIdentity(before.installationId, next) + } + + private fun validStored(key: String): String? = storage.read(key) + ?.takeIf(identifierPattern::matches) + + private fun newIdentifier(): String { + val normalized = generateId() + .replace(invalidCharacterPattern, "-") + .trim('-') + .take(128) + return normalized.takeIf(identifierPattern::matches) + ?: UUID.randomUUID().toString() + } + + private companion object { + const val INSTALLATION_KEY = "installation_id" + const val CONVERSATION_KEY = "conversation_id" + val identifierPattern = Regex("[A-Za-z0-9_.:-]{1,128}") + val invalidCharacterPattern = Regex("[^A-Za-z0-9_.:-]+") + } +} diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/Timeline.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/Timeline.kt new file mode 100644 index 0000000..b8eb9c7 --- /dev/null +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/realtime/Timeline.kt @@ -0,0 +1,203 @@ +package io.livekit.android.example.voiceassistant.realtime + +enum class MessageRole { + USER, + HERMES, + SYSTEM +} + +enum class MessageSource { + TEXT, + VOICE, + HERMES, + STATUS +} + +enum class DeliveryState { + PENDING, + SENT, + FAILED +} + +data class TimelineMessage( + val id: String, + val text: String, + val role: MessageRole, + val source: MessageSource, + val timestampMs: Long, + val isFinal: Boolean, + val delivery: DeliveryState, + val transportId: String? = null, + val statusType: String? = null +) { + val persistable: Boolean + get() = isFinal && + delivery == DeliveryState.SENT && + role in setOf(MessageRole.USER, MessageRole.HERMES) && + source != MessageSource.STATUS +} + +sealed interface TimelineUpdate { + data class LocalText( + val localId: String, + val text: String, + val timestampMs: Long + ) : TimelineUpdate + + data class TextSent( + val localId: String, + val transportId: String, + val timestampMs: Long + ) : TimelineUpdate + + data class TextFailed(val localId: String, val timestampMs: Long) : TimelineUpdate + + data class RemoteText( + val transportId: String, + val text: String, + val isFinal: Boolean, + val timestampMs: Long, + val isUser: Boolean = true, + val localId: String? = null + ) : TimelineUpdate + + data class Transcript( + val segmentId: String, + val text: String, + val isFinal: Boolean, + val isUser: Boolean, + val timestampMs: Long + ) : TimelineUpdate + + data class Status( + val eventId: String, + val text: String, + val statusType: String, + val timestampMs: Long + ) : TimelineUpdate +} + +fun reduceTimeline( + current: List, + update: TimelineUpdate +): List = when (update) { + is TimelineUpdate.LocalText -> current.upsert( + TimelineMessage( + id = localMessageId(update.localId), + text = update.text, + role = MessageRole.USER, + source = MessageSource.TEXT, + timestampMs = update.timestampMs, + isFinal = true, + delivery = DeliveryState.PENDING + ) + ) + is TimelineUpdate.TextSent -> current.updateFirst( + predicate = { it.id == localMessageId(update.localId) }, + transform = { + it.copy( + delivery = DeliveryState.SENT, + transportId = update.transportId + ) + } + ) + is TimelineUpdate.TextFailed -> current.updateFirst( + predicate = { it.id == localMessageId(update.localId) }, + transform = { it.copy(delivery = DeliveryState.FAILED) } + ) + is TimelineUpdate.RemoteText -> { + val existing = current.indexOfFirst { + it.transportId == update.transportId || + (update.localId != null && it.id == localMessageId(update.localId)) + } + if (existing >= 0) { + current.replaceAt( + existing, + current[existing].copy( + text = update.text, + isFinal = update.isFinal, + delivery = DeliveryState.SENT, + transportId = update.transportId + ) + ) + } else { + current.upsert( + TimelineMessage( + id = transportMessageId(update.transportId), + text = update.text, + role = if (update.isUser) MessageRole.USER else MessageRole.HERMES, + source = if (update.isUser) { + MessageSource.TEXT + } else { + MessageSource.HERMES + }, + timestampMs = update.timestampMs, + isFinal = update.isFinal, + delivery = DeliveryState.SENT, + transportId = update.transportId + ) + ) + } + } + is TimelineUpdate.Transcript -> current.upsert( + TimelineMessage( + id = transcriptMessageId(update.segmentId), + text = update.text, + role = if (update.isUser) MessageRole.USER else MessageRole.HERMES, + source = if (update.isUser) MessageSource.VOICE else MessageSource.HERMES, + timestampMs = update.timestampMs, + isFinal = update.isFinal, + delivery = DeliveryState.SENT, + transportId = update.segmentId + ) + ) + is TimelineUpdate.Status -> current.upsert( + TimelineMessage( + id = statusMessageId(update.eventId), + text = update.text, + role = MessageRole.SYSTEM, + source = MessageSource.STATUS, + timestampMs = update.timestampMs, + isFinal = true, + delivery = DeliveryState.SENT, + statusType = update.statusType + ) + ) +} + +private fun localMessageId(localId: String) = "local:$localId" + +private fun transportMessageId(transportId: String) = "transport:$transportId" + +private fun transcriptMessageId(segmentId: String) = "transcript:$segmentId" + +private fun statusMessageId(eventId: String) = "status:$eventId" + +private fun List.upsert(message: TimelineMessage): List { + val existing = indexOfFirst { it.id == message.id } + if (existing >= 0) { + return replaceAt( + existing, + message.copy(timestampMs = this[existing].timestampMs) + ) + } + + val insertion = indexOfFirst { it.timestampMs > message.timestampMs } + return toMutableList().apply { + if (insertion < 0) add(message) else add(insertion, message) + } +} + +private inline fun List.updateFirst( + predicate: (TimelineMessage) -> Boolean, + transform: (TimelineMessage) -> TimelineMessage +): List { + val index = indexOfFirst(predicate) + if (index < 0) return this + return replaceAt(index, transform(this[index])) +} + +private fun List.replaceAt( + index: Int, + message: TimelineMessage +): List = toMutableList().apply { this[index] = message } diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/screen/VoiceAssistantScreen.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/screen/VoiceAssistantScreen.kt index 231b6e9..51895ed 100644 --- a/app/src/main/java/io/livekit/android/example/voiceassistant/screen/VoiceAssistantScreen.kt +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/screen/VoiceAssistantScreen.kt @@ -1,26 +1,31 @@ package io.livekit.android.example.voiceassistant.screen -import android.app.Activity -import android.content.Context.MEDIA_PROJECTION_SERVICE -import android.media.projection.MediaProjectionManager import android.widget.Toast -import androidx.activity.compose.rememberLauncherForActivityResult -import androidx.activity.result.contract.ActivityResultContracts -import androidx.compose.animation.core.animateFloatAsState -import androidx.compose.animation.core.spring import androidx.compose.foundation.background -import androidx.compose.foundation.border -import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.Cameraswitch +import androidx.compose.material.icons.filled.Call +import androidx.compose.material.icons.filled.CallEnd +import androidx.compose.material.icons.filled.Mic +import androidx.compose.material.icons.filled.MicOff +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.Button +import androidx.compose.material3.ButtonDefaults import androidx.compose.material3.Icon import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect @@ -28,62 +33,175 @@ import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope -import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.alpha -import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.Color import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.testTag +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.unit.dp -import androidx.constraintlayout.compose.ChainStyle -import androidx.constraintlayout.compose.ConstrainScope -import androidx.constraintlayout.compose.ConstraintLayout -import androidx.constraintlayout.compose.ConstraintSet -import androidx.constraintlayout.compose.Dimension -import androidx.constraintlayout.compose.Visibility -import androidx.constraintlayout.compose.layoutId import com.google.accompanist.permissions.ExperimentalPermissionsApi import io.livekit.android.annotations.Beta import io.livekit.android.compose.local.SessionScope import io.livekit.android.compose.local.requireRoom +import io.livekit.android.compose.state.Agent +import io.livekit.android.compose.state.Session +import io.livekit.android.compose.state.SessionConnectOptions +import io.livekit.android.compose.state.SessionConnectTrackOptions +import io.livekit.android.compose.state.SessionMessages import io.livekit.android.compose.state.SessionOptions import io.livekit.android.compose.state.rememberAgent import io.livekit.android.compose.state.rememberLocalMedia import io.livekit.android.compose.state.rememberSession import io.livekit.android.compose.state.rememberSessionMessages -import io.livekit.android.compose.ui.VideoTrackView -import io.livekit.android.example.voiceassistant.rememberCanEnableCamera +import io.livekit.android.compose.types.LocalMedia +import io.livekit.android.compose.types.ReceivedAgentTranscriptionMessage +import io.livekit.android.compose.types.ReceivedChatMessage +import io.livekit.android.compose.types.ReceivedMessage +import io.livekit.android.compose.types.ReceivedUserTranscriptionMessage +import io.livekit.android.events.RoomEvent +import io.livekit.android.example.voiceassistant.APPROVAL_REQUEST_TOPIC +import io.livekit.android.example.voiceassistant.APPROVAL_RESPONSE_TOPIC +import io.livekit.android.example.voiceassistant.ApprovalRequest +import io.livekit.android.example.voiceassistant.approvalResponseJson +import io.livekit.android.example.voiceassistant.parseApprovalRequest +import io.livekit.android.example.voiceassistant.realtime.CONTROL_TOPIC +import io.livekit.android.example.voiceassistant.realtime.ControlPacket +import io.livekit.android.example.voiceassistant.realtime.HermesCommand +import io.livekit.android.example.voiceassistant.realtime.InputIntent +import io.livekit.android.example.voiceassistant.realtime.LatencyTracker +import io.livekit.android.example.voiceassistant.realtime.STATUS_TOPIC +import io.livekit.android.example.voiceassistant.realtime.StatusPacket +import io.livekit.android.example.voiceassistant.realtime.TimelineMessage +import io.livekit.android.example.voiceassistant.realtime.TimelineUpdate +import io.livekit.android.example.voiceassistant.realtime.controlPacketJson +import io.livekit.android.example.voiceassistant.realtime.parseInput +import io.livekit.android.example.voiceassistant.realtime.parseStatusPacket +import io.livekit.android.example.voiceassistant.realtime.reduceTimeline +import io.livekit.android.example.voiceassistant.realtime.suggestInputs import io.livekit.android.example.voiceassistant.rememberCanEnableMic import io.livekit.android.example.voiceassistant.requirePermissions -import io.livekit.android.example.voiceassistant.ui.AgentVisualization import io.livekit.android.example.voiceassistant.ui.ChatBar import io.livekit.android.example.voiceassistant.ui.ChatLog -import io.livekit.android.example.voiceassistant.ui.ControlBar import io.livekit.android.example.voiceassistant.viewmodel.VoiceAssistantViewModel -import io.livekit.android.room.track.screencapture.ScreenCaptureParams +import io.livekit.android.room.Room +import io.livekit.android.room.datastream.StreamTextOptions +import io.livekit.android.room.track.DataPublishReliability +import io.livekit.android.room.track.RemoteAudioTrack +import java.util.UUID import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.collect import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull import kotlinx.serialization.Serializable +private const val AGENT_NAME = "hermes-voice" + @Serializable data class VoiceAssistantRoute( val tokenServerId: String, val hardcodedUrl: String, - val hardcodedToken: String, - val homepageAgentEndpoint: String + val hardcodedToken: String ) +interface HermesSessionController { + val isConnected: Boolean + val isReconnecting: Boolean + + suspend fun start(microphoneEnabled: Boolean) + + suspend fun setMicrophoneEnabled(enabled: Boolean) + + suspend fun setAgentVolume(volume: Double) +} + +interface HermesChatTransport { + suspend fun sendMessage(text: String, localId: String, operationId: String): String + + suspend fun sendControl(packet: ControlPacket): Boolean +} + +private object NoOpHermesChatTransport : HermesChatTransport { + override suspend fun sendMessage( + text: String, + localId: String, + operationId: String + ): String = localId + + override suspend fun sendControl(packet: ControlPacket): Boolean = true +} + +@OptIn(Beta::class) +private class LiveKitHermesSessionController( + private val session: Session, + private val localMedia: LocalMedia, + private val agent: Agent +) : HermesSessionController { + override val isConnected: Boolean + get() = session.isConnected + override val isReconnecting: Boolean + get() = session.isReconnecting + + override suspend fun start(microphoneEnabled: Boolean) { + session.start( + SessionConnectOptions( + tracks = SessionConnectTrackOptions( + microphoneEnabled = microphoneEnabled, + usePreconnectBuffer = microphoneEnabled + ) + ) + ).getOrThrow() + } + + override suspend fun setMicrophoneEnabled(enabled: Boolean) { + localMedia.setMicrophoneEnabled(enabled) + } + + override suspend fun setAgentVolume(volume: Double) { + if (agent.audioTrack == null) { + withTimeoutOrNull(5_000) { agent.waitUntilMicrophone() } + } + (agent.audioTrack?.publication?.track as? RemoteAudioTrack)?.setVolume(volume) + } +} + +@OptIn(Beta::class) +private class LiveKitHermesChatTransport( + private val messages: SessionMessages, + private val room: Room +) : HermesChatTransport { + override suspend fun sendMessage( + text: String, + localId: String, + operationId: String + ): String { + val message = messages.send( + text, + StreamTextOptions( + topic = "lk.chat", + attributes = mapOf("local_id" to localId, "op_id" to operationId) + ) + ).getOrThrow() + return message.id + } + + override suspend fun sendControl(packet: ControlPacket): Boolean { + val payload = controlPacketJson(packet) ?: return false + return room.localParticipant.publishData( + payload.encodeToByteArray(), + reliability = DataPublishReliability.RELIABLE, + topic = CONTROL_TOPIC + ).isSuccess + } +} + @Composable -fun VoiceAssistantScreen( - viewModel: VoiceAssistantViewModel, - onEndCall: () -> Unit, -) { +fun VoiceAssistantScreen(viewModel: VoiceAssistantViewModel) { VoiceAssistant( viewModel = viewModel, - modifier = Modifier.fillMaxSize(), - onEndCall = onEndCall + modifier = Modifier.fillMaxSize() ) } @@ -91,297 +209,516 @@ fun VoiceAssistantScreen( @Composable fun VoiceAssistant( viewModel: VoiceAssistantViewModel, - modifier: Modifier = Modifier, - onEndCall: () -> Unit + modifier: Modifier = Modifier ) { - var requestedAudio by remember { mutableStateOf(true) } // Turn on audio by default. - var requestedVideo by remember { mutableStateOf(false) } - - requirePermissions(requestedAudio, requestedVideo) - + var requestedAudio by remember { mutableStateOf(false) } + requirePermissions(requestedAudio, false) val canEnableMic by rememberCanEnableMic() - val canEnableVideo by rememberCanEnableCamera() - val session = rememberSession( tokenSource = viewModel.tokenSource, options = SessionOptions( - room = viewModel.room + room = viewModel.room, + tokenRequestOptions = viewModel.tokenRequestOptions(AGENT_NAME) ) ) - val context = LocalContext.current - SessionScope(session = session) { session -> + SessionScope(session = session) { + DisposableEffect(Unit) { + onDispose { session.end() } + } - // Start the session when we have at least microphone permissions. - // Permission removals kill the app, so this is a one-way transition. - LaunchedEffect(canEnableMic) { - if (!canEnableMic) { - return@LaunchedEffect + val room = requireRoom() + val localMedia = rememberLocalMedia() + val agent = rememberAgent() + val sessionMessages = rememberSessionMessages() + val controller = remember(session, localMedia, agent) { + LiveKitHermesSessionController(session, localMedia, agent) + } + val chatTransport = remember(sessionMessages, room) { + LiveKitHermesChatTransport(sessionMessages, room) + } + val coroutineScope = rememberCoroutineScope { Dispatchers.IO } + val statusScope = rememberCoroutineScope() + var pendingApproval by remember { mutableStateOf(null) } + var statusUpdates by remember { mutableStateOf(emptyList()) } + val messageUpdates = remember(sessionMessages.messages) { + sessionMessages.messages.mapNotNull { message -> + message.toTimelineUpdate(room) } + } + val initialHistory = remember(viewModel.identity.conversationId) { + viewModel.historyRepository.load(viewModel.identity.conversationId) + } - val result = session.start() + LaunchedEffect(room) { + room.events.events.collect { event -> + if (event is RoomEvent.DataReceived && + event.topic == APPROVAL_REQUEST_TOPIC + ) { + parseApprovalRequest(event.data)?.let { pendingApproval = it } + } + } + } - // Handle if the session fails to connect. - if (result.isFailure) { - Toast.makeText(context, "Error connecting to the session.", Toast.LENGTH_SHORT).show() - onEndCall() + DisposableEffect(room) { + room.registerTextStreamHandler(STATUS_TOPIC) { receiver, _ -> + statusScope.launch { + val payload = StringBuilder() + receiver.flow.collect(payload::append) + parseStatusPacket(payload.toString().encodeToByteArray()) + ?.toTimelineUpdate( + eventId = receiver.info.id, + timestampMs = receiver.info.timestampMs + ) + ?.let { statusUpdates = statusUpdates + it } + } } + onDispose { room.unregisterTextStreamHandler(STATUS_TOPIC) } } - // End the session when leaving the screen. - DisposableEffect(Unit) { - onDispose { - session.end() + fun respondToApproval(choice: String) { + val request = pendingApproval ?: return + pendingApproval = null + val payload = approvalResponseJson(request.runId, choice) ?: return + coroutineScope.launch { + room.localParticipant.publishData( + payload.encodeToByteArray(), + reliability = DataPublishReliability.RELIABLE, + topic = APPROVAL_RESPONSE_TOPIC + ) } } - val room = requireRoom() - var chatVisible by remember { mutableStateOf(false) } + Box(modifier = modifier) { + HermesScreen( + controller = controller, + chatTransport = chatTransport, + incomingUpdates = messageUpdates + statusUpdates, + initialMessages = initialHistory, + canEnableMic = canEnableMic, + onRequestMicrophonePermission = { requestedAudio = true }, + onRotateConversation = { + val identity = viewModel.rotateConversation() + room.localParticipant.updateAttributes( + mapOf("hermes.conversation_id" to identity.conversationId) + ) + identity.conversationId + }, + onTimelineChanged = { + viewModel.historyRepository.save( + viewModel.identity.conversationId, + it + ) + }, + onConnectionError = { + Toast.makeText( + context, + "Hermes-এর সঙ্গে সংযোগ করা যায়নি।", + Toast.LENGTH_SHORT + ).show() + } + ) - // LocalMedia provides state information about the user's local devices - val localMedia = rememberLocalMedia() - val isMicEnabled by localMedia::isMicrophoneEnabled - val isCameraEnabled by localMedia::isCameraEnabled - val isScreenShareEnabled by localMedia::isScreenShareEnabled + pendingApproval?.let { request -> + HermesApprovalDialog( + request = request, + onConfirm = { respondToApproval("once") }, + onCancel = { respondToApproval("deny") } + ) + } + } + } +} - LaunchedEffect(canEnableMic, requestedAudio) { - session.waitUntilConnected() - localMedia.setMicrophoneEnabled(canEnableMic && requestedAudio) +@Composable +fun HermesScreen( + controller: HermesSessionController, + modifier: Modifier = Modifier, + chatTransport: HermesChatTransport = NoOpHermesChatTransport, + incomingUpdates: List = emptyList(), + initialMessages: List = emptyList(), + canEnableMic: Boolean = true, + onRequestMicrophonePermission: () -> Unit = {}, + onRotateConversation: () -> String = { "conversation-${UUID.randomUUID()}" }, + onTimelineChanged: (List) -> Unit = {}, + onConnectionError: (Throwable) -> Unit = {} +) { + val coroutineScope = rememberCoroutineScope() + var callActive by remember { mutableStateOf(false) } + var micEnabled by remember { mutableStateOf(false) } + var pendingCall by remember { mutableStateOf(false) } + var input by remember { mutableStateOf("") } + var timeline by remember { mutableStateOf(initialMessages) } + var working by remember { mutableStateOf(false) } + var activeLatency by remember { mutableStateOf(null) } + + LaunchedEffect(Unit) { + runCatching { + controller.start(microphoneEnabled = false) + controller.setAgentVolume(0.0) + }.onFailure(onConnectionError) + } + + LaunchedEffect(incomingUpdates) { + var next = timeline + incomingUpdates.forEach { update -> + next = reduceTimeline(next, update) + if (update is TimelineUpdate.Status) { + working = update.statusType in setOf( + "delegation.requested", + "tool.started", + "subagent.start" + ) + if (update.statusType in setOf( + "run.completed", + "run.failed", + "run.cancelled" + ) + ) { + working = false + } + } + if (update is TimelineUpdate.Transcript && !update.isUser) { + activeLatency?.let { tracker -> + if (tracker.durationMs("send_pressed", "first_ui_delta") == null) { + tracker.mark("first_ui_delta") + } + } + if (update.isFinal) working = false + } } + timeline = next + } - LaunchedEffect(canEnableVideo, requestedVideo) { - session.waitUntilConnected() - localMedia.setCameraEnabled(canEnableVideo && requestedVideo) + LaunchedEffect(timeline) { + onTimelineChanged(timeline) + } + + LaunchedEffect(canEnableMic, pendingCall) { + if (pendingCall && canEnableMic) { + controller.setAgentVolume(1.0) + controller.setMicrophoneEnabled(true) + callActive = true + micEnabled = true + pendingCall = false } + } - // SessionMessages handles all transcriptions and chat messages - val sessionMessages = rememberSessionMessages() + fun beginCall() { + if (!canEnableMic) { + pendingCall = true + onRequestMicrophonePermission() + return + } + coroutineScope.launch { + controller.setAgentVolume(1.0) + controller.setMicrophoneEnabled(true) + callActive = true + micEnabled = true + } + } - // Agent provides state information about the agent participant. - val agent = rememberAgent() + fun mute() { + coroutineScope.launch { + controller.setMicrophoneEnabled(false) + micEnabled = false + } + } - val constraints = getConstraints(chatVisible, isCameraEnabled, isScreenShareEnabled) - ConstraintLayout( - constraintSet = constraints, - modifier = modifier, - animateChangesSpec = spring() - ) { - val coroutineScope = rememberCoroutineScope { Dispatchers.IO } + fun unmute() { + if (!callActive) { + beginCall() + return + } + if (!canEnableMic) { + pendingCall = true + onRequestMicrophonePermission() + return + } + coroutineScope.launch { + controller.setMicrophoneEnabled(true) + micEnabled = true + } + } - ChatLog( - room = room, - messages = sessionMessages.messages, - modifier = Modifier.layoutId(LAYOUT_ID_CHAT_LOG) - ) + fun endCall() { + coroutineScope.launch { + controller.setMicrophoneEnabled(false) + controller.setAgentVolume(0.0) + callActive = false + micEnabled = false + } + } + + fun addStatus(text: String, type: String, operationId: String = newOperationId()) { + timeline = reduceTimeline( + timeline, + TimelineUpdate.Status(operationId, text, type, System.currentTimeMillis()) + ) + } - var message by rememberSaveable { - mutableStateOf("") + fun sendControl(command: HermesCommand, conversationId: String? = null) { + val operationId = newOperationId() + coroutineScope.launch { + if (!chatTransport.sendControl( + ControlPacket( + opId = operationId, + command = command.wireValue, + conversationId = conversationId + ) + ) + ) { + addStatus("Control delivery failed", "delivery.failed", operationId) } - ChatBar( - value = message, - onValueChange = { message = it }, - onChatSend = { msg -> - coroutineScope.launch { - sessionMessages.send(msg) - } - message = "" - }, - modifier = Modifier.layoutId(LAYOUT_ID_CHAT_BAR) - ) + } + } - // Amplitude visualization of the Assistant's voice track. - val agentBorderAlpha by animateFloatAsState(if (chatVisible) 1f else 0f, label = "agentBorderAlpha") - AgentVisualization( - agent = agent, - modifier = Modifier - .layoutId(LAYOUT_ID_AGENT) - .clip(RoundedCornerShape(8.dp)) - .border(1.dp, MaterialTheme.colorScheme.outline.copy(alpha = agentBorderAlpha), RoundedCornerShape(8.dp)) - ) + fun sendMessage(text: String) { + val localId = newOperationId() + val operationId = newOperationId() + val tracker = LatencyTracker(operationId).apply { mark("send_pressed") } + activeLatency = tracker + timeline = reduceTimeline( + timeline, + TimelineUpdate.LocalText(localId, text, System.currentTimeMillis()) + ) + working = true + coroutineScope.launch { + runCatching { + chatTransport.sendMessage(text, localId, operationId) + }.onSuccess { transportId -> + tracker.mark("packet_sent") + timeline = reduceTimeline( + timeline, + TimelineUpdate.TextSent( + localId, + transportId, + System.currentTimeMillis() + ) + ) + }.onFailure { + working = false + timeline = reduceTimeline( + timeline, + TimelineUpdate.TextFailed(localId, System.currentTimeMillis()) + ) + } + } + } - val context = LocalContext.current - val screenSharePermissionLauncher = - rememberLauncherForActivityResult(ActivityResultContracts.StartActivityForResult()) { result -> - val resultCode = result.resultCode - val data = result.data - if (resultCode != Activity.RESULT_OK || data == null) { - return@rememberLauncherForActivityResult - } - coroutineScope.launch { - // Agents only support one video stream at a time. - requestedVideo = false - localMedia.setScreenShareEnabled(true, ScreenCaptureParams(data)) - } + fun submit(raw: String) { + if (raw.isBlank()) return + input = "" + when (val intent = parseInput(raw.trim())) { + is InputIntent.Message -> sendMessage(intent.text) + is InputIntent.Control -> when (intent.command) { + HermesCommand.NEW -> { + val conversationId = onRotateConversation() + timeline = emptyList() + working = false + sendControl(HermesCommand.NEW, conversationId) + } + HermesCommand.STATUS -> { + addStatus( + if (controller.isConnected) "Connected · text ready" else "Connecting", + "local.status" + ) + sendControl(HermesCommand.STATUS) } + HermesCommand.STOP -> sendControl(HermesCommand.STOP) + else -> Unit + } + is InputIntent.Local -> when (intent.command) { + HermesCommand.MUTE -> mute() + HermesCommand.UNMUTE -> unmute() + HermesCommand.VOICE, + HermesCommand.CALL -> beginCall() + HermesCommand.ENDCALL -> endCall() + HermesCommand.HELP -> addStatus( + "Use text, @agent routes, or /new /status /stop /call /mute", + "local.help" + ) + else -> Unit + } + } + } - ControlBar( - isMicEnabled = isMicEnabled, - onMicClick = { requestedAudio = !requestedAudio }, - localAudioTrack = localMedia.microphoneTrack, - isCameraEnabled = isCameraEnabled, - onCameraClick = { - requestedVideo = !requestedVideo - if (requestedVideo) { - // Agents only support one video stream at a time. - coroutineScope.launch { localMedia.setScreenShareEnabled(false) } - } - }, - isScreenShareEnabled = isScreenShareEnabled, - onScreenShareClick = { - if (!isScreenShareEnabled) { - // Screenshare permission needs to be requested each time. - val mediaProjectionManager = context.getSystemService(MEDIA_PROJECTION_SERVICE) as MediaProjectionManager - screenSharePermissionLauncher.launch(mediaProjectionManager.createScreenCaptureIntent()) - } else { - coroutineScope.launch { localMedia.setScreenShareEnabled(false) } - } - }, - isChatEnabled = chatVisible, - onChatClick = { chatVisible = !chatVisible }, - onExitClick = onEndCall, - modifier = Modifier - .layoutId(LAYOUT_ID_CONTROL_BAR) + Column( + modifier = modifier + .fillMaxSize() + .background(MaterialTheme.colorScheme.background) + .padding(horizontal = 16.dp, vertical = 14.dp), + horizontalAlignment = Alignment.CenterHorizontally + ) { + Row( + modifier = Modifier.fillMaxWidth(), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.SpaceBetween + ) { + Text( + text = "HERMES", + style = MaterialTheme.typography.headlineSmall, + fontWeight = FontWeight.Bold ) - - val cameraAlpha by animateFloatAsState(targetValue = if (isCameraEnabled) 1f else 0f, label = "Camera Alpha") - Box( - modifier = Modifier - .layoutId(LAYOUT_ID_CAMERA) - .clickable { localMedia.switchCamera() } - .clip(RoundedCornerShape(8.dp)) - .alpha(cameraAlpha) + Surface( + color = MaterialTheme.colorScheme.primaryContainer, + shape = RoundedCornerShape(50) ) { - VideoTrackView( - trackReference = localMedia.cameraTrack, - modifier = Modifier.fillMaxSize() + Text( + text = when { + controller.isReconnecting -> "Reconnecting..." + controller.isConnected -> "Connected" + else -> "Connecting..." + }, + modifier = Modifier.padding(horizontal = 12.dp, vertical = 6.dp), + color = MaterialTheme.colorScheme.onPrimaryContainer, + style = MaterialTheme.typography.labelMedium ) + } + } - Box( - contentAlignment = Alignment.Center, - modifier = Modifier - .align(Alignment.BottomEnd) - .padding(end = 8.dp, bottom = 8.dp) - .background(Color.Black.copy(alpha = 0.5f), RoundedCornerShape(50)) - .fillMaxWidth(.35f) - .aspectRatio(1f) - ) { + ChatLog( + messages = timeline, + working = working, + modifier = Modifier + .fillMaxWidth() + .weight(1f) + .padding(vertical = 10.dp) + .testTag("conversation_timeline") + ) + + Text( + text = if (micEnabled) "মাইক্রোফোন চালু" else "মাইক্রোফোন বন্ধ", + modifier = Modifier.padding(bottom = 6.dp), + textAlign = TextAlign.Center, + style = MaterialTheme.typography.labelMedium + ) + + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(8.dp) + ) { + if (!callActive) { + OutlinedButton(onClick = ::beginCall) { + Icon(Icons.Default.Call, contentDescription = "Call Hermes") + Spacer(Modifier.size(6.dp)) + Text("CALL HERMES") + } + } else { + OutlinedButton(onClick = { if (micEnabled) mute() else unmute() }) { Icon( - Icons.Default.Cameraswitch, - tint = Color.White.copy(alpha = 0.7f), - contentDescription = "Flip Camera", - modifier = Modifier.fillMaxSize(0.6f) + if (micEnabled) Icons.Default.Mic else Icons.Default.MicOff, + contentDescription = "Toggle microphone" ) + Spacer(Modifier.size(6.dp)) + Text(if (micEnabled) "MUTE" else "UNMUTE") + } + Button( + onClick = ::endCall, + colors = ButtonDefaults.buttonColors(containerColor = Color.Red) + ) { + Icon(Icons.Default.CallEnd, contentDescription = "End call") + Spacer(Modifier.size(6.dp)) + Text("END CALL") } } - - val screenShareAlpha by animateFloatAsState(targetValue = if (isScreenShareEnabled) 1f else 0f, label = "Screen Share Alpha") - VideoTrackView( - trackReference = localMedia.screenShareTrack, - modifier = Modifier - .layoutId(LAYOUT_ID_SCREENSHARE) - .clip(RoundedCornerShape(8.dp)) - .alpha(screenShareAlpha) - ) } + + ChatBar( + value = input, + onValueChange = { input = it }, + onChatSend = ::submit, + suggestions = suggestInputs(input), + onSuggestionSelected = { input = "$it " }, + modifier = Modifier + .fillMaxWidth() + .padding(top = 8.dp) + ) } } - -private const val LAYOUT_ID_AGENT = "agentVisualizer" -private const val LAYOUT_ID_CHAT_LOG = "chatLog" -private const val LAYOUT_ID_CONTROL_BAR = "controlBar" -private const val LAYOUT_ID_CHAT_BAR = "chatBar" -private const val LAYOUT_ID_CAMERA = "camera" -private const val LAYOUT_ID_SCREENSHARE = "screenshare" - -private fun getConstraints(chatVisible: Boolean, cameraVisible: Boolean, screenShareVisible: Boolean) = ConstraintSet { - val (agentVisualizer, chatLog, controlBar, chatBar, camera, screenShare) = createRefsFor( - LAYOUT_ID_AGENT, - LAYOUT_ID_CHAT_LOG, - LAYOUT_ID_CONTROL_BAR, - LAYOUT_ID_CHAT_BAR, - LAYOUT_ID_CAMERA, - LAYOUT_ID_SCREENSHARE, +private fun ReceivedMessage.toTimelineUpdate(room: Room): TimelineUpdate = when (this) { + is ReceivedUserTranscriptionMessage -> TimelineUpdate.Transcript( + segmentId = attributes["lk.segment_id"] ?: id, + text = message, + isFinal = attributes["lk.transcription_final"] == "true", + isUser = true, + timestampMs = timestamp ) - val chatTopGuideline = createGuidelineFromTop(0.2f) - - constrain(chatLog) { - top.linkTo(chatTopGuideline) - bottom.linkTo(chatBar.top) - start.linkTo(parent.start) - end.linkTo(parent.end) - width = Dimension.fillToConstraints - height = Dimension.fillToConstraints - } - - constrain(chatBar) { - bottom.linkTo(controlBar.top, 16.dp) - start.linkTo(parent.start, 16.dp) - end.linkTo(parent.end, 16.dp) - width = Dimension.fillToConstraints - height = Dimension.wrapContent - } - - constrain(controlBar) { - bottom.linkTo(parent.bottom, 10.dp) - start.linkTo(parent.start, 16.dp) - end.linkTo(parent.end, 16.dp) + is ReceivedAgentTranscriptionMessage -> TimelineUpdate.Transcript( + segmentId = attributes["lk.segment_id"] ?: id, + text = message, + isFinal = attributes["lk.transcription_final"] == "true", + isUser = false, + timestampMs = timestamp + ) + is ReceivedChatMessage -> TimelineUpdate.RemoteText( + transportId = id, + text = message, + isFinal = true, + timestampMs = timestamp, + isUser = fromParticipant?.identity == room.localParticipant.identity, + localId = attributes["local_id"] + ) +} - width = Dimension.fillToConstraints - height = Dimension.value(60.dp) - } +private fun StatusPacket.toTimelineUpdate( + eventId: String, + timestampMs: Long +): TimelineUpdate? { + val text = when (type) { + "session.ready" -> "Hermes session ready" + "tool.started" -> tool?.let { "Using $it" } + "tool.completed" -> tool?.let { "$it completed" } + "subagent.start" -> status ?: "Specialist working" + "subagent.complete" -> status ?: "Specialist completed" + "delegation.requested" -> status ?: mention?.let { "$it assigned" } + "approval.request" -> "Confirmation required on this device" + "run.completed" -> "Completed" + "run.failed" -> "Hermes run failed" + "run.cancelled" -> "Stopped" + "first_hermes_delta" -> durationMs?.let { "First response · ${it}ms" } + else -> null + } ?: return null + return TimelineUpdate.Status(eventId, text, type, timestampMs) +} - if (chatVisible) { - val chain = createHorizontalChain(agentVisualizer, screenShare, camera, chainStyle = ChainStyle.Spread) +private fun newOperationId(): String = "op-${UUID.randomUUID()}" - constrain(chain) { - start.linkTo(parent.start) - end.linkTo(parent.end) - } - - fun ConstrainScope.itemConstraints(visible: Boolean = true) { - top.linkTo(parent.top) - bottom.linkTo(chatTopGuideline) - width = Dimension.percent(0.3f) - height = Dimension.fillToConstraints - visibility = if (visible) Visibility.Visible else Visibility.Gone - } - constrain(agentVisualizer) { - itemConstraints() - } - constrain(camera) { - itemConstraints(cameraVisible) - } - constrain(screenShare) { - itemConstraints(screenShareVisible) - } - } else { - constrain(agentVisualizer) { - top.linkTo(parent.top) - bottom.linkTo(parent.bottom) - start.linkTo(parent.start) - end.linkTo(parent.end) - height = Dimension.fillToConstraints - width = Dimension.fillToConstraints - } - constrain(camera) { - end.linkTo(parent.end, 16.dp) - bottom.linkTo(controlBar.top, 16.dp) - width = Dimension.percent(0.25f) - height = Dimension.percent(0.2f) - visibility = if (cameraVisible) Visibility.Visible else Visibility.Gone - } - constrain(screenShare) { - if (cameraVisible) { - end.linkTo(camera.start, 16.dp) - } else { - end.linkTo(parent.end, 16.dp) +@Composable +fun HermesApprovalDialog( + request: ApprovalRequest, + onConfirm: () -> Unit, + onCancel: () -> Unit +) { + AlertDialog( + onDismissRequest = onCancel, + title = { + Text( + "⚠ DESTRUCTIVE ACTION", + color = MaterialTheme.colorScheme.error, + fontWeight = FontWeight.Bold + ) + }, + text = { + Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { + Text("Agent: ${request.displayAgent}") + Text("Action: ${request.action}") + Text("Target: ${request.target}") + Text("Reason: ${request.reason}") + Text( + "Voice or text approval is not accepted. Confirm only by tapping this dialog.", + color = MaterialTheme.colorScheme.error, + fontWeight = FontWeight.SemiBold + ) } - bottom.linkTo(controlBar.top, 16.dp) - width = Dimension.percent(0.25f) - height = Dimension.percent(0.2f) - visibility = if (screenShareVisible) Visibility.Visible else Visibility.Gone + }, + confirmButton = { + Button(onClick = onConfirm) { Text("CONFIRM") } + }, + dismissButton = { + TextButton(onClick = onCancel) { Text("CANCEL") } } - } -} \ No newline at end of file + ) +} diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/ui/ChatBar.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/ui/ChatBar.kt index e83e526..bb75865 100644 --- a/app/src/main/java/io/livekit/android/example/voiceassistant/ui/ChatBar.kt +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/ui/ChatBar.kt @@ -1,19 +1,27 @@ package io.livekit.android.example.voiceassistant.ui -import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.horizontalScroll +import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.imePadding import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.layout.sizeIn +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.text.KeyboardActions +import androidx.compose.foundation.text.KeyboardOptions import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.ArrowUpward import androidx.compose.material3.Button import androidx.compose.material3.ButtonDefaults import androidx.compose.material3.Icon import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.material3.TextFieldDefaults import androidx.compose.runtime.Composable @@ -21,15 +29,13 @@ import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.Color -import androidx.compose.ui.platform.LocalDensity -import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.platform.testTag +import androidx.compose.ui.text.input.ImeAction import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp -import androidx.constraintlayout.compose.ConstraintLayout -import androidx.constraintlayout.compose.Dimension import io.livekit.android.example.voiceassistant.ui.theme.Blue500 @Composable @@ -37,83 +43,97 @@ fun ChatBar( value: String, onValueChange: (String) -> Unit, onChatSend: (String) -> Unit, + suggestions: List = emptyList(), + onSuggestionSelected: (String) -> Unit = {}, modifier: Modifier = Modifier ) { - val sendButtonColors = ButtonDefaults.buttonColors( - containerColor = Blue500, - contentColor = Color.White - ) - ConstraintLayout( - modifier = Modifier - .imePadding() - .sizeIn(minHeight = 48.dp) - .clip(RoundedCornerShape(with(LocalDensity.current) { 24.dp.toPx() })) - .background(MaterialTheme.colorScheme.surface) - .padding(8.dp) - .then(modifier) - ) { - val (sendButton, messageInput) = createRefs() - - LKTextField( - value = value, - onValueChange = onValueChange, - textStyle = TextStyle(color = MaterialTheme.colorScheme.onSurface), - colors = TextFieldDefaults.colors().copy( - disabledTextColor = Color.Transparent, - focusedContainerColor = Color.Transparent, - unfocusedContainerColor = Color.Transparent, - focusedIndicatorColor = Color.Transparent, - unfocusedIndicatorColor = Color.Transparent, - disabledIndicatorColor = Color.Transparent, - ), - maxLines = 3, - placeholder = { - Text("Message") - }, - modifier = Modifier - .constrainAs(messageInput) { - start.linkTo(parent.start, 8.dp) - end.linkTo(sendButton.start, 8.dp) - top.linkTo(parent.top) - bottom.linkTo(parent.bottom) - width = Dimension.fillToConstraints - height = Dimension.wrapContent - }, - ) - - Button( - colors = sendButtonColors, - shape = RoundedCornerShape(50), - onClick = { - onChatSend(value) - }, - enabled = value.isNotEmpty(), - contentPadding = PaddingValues(0.dp), - modifier = Modifier - .padding(0.dp) - .constrainAs(sendButton) { - end.linkTo(parent.end) - bottom.linkTo(parent.bottom) - width = Dimension.ratio("1:1") - height = Dimension.preferredValue(32.dp) + Column(modifier = modifier.imePadding()) { + if (suggestions.isNotEmpty()) { + Row( + modifier = Modifier + .fillMaxWidth() + .horizontalScroll(rememberScrollState()) + .padding(bottom = 8.dp), + horizontalArrangement = Arrangement.spacedBy(8.dp) + ) { + suggestions.forEach { suggestion -> + Surface( + color = MaterialTheme.colorScheme.secondaryContainer, + shape = RoundedCornerShape(50), + modifier = Modifier.clickable { + onSuggestionSelected(suggestion) + } + ) { + Text( + text = suggestion, + modifier = Modifier.padding(horizontal = 12.dp, vertical = 6.dp), + color = MaterialTheme.colorScheme.onSecondaryContainer + ) + } } + } + } + + Row( + modifier = Modifier.fillMaxWidth(), + verticalAlignment = Alignment.Bottom, + horizontalArrangement = Arrangement.spacedBy(8.dp) ) { - Icon(Icons.Default.ArrowUpward, contentDescription = "Send Message", tint = if(value.isEmpty()) MaterialTheme.colorScheme.onSurface else Color.White) + Surface( + modifier = Modifier.weight(1f), + shape = RoundedCornerShape(24.dp), + color = MaterialTheme.colorScheme.surfaceVariant + ) { + LKTextField( + value = value, + onValueChange = onValueChange, + colors = TextFieldDefaults.colors().copy( + focusedContainerColor = Color.Transparent, + unfocusedContainerColor = Color.Transparent, + focusedIndicatorColor = Color.Transparent, + unfocusedIndicatorColor = Color.Transparent + ), + minLines = 1, + maxLines = 3, + placeholder = { Text("Message Hermes Main") }, + keyboardOptions = KeyboardOptions(imeAction = ImeAction.Send), + keyboardActions = KeyboardActions( + onSend = { if (value.isNotBlank()) onChatSend(value) } + ), + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 14.dp, vertical = 10.dp) + .testTag("message_input") + ) + } + Button( + colors = ButtonDefaults.buttonColors( + containerColor = Blue500, + contentColor = Color.White + ), + shape = RoundedCornerShape(50), + onClick = { onChatSend(value) }, + enabled = value.isNotBlank(), + modifier = Modifier + .size(48.dp) + .testTag("send_button") + ) { + Icon(Icons.Default.ArrowUpward, contentDescription = "Send Message") + } } + Spacer(Modifier.height(2.dp)) } - } @Preview @Composable fun ChatWidgetPreview() { - Column { - var message by rememberSaveable { mutableStateOf("") } - ChatBar( - value = message, - onValueChange = { message = it }, - onChatSend = {}, - modifier = Modifier.fillMaxWidth() - ) - } -} \ No newline at end of file + var message by rememberSaveable { mutableStateOf("") } + ChatBar( + value = message, + onValueChange = { message = it }, + onChatSend = {}, + suggestions = listOf("@coder", "@computer-operator"), + modifier = Modifier.fillMaxWidth() + ) +} diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/ui/ChatLog.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/ui/ChatLog.kt index e6d4b44..e261656 100644 --- a/app/src/main/java/io/livekit/android/example/voiceassistant/ui/ChatLog.kt +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/ui/ChatLog.kt @@ -1,86 +1,119 @@ package io.livekit.android.example.voiceassistant.ui -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.fillMaxHeight +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.items import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier -import androidx.compose.ui.draw.drawWithContent -import androidx.compose.ui.graphics.BlendMode -import androidx.compose.ui.graphics.Brush -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.graphics.CompositingStrategy -import androidx.compose.ui.graphics.graphicsLayer +import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp -import io.livekit.android.compose.types.ReceivedMessage -import io.livekit.android.room.Room +import io.livekit.android.example.voiceassistant.realtime.DeliveryState +import io.livekit.android.example.voiceassistant.realtime.MessageRole +import io.livekit.android.example.voiceassistant.realtime.MessageSource +import io.livekit.android.example.voiceassistant.realtime.TimelineMessage @Composable -fun ChatLog(room: Room, messages: List, modifier: Modifier = Modifier) { - Box(modifier = modifier) { - // Get and display the transcriptions. - val displayTranscriptions = messages.asReversed() - val lazyListState = rememberLazyListState() +fun ChatLog( + messages: List, + working: Boolean, + modifier: Modifier = Modifier +) { + val listState = rememberLazyListState() + LaunchedEffect(messages.size, messages.lastOrNull()?.text) { + if (messages.isNotEmpty()) listState.animateScrollToItem(messages.lastIndex) + } - // Scroll to bottom when new transcriptions come in. - LaunchedEffect(messages.count()) { - lazyListState.animateScrollToItem(0) + LazyColumn( + state = listState, + modifier = modifier, + verticalArrangement = Arrangement.spacedBy(10.dp) + ) { + items(messages, key = TimelineMessage::id) { message -> + if (message.source == MessageSource.STATUS) { + StatusChip(message) + } else { + MessageBubble(message) + } + } + if (working) { + item(key = "working") { + Text( + text = "Hermes is working…", + modifier = Modifier.padding(horizontal = 8.dp, vertical = 4.dp), + color = MaterialTheme.colorScheme.primary, + style = MaterialTheme.typography.labelMedium + ) + } } - LazyColumn( - userScrollEnabled = true, - state = lazyListState, - reverseLayout = true, - modifier = modifier - .fillMaxWidth() - .fillMaxHeight() - .graphicsLayer(compositingStrategy = CompositingStrategy.Offscreen) - .drawWithContent { - // Fade top - val colors = arrayOf( - 0.0f to Color.Transparent, - 0.15f to Color.Black, - 1.0f to Color.Black, - ) - drawContent() - drawRect( - brush = Brush.verticalGradient( - colorStops = colors - ), - blendMode = BlendMode.DstIn - ) - } + } +} + +@Composable +private fun MessageBubble(message: TimelineMessage) { + val isUser = message.role == MessageRole.USER + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = if (isUser) Arrangement.End else Arrangement.Start + ) { + Surface( + color = if (isUser) { + MaterialTheme.colorScheme.primaryContainer + } else { + MaterialTheme.colorScheme.surfaceVariant + }, + shape = RoundedCornerShape(16.dp), + modifier = Modifier.fillMaxWidth(0.86f) ) { - items( - items = displayTranscriptions, - key = { transcription -> transcription.id }, - ) { message -> - Box( - modifier = Modifier - .fillMaxWidth() - .padding(16.dp) - .animateItem() - ) { - if (message.fromParticipant?.identity == room.localParticipant.identity) { - UserMessage( - message = message, - modifier = Modifier.align(Alignment.CenterEnd) + Column(modifier = Modifier.padding(horizontal = 14.dp, vertical = 10.dp)) { + Text(message.text, color = MaterialTheme.colorScheme.onSurface) + Text( + text = buildString { + append( + when (message.source) { + MessageSource.TEXT -> "TEXT" + MessageSource.VOICE -> "VOICE" + MessageSource.HERMES -> "HERMES" + MessageSource.STATUS -> "STATUS" + } ) - } else { - // Agent transcription or chat message - Text( - text = message.message, - modifier = Modifier.align(Alignment.CenterStart) - ) - } - } + if (!message.isFinal) append(" · streaming") + if (message.delivery == DeliveryState.PENDING) append(" · sending") + if (message.delivery == DeliveryState.FAILED) append(" · failed") + }, + modifier = Modifier.align(Alignment.End), + color = MaterialTheme.colorScheme.onSurfaceVariant, + fontWeight = FontWeight.SemiBold, + style = MaterialTheme.typography.labelSmall + ) } } } -} \ No newline at end of file +} + +@Composable +private fun StatusChip(message: TimelineMessage) { + Row(modifier = Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.Center) { + Surface( + color = MaterialTheme.colorScheme.secondaryContainer, + shape = RoundedCornerShape(50) + ) { + Text( + text = message.text, + modifier = Modifier.padding(horizontal = 12.dp, vertical = 5.dp), + color = MaterialTheme.colorScheme.onSecondaryContainer, + style = MaterialTheme.typography.labelMedium + ) + } + } +} diff --git a/app/src/main/java/io/livekit/android/example/voiceassistant/viewmodel/VoiceAssistantViewModel.kt b/app/src/main/java/io/livekit/android/example/voiceassistant/viewmodel/VoiceAssistantViewModel.kt index 4fa023d..5021f82 100644 --- a/app/src/main/java/io/livekit/android/example/voiceassistant/viewmodel/VoiceAssistantViewModel.kt +++ b/app/src/main/java/io/livekit/android/example/voiceassistant/viewmodel/VoiceAssistantViewModel.kt @@ -1,47 +1,71 @@ package io.livekit.android.example.voiceassistant.viewmodel import android.app.Application -import android.util.Log import androidx.lifecycle.AndroidViewModel -import androidx.lifecycle.SavedStateHandle -import androidx.navigation.toRoute import io.livekit.android.LiveKit -import io.livekit.android.example.voiceassistant.screen.VoiceAssistantRoute +import io.livekit.android.example.voiceassistant.hardcodedToken +import io.livekit.android.example.voiceassistant.hardcodedUrl +import io.livekit.android.example.voiceassistant.realtime.HistoryRepository +import io.livekit.android.example.voiceassistant.realtime.SessionIdentity +import io.livekit.android.example.voiceassistant.realtime.SessionIdentityStore +import io.livekit.android.example.voiceassistant.realtime.SharedPreferencesHistoryStorage +import io.livekit.android.example.voiceassistant.realtime.SharedPreferencesIdentityStorage +import io.livekit.android.example.voiceassistant.tokenServerId +import io.livekit.android.token.TokenRequestOptions import io.livekit.android.token.TokenSource -import java.net.URI +import io.livekit.android.token.cached /** * This ViewModel handles holding onto the Room object, so that it is * maintained across configuration changes, such as rotation. */ -class VoiceAssistantViewModel(application: Application, savedStateHandle: SavedStateHandle) : AndroidViewModel(application) { +class VoiceAssistantViewModel(application: Application) : AndroidViewModel(application) { val room = LiveKit.create(application) val tokenSource: TokenSource + private val preferences = application.getSharedPreferences( + "hermes_private_state", + Application.MODE_PRIVATE + ) + private val identityStore = SessionIdentityStore( + SharedPreferencesIdentityStorage(preferences) + ) + val historyRepository = HistoryRepository( + SharedPreferencesHistoryStorage(preferences) + ) - init { - val (tokenServerId, url, token, homepageAgentEndpoint) = savedStateHandle.toRoute() + var identity: SessionIdentity = identityStore.current() + private set + init { tokenSource = if (tokenServerId.isNotEmpty()) { TokenSource.fromDevelopmentTokenServer(tokenServerId = tokenServerId) - } else if (url.isNotEmpty() && token.isNotEmpty()) { - TokenSource.fromLiteral(url, token) + .cached() + } else if (hardcodedUrl.isNotEmpty() && hardcodedToken.isNotEmpty()) { + TokenSource.fromLiteral(hardcodedUrl, hardcodedToken) + .cached() } else { - if (url.isNotEmpty() || token.isNotEmpty()) { - Log.w(TAG, "hardcodedUrl and hardcodedToken must both be set; falling back to the homepage agent.") - } - TokenSource.fromEndpoint(URI(homepageAgentEndpoint).toURL()) + error("A LiveKit development token server ID or literal URL/token pair is required") } } + fun tokenRequestOptions(agentName: String): TokenRequestOptions = TokenRequestOptions( + agentName = agentName, + participantIdentity = "hermes-android-${identity.installationId}", + participantAttributes = mapOf( + "hermes.conversation_id" to identity.conversationId + ) + ) + + fun rotateConversation(): SessionIdentity { + identity = identityStore.rotateConversation() + return identity + } + override fun onCleared() { super.onCleared() room.disconnect() room.release() } - - companion object { - private const val TAG = "VoiceAssistantViewModel" - } -} \ No newline at end of file +} diff --git a/app/src/test/java/io/livekit/android/example/voiceassistant/ApprovalProtocolTest.kt b/app/src/test/java/io/livekit/android/example/voiceassistant/ApprovalProtocolTest.kt new file mode 100644 index 0000000..325bfad --- /dev/null +++ b/app/src/test/java/io/livekit/android/example/voiceassistant/ApprovalProtocolTest.kt @@ -0,0 +1,60 @@ +package io.livekit.android.example.voiceassistant + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class ApprovalProtocolTest { + @Test + fun parsesApprovalRequest() { + val request = parseApprovalRequest( + """{"runId":"run_1","target":"C:\\\\safe-test.txt","action":"Delete file","reason":"Requested by user"}""" + .encodeToByteArray() + ) + + assertEquals("run_1", request?.runId) + assertEquals("Delete file", request?.action) + } + + @Test + fun rejectsMalformedOrIncompleteRequest() { + assertNull(parseApprovalRequest("not json".encodeToByteArray())) + assertNull(parseApprovalRequest("""{"runId":"run_1"}""".encodeToByteArray())) + } + + @Test + fun responseAllowsOnlyOneShotConfirmOrDeny() { + assertEquals( + """{"runId":"run_1","choice":"once"}""", + approvalResponseJson("run_1", "once") + ) + assertEquals( + """{"runId":"run_1","choice":"deny"}""", + approvalResponseJson("run_1", "deny") + ) + assertNull(approvalResponseJson("run_1", "always")) + assertNull(approvalResponseJson("run_1", "yes")) + } + + @Test + fun approveAliasesRemainRejected() { + assertNull(approvalResponseJson("run-1", "approve")) + assertNull(approvalResponseJson("run-1", "yes")) + assertNull(approvalResponseJson("run-1", "হ্যাঁ")) + } + + @Test + fun optionalAgentDefaultsToHermesMain() { + val explicit = parseApprovalRequest( + """{"runId":"run-1","target":"fixture","action":"Delete","reason":"test","agent":"Computer Operator"}""" + .encodeToByteArray() + ) + val missing = parseApprovalRequest( + """{"runId":"run-2","target":"fixture","action":"Delete","reason":"test"}""" + .encodeToByteArray() + ) + + assertEquals("Computer Operator", explicit?.displayAgent) + assertEquals("Hermes Main", missing?.displayAgent) + } +} diff --git a/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/HermesInputTest.kt b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/HermesInputTest.kt new file mode 100644 index 0000000..fe00679 --- /dev/null +++ b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/HermesInputTest.kt @@ -0,0 +1,44 @@ +package io.livekit.android.example.voiceassistant.realtime + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class HermesInputTest { + @Test + fun muteIsLocalAndApproveDoesNotExist() { + assertEquals(InputIntent.Local(HermesCommand.MUTE), parseInput("/mute")) + assertEquals(InputIntent.Message("/approve"), parseInput("/approve")) + } + + @Test + fun newStopAndStatusUseControlChannel() { + assertEquals(InputIntent.Control(HermesCommand.NEW), parseInput("/new")) + assertEquals(InputIntent.Control(HermesCommand.STOP), parseInput("/stop")) + assertEquals(InputIntent.Control(HermesCommand.STATUS), parseInput("/status")) + } + + @Test + fun agentsTasksAndMemoryRemainHermesMessages() { + assertEquals(InputIntent.Message("/agents"), parseInput("/agents")) + assertEquals(InputIntent.Message("/tasks"), parseInput("/tasks")) + assertEquals(InputIntent.Message("/memory"), parseInput("/memory")) + } + + @Test + fun atSignSuggestsSupportedHermesRoutes() { + assertTrue( + suggestInputs("@c").containsAll(listOf("@coder", "@computer-operator")) + ) + assertEquals(listOf("@main"), suggestInputs("@mai")) + } + + @Test + fun slashSuggestionsAreExactAndApproveIsAbsent() { + val suggestions = suggestInputs("/") + + assertTrue(suggestions.containsAll(listOf("/new", "/call", "/memory"))) + assertTrue("/approve" !in suggestions) + assertEquals(listOf("/mute"), suggestInputs("/mu")) + } +} diff --git a/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/HistoryRepositoryTest.kt b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/HistoryRepositoryTest.kt new file mode 100644 index 0000000..2d94b7f --- /dev/null +++ b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/HistoryRepositoryTest.kt @@ -0,0 +1,104 @@ +package io.livekit.android.example.voiceassistant.realtime + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class HistoryRepositoryTest { + private class MemoryStorage : HistoryStorage { + val values = mutableMapOf() + + override fun read(key: String): String? = values[key] + + override fun write(key: String, value: String) { + values[key] = value + } + } + + @Test + fun historyRedactsCredentialsAndCapsFinalMessages() { + val storage = MemoryStorage() + val repository = HistoryRepository(storage) + val messages = (1..210).map { + finalText("m$it", "Bearer secret-token-$it") + } + + repository.save("conv-1", messages) + val restored = repository.load("conv-1") + + assertEquals(200, restored.size) + assertEquals("m11", restored.first().id) + assertTrue(restored.all { "secret-token" !in it.text }) + assertFalse(storage.values.values.single().contains("secret-token")) + } + + @Test + fun interimApprovalPendingAndFailedMessagesAreNotPersisted() { + val repository = HistoryRepository(MemoryStorage()) + val messages = listOf( + interimVoice(), + approvalStatus(), + finalText("pending", "pending", DeliveryState.PENDING), + finalText("failed", "failed", DeliveryState.FAILED), + finalText("final", "safe") + ) + + repository.save("conv-1", messages) + + assertEquals(listOf("final"), repository.load("conv-1").map { it.id }) + } + + @Test + fun redactorCoversAssignmentsJwtHexAndBase64Credentials() { + val text = """ + api_key=top-secret-value + eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyIn0.signature-value + 0123456789abcdef0123456789abcdef0123456789abcdef + QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo4ODg4ODg4ODg= + """.trimIndent() + + val redacted = redactForHistory(text) + + assertTrue("[REDACTED]" in redacted) + assertFalse("top-secret-value" in redacted) + assertFalse("eyJhbGci" in redacted) + assertFalse("0123456789abcdef" in redacted) + assertFalse("QUJDREV" in redacted) + } + + private fun finalText( + id: String, + text: String, + delivery: DeliveryState = DeliveryState.SENT + ) = TimelineMessage( + id = id, + text = text, + role = MessageRole.USER, + source = MessageSource.TEXT, + timestampMs = id.filter(Char::isDigit).toLongOrNull() ?: 1, + isFinal = true, + delivery = delivery + ) + + private fun interimVoice() = TimelineMessage( + id = "interim", + text = "partial", + role = MessageRole.USER, + source = MessageSource.VOICE, + timestampMs = 1, + isFinal = false, + delivery = DeliveryState.SENT + ) + + private fun approvalStatus() = TimelineMessage( + id = "approval", + text = "approval", + role = MessageRole.SYSTEM, + source = MessageSource.STATUS, + timestampMs = 2, + isFinal = true, + delivery = DeliveryState.SENT, + statusType = "approval.request" + ) +} diff --git a/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/LatencyTrackerTest.kt b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/LatencyTrackerTest.kt new file mode 100644 index 0000000..9bd348a --- /dev/null +++ b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/LatencyTrackerTest.kt @@ -0,0 +1,31 @@ +package io.livekit.android.example.voiceassistant.realtime + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class LatencyTrackerTest { + @Test + fun textFirstRenderLatencyUsesMonotonicMarks() { + val tracker = LatencyTracker("op-1") + tracker.mark("send_pressed", 1_000_000_000) + tracker.mark("first_ui_delta", 1_240_000_000) + + assertEquals(240, tracker.durationMs("send_pressed", "first_ui_delta")) + } + + @Test + fun firstMarkWinsAndPayloadContainsNoMessageContent() { + val tracker = LatencyTracker("op-1") + tracker.mark("send_pressed", 1_000_000_000) + tracker.mark("send_pressed", 9_000_000_000) + tracker.mark("packet_sent", 1_010_000_000) + + val payload = tracker.payload() + + assertEquals(10, payload.durationsMs["send_pressed_to_packet_sent"]) + assertTrue("text" !in payload.toString().lowercase()) + assertNull(tracker.durationMs("missing", "packet_sent")) + } +} diff --git a/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/RealtimeProtocolTest.kt b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/RealtimeProtocolTest.kt new file mode 100644 index 0000000..7a2c167 --- /dev/null +++ b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/RealtimeProtocolTest.kt @@ -0,0 +1,77 @@ +package io.livekit.android.example.voiceassistant.realtime + +import com.google.gson.JsonParser +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class RealtimeProtocolTest { + @Test + fun stopPacketContainsOnlyVersionOperationAndCommand() { + val json = controlPacketJson(ControlPacket(1, "op-1", "stop", null))!! + val fields = JsonParser.parseString(json).asJsonObject.keySet() + + assertEquals(setOf("version", "op_id", "command"), fields) + } + + @Test + fun newPacketIncludesValidatedConversationIdentifier() { + val json = controlPacketJson(ControlPacket(1, "op-2", "new", "conv-next"))!! + val fields = JsonParser.parseString(json).asJsonObject + + assertEquals("conv-next", fields["conversation_id"].asString) + assertNull(controlPacketJson(ControlPacket(1, "op-2", "new", "../bad"))) + } + + @Test + fun privilegedAndUnknownCommandsCannotBeEncoded() { + assertNull(controlPacketJson(ControlPacket(1, "op-1", "approve"))) + assertNull(controlPacketJson(ControlPacket(2, "op-1", "stop"))) + assertNull(controlPacketJson(ControlPacket(1, "../bad", "stop"))) + } + + @Test + fun statusParserAcceptsVersionedSafeFields() { + val packet = parseStatusPacket( + """{"version":1,"type":"tool.started","tool":"computer"}""" + .encodeToByteArray() + ) + + assertEquals("tool.started", packet?.type) + assertEquals("computer", packet?.tool) + } + + @Test + fun statusParserRejectsUnknownFieldsAndOversizePayload() { + assertNull( + parseStatusPacket( + """{"version":1,"type":"tool.started","tool":"computer","args":"private"}""" + .encodeToByteArray() + ) + ) + assertNull(parseStatusPacket(ByteArray(4097) { 'x'.code.toByte() })) + } + + @Test + fun latencyStatusContainsOnlyNumericDurations() { + val packet = parseStatusPacket( + """{"version":1,"type":"latency","op_id":"op-1","durations_ms":{"send_to_first":240}}""" + .encodeToByteArray() + ) + + assertEquals(240, packet?.durationsMs?.get("send_to_first")) + assertTrue(packet?.durationsMs?.values?.all { it >= 0 } == true) + } + + @Test + fun delegationRequestedStatusUsesExplicitSafeFields() { + val packet = parseStatusPacket( + """{"version":1,"type":"delegation.requested","mention":"coder","status":"Coder assigned"}""" + .encodeToByteArray() + ) + + assertEquals("delegation.requested", packet?.type) + assertEquals("coder", packet?.mention) + } +} diff --git a/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/SessionIdentityTest.kt b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/SessionIdentityTest.kt new file mode 100644 index 0000000..ef32d2d --- /dev/null +++ b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/SessionIdentityTest.kt @@ -0,0 +1,42 @@ +package io.livekit.android.example.voiceassistant.realtime + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class SessionIdentityTest { + private class MemoryStorage : IdentityStorage { + private val values = mutableMapOf() + + override fun read(key: String): String? = values[key] + + override fun write(key: String, value: String) { + values[key] = value + } + } + + @Test + fun conversationRotationPreservesInstallationIdentity() { + var next = 0 + val identities = SessionIdentityStore(MemoryStorage()) { "generated-${++next}" } + + val before = identities.current() + val after = identities.rotateConversation() + + assertEquals(before.installationId, after.installationId) + assertNotEquals(before.conversationId, after.conversationId) + assertTrue(Regex("[A-Za-z0-9_.:-]{1,128}").matches(after.conversationId)) + } + + @Test + fun identitySurvivesStoreRecreationAndInvalidValuesAreReplaced() { + val storage = MemoryStorage() + storage.write("installation_id", "../../bad") + val first = SessionIdentityStore(storage) { "safe-id" }.current() + val second = SessionIdentityStore(storage) { "unused-id" }.current() + + assertEquals("safe-id", first.installationId) + assertEquals(first, second) + } +} diff --git a/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/TimelineTest.kt b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/TimelineTest.kt new file mode 100644 index 0000000..70bf6b0 --- /dev/null +++ b/app/src/test/java/io/livekit/android/example/voiceassistant/realtime/TimelineTest.kt @@ -0,0 +1,142 @@ +package io.livekit.android.example.voiceassistant.realtime + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class TimelineTest { + @Test + fun optimisticTextReconcilesWithoutDuplicate() { + val optimistic = reduceTimeline( + emptyList(), + TimelineUpdate.LocalText("local-1", "hello", 10) + ) + val sent = reduceTimeline( + optimistic, + TimelineUpdate.TextSent("local-1", "stream-7", 12) + ) + + assertEquals(1, sent.size) + assertEquals(DeliveryState.SENT, sent.single().delivery) + assertEquals("stream-7", sent.single().transportId) + assertTrue(sent.single().persistable) + } + + @Test + fun finalVoiceTranscriptReplacesInterimSegment() { + val interim = reduceTimeline( + emptyList(), + TimelineUpdate.Transcript("seg-1", "হারমিস", false, true, 20) + ) + val final = reduceTimeline( + interim, + TimelineUpdate.Transcript("seg-1", "হারমিস শুনো", true, true, 25) + ) + + assertEquals(1, final.size) + assertEquals("হারমিস শুনো", final.single().text) + assertTrue(final.single().isFinal) + assertTrue(final.single().persistable) + } + + @Test + fun voiceAndTextShareTimestampOrderedTimeline() { + val updates = listOf( + TimelineUpdate.Transcript("v1", "voice", true, true, 30), + TimelineUpdate.LocalText("t1", "text", 40), + TimelineUpdate.Transcript("a1", "reply", true, false, 50) + ) + + val result = updates.fold(emptyList(), ::reduceTimeline) + + assertEquals( + listOf(MessageSource.VOICE, MessageSource.TEXT, MessageSource.HERMES), + result.map { it.source } + ) + } + + @Test + fun streamingHermesTextUpdatesOneBubbleInPlace() { + val first = reduceTimeline( + emptyList(), + TimelineUpdate.Transcript("agent-1", "আমি ", false, false, 50) + ) + val completed = reduceTimeline( + first, + TimelineUpdate.Transcript("agent-1", "আমি দেখছি", true, false, 55) + ) + + assertEquals(1, completed.size) + assertEquals("আমি দেখছি", completed.single().text) + assertEquals(MessageRole.HERMES, completed.single().role) + assertEquals(50, completed.single().timestampMs) + } + + @Test + fun statusInterimPendingAndFailedMessagesAreNotPersistable() { + var timeline = reduceTimeline( + emptyList(), + TimelineUpdate.LocalText("pending", "pending", 10) + ) + timeline = reduceTimeline( + timeline, + TimelineUpdate.Transcript("interim", "partial", false, true, 20) + ) + timeline = reduceTimeline( + timeline, + TimelineUpdate.Status("op-1", "Working", "tool.started", 30) + ) + timeline = reduceTimeline( + timeline, + TimelineUpdate.LocalText("failed", "failed", 40) + ) + timeline = reduceTimeline(timeline, TimelineUpdate.TextFailed("failed", 45)) + + assertFalse(timeline.any { it.persistable }) + } + + @Test + fun transportEchoReconcilesWithOptimisticMessage() { + var timeline = reduceTimeline( + emptyList(), + TimelineUpdate.LocalText("local-1", "hello", 10) + ) + timeline = reduceTimeline( + timeline, + TimelineUpdate.TextSent("local-1", "stream-7", 12) + ) + timeline = reduceTimeline( + timeline, + TimelineUpdate.RemoteText("stream-7", "hello", true, 15) + ) + + assertEquals(1, timeline.size) + assertEquals(DeliveryState.SENT, timeline.single().delivery) + } + + @Test + fun transportEchoCanArriveBeforeSendCompletionWithoutDuplicate() { + var timeline = reduceTimeline( + emptyList(), + TimelineUpdate.LocalText("local-1", "hello", 10) + ) + timeline = reduceTimeline( + timeline, + TimelineUpdate.RemoteText( + transportId = "stream-7", + text = "hello", + isFinal = true, + timestampMs = 11, + localId = "local-1" + ) + ) + timeline = reduceTimeline( + timeline, + TimelineUpdate.TextSent("local-1", "stream-7", 12) + ) + + assertEquals(1, timeline.size) + assertEquals("stream-7", timeline.single().transportId) + } +}