diff --git a/Package.swift b/Package.swift index e43e6b4..a24d6bd 100644 --- a/Package.swift +++ b/Package.swift @@ -69,6 +69,9 @@ let package = Package( "BestASRKit", "BestASRMCPCore", "BestASRGUICore", + // CLI parse regression tests (#101: ArgumentParser negative-value + // form) — SwiftPM links executable targets into tests since 5.5. + "bestasr", .product(name: "WhisperKit", package: "WhisperKit"), ], swiftSettings: [.swiftLanguageMode(.v5)] diff --git a/README.md b/README.md index de47974..580b1f4 100644 --- a/README.md +++ b/README.md @@ -411,6 +411,26 @@ harmless no-op for backends that never emit those strings. Pass `--hallucination-filter off` (CLI) or `hallucination_filter: "off"` (the MCP `transcribe` tool) to disable it. +`--hallucination-filter full` adds confidence gating on top of the denylist: +cues are also dropped by Whisper's per-segment signals — the joint silence rule +(`no_speech_prob > 0.6` **and** `avg_logprob < -1.0`, openai-whisper semantics) +and the repetition rule (`compression_ratio > 2.4`). WhisperKit populates the +signals; backends that don't (whisper.cpp, Parakeet) sail through untouched, so +`full` degrades to `denylist` per backend automatically. + +Complementary decode-side knobs (WhisperKit only; CLI-only for now — the MCP +tool and GUI don't expose them) suppress hallucinations at the source instead +of filtering them afterwards — unset, they ride WhisperKit's own defaults. With +`--decode-deterministic` (no fallback retries), aggressive thresholds mark or +skip segments outright instead of triggering a re-decode: + +```bash +bestasr transcribe meeting.m4a \ + --no-speech-threshold 0.5 \ + --compression-ratio-threshold 2.2 \ + --logprob-threshold -1.0 +``` + ### Transcribe any source (agent skill) `bestasr transcribe` takes a local audio file. The **`transcript` agent skill** diff --git a/Sources/BestASRKit/CommandCore.swift b/Sources/BestASRKit/CommandCore.swift index b256f3d..d77c1dd 100644 --- a/Sources/BestASRKit/CommandCore.swift +++ b/Sources/BestASRKit/CommandCore.swift @@ -286,7 +286,10 @@ public struct CommandCore: Sendable { formatName: String, outputPath: String?, diarize: Bool = false, - hallucinationFilter: HallucinationFilterMode = .denylist + hallucinationFilter: HallucinationFilterMode = .denylist, + noSpeechThreshold: Double? = nil, + compressionRatioThreshold: Double? = nil, + logProbThreshold: Double? = nil ) async throws -> TranscribeOutcome { let format = try TranscriptWriter.format(named: formatName) let audio = try AudioProber.probe( @@ -301,7 +304,10 @@ public struct CommandCore: Sendable { audioPath: audio.path, options: TranscribeOptions( model: rec.model, quantization: rec.quantization, - language: audio.language, prompt: context?.rendered.prompt) + language: audio.language, prompt: context?.rendered.prompt, + noSpeechThreshold: noSpeechThreshold, + compressionRatioThreshold: compressionRatioThreshold, + logProbThreshold: logProbThreshold) ) // Cue-level diarization (#25, spec diarization): acoustic turns from the diff --git a/Sources/BestASRKit/Engines/Engine.swift b/Sources/BestASRKit/Engines/Engine.swift index b13a17e..ca145bc 100644 --- a/Sources/BestASRKit/Engines/Engine.swift +++ b/Sources/BestASRKit/Engines/Engine.swift @@ -25,12 +25,21 @@ public struct RawTranscription: Sendable { public let end: Double public let text: String public let confidence: Double? + /// Whisper per-segment hallucination signals (#100); nil when the + /// backend does not compute them. + public let noSpeechProb: Double? + public let compressionRatio: Double? - public init(start: Double, end: Double, text: String, confidence: Double? = nil) { + public init( + start: Double, end: Double, text: String, confidence: Double? = nil, + noSpeechProb: Double? = nil, compressionRatio: Double? = nil + ) { self.start = start self.end = end self.text = text self.confidence = confidence + self.noSpeechProb = noSpeechProb + self.compressionRatio = compressionRatio } } @@ -109,7 +118,9 @@ extension Engine { start: seg.start, end: seg.end, text: seg.text, - confidence: seg.confidence + confidence: seg.confidence, + noSpeechProb: seg.noSpeechProb, + compressionRatio: seg.compressionRatio ) } let text = segments.map(\.text).joined().trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/Sources/BestASRKit/Engines/WhisperKitEngine.swift b/Sources/BestASRKit/Engines/WhisperKitEngine.swift index f4cd1a4..befa4ac 100644 --- a/Sources/BestASRKit/Engines/WhisperKitEngine.swift +++ b/Sources/BestASRKit/Engines/WhisperKitEngine.swift @@ -71,12 +71,21 @@ public struct WhisperKitEngine: Engine { /// segment text, polluting transcripts and inflating WER (#6 — caught by /// real-model verification against jfk.wav / OSR Harvard). static func makeDecodeOptions( - language: String?, promptTokens: [Int]?, deterministic: Bool = false + language: String?, promptTokens: [Int]?, deterministic: Bool = false, + noSpeechThreshold: Double? = nil, compressionRatioThreshold: Double? = nil, + logProbThreshold: Double? = nil ) -> DecodingOptions { var decodeOptions = DecodingOptions() decodeOptions.language = language decodeOptions.detectLanguage = language == nil decodeOptions.skipSpecialTokens = true + // Decode-param knobs (#101): apply only when set — nil keeps + // WhisperKit's own defaults byte-for-byte (pre-#101 behavior). + if let noSpeechThreshold { decodeOptions.noSpeechThreshold = Float(noSpeechThreshold) } + if let compressionRatioThreshold { + decodeOptions.compressionRatioThreshold = Float(compressionRatioThreshold) + } + if let logProbThreshold { decodeOptions.logProbThreshold = Float(logProbThreshold) } if deterministic { // #34 regression gate: WhisperKit defaults to 5 temperature- // fallback retries (0.2 increments) on low-quality segments — @@ -143,7 +152,10 @@ public struct WhisperKitEngine: Engine { } let decodeOptions = Self.makeDecodeOptions( language: options.language, promptTokens: promptTokens, - deterministic: options.deterministicDecode) + deterministic: options.deterministicDecode, + noSpeechThreshold: options.noSpeechThreshold, + compressionRatioThreshold: options.compressionRatioThreshold, + logProbThreshold: options.logProbThreshold) let results = try await pipe.transcribe(audioPath: audioPath, decodeOptions: decodeOptions) let segments = results.flatMap(\.segments).map { seg in @@ -151,7 +163,9 @@ public struct WhisperKitEngine: Engine { start: Double(seg.start), end: Double(seg.end), text: seg.text, - confidence: Double(seg.avgLogprob) + confidence: Double(seg.avgLogprob), + noSpeechProb: Double(seg.noSpeechProb), + compressionRatio: Double(seg.compressionRatio) ) } return RawTranscription( diff --git a/Sources/BestASRKit/Models/DataModels.swift b/Sources/BestASRKit/Models/DataModels.swift index 224af93..35acee1 100644 --- a/Sources/BestASRKit/Models/DataModels.swift +++ b/Sources/BestASRKit/Models/DataModels.swift @@ -81,16 +81,28 @@ public struct TranscribeOptions: Sendable, Equatable { /// needs reproducibility more than the occasional rescue; normal /// transcription keeps the fallback. public let deterministicDecode: Bool + /// WhisperKit decode-param knobs (#101): nil rides WhisperKit's own + /// defaults. Other backends never read these — the knobs suppress + /// hallucinations at decode time and only WhisperKit's decoder has them + /// (complementary to the backend-agnostic post-decode filter, #98/#100). + public let noSpeechThreshold: Double? + public let compressionRatioThreshold: Double? + public let logProbThreshold: Double? public init( model: String, quantization: String, language: String? = nil, prompt: String? = nil, - deterministicDecode: Bool = false + deterministicDecode: Bool = false, + noSpeechThreshold: Double? = nil, compressionRatioThreshold: Double? = nil, + logProbThreshold: Double? = nil ) { self.model = model self.quantization = quantization self.language = language self.prompt = prompt self.deterministicDecode = deterministicDecode + self.noSpeechThreshold = noSpeechThreshold + self.compressionRatioThreshold = compressionRatioThreshold + self.logProbThreshold = logProbThreshold } } @@ -100,6 +112,13 @@ public struct TranscriptSegment: Sendable, Equatable { public let end: Double public let text: String public let confidence: Double? + /// Whisper per-segment hallucination signals (#100): probability the + /// segment is silence, and the text's gzip compression ratio (repetition + /// marker). nil when the backend does not compute them (whisper.cpp / + /// Parakeet) — a nil signal can never trip the `full` filter's thresholds, + /// which is exactly the per-backend degradation the spec asks for. + public let noSpeechProb: Double? + public let compressionRatio: Double? /// Cue-level diarization label (`SPEAKER_1`-based, order of first appearance; /// #25). nil when diarization did not run or no turn overlapped this segment /// — absent means "unknown", never a fabricated speaker (spec diarization). @@ -107,6 +126,7 @@ public struct TranscriptSegment: Sendable, Equatable { public init( id: Int, start: Double, end: Double, text: String, confidence: Double? = nil, + noSpeechProb: Double? = nil, compressionRatio: Double? = nil, speaker: String? = nil ) { self.id = id @@ -114,6 +134,8 @@ public struct TranscriptSegment: Sendable, Equatable { self.end = end self.text = text self.confidence = confidence + self.noSpeechProb = noSpeechProb + self.compressionRatio = compressionRatio self.speaker = speaker } @@ -121,6 +143,7 @@ public struct TranscriptSegment: Sendable, Equatable { public func withSpeaker(_ speaker: String?) -> TranscriptSegment { TranscriptSegment( id: id, start: start, end: end, text: text, confidence: confidence, + noSpeechProb: noSpeechProb, compressionRatio: compressionRatio, speaker: speaker) } } diff --git a/Sources/BestASRKit/Output/HallucinationFilter.swift b/Sources/BestASRKit/Output/HallucinationFilter.swift index b3b0377..3f5e735 100644 --- a/Sources/BestASRKit/Output/HallucinationFilter.swift +++ b/Sources/BestASRKit/Output/HallucinationFilter.swift @@ -10,6 +10,12 @@ public enum HallucinationFilterMode: String, Sendable, CaseIterable { /// adjacent-duplicate cues. Backend-agnostic; the denylist content is /// Whisper-family, so it is a no-op for backends that never emit it. case denylist + /// Everything `denylist` does, plus confidence-gated drops on Whisper's + /// per-segment signals (#100): the joint silence rule (noSpeechProb AND + /// avgLogprob past threshold, openai-whisper semantics) and the repetition + /// rule (compressionRatio past threshold). Backends that don't populate the + /// signals degrade to `denylist` behavior — nil never trips a threshold. + case full } /// Post-decode cleanup pass. A pure function over a `Transcript` that never @@ -19,6 +25,26 @@ public enum HallucinationFilterMode: String, Sendable, CaseIterable { /// diarization, which makes it backend-agnostic and preserves speaker labels on /// the cues that survive. public enum HallucinationFilter { + /// Whisper's canonical thresholds (openai/whisper defaults). The silence + /// rule is deliberately a conjunction — a high no-speech probability alone + /// (or a low logprob alone) also occurs on quiet-but-real speech. + static let noSpeechThreshold = 0.6 + static let logProbThreshold = -1.0 + static let compressionRatioThreshold = 2.4 + + /// True when the segment's confidence signals mark it as a hallucination + /// (`full` mode only). nil signals never trip a rule. + static func isConfidenceFlagged(_ segment: TranscriptSegment) -> Bool { + if let noSpeech = segment.noSpeechProb, let logProb = segment.confidence, + noSpeech > noSpeechThreshold, logProb < logProbThreshold { + return true + } + if let compression = segment.compressionRatio, compression > compressionRatioThreshold { + return true + } + return false + } + /// Return `transcript` with hallucination cues removed per `mode`. /// A no-op (`mode == .off`, or nothing matched) returns the input untouched /// so ids and flat text stay byte-identical. @@ -36,6 +62,8 @@ public enum HallucinationFilter { if trimmed.isEmpty { continue } // Known-boilerplate hallucination. if denylist.matches(segment.text) { continue } + // Confidence-gated drop (full mode only, #100). + if mode == .full, isConfidenceFlagged(segment) { continue } // Adjacent exact-duplicate cue (rolling caption / token echo). if let previous = kept.last, previous.text.trimmingCharacters(in: .whitespacesAndNewlines) == trimmed { @@ -54,7 +82,9 @@ public enum HallucinationFilter { let reindexed = kept.enumerated().map { index, segment in TranscriptSegment( id: index + 1, start: segment.start, end: segment.end, - text: segment.text, confidence: segment.confidence, speaker: segment.speaker) + text: segment.text, confidence: segment.confidence, + noSpeechProb: segment.noSpeechProb, compressionRatio: segment.compressionRatio, + speaker: segment.speaker) } let rebuiltText = reindexed.map(\.text).joined() .trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/Sources/bestasr/BestASRCommand.swift b/Sources/bestasr/BestASRCommand.swift index 54ffba0..5042bce 100644 --- a/Sources/bestasr/BestASRCommand.swift +++ b/Sources/bestasr/BestASRCommand.swift @@ -106,9 +106,26 @@ struct Transcribe: AsyncParsableCommand { @Flag(help: "Label each cue with an acoustic speaker (SPEAKER_1…); downloads CoreML diarization models on first use") var diarize = false - @Option(help: "Strip decoder hallucinations before writing: off | denylist (default denylist)") + @Option(help: "Strip decoder hallucinations before writing: off | denylist | full (confidence-gated, WhisperKit signals; default denylist)") var hallucinationFilter: HallucinationFilterMode = .denylist + @Option(help: "Decode knob (WhisperKit only): mark segments above this no-speech probability as silence (default: WhisperKit's)") + var noSpeechThreshold: Double? + + @Option(help: "Decode knob (WhisperKit only): treat segments above this compression ratio as failed/repetitive (default: WhisperKit's)") + var compressionRatioThreshold: Double? + + // parsing: .unconditional — the meaningful domain is all-negative (it's a + // log-prob floor), and ArgumentParser's default strategy rejects a leading + // dash ("--logprob-threshold -1.0" → 'Missing value'). Unconditional + // consumes the next token as the value (verify #101 HIGH). + @Option( + parsing: .unconditional, + help: + "Decode knob (WhisperKit only): average-logprob floor below which a segment decode is retried/marked (negative, e.g. -1.0; default: WhisperKit's). Note: with --decode-deterministic, threshold trips mark/skip segments instead of triggering a fallback retry" + ) + var logprobThreshold: Double? + func run() async throws { try await runMapped { let result = try await CommandCore.live().transcribe( @@ -117,7 +134,10 @@ struct Transcribe: AsyncParsableCommand { formatName: format, outputPath: output, diarize: diarize, - hallucinationFilter: hallucinationFilter + hallucinationFilter: hallucinationFilter, + noSpeechThreshold: noSpeechThreshold, + compressionRatioThreshold: compressionRatioThreshold, + logProbThreshold: logprobThreshold ) print("Wrote \(result.format) transcript to \(result.outputPath)") if explain { diff --git a/Tests/BestASRKitTests/DecodeKnobsTests.swift b/Tests/BestASRKitTests/DecodeKnobsTests.swift new file mode 100644 index 0000000..bdeaf32 --- /dev/null +++ b/Tests/BestASRKitTests/DecodeKnobsTests.swift @@ -0,0 +1,52 @@ +import Foundation +import Testing +import WhisperKit + +@testable import BestASRKit +@testable import bestasr + +/// spec asr-engine (#101): WhisperKit decode-param knobs. nil = ride +/// WhisperKit's own defaults (the pre-#101 behavior, byte-for-byte). +struct DecodeKnobsTests { + @Test func `set thresholds reach DecodingOptions`() { + let options = WhisperKitEngine.makeDecodeOptions( + language: "en", promptTokens: nil, + noSpeechThreshold: 0.5, compressionRatioThreshold: 2.0, logProbThreshold: -0.8) + #expect(options.noSpeechThreshold == 0.5) + #expect(options.compressionRatioThreshold == 2.0) + #expect(options.logProbThreshold == -0.8) + } + + @Test func `unset thresholds keep WhisperKit defaults`() { + let ours = WhisperKitEngine.makeDecodeOptions(language: "en", promptTokens: nil) + let stock = DecodingOptions() + #expect(ours.noSpeechThreshold == stock.noSpeechThreshold) + #expect(ours.compressionRatioThreshold == stock.compressionRatioThreshold) + #expect(ours.logProbThreshold == stock.logProbThreshold) + } + + // verify #101 HIGH: the log-prob domain is all-negative; the documented + // space form ("--logprob-threshold -1.0") must parse (parsing: .unconditional). + @Test func `negative logprob threshold parses in space form`() throws { + let command = try Transcribe.parse(["in.wav", "--logprob-threshold", "-1.0"]) + #expect(command.logprobThreshold == -1.0) + // The positive-domain knobs stay on the default strategy and parse too. + let both = try Transcribe.parse( + ["in.wav", "--no-speech-threshold", "0.5", "--compression-ratio-threshold", "2.2"]) + #expect(both.noSpeechThreshold == 0.5) + #expect(both.compressionRatioThreshold == 2.2) + } + + @Test func `TranscribeOptions carries the knobs with nil defaults`() { + let plain = TranscribeOptions(model: "base", quantization: "default") + #expect(plain.noSpeechThreshold == nil) + #expect(plain.compressionRatioThreshold == nil) + #expect(plain.logProbThreshold == nil) + + let tuned = TranscribeOptions( + model: "base", quantization: "default", + noSpeechThreshold: 0.7, compressionRatioThreshold: 2.2, logProbThreshold: -1.2) + #expect(tuned.noSpeechThreshold == 0.7) + #expect(tuned != plain) + } +} diff --git a/Tests/BestASRKitTests/HallucinationFullModeTests.swift b/Tests/BestASRKitTests/HallucinationFullModeTests.swift new file mode 100644 index 0000000..81dc416 --- /dev/null +++ b/Tests/BestASRKitTests/HallucinationFullModeTests.swift @@ -0,0 +1,96 @@ +import Foundation +import Testing + +@testable import BestASRKit + +/// spec transcript-output (#100): the confidence-gated `full` filter mode. +/// Signals ride Whisper's canonical thresholds — the joint silence rule +/// (noSpeechProb > 0.6 AND avgLogprob < -1.0) and the repetition rule +/// (compressionRatio > 2.4). nil signals can never trip a rule, so backends +/// that don't populate them degrade to denylist behavior with no branching. +struct HallucinationFullModeTests { + private func seg( + _ id: Int, _ text: String, confidence: Double? = nil, + noSpeech: Double? = nil, compression: Double? = nil + ) -> TranscriptSegment { + TranscriptSegment( + id: id, start: Double(id), end: Double(id) + 1, text: text, + confidence: confidence, noSpeechProb: noSpeech, compressionRatio: compression) + } + + private func make(_ segments: [TranscriptSegment]) -> Transcript { + let text = segments.map(\.text).joined().trimmingCharacters(in: .whitespacesAndNewlines) + return Transcript( + text: text, language: "zh", duration: segments.last?.end, + backend: "whisperkit", model: "large-v3-turbo", segments: segments) + } + + // The joint silence rule needs BOTH signals past threshold (openai-whisper + // semantics): high no-speech alone or low logprob alone must not drop — + // that would eat quiet-but-real speech. + @Test func `silence rule is a conjunction of noSpeech and logprob`() { + let input = make([ + seg(1, "真的有人在說話", confidence: -0.2, noSpeech: 0.9), // high ns, good lp → keep + seg(2, "模糊但真實的話", confidence: -1.5, noSpeech: 0.3), // low ns, bad lp → keep + seg(3, "幻覺字幕", confidence: -1.5, noSpeech: 0.9), // both → drop + seg(4, "正常收尾"), + ]) + let out = HallucinationFilter.filter(input, mode: .full) + #expect(out.segments.map(\.text) == ["真的有人在說話", "模糊但真實的話", "正常收尾"]) + } + + @Test func `repetition rule drops above the compression threshold only`() { + let input = make([ + seg(1, "重複重複重複重複重複重複", compression: 2.5), // > 2.4 → drop + seg(2, "臨界值上的正常句子", compression: 2.4), // == threshold → keep + seg(3, "一般句子", compression: 1.1), + ]) + let out = HallucinationFilter.filter(input, mode: .full) + #expect(out.segments.map(\.text) == ["臨界值上的正常句子", "一般句子"]) + } + + // Backends that never populate the signals (whisper.cpp / Parakeet) sail + // through full mode untouched — full degrades to denylist for free. + @Test func `nil signals never trip confidence rules`() { + let input = make([ + seg(1, "無訊號後端的句子", confidence: -2.0), // terrible logprob but no noSpeech signal + seg(2, "另一句", confidence: nil), + ]) + let out = HallucinationFilter.filter(input, mode: .full) + #expect(out.segments.count == 2) + } + + @Test func `full still applies the denylist and dedup passes`() { + let input = make([ + seg(1, "感謝觀看"), // standalone boilerplate → dropped + seg(2, "實際內容", noSpeech: 0.1), + seg(3, "實際內容"), // adjacent duplicate → dropped + ]) + let out = HallucinationFilter.filter(input, mode: .full) + #expect(out.segments.map(\.text) == ["實際內容"]) + } + + // denylist mode must NOT gain confidence-gating (mode contract stays put). + @Test func `denylist mode ignores confidence signals`() { + let input = make([ + seg(1, "低信心但 denylist 模式", confidence: -3.0, noSpeech: 0.99, compression: 9.9) + ]) + let out = HallucinationFilter.filter(input, mode: .denylist) + #expect(out.segments.count == 1) + } + + // The reindex path must forward the new fields — otherwise the signals die + // mid-pipeline and downstream consumers (future) see nil. + @Test func `reindex preserves confidence signals on survivors`() { + let input = make([ + seg(1, "感謝觀看"), // dropped → forces the reindex path + seg(2, "留下的句子", confidence: -0.3, noSpeech: 0.2, compression: 1.7), + ]) + let out = HallucinationFilter.filter(input, mode: .full) + let survivor = out.segments[0] + #expect(survivor.id == 1) + #expect(survivor.noSpeechProb == 0.2) + #expect(survivor.compressionRatio == 1.7) + #expect(survivor.confidence == -0.3) + } +}