From 6c2e5d9ad85a8d72cd7b217c8ca707cbee2b42c3 Mon Sep 17 00:00:00 2001 From: niksedk Date: Wed, 8 Jul 2026 11:02:41 +0200 Subject: [PATCH] Add Video OCR: extract burned-in (hardcoded) subtitles via OCR New "Video -> OCR burned-in subtitle..." feature (#10944), inspired by timminator's VideOCR pipeline: - ffmpeg samples frames from a user-selected scan area (draggable/resizable crop rectangle over a video frame preview, with bottom-third default and presets) at a configurable rate - consecutive near-identical frames are collapsed before OCR by comparing bright-pixel masks (Jaccard overlap), so each on-screen subtitle is OCR'ed once; frames with no bright pixels are skipped as blank - OCR engines: Paddle OCR (standalone/Python, reusing the existing batch engine via a new decode-free file input path), Ollama vision (e.g. glm-ocr), and a new GLM API engine (Z.ai / bigmodel.cn, OpenAI-compatible) - consecutive OCR results are merged into subtitle lines by Levenshtein text similarity with majority-vote text selection, max-gap bridging and minimum-duration filtering; vision-LLM hallucinations on textless frames are sanitized away - optional ASSA position tag ({\anX}) derived from the scan area position - shared Paddle install prompts extracted to PaddleOcrInstallHelper Verified end-to-end against a synthetic video with burned-in subtitles over a moving background (Ollama + glm-ocr): exact texts and timings. Co-Authored-By: Claude Fable 5 --- docs/features/video-ocr.md | 47 ++ docs/index.md | 1 + src/ui/DependencyInjectionExtensions.cs | 2 + src/ui/Features/Main/Layout/InitMenu.cs | 5 + .../Features/Main/Layout/InitNativeMacMenu.cs | 1 + src/ui/Features/Main/MainViewModel.cs | 65 ++ .../Ocr/Download/PaddleOcrInstallHelper.cs | 109 +++ src/ui/Features/Ocr/Engines/GlmOcr.cs | 122 +++ src/ui/Features/Ocr/Engines/OcrEngineType.cs | 1 + src/ui/Features/Ocr/Engines/PaddleOcr.cs | 19 + .../Ocr/Engines/PaddleOcrBatchInput.cs | 6 + .../Video/VideoOcr/CropAreaSelector.cs | 395 +++++++++ .../Video/VideoOcr/VideoOcrEngineItem.cs | 40 + .../Video/VideoOcr/VideoOcrFrameGroup.cs | 33 + .../Video/VideoOcr/VideoOcrFrameGrouper.cs | 224 +++++ .../Video/VideoOcr/VideoOcrLineBuilder.cs | 231 +++++ .../Video/VideoOcr/VideoOcrLineItem.cs | 14 + .../Video/VideoOcr/VideoOcrViewModel.cs | 792 ++++++++++++++++++ .../Features/Video/VideoOcr/VideoOcrWindow.cs | 369 ++++++++ .../Config/Language/Main/LanguageMainMenu.cs | 2 + .../Options/LanguageSettingsShortcuts.cs | 2 + .../Config/Language/Video/LanguageVideo.cs | 1 + .../Config/Language/Video/LanguageVideoOcr.cs | 71 ++ src/ui/Logic/Config/SeVideo.cs | 2 + src/ui/Logic/Config/SeVideoOcr.cs | 57 ++ src/ui/Logic/ShortcutsMain.cs | 2 + tests/UI/Features/VideoOcrTests.cs | 224 +++++ 27 files changed, 2837 insertions(+) create mode 100644 docs/features/video-ocr.md create mode 100644 src/ui/Features/Ocr/Download/PaddleOcrInstallHelper.cs create mode 100644 src/ui/Features/Ocr/Engines/GlmOcr.cs create mode 100644 src/ui/Features/Video/VideoOcr/CropAreaSelector.cs create mode 100644 src/ui/Features/Video/VideoOcr/VideoOcrEngineItem.cs create mode 100644 src/ui/Features/Video/VideoOcr/VideoOcrFrameGroup.cs create mode 100644 src/ui/Features/Video/VideoOcr/VideoOcrFrameGrouper.cs create mode 100644 src/ui/Features/Video/VideoOcr/VideoOcrLineBuilder.cs create mode 100644 src/ui/Features/Video/VideoOcr/VideoOcrLineItem.cs create mode 100644 src/ui/Features/Video/VideoOcr/VideoOcrViewModel.cs create mode 100644 src/ui/Features/Video/VideoOcr/VideoOcrWindow.cs create mode 100644 src/ui/Logic/Config/Language/Video/LanguageVideoOcr.cs create mode 100644 src/ui/Logic/Config/SeVideoOcr.cs create mode 100644 tests/UI/Features/VideoOcrTests.cs diff --git a/docs/features/video-ocr.md b/docs/features/video-ocr.md new file mode 100644 index 00000000000..6439c1a87b4 --- /dev/null +++ b/docs/features/video-ocr.md @@ -0,0 +1,47 @@ +# OCR Burned-in Subtitle (Video OCR) + +Extract hardcoded (burned-in) subtitles from a video into editable text lines using OCR. + +- **Menu:** Video → OCR burned-in subtitle... + +## How to Use + +1. Open **Video → OCR burned-in subtitle...** (a video file is required) +2. Use the preview slider to find a frame that shows a subtitle +3. Adjust the scan area rectangle so it covers where the subtitles appear (default: bottom third) — drag to move/resize, or use the preset buttons +4. Pick an OCR engine and language +5. Click **Start OCR** — lines appear in the list as they are recognized +6. Click **OK** to load the result into the main window + +## OCR Engines + +- **Paddle OCR** — local, fast and accurate; downloaded automatically (Windows/Linux). Recommended. +- **Paddle OCR Python** — local, via a Python `paddleocr` installation (also works on macOS) +- **Ollama vision** — local vision model via [Ollama](https://ollama.com), e.g. `glm-ocr` +- **GLM API** — GLM vision model via the Z.ai / bigmodel.cn cloud API (requires an API key) + +## How It Works + +Frames are sampled from the scan area at a few frames per second with ffmpeg. Consecutive +near-identical frames are collapsed so each on-screen subtitle is OCR'ed only once, then +consecutive OCR results with near-identical text are merged into one line with correct start/end +times (the text variant shown the longest wins). + +## Settings + +- **Frames per second** — how many frames per second to sample (higher = more precise timing, slower) +- **Text brightness minimum** — pixels darker than this are ignored when comparing frames, so the + comparison follows the (bright) subtitle text instead of the moving video behind it; frames with + no bright pixels are skipped entirely. Set to 0 to disable (e.g. for dark subtitle text). +- **Merge lines with similarity (%)** — how similar the text of two consecutive OCR results must be to merge into one line +- **Max gap between lines (ms)** — maximum time gap allowed when merging +- **Minimum duration (ms)** — lines shorter than this are dropped (removes OCR blips/false positives) +- **Add ASSA position tag** — prepend an alignment tag (e.g. `{\an8}` for a top scan area) based on + the scan area position, for Advanced Sub Station Alpha output + +## Tips + +- Scan only the region where subtitles actually appear — a smaller area is faster and has fewer false positives +- For subtitles at the top of the frame (e.g. sign translations), move the scan area up and enable the ASSA position tag +- If lines are duplicated with small OCR differences, raise **Merge lines with similarity** tolerance by lowering the percentage +- If short random text blips appear, raise **Minimum duration** diff --git a/docs/index.md b/docs/index.md index 23feee72f88..bd1f1f6201f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -56,6 +56,7 @@ Subtitle Edit is a free, open-source editor for video subtitles. This is the doc - [Video Player](features/video-player.md) — Video playback controls - [Audio Visualizer / Waveform](features/audio-visualizer.md) — Waveform display and editing - [Speech to Text](features/speech-to-text.md) — Automatic speech recognition with Whisper, Qwen3 ASR, Crisp ASR, and related engines +- [Video OCR](features/video-ocr.md) — Extract burned-in (hardcoded) subtitles via OCR - [Text to Speech](features/text-to-speech.md) — Generate speech from subtitles - [Burn-In Subtitles](features/burn-in.md) — Hardcode subtitles into video - [Embedded Subtitles](features/embedded-subtitles.md) — Add, remove, preview, and edit subtitle tracks in Matroska/WebM files diff --git a/src/ui/DependencyInjectionExtensions.cs b/src/ui/DependencyInjectionExtensions.cs index 062b03473b5..99d1113df61 100644 --- a/src/ui/DependencyInjectionExtensions.cs +++ b/src/ui/DependencyInjectionExtensions.cs @@ -140,6 +140,7 @@ using Nikse.SubtitleEdit.Features.Video.ShotChanges; using Nikse.SubtitleEdit.Features.Video.TextToSpeech; using Nikse.SubtitleEdit.Features.Video.TextToSpeech.ActorVoices; +using Nikse.SubtitleEdit.Features.Video.VideoOcr; using Nikse.SubtitleEdit.Features.Video.TextToSpeech.AdvancedTtsSettings; using Nikse.SubtitleEdit.Features.Video.TextToSpeech.DownloadTts; using Nikse.SubtitleEdit.Features.Video.TextToSpeech.ElevenLabsSettings; @@ -339,6 +340,7 @@ public static void AddSubtitleEditServices(this IServiceCollection collection) collection.AddTransient(); collection.AddTransient(); collection.AddTransient(); + collection.AddTransient(); collection.AddTransient(); collection.AddTransient(); collection.AddTransient(); diff --git a/src/ui/Features/Main/Layout/InitMenu.cs b/src/ui/Features/Main/Layout/InitMenu.cs index fb143ba5f26..bb4cb58d349 100644 --- a/src/ui/Features/Main/Layout/InitMenu.cs +++ b/src/ui/Features/Main/Layout/InitMenu.cs @@ -681,6 +681,11 @@ public static void Make(MainViewModel vm) Header = l.TextToSpeech, Command = vm.ShowVideoTextToSpeechCommand, }, + new MenuItem + { + Header = l.VideoOcr, + Command = vm.ShowVideoOcrCommand, + }, new Separator(), new MenuItem { diff --git a/src/ui/Features/Main/Layout/InitNativeMacMenu.cs b/src/ui/Features/Main/Layout/InitNativeMacMenu.cs index 6f440982b67..f1500251d0f 100644 --- a/src/ui/Features/Main/Layout/InitNativeMacMenu.cs +++ b/src/ui/Features/Main/Layout/InitNativeMacMenu.cs @@ -331,6 +331,7 @@ public static void MakeStructure(NativeMenu root, Window window) videoItems.Items.Add(new NativeMenuItemSeparator()); videoItems.Items.Add(Item(Clean(l.SpeechToText), v => v.ShowSpeechToTextWhisperCommand)); videoItems.Items.Add(Item(Clean(l.TextToSpeech), v => v.ShowVideoTextToSpeechCommand)); + videoItems.Items.Add(Item(Clean(l.VideoOcr), v => v.ShowVideoOcrCommand)); videoItems.Items.Add(new NativeMenuItemSeparator()); videoItems.Items.Add(Item(Clean(l.GenerateBurnIn), v => v.ShowVideoBurnInCommand)); videoItems.Items.Add(Item(Clean(l.GenerateTransparent), v => v.ShowVideoTransparentSubtitlesCommand)); diff --git a/src/ui/Features/Main/MainViewModel.cs b/src/ui/Features/Main/MainViewModel.cs index d2c0a594ece..e16be62e487 100644 --- a/src/ui/Features/Main/MainViewModel.cs +++ b/src/ui/Features/Main/MainViewModel.cs @@ -145,6 +145,7 @@ using Nikse.SubtitleEdit.Features.Video.ShotChanges; using Nikse.SubtitleEdit.Features.Video.SpeechToText; using Nikse.SubtitleEdit.Features.Video.SpeechToText.OpenAiCompatible; +using Nikse.SubtitleEdit.Features.Video.VideoOcr; using Nikse.SubtitleEdit.Features.Video.TextToSpeech; using Nikse.SubtitleEdit.Features.Video.TextToSpeech.ReviewSpeech; using Nikse.SubtitleEdit.Features.Video.TransparentSubtitles; @@ -6014,6 +6015,70 @@ private async Task ShowSpeechToTextWhisper() } } + [RelayCommand] + private async Task ShowVideoOcr() + { + if (Window == null) + { + return; + } + + var ffmpegOk = await RequireFfmpegOk(); + if (!ffmpegOk) + { + return; + } + + var doContinue = await HasChangesContinue(); + if (!doContinue) + { + return; + } + + // Frame extraction needs a local file - for URL sources, ask for a file instead. + var videoFileName = !string.IsNullOrEmpty(_videoFileName) && File.Exists(_videoFileName) ? _videoFileName : string.Empty; + if (string.IsNullOrEmpty(videoFileName)) + { + videoFileName = await _fileHelper.PickOpenVideoFile(Window!, Se.Language.General.OpenVideoFileTitle); + if (string.IsNullOrEmpty(videoFileName)) + { + return; + } + } + + var result = await ShowDialogAsync(vm => { vm.Initialize(videoFileName); }); + + if (result.OkPressed && result.ResultSubtitle.Paragraphs.Count > 0) + { + // Open the video before setting the subtitle state - VideoOpenFile persists + // the current subtitle file name to the recent-files list, and the OCR result + // has not been saved to disk yet. + if (_videoFileName != videoFileName) + { + await VideoOpenFile(videoFileName); + } + + ResetSubtitle(); + + _subtitle = result.ResultSubtitle; + + if (SelectedSubtitleFormat is AdvancedSubStationAlpha) + { + foreach (var p in _subtitle.Paragraphs) + { + p.Text = AdvancedSubStationAlpha.FormatText(p.Text); + } + } + + _subtitleFileName = Path.ChangeExtension(videoFileName, SelectedSubtitleFormat.Extension); + _converted = true; + + SetSubtitles(_subtitle); + SelectAndScrollToRow(0); + ShowStatus(string.Format(Se.Language.Video.VideoOcr.LinesFoundX, _subtitle.Paragraphs.Count)); + } + } + [RelayCommand] private async Task SpeechToTextSelectedLines() { diff --git a/src/ui/Features/Ocr/Download/PaddleOcrInstallHelper.cs b/src/ui/Features/Ocr/Download/PaddleOcrInstallHelper.cs new file mode 100644 index 00000000000..14fe6734d0d --- /dev/null +++ b/src/ui/Features/Ocr/Download/PaddleOcrInstallHelper.cs @@ -0,0 +1,109 @@ +using Avalonia.Controls; +using Nikse.SubtitleEdit.Core.Common; +using Nikse.SubtitleEdit.Features.Ocr.Engines; +using Nikse.SubtitleEdit.Features.Shared; +using Nikse.SubtitleEdit.Logic; +using Nikse.SubtitleEdit.Logic.Config; +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Nikse.SubtitleEdit.Features.Ocr.Download; + +/// +/// Prompts for and runs the Paddle OCR engine/models download when missing. +/// Shared by the features that run Paddle OCR (image OCR, batch convert, video OCR). +/// +public static class PaddleOcrInstallHelper +{ + /// + /// Makes sure the Paddle OCR engine (standalone only) and models are installed, + /// prompting the user to download what is missing. Returns false if the user cancelled. + /// + public static async Task EnsureInstalled(Window window, IWindowService windowService, OcrEngineType engineType) + { + if (engineType == OcrEngineType.PaddleOcrStandalone) + { + if (Configuration.IsRunningOnWindows && !File.Exists(Path.Combine(Se.PaddleOcrFolder, "paddleocr.exe"))) + { + var answer = await MessageBox.Show( + window, + "Download Paddle OCR?", + $"{Environment.NewLine}\"Paddle OCR\" requires downloading Paddle OCR.{Environment.NewLine}{Environment.NewLine}Download and use Paddle OCR?", + MessageBoxButtons.Cancel, + MessageBoxIcon.Question, + "CPU", + "GPU CUDA 11", + "GPU CUDA 12"); + + if (answer == MessageBoxResult.Cancel) + { + return false; + } + + var result = await windowService.ShowDialogAsync(window, + vm => + { + var engine = PaddleOcrDownloadType.EngineCpu; + if (answer == MessageBoxResult.Custom2) + { + engine = PaddleOcrDownloadType.EngineGpu11; + } + else if (answer == MessageBoxResult.Custom3) + { + engine = PaddleOcrDownloadType.EngineGpu12; + } + + vm.Initialize(engine); + }); + + if (!result.OkPressed) + { + return false; + } + } + else if (Configuration.IsRunningOnLinux && !File.Exists(Path.Combine(Se.PaddleOcrFolder, "paddleocr.bin"))) + { + var answer = await MessageBox.Show( + window, + "Download Paddle OCR?", + $"{Environment.NewLine}\"Paddle OCR\" requires downloading Paddle OCR.{Environment.NewLine}{Environment.NewLine}Download and use Paddle OCR?", + MessageBoxButtons.Cancel, + MessageBoxIcon.Question, + "CPU", + "GPU CUDA"); + + if (answer == MessageBoxResult.Cancel) + { + return false; + } + + var result = await windowService.ShowDialogAsync(window, + vm => + { + vm.Initialize(answer == MessageBoxResult.Custom1 + ? PaddleOcrDownloadType.EngineCpuLinux + : PaddleOcrDownloadType.EngineGpuLinux); + }); + + if (!result.OkPressed) + { + return false; + } + } + } + + if (!Directory.Exists(Se.PaddleOcrModelsFolder)) + { + var result = await windowService.ShowDialogAsync(window, + vm => { vm.Initialize(PaddleOcrDownloadType.Models); }); + + if (!result.OkPressed) + { + return false; + } + } + + return true; + } +} diff --git a/src/ui/Features/Ocr/Engines/GlmOcr.cs b/src/ui/Features/Ocr/Engines/GlmOcr.cs new file mode 100644 index 00000000000..7a19abd8d2a --- /dev/null +++ b/src/ui/Features/Ocr/Engines/GlmOcr.cs @@ -0,0 +1,122 @@ +using Nikse.SubtitleEdit.Core.Common; +using Nikse.SubtitleEdit.Logic; +using Nikse.SubtitleEdit.Logic.Config; +using SkiaSharp; +using System; +using System.IO; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Nikse.SubtitleEdit.Features.Ocr.Engines; + +/// +/// OCR via GLM vision models (Z.ai / bigmodel.cn) through the OpenAI-compatible +/// chat completions API - e.g. glm-4.5v or glm-4.6v with an image + prompt. +/// +public class GlmOcr +{ + public const string DefaultUrl = "https://api.z.ai/api/paas/v4/chat/completions"; + public const string DefaultModel = "glm-4.5v"; + + private readonly HttpClient _httpClient; + + public string Error { get; set; } + + public GlmOcr(string apiKey, int timeoutMinutes = 5) + { + Error = string.Empty; + _httpClient = new HttpClient(); + _httpClient.DefaultRequestHeaders.TryAddWithoutValidation("accept", "application/json"); + if (!string.IsNullOrWhiteSpace(apiKey)) + { + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey.Trim()); + } + + _httpClient.Timeout = TimeSpan.FromMinutes(Math.Max(1, timeoutMinutes)); + } + + public async Task Ocr(SKBitmap bitmap, string url, string model, string language, CancellationToken cancellationToken) + { + return await Ocr(bitmap.ToBase64String(), "image/png", url, model, language, cancellationToken); + } + + /// OCR an image file as-is - no decode/re-encode roundtrip. + public async Task Ocr(string imageFileName, string url, string model, string language, CancellationToken cancellationToken) + { + var bytes = await File.ReadAllBytesAsync(imageFileName, cancellationToken); + var mimeType = imageFileName.EndsWith(".png", StringComparison.OrdinalIgnoreCase) ? "image/png" : "image/jpeg"; + return await Ocr(Convert.ToBase64String(bytes), mimeType, url, model, language, cancellationToken); + } + + private async Task Ocr(string base64Image, string mimeType, string url, string model, string language, CancellationToken cancellationToken) + { + try + { + var prompt = string.Format("You are an OCR engine. The language is {0}. Output only the exact text visible in the image, nothing else. If the image contains no text, output nothing. Separate two lines with a single newline.", language); + var modelName = string.IsNullOrWhiteSpace(model) ? DefaultModel : model.Trim(); + + // "thinking" disabled: GLM vision models are thinking models by default, and + // reasoning output would drown the short transcription we want. + var input = + "{ \"model\": \"" + EscapeJson(modelName) + "\", \"temperature\": 0, " + + "\"thinking\": { \"type\": \"disabled\" }, " + + "\"messages\": [ { \"role\": \"user\", \"content\": [ " + + "{ \"type\": \"text\", \"text\": \"" + EscapeJson(prompt) + "\" }, " + + "{ \"type\": \"image_url\", \"image_url\": { \"url\": \"data:" + mimeType + ";base64," + base64Image + "\" } } " + + "] } ] }"; + + var content = new StringContent(input, Encoding.UTF8); + content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json"); + + var requestUrl = string.IsNullOrWhiteSpace(url) ? DefaultUrl : url.Trim(); + var result = await _httpClient.PostAsync(requestUrl, content, cancellationToken); + var bytes = await result.Content.ReadAsByteArrayAsync(cancellationToken); + var json = Encoding.UTF8.GetString(bytes).Trim(); + if (!result.IsSuccessStatusCode) + { + Error = json; + SeLogger.Error("Error calling GLM for OCR: Status code=" + result.StatusCode + Environment.NewLine + json); + } + + result.EnsureSuccessStatusCode(); + + var parser = new SeJsonParser(); + var outputTexts = parser.GetAllTagsByNameAsStrings(json, "content"); + var resultText = string.Join(string.Empty, outputTexts).Trim(); + + resultText = resultText.Replace("\\n", Environment.NewLine); + resultText = OllamaOcr.CleanOcrText(resultText, prompt); + resultText = resultText.Replace("\\\"", "\""); + + return resultText.Trim(); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (OperationCanceledException ex) + { + Error = "Request timed out"; + SeLogger.Error(ex, "GLM OCR request timed out"); + return string.Empty; + } + catch (Exception ex) + { + Error = ex.Message; + SeLogger.Error(ex, "Error calling GLM for OCR"); + return string.Empty; + } + } + + private static string EscapeJson(string text) + { + return text + .Replace("\\", "\\\\") + .Replace("\"", "\\\"") + .Replace("\r", " ") + .Replace("\n", " "); + } +} diff --git a/src/ui/Features/Ocr/Engines/OcrEngineType.cs b/src/ui/Features/Ocr/Engines/OcrEngineType.cs index c9ac8607f5b..06b9da82e8d 100644 --- a/src/ui/Features/Ocr/Engines/OcrEngineType.cs +++ b/src/ui/Features/Ocr/Engines/OcrEngineType.cs @@ -15,4 +15,5 @@ public enum OcrEngineType AmazonRekognition, Mistral, BinaryImageCompare, + Glm, } \ No newline at end of file diff --git a/src/ui/Features/Ocr/Engines/PaddleOcr.cs b/src/ui/Features/Ocr/Engines/PaddleOcr.cs index 9eb92ccdd9d..434d8a6db30 100644 --- a/src/ui/Features/Ocr/Engines/PaddleOcr.cs +++ b/src/ui/Features/Ocr/Engines/PaddleOcr.cs @@ -215,6 +215,25 @@ public async Task OcrBatch(OcrEngineType engineType, List b await Parallel.ForEachAsync(bitmaps, parallelOptions, async (input, ct) => { + if (input.Bitmap == null && !string.IsNullOrEmpty(input.SourceFileName)) + { + // Image is already a file on disk (e.g. a video frame) - just copy it. + try + { + var tempImageFromFile = Path.Combine(folder, + input.Index.ToString("0000") + Path.GetExtension(input.SourceFileName)); + File.Copy(input.SourceFileName, tempImageFromFile, true); + input.FileName = tempImageFromFile; + batchFileNamesList.Add(input); + } + catch + { + // ignore + } + + return; + } + SKBitmap? bitmap = null; SKBitmap? borderedBitmap = null; try diff --git a/src/ui/Features/Ocr/Engines/PaddleOcrBatchInput.cs b/src/ui/Features/Ocr/Engines/PaddleOcrBatchInput.cs index fe85192dd43..a775a69e47d 100644 --- a/src/ui/Features/Ocr/Engines/PaddleOcrBatchInput.cs +++ b/src/ui/Features/Ocr/Engines/PaddleOcrBatchInput.cs @@ -9,4 +9,10 @@ public class PaddleOcrBatchInput public string Text { get; set; } = string.Empty; public string FileName { get; set; } = string.Empty; public Ocr.OcrSubtitleItem? Item { get; set; } + + /// + /// Optional image file to OCR as-is. When set (and Bitmap is null) the file is copied + /// into the batch folder instead of encoding a bitmap - no decode/encode roundtrip. + /// + public string SourceFileName { get; set; } = string.Empty; } diff --git a/src/ui/Features/Video/VideoOcr/CropAreaSelector.cs b/src/ui/Features/Video/VideoOcr/CropAreaSelector.cs new file mode 100644 index 00000000000..ab4d035be25 --- /dev/null +++ b/src/ui/Features/Video/VideoOcr/CropAreaSelector.cs @@ -0,0 +1,395 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Input; +using Avalonia.Media; +using Avalonia.Media.Immutable; +using System; + +namespace Nikse.SubtitleEdit.Features.Video.VideoOcr; + +/// +/// Overlay control for selecting the scan area on top of a video frame preview. +/// The selection is kept in video pixel coordinates; the control maps to/from the +/// letterboxed display rectangle of the underlying Image (Stretch=Uniform). +/// The area outside the selection is dimmed; the selection can be moved, resized +/// via edges/corners, or redrawn by dragging on the outside. +/// +public class CropAreaSelector : Control +{ + public static readonly StyledProperty VideoWidthProperty = + AvaloniaProperty.Register(nameof(VideoWidth), 1920); + + public static readonly StyledProperty VideoHeightProperty = + AvaloniaProperty.Register(nameof(VideoHeight), 1080); + + public static readonly StyledProperty SelectionXProperty = + AvaloniaProperty.Register(nameof(SelectionX)); + + public static readonly StyledProperty SelectionYProperty = + AvaloniaProperty.Register(nameof(SelectionY), 720); + + public static readonly StyledProperty SelectionWidthProperty = + AvaloniaProperty.Register(nameof(SelectionWidth), 1920); + + public static readonly StyledProperty SelectionHeightProperty = + AvaloniaProperty.Register(nameof(SelectionHeight), 360); + + private const double HandleHitSize = 8; + private const int MinSelectionSize = 16; + + private enum DragMode + { + None, + Draw, + Move, + ResizeLeft, + ResizeRight, + ResizeTop, + ResizeBottom, + ResizeTopLeft, + ResizeTopRight, + ResizeBottomLeft, + ResizeBottomRight, + } + + private DragMode _dragMode = DragMode.None; + private Point _dragStartDisplay; + private Rect _dragStartSelectionVideo; + + private static readonly IImmutableSolidColorBrush DimBrush = new ImmutableSolidColorBrush(Colors.Black, 0.55); + private static readonly IImmutableSolidColorBrush AccentBrush = new ImmutableSolidColorBrush(Color.FromRgb(0x2e, 0x9b, 0xe0)); + private static readonly ImmutablePen BorderPen = new(AccentBrush, 2); + + public int VideoWidth + { + get => GetValue(VideoWidthProperty); + set => SetValue(VideoWidthProperty, value); + } + + public int VideoHeight + { + get => GetValue(VideoHeightProperty); + set => SetValue(VideoHeightProperty, value); + } + + public int SelectionX + { + get => GetValue(SelectionXProperty); + set => SetValue(SelectionXProperty, value); + } + + public int SelectionY + { + get => GetValue(SelectionYProperty); + set => SetValue(SelectionYProperty, value); + } + + public int SelectionWidth + { + get => GetValue(SelectionWidthProperty); + set => SetValue(SelectionWidthProperty, value); + } + + public int SelectionHeight + { + get => GetValue(SelectionHeightProperty); + set => SetValue(SelectionHeightProperty, value); + } + + static CropAreaSelector() + { + AffectsRender( + VideoWidthProperty, + VideoHeightProperty, + SelectionXProperty, + SelectionYProperty, + SelectionWidthProperty, + SelectionHeightProperty); + } + + public CropAreaSelector() + { + ClipToBounds = true; + } + + /// The area (in control coordinates) where the video frame is drawn (uniform fit, centered). + private Rect GetDisplayRect() + { + if (VideoWidth <= 0 || VideoHeight <= 0 || Bounds.Width <= 0 || Bounds.Height <= 0) + { + return new Rect(); + } + + var scale = Math.Min(Bounds.Width / VideoWidth, Bounds.Height / VideoHeight); + var width = VideoWidth * scale; + var height = VideoHeight * scale; + return new Rect((Bounds.Width - width) / 2.0, (Bounds.Height - height) / 2.0, width, height); + } + + private Rect VideoToDisplay(Rect videoRect) + { + var display = GetDisplayRect(); + if (display.Width <= 0) + { + return new Rect(); + } + + var scale = display.Width / VideoWidth; + return new Rect( + display.X + videoRect.X * scale, + display.Y + videoRect.Y * scale, + videoRect.Width * scale, + videoRect.Height * scale); + } + + private Point DisplayToVideo(Point displayPoint) + { + var display = GetDisplayRect(); + if (display.Width <= 0) + { + return new Point(); + } + + var scale = VideoWidth / display.Width; + return new Point((displayPoint.X - display.X) * scale, (displayPoint.Y - display.Y) * scale); + } + + public override void Render(DrawingContext context) + { + base.Render(context); + + var display = GetDisplayRect(); + if (display.Width <= 0 || display.Height <= 0) + { + return; + } + + var selection = VideoToDisplay(new Rect(SelectionX, SelectionY, SelectionWidth, SelectionHeight)); + + // Dim everything outside the selection. + context.FillRectangle(DimBrush, new Rect(display.X, display.Y, display.Width, Math.Max(0, selection.Y - display.Y))); + context.FillRectangle(DimBrush, new Rect(display.X, selection.Bottom, display.Width, Math.Max(0, display.Bottom - selection.Bottom))); + context.FillRectangle(DimBrush, new Rect(display.X, selection.Y, Math.Max(0, selection.X - display.X), selection.Height)); + context.FillRectangle(DimBrush, new Rect(selection.Right, selection.Y, Math.Max(0, display.Right - selection.Right), selection.Height)); + + // Selection border + corner handles. + context.DrawRectangle(BorderPen, selection); + + foreach (var p in new[] + { + selection.TopLeft, selection.TopRight, selection.BottomLeft, selection.BottomRight, + new Point(selection.Center.X, selection.Y), new Point(selection.Center.X, selection.Bottom), + new Point(selection.X, selection.Center.Y), new Point(selection.Right, selection.Center.Y), + }) + { + context.FillRectangle(AccentBrush, new Rect(p.X - 3.5, p.Y - 3.5, 7, 7)); + } + } + + private DragMode HitTest(Point point) + { + var selection = VideoToDisplay(new Rect(SelectionX, SelectionY, SelectionWidth, SelectionHeight)); + + bool Near(double a, double b) => Math.Abs(a - b) <= HandleHitSize; + var nearLeft = Near(point.X, selection.X); + var nearRight = Near(point.X, selection.Right); + var nearTop = Near(point.Y, selection.Y); + var nearBottom = Near(point.Y, selection.Bottom); + var insideX = point.X > selection.X - HandleHitSize && point.X < selection.Right + HandleHitSize; + var insideY = point.Y > selection.Y - HandleHitSize && point.Y < selection.Bottom + HandleHitSize; + + if (nearTop && nearLeft) + { + return DragMode.ResizeTopLeft; + } + + if (nearTop && nearRight) + { + return DragMode.ResizeTopRight; + } + + if (nearBottom && nearLeft) + { + return DragMode.ResizeBottomLeft; + } + + if (nearBottom && nearRight) + { + return DragMode.ResizeBottomRight; + } + + if (nearLeft && insideY) + { + return DragMode.ResizeLeft; + } + + if (nearRight && insideY) + { + return DragMode.ResizeRight; + } + + if (nearTop && insideX) + { + return DragMode.ResizeTop; + } + + if (nearBottom && insideX) + { + return DragMode.ResizeBottom; + } + + if (point.X > selection.X && point.X < selection.Right && point.Y > selection.Y && point.Y < selection.Bottom) + { + return DragMode.Move; + } + + return DragMode.Draw; + } + + protected override void OnPointerPressed(PointerPressedEventArgs e) + { + base.OnPointerPressed(e); + + var point = e.GetPosition(this); + var display = GetDisplayRect(); + if (display.Width <= 0 || !display.Inflate(HandleHitSize).Contains(point)) + { + return; + } + + _dragMode = HitTest(point); + _dragStartDisplay = point; + _dragStartSelectionVideo = new Rect(SelectionX, SelectionY, SelectionWidth, SelectionHeight); + e.Pointer.Capture(this); + e.Handled = true; + } + + protected override void OnPointerMoved(PointerEventArgs e) + { + base.OnPointerMoved(e); + + var point = e.GetPosition(this); + + if (_dragMode == DragMode.None) + { + Cursor = GetCursorFor(HitTest(point)); + return; + } + + var startVideo = DisplayToVideo(_dragStartDisplay); + var currentVideo = DisplayToVideo(point); + var deltaX = currentVideo.X - startVideo.X; + var deltaY = currentVideo.Y - startVideo.Y; + var start = _dragStartSelectionVideo; + + double x = start.X, y = start.Y, width = start.Width, height = start.Height; + + switch (_dragMode) + { + case DragMode.Draw: + x = Math.Min(startVideo.X, currentVideo.X); + y = Math.Min(startVideo.Y, currentVideo.Y); + width = Math.Abs(currentVideo.X - startVideo.X); + height = Math.Abs(currentVideo.Y - startVideo.Y); + break; + case DragMode.Move: + x = start.X + deltaX; + y = start.Y + deltaY; + break; + case DragMode.ResizeLeft: + x = start.X + deltaX; + width = start.Width - deltaX; + break; + case DragMode.ResizeRight: + width = start.Width + deltaX; + break; + case DragMode.ResizeTop: + y = start.Y + deltaY; + height = start.Height - deltaY; + break; + case DragMode.ResizeBottom: + height = start.Height + deltaY; + break; + case DragMode.ResizeTopLeft: + x = start.X + deltaX; + width = start.Width - deltaX; + y = start.Y + deltaY; + height = start.Height - deltaY; + break; + case DragMode.ResizeTopRight: + width = start.Width + deltaX; + y = start.Y + deltaY; + height = start.Height - deltaY; + break; + case DragMode.ResizeBottomLeft: + x = start.X + deltaX; + width = start.Width - deltaX; + height = start.Height + deltaY; + break; + case DragMode.ResizeBottomRight: + width = start.Width + deltaX; + height = start.Height + deltaY; + break; + } + + SetSelection(x, y, width, height); + e.Handled = true; + } + + protected override void OnPointerReleased(PointerReleasedEventArgs e) + { + base.OnPointerReleased(e); + + if (_dragMode != DragMode.None) + { + _dragMode = DragMode.None; + e.Pointer.Capture(null); + e.Handled = true; + } + } + + private Cursor GetCursorFor(DragMode mode) + { + return mode switch + { + DragMode.Move => new Cursor(StandardCursorType.SizeAll), + DragMode.ResizeLeft or DragMode.ResizeRight => new Cursor(StandardCursorType.SizeWestEast), + DragMode.ResizeTop or DragMode.ResizeBottom => new Cursor(StandardCursorType.SizeNorthSouth), + DragMode.ResizeTopLeft or DragMode.ResizeBottomRight => new Cursor(StandardCursorType.TopLeftCorner), + DragMode.ResizeTopRight or DragMode.ResizeBottomLeft => new Cursor(StandardCursorType.TopRightCorner), + _ => Cursor.Default, + }; + } + + private void SetSelection(double x, double y, double width, double height) + { + if (width < 0) + { + x += width; + width = -width; + } + + if (height < 0) + { + y += height; + height = -height; + } + + width = Math.Clamp(width, MinSelectionSize, VideoWidth); + height = Math.Clamp(height, MinSelectionSize, VideoHeight); + x = Math.Clamp(x, 0, VideoWidth - width); + y = Math.Clamp(y, 0, VideoHeight - height); + + SelectionX = (int)Math.Round(x); + SelectionY = (int)Math.Round(y); + SelectionWidth = (int)Math.Round(width); + SelectionHeight = (int)Math.Round(height); + } + + /// Sets the selection from video coordinates (clamped), e.g. for presets. + public void SetSelectionVideoRect(int x, int y, int width, int height) + { + SetSelection(x, y, width, height); + InvalidateVisual(); + } +} diff --git a/src/ui/Features/Video/VideoOcr/VideoOcrEngineItem.cs b/src/ui/Features/Video/VideoOcr/VideoOcrEngineItem.cs new file mode 100644 index 00000000000..65b35408404 --- /dev/null +++ b/src/ui/Features/Video/VideoOcr/VideoOcrEngineItem.cs @@ -0,0 +1,40 @@ +using Nikse.SubtitleEdit.Features.Ocr.Engines; +using System; +using System.Collections.Generic; + +namespace Nikse.SubtitleEdit.Features.Video.VideoOcr; + +public class VideoOcrEngineItem +{ + public string Name { get; set; } + public OcrEngineType EngineType { get; set; } + public string Description { get; set; } + + public VideoOcrEngineItem(string name, OcrEngineType engineType, string description) + { + Name = name; + EngineType = engineType; + Description = description; + } + + public override string ToString() + { + return Name; + } + + public static List GetEngines() + { + var list = new List(); + + if (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) + { + list.Add(new("Paddle OCR", OcrEngineType.PaddleOcrStandalone, "Local OCR engine (downloaded automatically) - fast and accurate")); + } + + list.Add(new("Paddle OCR Python", OcrEngineType.PaddleOcrPython, "Local OCR engine via Python (requires \"pip install paddleocr\")")); + list.Add(new("Ollama vision", OcrEngineType.Ollama, "Local vision model via Ollama - e.g. glm-ocr")); + list.Add(new("GLM API", OcrEngineType.Glm, "GLM vision model via Z.ai / bigmodel.cn API (requires API key)")); + + return list; + } +} diff --git a/src/ui/Features/Video/VideoOcr/VideoOcrFrameGroup.cs b/src/ui/Features/Video/VideoOcr/VideoOcrFrameGroup.cs new file mode 100644 index 00000000000..949c0ad5aeb --- /dev/null +++ b/src/ui/Features/Video/VideoOcr/VideoOcrFrameGroup.cs @@ -0,0 +1,33 @@ +namespace Nikse.SubtitleEdit.Features.Video.VideoOcr; + +/// +/// A run of consecutive video frames that look alike (same subtitle on screen, or all blank). +/// Only one representative frame per group is sent to the OCR engine. +/// +public class VideoOcrFrameGroup +{ + /// First frame index of the group (0-based, in sampled frames). + public int StartFrame { get; set; } + + /// Last frame index of the group (inclusive). + public int EndFrame { get; set; } + + /// File name of the frame image used for OCR (middle of the group). + public string RepresentativeFileName { get; set; } = string.Empty; + + /// True when the group has no pixels above the brightness minimum - no OCR needed. + public bool IsBlank { get; set; } + + /// OCR result for the representative frame. + public string Text { get; set; } = string.Empty; + + public double GetStartMs(double framesPerSecond) + { + return StartFrame * 1000.0 / framesPerSecond; + } + + public double GetEndMs(double framesPerSecond) + { + return (EndFrame + 1) * 1000.0 / framesPerSecond; + } +} diff --git a/src/ui/Features/Video/VideoOcr/VideoOcrFrameGrouper.cs b/src/ui/Features/Video/VideoOcr/VideoOcrFrameGrouper.cs new file mode 100644 index 00000000000..e2d6d203d66 --- /dev/null +++ b/src/ui/Features/Video/VideoOcr/VideoOcrFrameGrouper.cs @@ -0,0 +1,224 @@ +using SkiaSharp; +using System; +using System.Collections.Generic; +using System.Threading; + +namespace Nikse.SubtitleEdit.Features.Video.VideoOcr; + +/// +/// Collapses the sampled video frames into groups of consecutive near-identical frames so +/// only one frame per on-screen subtitle needs OCR. Frames are compared on a small +/// grayscale thumbnail; with a brightness minimum set, pixels below it are zeroed first, +/// which makes the comparison follow the (bright) subtitle text instead of the moving +/// video behind it - and makes frames without any bright pixels skippable as blank. +/// +public static class VideoOcrFrameGrouper +{ + private const int ThumbnailWidth = 96; + + // Less than this fraction of bright pixels counts as "no text on screen". + private const double BlankFraction = 0.002; + + public static List Group( + IReadOnlyList frameFileNames, + int brightnessMinimum, + int imageSimilarityPercent, + Action? progress, + CancellationToken cancellationToken) + { + var groups = new List(); + byte[]? lastThumbnail = null; + VideoOcrFrameGroup? current = null; + var currentFileList = new List(); + + for (var index = 0; index < frameFileNames.Count; index++) + { + cancellationToken.ThrowIfCancellationRequested(); + progress?.Invoke(index + 1, frameFileNames.Count); + + var thumbnail = MakeThumbnail(frameFileNames[index], brightnessMinimum); + if (thumbnail == null) + { + continue; // unreadable frame - treat as part of the current group + } + + var isBlank = brightnessMinimum > 0 && IsBlank(thumbnail); + + var isSameAsLast = current != null && + lastThumbnail != null && + lastThumbnail.Length == thumbnail.Length && + current.IsBlank == isBlank && + (current.IsBlank || + (brightnessMinimum > 0 + ? GetMaskSimilarityPercent(lastThumbnail, thumbnail) + : GetSimilarityPercent(lastThumbnail, thumbnail)) >= imageSimilarityPercent); + + if (isSameAsLast) + { + current!.EndFrame = index; + currentFileList.Add(frameFileNames[index]); + } + else + { + CloseGroup(groups, current, currentFileList); + current = new VideoOcrFrameGroup + { + StartFrame = index, + EndFrame = index, + IsBlank = isBlank, + }; + currentFileList = new List { frameFileNames[index] }; + } + + lastThumbnail = thumbnail; + } + + CloseGroup(groups, current, currentFileList); + + return groups; + } + + private static void CloseGroup(List groups, VideoOcrFrameGroup? group, List files) + { + if (group == null || files.Count == 0) + { + return; + } + + // The middle frame avoids fade-in/fade-out edges of the subtitle. + group.RepresentativeFileName = files[files.Count / 2]; + groups.Add(group); + } + + private static bool IsBlank(byte[] thumbnail) + { + var bright = 0; + foreach (var b in thumbnail) + { + if (b > 0) + { + bright++; + } + } + + return bright < thumbnail.Length * BlankFraction; + } + + /// + /// Overlap (Jaccard) similarity of two bright-pixel masks in percent. Unlike a plain + /// pixel difference this is relative to the amount of bright pixels, so a subtitle text + /// change registers as a big change even when a bright background dominates the area. + /// + internal static int GetMaskSimilarityPercent(byte[] a, byte[] b) + { + if (a.Length == 0 || a.Length != b.Length) + { + return 0; + } + + var intersection = 0; + var union = 0; + for (var i = 0; i < a.Length; i++) + { + var inA = a[i] > 0; + var inB = b[i] > 0; + if (inA || inB) + { + union++; + if (inA && inB) + { + intersection++; + } + } + } + + if (union == 0) + { + return 100; + } + + return (int)Math.Round(intersection * 100.0 / union); + } + + internal static int GetSimilarityPercent(byte[] a, byte[] b) + { + if (a.Length == 0 || a.Length != b.Length) + { + return 0; + } + + long diff = 0; + for (var i = 0; i < a.Length; i++) + { + diff += Math.Abs(a[i] - b[i]); + } + + var meanDiff = diff / (double)a.Length; + return (int)Math.Round(100.0 - meanDiff * 100.0 / 255.0); + } + + private static byte[]? MakeThumbnail(string fileName, int brightnessMinimum) + { + try + { + using var bitmap = DecodeScaledDown(fileName); + if (bitmap == null || bitmap.Width == 0 || bitmap.Height == 0) + { + return null; + } + + var height = Math.Max(1, (int)Math.Round(bitmap.Height * ThumbnailWidth / (double)bitmap.Width)); + using var small = bitmap.Resize(new SKImageInfo(ThumbnailWidth, height), new SKSamplingOptions(SKFilterMode.Linear, SKMipmapMode.None)); + if (small == null) + { + return null; + } + + var pixels = small.Pixels; + var result = new byte[pixels.Length]; + for (var i = 0; i < pixels.Length; i++) + { + var c = pixels[i]; + var luma = (byte)((c.Red * 299 + c.Green * 587 + c.Blue * 114) / 1000); + if (brightnessMinimum > 0) + { + result[i] = luma >= brightnessMinimum ? (byte)255 : (byte)0; + } + else + { + result[i] = luma; + } + } + + return result; + } + catch + { + return null; + } + } + + /// + /// Decodes an image at reduced size when the codec supports it (JPEG decodes natively + /// at 1/2, 1/4, 1/8 scale) - much cheaper than a full decode for thumbnail use. + /// + private static SKBitmap? DecodeScaledDown(string fileName) + { + try + { + using var codec = SKCodec.Create(fileName); + if (codec == null || codec.Info.Width <= 0) + { + return SKBitmap.Decode(fileName); + } + + var scaled = codec.GetScaledDimensions(ThumbnailWidth / (float)codec.Info.Width); + var info = new SKImageInfo(scaled.Width, scaled.Height); + return SKBitmap.Decode(codec, info) ?? SKBitmap.Decode(fileName); + } + catch + { + return SKBitmap.Decode(fileName); + } + } +} diff --git a/src/ui/Features/Video/VideoOcr/VideoOcrLineBuilder.cs b/src/ui/Features/Video/VideoOcr/VideoOcrLineBuilder.cs new file mode 100644 index 00000000000..f08fe513ac4 --- /dev/null +++ b/src/ui/Features/Video/VideoOcr/VideoOcrLineBuilder.cs @@ -0,0 +1,231 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Nikse.SubtitleEdit.Features.Video.VideoOcr; + +/// +/// Turns OCR'ed frame groups into subtitle lines: merges consecutive groups whose text is +/// near-identical (OCR jitter), picks the text variant that was on screen the longest +/// (majority vote), and drops blips shorter than the minimum duration. +/// +public static class VideoOcrLineBuilder +{ + public class OcrLine + { + public double StartMs { get; set; } + public double EndMs { get; set; } + public string Text { get; set; } = string.Empty; + } + + private class WorkLine + { + public double StartMs; + public double EndMs; + public readonly Dictionary TextDurations = new(); + + public void AddText(string text, double durationMs) + { + TextDurations.TryGetValue(text, out var current); + TextDurations[text] = current + durationMs; + } + + public string GetMajorityText() + { + return TextDurations.OrderByDescending(p => p.Value).First().Key; + } + } + + public static List Build( + IReadOnlyList groups, + double framesPerSecond, + int textSimilarityPercent, + int maxGapMs, + int minDurationMs) + { + var work = new List(); + WorkLine? current = null; + + foreach (var group in groups.OrderBy(p => p.StartFrame)) + { + var text = (group.Text ?? string.Empty).Trim(); + if (group.IsBlank || text.Length == 0) + { + // Empty stretches are just skipped: the max-gap check below bridges a + // short flicker (e.g. one frame dipping below the brightness minimum) + // while a longer empty stretch still ends the line. + continue; + } + + var startMs = group.GetStartMs(framesPerSecond); + var endMs = group.GetEndMs(framesPerSecond); + + if (current != null && + startMs - current.EndMs <= maxGapMs && + GetTextSimilarityPercent(current.GetMajorityText(), text) >= textSimilarityPercent) + { + current.EndMs = endMs; + current.AddText(text, endMs - startMs); + } + else + { + current = new WorkLine { StartMs = startMs, EndMs = endMs }; + current.AddText(text, endMs - startMs); + work.Add(current); + } + } + + return work + .Where(p => p.EndMs - p.StartMs >= minDurationMs) + .Select(p => new OcrLine + { + StartMs = p.StartMs, + EndMs = p.EndMs, + Text = p.GetMajorityText(), + }) + .ToList(); + } + + /// + /// Cleans a raw OCR result for a video frame. Frames without subtitle text make + /// vision LLMs hallucinate (markdown fences, prompt echoes, repeated symbol lines) - + /// after cleaning, such results become empty, which correctly means "no subtitle". + /// + public static string CleanOcrResult(string text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return string.Empty; + } + + var kept = new List(); + string? lastLine = null; + foreach (var raw in text.Replace("\r\n", "\n").Replace('\r', '\n').Split('\n')) + { + var line = raw.Trim(); + if (line.Length == 0 || + line.StartsWith("```", StringComparison.Ordinal) || + line.StartsWith("You are an OCR engine", StringComparison.OrdinalIgnoreCase) || + line == lastLine || + !line.Any(char.IsLetterOrDigit)) + { + continue; + } + + lastLine = line; + kept.Add(line); + + if (kept.Count >= 4) // a subtitle is at most a few short lines + { + break; + } + } + + return string.Join("\n", kept); + } + + /// + /// Similarity of two texts as Levenshtein ratio in percent, ignoring case and + /// whitespace so OCR spacing/line-break jitter does not split one subtitle in two. + /// + public static int GetTextSimilarityPercent(string a, string b) + { + var s1 = Normalize(a); + var s2 = Normalize(b); + + if (s1.Length == 0 && s2.Length == 0) + { + return 100; + } + + if (s1.Length == 0 || s2.Length == 0) + { + return 0; + } + + var distance = GetLevenshteinDistance(s1, s2); + var maxLength = Math.Max(s1.Length, s2.Length); + return (int)Math.Round(100.0 * (maxLength - distance) / maxLength); + } + + private static string Normalize(string text) + { + if (string.IsNullOrEmpty(text)) + { + return string.Empty; + } + + var sb = new System.Text.StringBuilder(text.Length); + foreach (var ch in text) + { + if (!char.IsWhiteSpace(ch)) + { + sb.Append(char.ToLowerInvariant(ch)); + } + } + + return sb.ToString(); + } + + private static int GetLevenshteinDistance(string a, string b) + { + var previous = new int[b.Length + 1]; + var current = new int[b.Length + 1]; + + for (var j = 0; j <= b.Length; j++) + { + previous[j] = j; + } + + for (var i = 1; i <= a.Length; i++) + { + current[0] = i; + for (var j = 1; j <= b.Length; j++) + { + var cost = a[i - 1] == b[j - 1] ? 0 : 1; + current[j] = Math.Min(Math.Min(current[j - 1] + 1, previous[j] + 1), previous[j - 1] + cost); + } + + (previous, current) = (current, previous); + } + + return previous[b.Length]; + } + + /// + /// Maps the center of the scan area to an ASSA alignment tag ("an1".."an9"), or an + /// empty string for the default bottom-center position (no tag needed). + /// + public static string GetAssaAlignmentTag(double relativeX, double relativeY) + { + int horizontal; // 0=left, 1=center, 2=right + if (relativeX < 0.33) + { + horizontal = 0; + } + else if (relativeX > 0.67) + { + horizontal = 2; + } + else + { + horizontal = 1; + } + + int alignment; + if (relativeY < 0.33) + { + alignment = 7 + horizontal; // top row: an7, an8, an9 + } + else if (relativeY > 0.67) + { + alignment = 1 + horizontal; // bottom row: an1, an2, an3 + } + else + { + alignment = 4 + horizontal; // middle row: an4, an5, an6 + } + + return alignment == 2 ? string.Empty : $"{{\\an{alignment}}}"; + } +} diff --git a/src/ui/Features/Video/VideoOcr/VideoOcrLineItem.cs b/src/ui/Features/Video/VideoOcr/VideoOcrLineItem.cs new file mode 100644 index 00000000000..336e5f224a1 --- /dev/null +++ b/src/ui/Features/Video/VideoOcr/VideoOcrLineItem.cs @@ -0,0 +1,14 @@ +using CommunityToolkit.Mvvm.ComponentModel; +using System; + +namespace Nikse.SubtitleEdit.Features.Video.VideoOcr; + +public partial class VideoOcrLineItem : ObservableObject +{ + [ObservableProperty] private int _number; + [ObservableProperty] private TimeSpan _startTime; + [ObservableProperty] private TimeSpan _endTime; + [ObservableProperty] private string _text = string.Empty; + + public TimeSpan Duration => EndTime - StartTime; +} diff --git a/src/ui/Features/Video/VideoOcr/VideoOcrViewModel.cs b/src/ui/Features/Video/VideoOcr/VideoOcrViewModel.cs new file mode 100644 index 00000000000..b14008d4bd6 --- /dev/null +++ b/src/ui/Features/Video/VideoOcr/VideoOcrViewModel.cs @@ -0,0 +1,792 @@ +using Avalonia.Controls; +using Avalonia.Input; +using Avalonia.Media.Imaging; +using Avalonia.Threading; +using CommunityToolkit.Mvvm.ComponentModel; +using CommunityToolkit.Mvvm.Input; +using Nikse.SubtitleEdit.Core.Common; +using Nikse.SubtitleEdit.Features.Ocr; +using Nikse.SubtitleEdit.Features.Ocr.Download; +using Nikse.SubtitleEdit.Features.Ocr.Engines; +using Nikse.SubtitleEdit.Features.Shared; +using Nikse.SubtitleEdit.Logic; +using Nikse.SubtitleEdit.Logic.Config; +using Nikse.SubtitleEdit.Logic.Media; +using SkiaSharp; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace Nikse.SubtitleEdit.Features.Video.VideoOcr; + +public partial class VideoOcrViewModel : ObservableObject +{ + [ObservableProperty] private ObservableCollection _engines; + [ObservableProperty] private VideoOcrEngineItem _selectedEngine; + [ObservableProperty] private bool _isPaddleEngine; + [ObservableProperty] private bool _isOllamaEngine; + [ObservableProperty] private bool _isGlmEngine; + [ObservableProperty] private ObservableCollection _paddleLanguages; + [ObservableProperty] private OcrLanguage2? _selectedPaddleLanguage; + [ObservableProperty] private string _ollamaUrl; + [ObservableProperty] private string _ollamaModel; + [ObservableProperty] private string _ollamaLanguage; + [ObservableProperty] private string _glmUrl; + [ObservableProperty] private string _glmModel; + [ObservableProperty] private string _glmApiKey; + [ObservableProperty] private string _glmLanguage; + [ObservableProperty] private int _framesPerSecond; + [ObservableProperty] private int _brightnessMinimum; + [ObservableProperty] private int _textSimilarityPercent; + [ObservableProperty] private int _maxGapMs; + [ObservableProperty] private int _minDurationMs; + [ObservableProperty] private bool _addAssaPositionTag; + [ObservableProperty] private ObservableCollection _lines; + [ObservableProperty] private bool _isRunning; + [ObservableProperty] private bool _isOkEnabled; + [ObservableProperty] private double _progressValue; + [ObservableProperty] private string _progressText; + [ObservableProperty] private Bitmap? _previewBitmap; + [ObservableProperty] private double _previewPositionSeconds; + [ObservableProperty] private double _durationSeconds; + [ObservableProperty] private string _previewPositionText; + [ObservableProperty] private int _videoWidth; + [ObservableProperty] private int _videoHeight; + [ObservableProperty] private int _selectionX; + [ObservableProperty] private int _selectionY; + [ObservableProperty] private int _selectionWidth; + [ObservableProperty] private int _selectionHeight; + [ObservableProperty] private string _scanAreaText; + + public Window? Window { get; set; } + public bool OkPressed { get; private set; } + public Subtitle ResultSubtitle { get; private set; } = new(); + public CropAreaSelector? CropSelector { get; set; } + + private string _videoFileName = string.Empty; + private CancellationTokenSource _cancellationTokenSource = new(); + private Process? _ffmpegProcess; + private long _extractedFrames; + private readonly DispatcherTimer _previewTimer; + private bool _previewLoading; + private bool _previewLoadQueued; + + private static readonly Regex FrameFinderRegex = new(@"[Ff]rame=\s*\d+", RegexOptions.Compiled); + + private readonly IWindowService _windowService; + + public VideoOcrViewModel(IWindowService windowService) + { + _windowService = windowService; + + Engines = new ObservableCollection(VideoOcrEngineItem.GetEngines()); + SelectedEngine = Engines[0]; + PaddleLanguages = new ObservableCollection(PaddleOcr.GetLanguages()); + SelectedPaddleLanguage = PaddleLanguages.FirstOrDefault(p => p.Code == "en"); + Lines = new ObservableCollection(); + + OllamaUrl = string.Empty; + OllamaModel = string.Empty; + OllamaLanguage = string.Empty; + GlmUrl = string.Empty; + GlmModel = string.Empty; + GlmApiKey = string.Empty; + GlmLanguage = string.Empty; + ProgressText = string.Empty; + PreviewPositionText = string.Empty; + ScanAreaText = string.Empty; + + // One-shot debounce for preview loading: restarted on every slider change. + _previewTimer = new DispatcherTimer { Interval = TimeSpan.FromMilliseconds(300) }; + _previewTimer.Tick += (s, e) => + { + _previewTimer.Stop(); + LoadPreview(); + }; + + LoadSettings(); + } + + public void Initialize(string videoFileName) + { + _videoFileName = videoFileName; + } + + internal void OnLoaded() + { + UiUtil.RestoreWindowPosition(Window); + + // Media probing can be slow (network shares) - keep it off the UI thread. + Task.Run(() => + { + FfmpegMediaInfo2? mediaInfo = null; + try + { + mediaInfo = FfmpegMediaInfo2.Parse(_videoFileName); + } + catch (Exception exception) + { + Se.LogError(exception, "Video OCR: could not read video info from " + _videoFileName); + } + + Dispatcher.UIThread.Post(async () => + { + if (mediaInfo == null || mediaInfo.Dimension.Width <= 0 || mediaInfo.Dimension.Height <= 0 || + mediaInfo.Duration == null) + { + await MessageBox.Show( + Window!, + Se.Language.Video.VideoOcr.UnableToReadVideoTitle, + string.Format(Se.Language.Video.VideoOcr.UnableToReadVideoMessage, _videoFileName), + MessageBoxButtons.OK, + MessageBoxIcon.Error); + + Window?.Close(); + return; + } + + VideoWidth = mediaInfo.Dimension.Width; + VideoHeight = mediaInfo.Dimension.Height; + DurationSeconds = mediaInfo.Duration.TotalSeconds; + + var settings = Se.Settings.Video.VideoOcr; + SelectionX = (int)Math.Round(settings.CropXPercent * VideoWidth / 100.0); + SelectionY = (int)Math.Round(settings.CropYPercent * VideoHeight / 100.0); + SelectionWidth = (int)Math.Round(settings.CropWidthPercent * VideoWidth / 100.0); + SelectionHeight = (int)Math.Round(settings.CropHeightPercent * VideoHeight / 100.0); + ClampSelection(); + + PreviewPositionSeconds = Math.Min(DurationSeconds * 0.2, 120); + LoadPreview(); + }); + }); + } + + partial void OnPreviewPositionSecondsChanged(double value) + { + PreviewPositionText = TimeSpan.FromSeconds(value).ToString(@"h\:mm\:ss"); + _previewTimer.Stop(); + _previewTimer.Start(); + } + + partial void OnSelectionXChanged(int value) => UpdateScanAreaText(); + partial void OnSelectionYChanged(int value) => UpdateScanAreaText(); + partial void OnSelectionWidthChanged(int value) => UpdateScanAreaText(); + partial void OnSelectionHeightChanged(int value) => UpdateScanAreaText(); + + private void UpdateScanAreaText() + { + ScanAreaText = $"{SelectionWidth}x{SelectionHeight} ({SelectionX},{SelectionY})"; + } + + partial void OnSelectedEngineChanged(VideoOcrEngineItem value) + { + IsPaddleEngine = value.EngineType is OcrEngineType.PaddleOcrStandalone or OcrEngineType.PaddleOcrPython; + IsOllamaEngine = value.EngineType == OcrEngineType.Ollama; + IsGlmEngine = value.EngineType == OcrEngineType.Glm; + } + + private void LoadPreview() + { + if (string.IsNullOrEmpty(_videoFileName) || DurationSeconds <= 0) + { + return; + } + + if (_previewLoading) + { + _previewLoadQueued = true; // load again when the current load finishes + return; + } + + _previewLoading = true; + var seconds = PreviewPositionSeconds; + + Task.Run(() => + { + try + { + var fileName = FfmpegGenerator.GetScreenShot(_videoFileName, seconds.ToString("0.###", CultureInfo.InvariantCulture)); + if (File.Exists(fileName) && new FileInfo(fileName).Length > 0) + { + var bitmap = new Bitmap(fileName); + Dispatcher.UIThread.Post(() => + { + var oldBitmap = PreviewBitmap; + PreviewBitmap = bitmap; + oldBitmap?.Dispose(); + CropSelector?.InvalidateVisual(); + }); + + try + { + File.Delete(fileName); + } + catch + { + // ignore + } + } + } + catch (Exception exception) + { + Se.LogError(exception, "Video OCR: could not load preview frame"); + } + finally + { + _previewLoading = false; + if (_previewLoadQueued) + { + _previewLoadQueued = false; + Dispatcher.UIThread.Post(LoadPreview); + } + } + }); + } + + [RelayCommand] + private void SetScanAreaBottomThird() + { + CropSelector?.SetSelectionVideoRect(0, VideoHeight * 2 / 3, VideoWidth, VideoHeight / 3); + } + + [RelayCommand] + private void SetScanAreaBottomHalf() + { + CropSelector?.SetSelectionVideoRect(0, VideoHeight / 2, VideoWidth, VideoHeight / 2); + } + + [RelayCommand] + private void SetScanAreaFullFrame() + { + CropSelector?.SetSelectionVideoRect(0, 0, VideoWidth, VideoHeight); + } + + [RelayCommand] + private async Task StartOcr() + { + if (IsRunning || string.IsNullOrEmpty(_videoFileName) || VideoWidth <= 0 || VideoHeight <= 0) + { + return; + } + + var engineOk = await EnsureEngineIsAvailable(); + if (!engineOk) + { + return; + } + + ClampSelection(); + SaveSettings(); + + _cancellationTokenSource = new CancellationTokenSource(); + var cancellationToken = _cancellationTokenSource.Token; + + IsRunning = true; + IsOkEnabled = false; + ProgressValue = 0; + Lines.Clear(); + + var framesFolder = Path.Combine(Path.GetTempPath(), "se_video_ocr_" + Guid.NewGuid()); + Directory.CreateDirectory(framesFolder); + + try + { + await ExtractFrames(framesFolder, cancellationToken); + + var frameFileNames = Directory.GetFiles(framesFolder, "*.jpg").OrderBy(p => p, StringComparer.Ordinal).ToList(); + if (frameFileNames.Count == 0) + { + throw new Exception("No frames were extracted from the video - see log for the ffmpeg command line."); + } + + var lastAnalyzeUpdate = 0L; + var groups = await Task.Run(() => VideoOcrFrameGrouper.Group( + frameFileNames, + BrightnessMinimum, + Se.Settings.Video.VideoOcr.ImageSimilarityPercent, + (current, total) => + { + var now = Environment.TickCount64; + if (now - lastAnalyzeUpdate > 200 || current == total) + { + lastAnalyzeUpdate = now; + Dispatcher.UIThread.Post(() => + { + ProgressText = string.Format(Se.Language.Video.VideoOcr.AnalyzingFramesXY, current, total); + ProgressValue = total == 0 ? 0 : current * 100.0 / total; + }); + } + }, + cancellationToken), cancellationToken); + + await RunOcr(groups, cancellationToken); + + var mergedLines = VideoOcrLineBuilder.Build(groups, FramesPerSecond, TextSimilarityPercent, MaxGapMs, MinDurationMs); + + var positionTag = string.Empty; + if (AddAssaPositionTag) + { + var relativeX = (SelectionX + SelectionWidth / 2.0) / VideoWidth; + var relativeY = (SelectionY + SelectionHeight / 2.0) / VideoHeight; + positionTag = VideoOcrLineBuilder.GetAssaAlignmentTag(relativeX, relativeY); + } + + Lines.Clear(); + var number = 1; + foreach (var line in mergedLines) + { + Lines.Add(new VideoOcrLineItem + { + Number = number++, + StartTime = TimeSpan.FromMilliseconds(line.StartMs), + EndTime = TimeSpan.FromMilliseconds(line.EndMs), + Text = positionTag + line.Text, + }); + } + + IsRunning = false; + IsOkEnabled = Lines.Count > 0; + ProgressValue = 0; + ProgressText = string.Format(Se.Language.Video.VideoOcr.LinesFoundX, Lines.Count); + + if (Lines.Count == 0) + { + await MessageBox.Show( + Window!, + Se.Language.Video.VideoOcr.NoLinesFoundTitle, + Se.Language.Video.VideoOcr.NoLinesFoundMessage, + MessageBoxButtons.OK, + MessageBoxIcon.Information); + } + } + catch (OperationCanceledException) + { + IsRunning = false; + IsOkEnabled = Lines.Count > 0; + ProgressValue = 0; + ProgressText = string.Empty; + } + catch (Exception exception) + { + Se.LogError(exception, "Video OCR failed"); + + IsRunning = false; + ProgressValue = 0; + ProgressText = string.Empty; + + await MessageBox.Show( + Window!, + Se.Language.General.Error, + exception.Message, + MessageBoxButtons.OK, + MessageBoxIcon.Error); + } + finally + { + try + { + Directory.Delete(framesFolder, true); + } + catch + { + // ignore + } + } + } + + private async Task ExtractFrames(string framesFolder, CancellationToken cancellationToken) + { + var scale = string.Empty; + var maxImageWidth = Se.Settings.Video.VideoOcr.MaxImageWidth; + if (maxImageWidth > 0 && SelectionWidth > maxImageWidth) + { + scale = $",scale={maxImageWidth}:-2"; + } + + // JPEG (near-lossless q=2) instead of PNG: a long video at 5 fps produces tens of + // thousands of frames, and PNG would need gigabytes of temp disk space. + var outputPattern = Path.Combine(framesFolder, "img%06d.jpg"); + var arguments = $"-nostdin -y -i \"{_videoFileName}\" " + + $"-vf \"fps={FramesPerSecond.ToString(CultureInfo.InvariantCulture)}," + + $"crop={SelectionWidth}:{SelectionHeight}:{SelectionX}:{SelectionY}{scale}\" " + + $"-q:v 2 -start_number 0 \"{outputPattern}\""; + + _extractedFrames = 0; + var totalFrames = Math.Max(1, (long)Math.Round(DurationSeconds * FramesPerSecond)); + + Se.WriteToolsLog("Video OCR: extracting frames - ffmpeg " + arguments); + _ffmpegProcess = FfmpegGenerator.GetProcess(arguments, ExtractFramesOutputHandler); + +#pragma warning disable CA1416 // Validate platform compatibility + _ffmpegProcess.Start(); +#pragma warning restore CA1416 + _ffmpegProcess.BeginOutputReadLine(); + _ffmpegProcess.BeginErrorReadLine(); + + using var timer = new System.Timers.Timer(200); + timer.Elapsed += (s, e) => + { + var percentage = Math.Clamp(_extractedFrames * 100.0 / totalFrames, 0, 100); + Dispatcher.UIThread.Post(() => + { + ProgressValue = percentage; + ProgressText = string.Format(Se.Language.Video.VideoOcr.ExtractingFramesX, (int)percentage); + }); + }; + timer.Start(); + + try + { + await _ffmpegProcess.WaitForExitAsync(cancellationToken); + } + catch (OperationCanceledException) + { + try + { + if (!_ffmpegProcess.HasExited) + { + _ffmpegProcess.Kill(true); + } + } + catch + { + // ignore + } + + throw; + } + finally + { + timer.Stop(); + } + } + + private void ExtractFramesOutputHandler(object sendingProcess, DataReceivedEventArgs outLine) + { + if (string.IsNullOrWhiteSpace(outLine.Data)) + { + return; + } + + var match = FrameFinderRegex.Match(outLine.Data); + if (!match.Success) + { + return; + } + + var arr = match.Value.Split('='); + if (arr.Length == 2 && long.TryParse(arr[1].Trim(), out var f)) + { + _extractedFrames = f; + } + } + + private async Task RunOcr(List groups, CancellationToken cancellationToken) + { + var ocrGroups = groups.Where(p => !p.IsBlank && !string.IsNullOrEmpty(p.RepresentativeFileName)).ToList(); + if (ocrGroups.Count == 0) + { + return; + } + + var done = 0; + void ReportOcrProgress() + { + var current = Interlocked.Increment(ref done); + Dispatcher.UIThread.Post(() => + { + ProgressText = string.Format(Se.Language.Video.VideoOcr.RunningOcrXY, current, ocrGroups.Count); + ProgressValue = current * 100.0 / ocrGroups.Count; + }); + } + + void AddPreviewLine(VideoOcrFrameGroup group) + { + if (string.IsNullOrWhiteSpace(group.Text)) + { + return; + } + + Dispatcher.UIThread.Post(() => + { + Lines.Add(new VideoOcrLineItem + { + Number = Lines.Count + 1, + StartTime = TimeSpan.FromMilliseconds(group.GetStartMs(FramesPerSecond)), + EndTime = TimeSpan.FromMilliseconds(group.GetEndMs(FramesPerSecond)), + Text = group.Text, + }); + }); + } + + Dispatcher.UIThread.Post(() => + { + ProgressValue = 0; + ProgressText = string.Format(Se.Language.Video.VideoOcr.RunningOcrXY, 0, ocrGroups.Count); + }); + + var engineType = SelectedEngine.EngineType; + if (engineType is OcrEngineType.PaddleOcrStandalone or OcrEngineType.PaddleOcrPython) + { + var language = SelectedPaddleLanguage?.Code ?? "en"; + var mode = Se.Settings.Ocr.PaddleOcrMode; + if (string.IsNullOrEmpty(mode)) + { + mode = "mobile"; + } + + var progress = new Progress(p => + { + var group = ocrGroups.ElementAtOrDefault(p.Index); + if (group != null) + { + group.Text = VideoOcrLineBuilder.CleanOcrResult(p.Text); + ReportOcrProgress(); + AddPreviewLine(group); + } + }); + + // The frames are already image files on disk, so pass them by file name - + // one batch, no per-image decode/encode, memory stays flat. + var batch = ocrGroups + .Select((g, i) => new PaddleOcrBatchInput { Index = i, SourceFileName = g.RepresentativeFileName }) + .ToList(); + + var paddleOcr = new PaddleOcr(); + await paddleOcr.OcrBatch(engineType, batch, language, mode, progress, cancellationToken); + if (!string.IsNullOrEmpty(paddleOcr.Error) && ocrGroups.All(p => string.IsNullOrEmpty(p.Text))) + { + throw new Exception("Paddle OCR failed: " + paddleOcr.Error); + } + } + else if (engineType == OcrEngineType.Ollama) + { + var ollamaOcr = new OllamaOcr(Se.Settings.Ocr.OllamaOcrTimeoutMinutes); + await RunLlmOcr(ocrGroups, group => OcrWithBitmap(group, bitmap => + ollamaOcr.Ocr(bitmap, OllamaUrl, OllamaModel, OllamaLanguage, cancellationToken)), + () => ollamaOcr.Error, ReportOcrProgress, AddPreviewLine, cancellationToken); + } + else if (engineType == OcrEngineType.Glm) + { + var glmOcr = new GlmOcr(GlmApiKey); + await RunLlmOcr(ocrGroups, group => + glmOcr.Ocr(group.RepresentativeFileName, GlmUrl, GlmModel, GlmLanguage, cancellationToken), + () => glmOcr.Error, ReportOcrProgress, AddPreviewLine, cancellationToken); + } + } + + private static async Task OcrWithBitmap(VideoOcrFrameGroup group, Func> ocr) + { + using var bitmap = SKBitmap.Decode(group.RepresentativeFileName); + if (bitmap == null) + { + return string.Empty; + } + + return await ocr(bitmap); + } + + private static async Task RunLlmOcr( + List ocrGroups, + Func> ocr, + Func getError, + Action reportProgress, + Action addPreviewLine, + CancellationToken cancellationToken) + { + var isFirst = true; + foreach (var group in ocrGroups) + { + cancellationToken.ThrowIfCancellationRequested(); + + group.Text = VideoOcrLineBuilder.CleanOcrResult(await ocr(group)); + + // Fail fast on a broken engine (wrong API key/URL) instead of grinding + // through the whole video and reporting "no subtitles found". + var error = getError(); + if (isFirst && string.IsNullOrEmpty(group.Text) && !string.IsNullOrEmpty(error)) + { + throw new Exception(error); + } + + isFirst = false; + reportProgress(); + addPreviewLine(group); + } + } + + private async Task EnsureEngineIsAvailable() + { + var engineType = SelectedEngine.EngineType; + + if (engineType == OcrEngineType.Glm && string.IsNullOrWhiteSpace(GlmApiKey)) + { + await MessageBox.Show( + Window!, + Se.Language.General.Error, + "An API key is required for the GLM API engine.", + MessageBoxButtons.OK, + MessageBoxIcon.Error); + return false; + } + + if (engineType is OcrEngineType.PaddleOcrStandalone or OcrEngineType.PaddleOcrPython) + { + return await PaddleOcrInstallHelper.EnsureInstalled(Window!, _windowService, engineType); + } + + return true; + } + + private void ClampSelection() + { + if (VideoWidth <= 0 || VideoHeight <= 0) + { + return; + } + + SelectionWidth = Math.Clamp(SelectionWidth, 16, VideoWidth); + SelectionHeight = Math.Clamp(SelectionHeight, 16, VideoHeight); + SelectionX = Math.Clamp(SelectionX, 0, VideoWidth - SelectionWidth); + SelectionY = Math.Clamp(SelectionY, 0, VideoHeight - SelectionHeight); + } + + private void LoadSettings() + { + var settings = Se.Settings.Video.VideoOcr; + SelectedEngine = Engines.FirstOrDefault(p => p.EngineType.ToString() == settings.Engine) ?? Engines[0]; + OnSelectedEngineChanged(SelectedEngine); + SelectedPaddleLanguage = PaddleLanguages.FirstOrDefault(p => p.Code == settings.PaddleLanguage) ?? + PaddleLanguages.FirstOrDefault(p => p.Code == "en"); + OllamaUrl = settings.OllamaUrl; + OllamaModel = settings.OllamaModel; + OllamaLanguage = settings.OllamaLanguage; + GlmUrl = settings.GlmUrl; + GlmModel = settings.GlmModel; + GlmApiKey = settings.GlmApiKey; + GlmLanguage = settings.GlmLanguage; + FramesPerSecond = Math.Clamp(settings.FramesPerSecond, 1, 30); + BrightnessMinimum = Math.Clamp(settings.BrightnessMinimum, 0, 255); + TextSimilarityPercent = Math.Clamp(settings.TextSimilarityPercent, 0, 100); + MaxGapMs = Math.Clamp(settings.MaxGapMs, 0, 10_000); + MinDurationMs = Math.Clamp(settings.MinDurationMs, 0, 10_000); + AddAssaPositionTag = settings.AddAssaPositionTag; + } + + private void SaveSettings() + { + var settings = Se.Settings.Video.VideoOcr; + settings.Engine = SelectedEngine.EngineType.ToString(); + settings.PaddleLanguage = SelectedPaddleLanguage?.Code ?? "en"; + settings.OllamaUrl = OllamaUrl; + settings.OllamaModel = OllamaModel; + settings.OllamaLanguage = OllamaLanguage; + settings.GlmUrl = GlmUrl; + settings.GlmModel = GlmModel; + settings.GlmApiKey = GlmApiKey; + settings.GlmLanguage = GlmLanguage; + settings.FramesPerSecond = FramesPerSecond; + settings.BrightnessMinimum = BrightnessMinimum; + settings.TextSimilarityPercent = TextSimilarityPercent; + settings.MaxGapMs = MaxGapMs; + settings.MinDurationMs = MinDurationMs; + settings.AddAssaPositionTag = AddAssaPositionTag; + + if (VideoWidth > 0 && VideoHeight > 0) + { + settings.CropXPercent = SelectionX * 100.0 / VideoWidth; + settings.CropYPercent = SelectionY * 100.0 / VideoHeight; + settings.CropWidthPercent = SelectionWidth * 100.0 / VideoWidth; + settings.CropHeightPercent = SelectionHeight * 100.0 / VideoHeight; + } + + Se.SaveSettings(); + } + + [RelayCommand] + private void Ok() + { + if (IsRunning || Lines.Count == 0) + { + return; + } + + var subtitle = new Subtitle(); + foreach (var line in Lines.OrderBy(p => p.StartTime)) + { + subtitle.Paragraphs.Add(new Paragraph(line.Text, line.StartTime.TotalMilliseconds, line.EndTime.TotalMilliseconds)); + } + + subtitle.Renumber(); + ResultSubtitle = subtitle; + + SaveSettings(); + OkPressed = true; + Window?.Close(); + } + + [RelayCommand] + private async Task Cancel() + { + if (IsRunning) + { + var answer = await MessageBox.Show( + Window!, + Se.Language.Video.VideoOcr.AbortOcrTitle, + Se.Language.Video.VideoOcr.AbortOcrMessage, + MessageBoxButtons.YesNo, + MessageBoxIcon.Question); + + if (answer == MessageBoxResult.Yes) + { + _cancellationTokenSource.Cancel(); + } + + return; + } + + Window?.Close(); + } + + internal void OnKeyDownHandler(object? sender, KeyEventArgs e) + { + if (e.Key == Key.Escape) + { + e.Handled = true; + _ = Cancel(); + } + else if (e.Key == Key.F1) + { + e.Handled = true; + UiUtil.ShowHelp("features/video-ocr"); + } + } + + internal void OnClosing() + { + _previewTimer.Stop(); + _cancellationTokenSource.Cancel(); + + try + { + if (_ffmpegProcess != null && !_ffmpegProcess.HasExited) + { + _ffmpegProcess.Kill(true); + } + } + catch + { + // ignore + } + + UiUtil.SaveWindowPosition(Window); + } +} diff --git a/src/ui/Features/Video/VideoOcr/VideoOcrWindow.cs b/src/ui/Features/Video/VideoOcr/VideoOcrWindow.cs new file mode 100644 index 00000000000..fab25691ab2 --- /dev/null +++ b/src/ui/Features/Video/VideoOcr/VideoOcrWindow.cs @@ -0,0 +1,369 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Data; +using Avalonia.Interactivity; +using Avalonia.Layout; +using Avalonia.Media; +using Nikse.SubtitleEdit.Logic; +using Nikse.SubtitleEdit.Logic.Config; +using Nikse.SubtitleEdit.Logic.ValueConverters; + +namespace Nikse.SubtitleEdit.Features.Video.VideoOcr; + +public class VideoOcrWindow : Window +{ + public VideoOcrWindow(VideoOcrViewModel vm) + { + UiUtil.InitializeWindow(this, GetType().Name); + Title = Se.Language.Video.VideoOcr.Title; + CanResize = true; + Width = 1100; + Height = 800; + MinWidth = 900; + MinHeight = 650; + vm.Window = this; + DataContext = vm; + + var previewView = MakePreviewView(vm); + var settingsView = MakeSettingsView(vm); + var linesView = MakeLinesView(vm); + var progressView = MakeProgressView(vm); + + var buttonStart = UiUtil.MakeButton(Se.Language.Video.VideoOcr.StartOcr, vm.StartOcrCommand) + .WithBindEnabled(nameof(vm.IsRunning), new InverseBooleanConverter()); + var buttonOk = UiUtil.MakeButtonOk(vm.OkCommand) + .WithBindEnabled(nameof(vm.IsOkEnabled)); + var buttonPanel = UiUtil.MakeButtonBar( + buttonStart, + buttonOk, + UiUtil.MakeButtonCancel(vm.CancelCommand)); + + var grid = new Grid + { + RowDefinitions = + { + new RowDefinition { Height = new GridLength(3, GridUnitType.Star) }, // preview + settings + new RowDefinition { Height = new GridLength(2, GridUnitType.Star) }, // result lines + new RowDefinition { Height = new GridLength(1, GridUnitType.Auto) }, // progress + new RowDefinition { Height = new GridLength(1, GridUnitType.Auto) }, // buttons + }, + ColumnDefinitions = + { + new ColumnDefinition { Width = new GridLength(1, GridUnitType.Star) }, // preview + new ColumnDefinition { Width = new GridLength(1, GridUnitType.Auto) }, // settings + }, + Margin = UiUtil.MakeWindowMargin(), + HorizontalAlignment = HorizontalAlignment.Stretch, + ColumnSpacing = 5, + RowSpacing = 5, + }; + + grid.Add(previewView, 0, 0); + grid.Add(settingsView, 0, 1); + grid.Add(linesView, 1, 0, 1, 2); + grid.Add(progressView, 2, 0, 1, 2); + grid.Add(buttonPanel, 3, 0, 1, 2); + + Content = grid; + + Activated += delegate { buttonStart.Focus(); }; // hack to make OnKeyDown work + Loaded += (s, e) => vm.OnLoaded(); + Closing += (s, e) => vm.OnClosing(); + AddHandler(KeyDownEvent, vm.OnKeyDownHandler, RoutingStrategies.Tunnel | RoutingStrategies.Bubble, handledEventsToo: false); + } + + private static Border MakePreviewView(VideoOcrViewModel vm) + { + var image = new Image + { + Stretch = Stretch.Uniform, + HorizontalAlignment = HorizontalAlignment.Stretch, + VerticalAlignment = VerticalAlignment.Stretch, + }; + image.Bind(Image.SourceProperty, new Binding(nameof(vm.PreviewBitmap)) { Source = vm }); + + var cropSelector = new CropAreaSelector + { + HorizontalAlignment = HorizontalAlignment.Stretch, + VerticalAlignment = VerticalAlignment.Stretch, + }; + cropSelector.Bind(CropAreaSelector.VideoWidthProperty, new Binding(nameof(vm.VideoWidth)) { Source = vm }); + cropSelector.Bind(CropAreaSelector.VideoHeightProperty, new Binding(nameof(vm.VideoHeight)) { Source = vm }); + cropSelector.Bind(CropAreaSelector.SelectionXProperty, new Binding(nameof(vm.SelectionX)) { Source = vm, Mode = BindingMode.TwoWay }); + cropSelector.Bind(CropAreaSelector.SelectionYProperty, new Binding(nameof(vm.SelectionY)) { Source = vm, Mode = BindingMode.TwoWay }); + cropSelector.Bind(CropAreaSelector.SelectionWidthProperty, new Binding(nameof(vm.SelectionWidth)) { Source = vm, Mode = BindingMode.TwoWay }); + cropSelector.Bind(CropAreaSelector.SelectionHeightProperty, new Binding(nameof(vm.SelectionHeight)) { Source = vm, Mode = BindingMode.TwoWay }); + vm.CropSelector = cropSelector; + + var imageArea = new Panel + { + Background = Brushes.Black, + Children = { image, cropSelector }, + }; + + var slider = new Slider + { + Minimum = 0, + HorizontalAlignment = HorizontalAlignment.Stretch, + VerticalAlignment = VerticalAlignment.Center, + }; + slider.Bind(Slider.MaximumProperty, new Binding(nameof(vm.DurationSeconds)) { Source = vm }); + slider.Bind(Slider.ValueProperty, new Binding(nameof(vm.PreviewPositionSeconds)) { Source = vm, Mode = BindingMode.TwoWay }); + + var positionText = new TextBlock + { + VerticalAlignment = VerticalAlignment.Center, + MinWidth = 60, + Margin = new Thickness(5, 0, 0, 0), + }; + positionText.Bind(TextBlock.TextProperty, new Binding(nameof(vm.PreviewPositionText)) { Source = vm }); + + var sliderRow = new Grid + { + ColumnDefinitions = + { + new ColumnDefinition { Width = new GridLength(1, GridUnitType.Auto) }, + new ColumnDefinition { Width = new GridLength(1, GridUnitType.Star) }, + new ColumnDefinition { Width = new GridLength(1, GridUnitType.Auto) }, + }, + }; + sliderRow.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.PreviewPosition), 0, 0); + sliderRow.Add(slider, 0, 1); + sliderRow.Add(positionText, 0, 2); + + var scanAreaText = new TextBlock + { + VerticalAlignment = VerticalAlignment.Center, + Opacity = 0.7, + Margin = new Thickness(10, 0, 0, 0), + }; + scanAreaText.Bind(TextBlock.TextProperty, new Binding(nameof(vm.ScanAreaText)) { Source = vm }); + + var scanAreaRow = new StackPanel + { + Orientation = Orientation.Horizontal, + Spacing = 5, + Children = + { + UiUtil.MakeLabel(Se.Language.Video.VideoOcr.ScanArea), + UiUtil.MakeButton(Se.Language.Video.VideoOcr.BottomThird, vm.SetScanAreaBottomThirdCommand), + UiUtil.MakeButton(Se.Language.Video.VideoOcr.BottomHalf, vm.SetScanAreaBottomHalfCommand), + UiUtil.MakeButton(Se.Language.Video.VideoOcr.FullFrame, vm.SetScanAreaFullFrameCommand), + scanAreaText, + new TextBlock + { + Text = Se.Language.Video.VideoOcr.ScanAreaInfo, + VerticalAlignment = VerticalAlignment.Center, + Opacity = 0.7, + Margin = new Thickness(10, 0, 0, 0), + }, + }, + }; + + var grid = new Grid + { + RowDefinitions = + { + new RowDefinition { Height = new GridLength(1, GridUnitType.Star) }, + new RowDefinition { Height = new GridLength(1, GridUnitType.Auto) }, + new RowDefinition { Height = new GridLength(1, GridUnitType.Auto) }, + }, + RowSpacing = 5, + }; + grid.Add(imageArea, 0, 0); + grid.Add(sliderRow, 1, 0); + grid.Add(scanAreaRow, 2, 0); + + return UiUtil.MakeBorderForControl(grid); + } + + private static Border MakeSettingsView(VideoOcrViewModel vm) + { + var comboEngine = UiUtil.MakeComboBox(vm.Engines, vm, nameof(vm.SelectedEngine)).WithWidth(220); + + var comboPaddleLanguage = UiUtil.MakeComboBox(vm.PaddleLanguages, vm, nameof(vm.SelectedPaddleLanguage)).WithWidth(220); + + var panel = new StackPanel + { + Orientation = Orientation.Vertical, + Spacing = 4, + Width = 350, + }; + + panel.Children.Add(MakeHeader(Se.Language.Video.VideoOcr.Engine, isFirst: true)); + panel.Children.Add(comboEngine); + + // Paddle OCR settings + var paddlePanel = new StackPanel { Orientation = Orientation.Vertical, Spacing = 4 }; + paddlePanel.Children.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.Language)); + paddlePanel.Children.Add(comboPaddleLanguage); + paddlePanel.Bind(StackPanel.IsVisibleProperty, new Binding(nameof(vm.IsPaddleEngine)) { Source = vm }); + panel.Children.Add(paddlePanel); + + // Ollama settings + var ollamaPanel = new StackPanel { Orientation = Orientation.Vertical, Spacing = 4 }; + ollamaPanel.Children.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.Url)); + ollamaPanel.Children.Add(UiUtil.MakeTextBox(330, vm, nameof(vm.OllamaUrl))); + ollamaPanel.Children.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.Model)); + ollamaPanel.Children.Add(UiUtil.MakeTextBox(330, vm, nameof(vm.OllamaModel))); + ollamaPanel.Children.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.Language)); + ollamaPanel.Children.Add(UiUtil.MakeTextBox(330, vm, nameof(vm.OllamaLanguage))); + ollamaPanel.Bind(StackPanel.IsVisibleProperty, new Binding(nameof(vm.IsOllamaEngine)) { Source = vm }); + panel.Children.Add(ollamaPanel); + + // GLM settings + var glmPanel = new StackPanel { Orientation = Orientation.Vertical, Spacing = 4 }; + glmPanel.Children.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.Url)); + glmPanel.Children.Add(UiUtil.MakeTextBox(330, vm, nameof(vm.GlmUrl))); + glmPanel.Children.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.Model)); + glmPanel.Children.Add(UiUtil.MakeTextBox(330, vm, nameof(vm.GlmModel))); + glmPanel.Children.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.ApiKey)); + glmPanel.Children.Add(UiUtil.MakeApiKeyTextBox(290, vm, nameof(vm.GlmApiKey))); + glmPanel.Children.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.Language)); + glmPanel.Children.Add(UiUtil.MakeTextBox(330, vm, nameof(vm.GlmLanguage))); + glmPanel.Bind(StackPanel.IsVisibleProperty, new Binding(nameof(vm.IsGlmEngine)) { Source = vm }); + panel.Children.Add(glmPanel); + + // Scan settings + panel.Children.Add(MakeHeader(Se.Language.Video.VideoOcr.Scan)); + panel.Children.Add(MakeSettingRow( + Se.Language.Video.VideoOcr.FramesPerSecond, + UiUtil.MakeNumericUpDownInt(1, 30, 5, 120, vm, nameof(vm.FramesPerSecond)))); + panel.Children.Add(MakeSettingRow( + Se.Language.Video.VideoOcr.TextBrightnessMinimum, + UiUtil.MakeNumericUpDownInt(0, 255, 190, 120, vm, nameof(vm.BrightnessMinimum)))); + + // Post-processing settings + panel.Children.Add(MakeHeader(Se.Language.Video.VideoOcr.PostProcessing)); + panel.Children.Add(MakeSettingRow( + Se.Language.Video.VideoOcr.TextSimilarityPercent, + UiUtil.MakeNumericUpDownInt(0, 100, 80, 120, vm, nameof(vm.TextSimilarityPercent)))); + panel.Children.Add(MakeSettingRow( + Se.Language.Video.VideoOcr.MaxGapMs, + UiUtil.MakeNumericUpDownInt(0, 10_000, 250, 120, vm, nameof(vm.MaxGapMs)))); + panel.Children.Add(MakeSettingRow( + Se.Language.Video.VideoOcr.MinDurationMs, + UiUtil.MakeNumericUpDownInt(0, 10_000, 250, 120, vm, nameof(vm.MinDurationMs)))); + panel.Children.Add(UiUtil.MakeCheckBox(Se.Language.Video.VideoOcr.AddAssaPositionTag, vm, nameof(vm.AddAssaPositionTag))); + + var scrollViewer = new ScrollViewer + { + Content = panel, + HorizontalScrollBarVisibility = Avalonia.Controls.Primitives.ScrollBarVisibility.Disabled, + }; + + return UiUtil.MakeBorderForControl(scrollViewer); + } + + private static TextBlock MakeHeader(string text, bool isFirst = false) + { + return new TextBlock + { + Text = text, + FontWeight = FontWeight.Bold, + Margin = new Thickness(0, isFirst ? 0 : 12, 0, 2), + }; + } + + private static Grid MakeSettingRow(string label, Control control) + { + var grid = new Grid + { + ColumnDefinitions = + { + new ColumnDefinition { Width = new GridLength(1, GridUnitType.Star) }, + new ColumnDefinition { Width = new GridLength(1, GridUnitType.Auto) }, + }, + }; + grid.Add(UiUtil.MakeLabel(label), 0, 0); + grid.Add(control, 0, 1); + return grid; + } + + private static Border MakeLinesView(VideoOcrViewModel vm) + { + var fullTimeConverter = new TimeSpanToDisplayFullConverter(); + var shortTimeConverter = new TimeSpanToDisplayShortConverter(); + var dataGrid = new DataGrid + { + AutoGenerateColumns = false, + SelectionMode = DataGridSelectionMode.Extended, + CanUserResizeColumns = true, + HorizontalAlignment = HorizontalAlignment.Stretch, + VerticalAlignment = VerticalAlignment.Stretch, + IsReadOnly = true, + DataContext = vm, + ItemsSource = vm.Lines, + Columns = + { + new DataGridTextColumn + { + Header = Se.Language.General.NumberSymbol, + CellTheme = UiUtil.DataGridNoBorderNoPaddingCellTheme, + Binding = new Binding(nameof(VideoOcrLineItem.Number)), + IsReadOnly = true, + }, + new DataGridTextColumn + { + Header = Se.Language.General.Show, + CellTheme = UiUtil.DataGridNoBorderNoPaddingCellTheme, + Binding = new Binding(nameof(VideoOcrLineItem.StartTime)) { Converter = fullTimeConverter }, + IsReadOnly = true, + }, + new DataGridTextColumn + { + Header = Se.Language.General.Hide, + CellTheme = UiUtil.DataGridNoBorderNoPaddingCellTheme, + Binding = new Binding(nameof(VideoOcrLineItem.EndTime)) { Converter = fullTimeConverter }, + IsReadOnly = true, + }, + new DataGridTextColumn + { + Header = Se.Language.General.Duration, + CellTheme = UiUtil.DataGridNoBorderNoPaddingCellTheme, + Binding = new Binding(nameof(VideoOcrLineItem.Duration)) { Converter = shortTimeConverter }, + IsReadOnly = true, + }, + new DataGridTextColumn + { + Header = Se.Language.General.Text, + CellTheme = UiUtil.DataGridNoBorderNoPaddingCellTheme, + Binding = new Binding(nameof(VideoOcrLineItem.Text)), + IsReadOnly = true, + Width = new DataGridLength(1, DataGridLengthUnitType.Star), + }, + }, + }; + + return UiUtil.MakeBorderForControl(dataGrid); + } + + private static Grid MakeProgressView(VideoOcrViewModel vm) + { + var progressBar = UiUtil.MakeProgressBar(); + progressBar.Bind(ProgressBar.ValueProperty, new Binding(nameof(vm.ProgressValue))); + progressBar.Bind(ProgressBar.IsVisibleProperty, new Binding(nameof(vm.IsRunning))); + progressBar.DataContext = vm; + + var statusText = new TextBlock + { + Margin = new Thickness(5, 20, 0, 0), + DataContext = vm, + }; + statusText.Bind(TextBlock.TextProperty, new Binding(nameof(vm.ProgressText))); + + var grid = new Grid + { + RowDefinitions = + { + new RowDefinition { Height = new GridLength(1, GridUnitType.Star) }, + }, + HorizontalAlignment = HorizontalAlignment.Stretch, + }; + + grid.Add(progressBar, 0, 0); + grid.Add(statusText, 0, 0); + + return grid; + } +} diff --git a/src/ui/Logic/Config/Language/Main/LanguageMainMenu.cs b/src/ui/Logic/Config/Language/Main/LanguageMainMenu.cs index a2b94bbf75b..2db2c1f141a 100644 --- a/src/ui/Logic/Config/Language/Main/LanguageMainMenu.cs +++ b/src/ui/Logic/Config/Language/Main/LanguageMainMenu.cs @@ -97,6 +97,7 @@ public class LanguageMainMenu public string AudioTracks { get; set; } public string SpeechToText { get; set; } public string TextToSpeech { get; set; } + public string VideoOcr { get; set; } public string SetVideoOffset { get; set; } public string UpdateVideoOffsetX { get; set; } public string SmpteTiming { get; set; } @@ -234,6 +235,7 @@ public LanguageMainMenu() AudioTracks = "_Audio tracks"; SpeechToText = "_Speech to text..."; TextToSpeech = "_Text to speech..."; + VideoOcr = "OCR burned-i_n subtitle..."; UndockVideoControls = "_Undock video controls"; ListShotChanges = "List s_hot changes..."; GenerateImportShotChanges = "Generate/import s_hot changes..."; diff --git a/src/ui/Logic/Config/Language/Options/LanguageSettingsShortcuts.cs b/src/ui/Logic/Config/Language/Options/LanguageSettingsShortcuts.cs index 8db712f35dc..c95ccdec696 100644 --- a/src/ui/Logic/Config/Language/Options/LanguageSettingsShortcuts.cs +++ b/src/ui/Logic/Config/Language/Options/LanguageSettingsShortcuts.cs @@ -207,6 +207,7 @@ public class LanguageSettingsShortcuts public string CloseVideo { get; set; } public string SpeechToText { get; set; } public string TextToSpeech { get; set; } + public string VideoOcr { get; set; } public string BurnIn { get; set; } public string GenerateTransparent { get; set; } public string UndockVideoControls { get; set; } @@ -518,6 +519,7 @@ public LanguageSettingsShortcuts() CloseVideo = "Close video"; SpeechToText = "Speech to text (Whisper)"; TextToSpeech = "Text to speech"; + VideoOcr = "OCR burned-in subtitle"; BurnIn = "Generate video with burned-in subtitles"; GenerateTransparent = "Generate transparent video with subtitles"; UndockVideoControls = "Un-dock video controls"; diff --git a/src/ui/Logic/Config/Language/Video/LanguageVideo.cs b/src/ui/Logic/Config/Language/Video/LanguageVideo.cs index 5587bd89403..80017c8b0d3 100644 --- a/src/ui/Logic/Config/Language/Video/LanguageVideo.cs +++ b/src/ui/Logic/Config/Language/Video/LanguageVideo.cs @@ -9,6 +9,7 @@ public class LanguageVideo public LanguageAudioToText AudioToText { get; set; } = new(); public LanguageTextToSpeech TextToSpeech { get; set; } = new(); public LanguageShotChanges ShotChanges { get; set; } = new(); + public LanguageVideoOcr VideoOcr { get; set; } = new(); public string GoToVideoPosition { get; set; } public string GenerateBlankVideoDotDotDot { get; set; } public string GenerateBlankVideoTitle { get; set; } diff --git a/src/ui/Logic/Config/Language/Video/LanguageVideoOcr.cs b/src/ui/Logic/Config/Language/Video/LanguageVideoOcr.cs new file mode 100644 index 00000000000..e69a693b016 --- /dev/null +++ b/src/ui/Logic/Config/Language/Video/LanguageVideoOcr.cs @@ -0,0 +1,71 @@ +namespace Nikse.SubtitleEdit.Logic.Config.Language.Tools; + +public class LanguageVideoOcr +{ + public string Title { get; set; } + public string ScanArea { get; set; } + public string ScanAreaInfo { get; set; } + public string BottomThird { get; set; } + public string BottomHalf { get; set; } + public string FullFrame { get; set; } + public string Engine { get; set; } + public string Language { get; set; } + public string Url { get; set; } + public string Model { get; set; } + public string ApiKey { get; set; } + public string Scan { get; set; } + public string FramesPerSecond { get; set; } + public string TextBrightnessMinimum { get; set; } + public string PostProcessing { get; set; } + public string TextSimilarityPercent { get; set; } + public string MaxGapMs { get; set; } + public string MinDurationMs { get; set; } + public string AddAssaPositionTag { get; set; } + public string StartOcr { get; set; } + public string ExtractingFramesX { get; set; } + public string AnalyzingFramesXY { get; set; } + public string RunningOcrXY { get; set; } + public string NoLinesFoundTitle { get; set; } + public string NoLinesFoundMessage { get; set; } + public string LinesFoundX { get; set; } + public string PreviewPosition { get; set; } + public string UnableToReadVideoTitle { get; set; } + public string UnableToReadVideoMessage { get; set; } + public string AbortOcrTitle { get; set; } + public string AbortOcrMessage { get; set; } + + public LanguageVideoOcr() + { + Title = "OCR burned-in subtitle"; + ScanArea = "Scan area"; + ScanAreaInfo = "Drag on the video frame to select where the subtitles are"; + BottomThird = "Bottom third"; + BottomHalf = "Bottom half"; + FullFrame = "Full frame"; + Engine = "OCR engine"; + Language = "Language"; + Url = "URL"; + Model = "Model"; + ApiKey = "API key"; + Scan = "Scan"; + FramesPerSecond = "Frames per second"; + TextBrightnessMinimum = "Text brightness minimum (0=off)"; + PostProcessing = "Post-processing"; + TextSimilarityPercent = "Merge lines with similarity (%)"; + MaxGapMs = "Max gap between lines (ms)"; + MinDurationMs = "Minimum duration (ms)"; + AddAssaPositionTag = "Add ASSA position tag (e.g. {\\an8})"; + StartOcr = "Start OCR"; + ExtractingFramesX = "Extracting image frames... {0}%"; + AnalyzingFramesXY = "Analyzing frames... {0}/{1}"; + RunningOcrXY = "Running OCR... {0}/{1}"; + NoLinesFoundTitle = "No subtitles found"; + NoLinesFoundMessage = "No text was found in the scan area - try adjusting the scan area, engine, or brightness minimum."; + LinesFoundX = "{0} lines found"; + PreviewPosition = "Preview position"; + UnableToReadVideoTitle = "Unable to read video"; + UnableToReadVideoMessage = "Could not read video info from: {0}"; + AbortOcrTitle = "Abort OCR?"; + AbortOcrMessage = "Do you want to abort the running OCR?"; + } +} diff --git a/src/ui/Logic/Config/SeVideo.cs b/src/ui/Logic/Config/SeVideo.cs index 0911b426e10..6cd11581e9d 100644 --- a/src/ui/Logic/Config/SeVideo.cs +++ b/src/ui/Logic/Config/SeVideo.cs @@ -9,6 +9,7 @@ public class SeVideo public SeVideoBurnIn BurnIn { get; set; } public SeVideoTransparent Transparent { get; set; } public SeVideoTextToSpeech TextToSpeech { get; set; } + public SeVideoOcr VideoOcr { get; set; } public string VideoPlayer { get; set; } public double Volume { get; set; } public bool ShowStopButton { get; set; } @@ -46,6 +47,7 @@ public SeVideo() BurnIn = new(); Transparent = new(); TextToSpeech = new(); + VideoOcr = new(); VideoPlayer = OperatingSystem.IsWindows() ? VideoPlayerName.MpvWid : VideoPlayerName.MpvOpenGl; Volume = 60; ShowStopButton = true; diff --git a/src/ui/Logic/Config/SeVideoOcr.cs b/src/ui/Logic/Config/SeVideoOcr.cs new file mode 100644 index 00000000000..634bdabe8ac --- /dev/null +++ b/src/ui/Logic/Config/SeVideoOcr.cs @@ -0,0 +1,57 @@ +using Nikse.SubtitleEdit.Features.Ocr.Engines; + +namespace Nikse.SubtitleEdit.Logic.Config; + +public class SeVideoOcr +{ + public string Engine { get; set; } + public string PaddleLanguage { get; set; } + public string OllamaUrl { get; set; } + public string OllamaModel { get; set; } + public string OllamaLanguage { get; set; } + public string GlmUrl { get; set; } + public string GlmModel { get; set; } + public string GlmApiKey { get; set; } + public string GlmLanguage { get; set; } + public int FramesPerSecond { get; set; } + public int ImageSimilarityPercent { get; set; } + public int TextSimilarityPercent { get; set; } + public int MaxGapMs { get; set; } + public int MinDurationMs { get; set; } + public int BrightnessMinimum { get; set; } + public int MaxImageWidth { get; set; } + public bool AddAssaPositionTag { get; set; } + public double CropXPercent { get; set; } + public double CropYPercent { get; set; } + public double CropWidthPercent { get; set; } + public double CropHeightPercent { get; set; } + + public SeVideoOcr() + { + Engine = System.OperatingSystem.IsWindows() || System.OperatingSystem.IsLinux() + ? OcrEngineType.PaddleOcrStandalone.ToString() + : OcrEngineType.PaddleOcrPython.ToString(); + PaddleLanguage = "en"; + OllamaUrl = "http://localhost:11434/api/chat"; + OllamaModel = "glm-ocr"; + OllamaLanguage = "English"; + GlmUrl = GlmOcr.DefaultUrl; + GlmModel = GlmOcr.DefaultModel; + GlmApiKey = string.Empty; + GlmLanguage = "English"; + FramesPerSecond = 5; + ImageSimilarityPercent = 92; + TextSimilarityPercent = 80; + MaxGapMs = 250; + MinDurationMs = 250; + BrightnessMinimum = 190; + MaxImageWidth = 720; + AddAssaPositionTag = false; + + // Default scan area: bottom third, full width. + CropXPercent = 0; + CropYPercent = 200.0 / 3.0; + CropWidthPercent = 100; + CropHeightPercent = 100.0 / 3.0; + } +} diff --git a/src/ui/Logic/ShortcutsMain.cs b/src/ui/Logic/ShortcutsMain.cs index 2888c8a05a7..25e43bdcf9c 100644 --- a/src/ui/Logic/ShortcutsMain.cs +++ b/src/ui/Logic/ShortcutsMain.cs @@ -172,6 +172,7 @@ private static Dictionary BuildCommandTranslations() { nameof(MainViewModel.CommandVideoCloseCommand), Se.Language.Options.Shortcuts.CloseVideo }, { nameof(MainViewModel.ShowSpeechToTextWhisperCommand), Se.Language.Options.Shortcuts.SpeechToText }, { nameof(MainViewModel.ShowVideoTextToSpeechCommand), Se.Language.Options.Shortcuts.TextToSpeech }, + { nameof(MainViewModel.ShowVideoOcrCommand), Se.Language.Options.Shortcuts.VideoOcr }, { nameof(MainViewModel.ShowVideoBurnInCommand), Se.Language.Options.Shortcuts.BurnIn }, { nameof(MainViewModel.ShowVideoTransparentSubtitlesCommand), Se.Language.Options.Shortcuts.GenerateTransparent }, { nameof(MainViewModel.ShowShotChangesSubtitlesCommand), Se.Language.General.GenerateImportShotChanges }, @@ -598,6 +599,7 @@ private static List GetAllAvailableShortcuts(MainViewModel vm AddShortcut(shortcuts, vm.CommandVideoCloseCommand, nameof(vm.CommandVideoCloseCommand), ShortcutCategory.General); AddShortcut(shortcuts, vm.ShowSpeechToTextWhisperCommand, nameof(vm.ShowSpeechToTextWhisperCommand), ShortcutCategory.General); AddShortcut(shortcuts, vm.ShowVideoTextToSpeechCommand, nameof(vm.ShowVideoTextToSpeechCommand), ShortcutCategory.General); + AddShortcut(shortcuts, vm.ShowVideoOcrCommand, nameof(vm.ShowVideoOcrCommand), ShortcutCategory.General); AddShortcut(shortcuts, vm.ShowVideoBurnInCommand, nameof(vm.ShowVideoBurnInCommand), ShortcutCategory.General); AddShortcut(shortcuts, vm.ShowVideoTransparentSubtitlesCommand, nameof(vm.ShowVideoTransparentSubtitlesCommand), ShortcutCategory.General); AddShortcut(shortcuts, vm.ShowShotChangesSubtitlesCommand, nameof(vm.ShowShotChangesSubtitlesCommand), ShortcutCategory.General); diff --git a/tests/UI/Features/VideoOcrTests.cs b/tests/UI/Features/VideoOcrTests.cs new file mode 100644 index 00000000000..3758beec0b8 --- /dev/null +++ b/tests/UI/Features/VideoOcrTests.cs @@ -0,0 +1,224 @@ +using Nikse.SubtitleEdit.Features.Video.VideoOcr; +using System.Collections.Generic; +using System.Linq; + +namespace UITests.Features; + +public class VideoOcrTests +{ + private static VideoOcrFrameGroup MakeGroup(int startFrame, int endFrame, string text, bool isBlank = false) + { + return new VideoOcrFrameGroup + { + StartFrame = startFrame, + EndFrame = endFrame, + Text = text, + IsBlank = isBlank, + }; + } + + [Fact] + public void Build_SimilarConsecutiveTexts_MergedIntoOneLine() + { + var groups = new List + { + MakeGroup(0, 4, "Hello world"), + MakeGroup(5, 14, "Hello world"), + MakeGroup(15, 19, "Hello worId"), // OCR glitch: capital I instead of l + }; + + var lines = VideoOcrLineBuilder.Build(groups, 5, 80, 250, 250); + + Assert.Single(lines); + Assert.Equal("Hello world", lines[0].Text); // majority text wins + Assert.Equal(0, lines[0].StartMs); + Assert.Equal(4000, lines[0].EndMs); + } + + [Fact] + public void Build_DifferentTexts_SeparateLines() + { + var groups = new List + { + MakeGroup(0, 9, "First subtitle"), + MakeGroup(10, 19, "A completely different text"), + }; + + var lines = VideoOcrLineBuilder.Build(groups, 5, 80, 250, 250); + + Assert.Equal(2, lines.Count); + Assert.Equal("First subtitle", lines[0].Text); + Assert.Equal("A completely different text", lines[1].Text); + } + + [Fact] + public void Build_LongBlankStretch_SameTextBecomesTwoLines() + { + var groups = new List + { + MakeGroup(0, 9, "Hello"), + MakeGroup(10, 19, string.Empty, isBlank: true), // 2000 ms blank > max gap + MakeGroup(20, 29, "Hello"), + }; + + var lines = VideoOcrLineBuilder.Build(groups, 5, 80, 250, 250); + + Assert.Equal(2, lines.Count); + } + + [Fact] + public void Build_OneFrameBlankFlicker_BridgedByMaxGap() + { + var groups = new List + { + MakeGroup(0, 9, "Hello"), + MakeGroup(10, 10, string.Empty, isBlank: true), // 200 ms flicker <= 250 ms max gap + MakeGroup(11, 20, "Hello"), + }; + + var lines = VideoOcrLineBuilder.Build(groups, 5, 80, 250, 250); + + Assert.Single(lines); + Assert.Equal(0, lines[0].StartMs); + Assert.Equal(4200, lines[0].EndMs); + } + + [Fact] + public void Build_ShortBlip_Dropped() + { + var groups = new List + { + MakeGroup(0, 0, "logo"), // single frame at 5 fps = 200 ms + MakeGroup(5, 14, "Real subtitle text"), + }; + + var lines = VideoOcrLineBuilder.Build(groups, 5, 80, 250, 250); + + Assert.Single(lines); + Assert.Equal("Real subtitle text", lines[0].Text); + } + + [Fact] + public void Build_EmptyOcrResults_NoLines() + { + var groups = new List + { + MakeGroup(0, 9, " "), + MakeGroup(10, 19, string.Empty), + }; + + var lines = VideoOcrLineBuilder.Build(groups, 5, 80, 250, 250); + + Assert.Empty(lines); + } + + [Fact] + public void Build_MajorityVote_LongestShownTextWins() + { + var groups = new List + { + MakeGroup(0, 1, "He1lo there"), + MakeGroup(2, 20, "Hello there"), + MakeGroup(21, 22, "Hell0 there"), + }; + + var lines = VideoOcrLineBuilder.Build(groups, 5, 80, 250, 250); + + Assert.Single(lines); + Assert.Equal("Hello there", lines[0].Text); + } + + [Theory] + [InlineData("Hello world", "Hello world", 100)] + [InlineData("Hello world", "HELLO WORLD", 100)] // case and whitespace ignored + [InlineData("Hello world", "", 0)] + [InlineData("", "", 100)] + public void GetTextSimilarityPercent_KnownValues(string a, string b, int expected) + { + Assert.Equal(expected, VideoOcrLineBuilder.GetTextSimilarityPercent(a, b)); + } + + [Fact] + public void GetTextSimilarityPercent_SmallOcrJitter_IsHigh() + { + var similarity = VideoOcrLineBuilder.GetTextSimilarityPercent("My mommy always said", "My mornmy a1ways said"); + Assert.True(similarity >= 80, $"Expected >= 80 but was {similarity}"); + } + + [Fact] + public void GetTextSimilarityPercent_DifferentTexts_IsLow() + { + var similarity = VideoOcrLineBuilder.GetTextSimilarityPercent("My mommy always said", "Life is like a box of chocolates"); + Assert.True(similarity < 50, $"Expected < 50 but was {similarity}"); + } + + [Theory] + [InlineData(0.5, 0.9, "")] // bottom center - default, no tag + [InlineData(0.5, 0.1, "{\\an8}")] // top center + [InlineData(0.1, 0.9, "{\\an1}")] // bottom left + [InlineData(0.9, 0.5, "{\\an6}")] // middle right + public void GetAssaAlignmentTag_KnownPositions(double relativeX, double relativeY, string expected) + { + Assert.Equal(expected, VideoOcrLineBuilder.GetAssaAlignmentTag(relativeX, relativeY)); + } + + [Fact] + public void FrameGroup_Timing_UsesFrameRate() + { + var group = MakeGroup(10, 19, "x"); + Assert.Equal(2000, group.GetStartMs(5)); + Assert.Equal(4000, group.GetEndMs(5)); + } + + [Theory] + [InlineData("Hello world", "Hello world")] + [InlineData("Hello\nworld", "Hello\nworld")] + [InlineData("```markdown\nHello world\n```", "Hello world")] // markdown fences stripped + [InlineData("Hello world\nHello world\nHello world", "Hello world")] // repeated lines deduped + [InlineData("```markdown\n\n```\n+\n```\n```", "")] // pure hallucination becomes empty + [InlineData("You are an OCR engine. The language is English.", "")] // prompt echo removed + [InlineData("", "")] + public void CleanOcrResult_KnownValues(string input, string expected) + { + Assert.Equal(expected, VideoOcrLineBuilder.CleanOcrResult(input)); + } + + [Fact] + public void GetMaskSimilarityPercent_SameMask_Is100() + { + var a = new byte[] { 0, 255, 255, 0, 0, 0, 0, 0 }; + Assert.Equal(100, VideoOcrFrameGrouper.GetMaskSimilarityPercent(a, (byte[])a.Clone())); + } + + [Fact] + public void GetMaskSimilarityPercent_RelativeToMaskSizeNotImageSize() + { + // 2 of 3 bright pixels overlap in a 12-pixel image: a plain pixel diff would say + // ~83% similar, but relative to the bright mask it is 50% (2 of 4 union pixels). + var a = new byte[] { 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + var b = new byte[] { 0, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0 }; + Assert.Equal(50, VideoOcrFrameGrouper.GetMaskSimilarityPercent(a, b)); + } + + [Fact] + public void GetMaskSimilarityPercent_BothEmpty_Is100() + { + var a = new byte[8]; + Assert.Equal(100, VideoOcrFrameGrouper.GetMaskSimilarityPercent(a, (byte[])a.Clone())); + } + + [Fact] + public void GetSimilarityPercent_IdenticalThumbnails_Is100() + { + var a = new byte[] { 0, 255, 128, 0 }; + Assert.Equal(100, VideoOcrFrameGrouper.GetSimilarityPercent(a, (byte[])a.Clone())); + } + + [Fact] + public void GetSimilarityPercent_OppositeThumbnails_Is0() + { + var a = new byte[] { 0, 0, 0, 0 }; + var b = new byte[] { 255, 255, 255, 255 }; + Assert.Equal(0, VideoOcrFrameGrouper.GetSimilarityPercent(a, b)); + } +}