Add Video OCR: extract burned-in (hardcoded) subtitles

docs/features/video-ocr.md

@@ -0,0 +1,47 @@
+# OCR Burned-in Subtitle (Video OCR)
+
+Extract hardcoded (burned-in) subtitles from a video into editable text lines using OCR.
+
+- **Menu:** Video → OCR burned-in subtitle...
+
+## How to Use
+
+1. Open **Video → OCR burned-in subtitle...** (a video file is required)
+2. Use the preview slider to find a frame that shows a subtitle
+3. Adjust the scan area rectangle so it covers where the subtitles appear (default: bottom third) — drag to move/resize, or use the preset buttons
+4. Pick an OCR engine and language
+5. Click **Start OCR** — lines appear in the list as they are recognized
+6. Click **OK** to load the result into the main window
+
+## OCR Engines
+
+- **Paddle OCR** — local, fast and accurate; downloaded automatically (Windows/Linux). Recommended.
+- **Paddle OCR Python** — local, via a Python `paddleocr` installation (also works on macOS)
+- **Ollama vision** — local vision model via [Ollama](https://ollama.com), e.g. `glm-ocr`
+- **GLM API** — GLM vision model via the Z.ai / bigmodel.cn cloud API (requires an API key)
+
+## How It Works
+
+Frames are sampled from the scan area at a few frames per second with ffmpeg. Consecutive
+near-identical frames are collapsed so each on-screen subtitle is OCR'ed only once, then
+consecutive OCR results with near-identical text are merged into one line with correct start/end
+times (the text variant shown the longest wins).
+
+## Settings
+
+- **Frames per second** — how many frames per second to sample (higher = more precise timing, slower)
+- **Text brightness minimum** — pixels darker than this are ignored when comparing frames, so the
+  comparison follows the (bright) subtitle text instead of the moving video behind it; frames with
+  no bright pixels are skipped entirely. Set to 0 to disable (e.g. for dark subtitle text).
+- **Merge lines with similarity (%)** — how similar the text of two consecutive OCR results must be to merge into one line
+- **Max gap between lines (ms)** — maximum time gap allowed when merging
+- **Minimum duration (ms)** — lines shorter than this are dropped (removes OCR blips/false positives)
+- **Add ASSA position tag** — prepend an alignment tag (e.g. `{\an8}` for a top scan area) based on
+  the scan area position, for Advanced Sub Station Alpha output
+
+## Tips
+
+- Scan only the region where subtitles actually appear — a smaller area is faster and has fewer false positives
+- For subtitles at the top of the frame (e.g. sign translations), move the scan area up and enable the ASSA position tag
+- If lines are duplicated with small OCR differences, raise **Merge lines with similarity** tolerance by lowering the percentage
+- If short random text blips appear, raise **Minimum duration**

docs/index.md

@@ -56,6 +56,7 @@ Subtitle Edit is a free, open-source editor for video subtitles. This is the doc
 - [Video Player](features/video-player.md) — Video playback controls
 - [Audio Visualizer / Waveform](features/audio-visualizer.md) — Waveform display and editing
 - [Speech to Text](features/speech-to-text.md) — Automatic speech recognition with Whisper, Qwen3 ASR, Crisp ASR, and related engines
+- [Video OCR](features/video-ocr.md) — Extract burned-in (hardcoded) subtitles via OCR
 - [Text to Speech](features/text-to-speech.md) — Generate speech from subtitles
 - [Burn-In Subtitles](features/burn-in.md) — Hardcode subtitles into video
 - [Embedded Subtitles](features/embedded-subtitles.md) — Add, remove, preview, and edit subtitle tracks in Matroska/WebM files

src/ui/DependencyInjectionExtensions.cs

@@ -140,6 +140,7 @@
 using Nikse.SubtitleEdit.Features.Video.ShotChanges;
 using Nikse.SubtitleEdit.Features.Video.TextToSpeech;
 using Nikse.SubtitleEdit.Features.Video.TextToSpeech.ActorVoices;
+using Nikse.SubtitleEdit.Features.Video.VideoOcr;
 using Nikse.SubtitleEdit.Features.Video.TextToSpeech.AdvancedTtsSettings;
 using Nikse.SubtitleEdit.Features.Video.TextToSpeech.DownloadTts;
 using Nikse.SubtitleEdit.Features.Video.TextToSpeech.ElevenLabsSettings;
@@ -339,6 +340,7 @@ public static void AddSubtitleEditServices(this IServiceCollection collection)
         collection.AddTransient<CopyPasteTranslateViewModel>();
         collection.AddTransient<CustomContinuationStyleViewModel>();
         collection.AddTransient<CutVideoViewModel>();
+        collection.AddTransient<VideoOcrViewModel>();
         collection.AddTransient<DCinemaSmptePropertiesViewModel>();
         collection.AddTransient<DownloadFfmpegViewModel>();
         collection.AddTransient<DownloadGoogleLensOcrViewModel>();

src/ui/Features/Main/Layout/InitMenu.cs

@@ -681,6 +681,11 @@ public static void Make(MainViewModel vm)
                     Header = l.TextToSpeech,
                     Command = vm.ShowVideoTextToSpeechCommand,
                 },
+                new MenuItem
+                {
+                    Header = l.VideoOcr,
+                    Command = vm.ShowVideoOcrCommand,
+                },
                 new Separator(),
                 new MenuItem
                 {

src/ui/Features/Main/Layout/InitNativeMacMenu.cs

@@ -331,6 +331,7 @@ public static void MakeStructure(NativeMenu root, Window window)
         videoItems.Items.Add(new NativeMenuItemSeparator());
         videoItems.Items.Add(Item(Clean(l.SpeechToText), v => v.ShowSpeechToTextWhisperCommand));
         videoItems.Items.Add(Item(Clean(l.TextToSpeech), v => v.ShowVideoTextToSpeechCommand));
+        videoItems.Items.Add(Item(Clean(l.VideoOcr), v => v.ShowVideoOcrCommand));
         videoItems.Items.Add(new NativeMenuItemSeparator());
         videoItems.Items.Add(Item(Clean(l.GenerateBurnIn), v => v.ShowVideoBurnInCommand));
         videoItems.Items.Add(Item(Clean(l.GenerateTransparent), v => v.ShowVideoTransparentSubtitlesCommand));

src/ui/Features/Main/MainViewModel.cs

@@ -145,6 +145,7 @@
 using Nikse.SubtitleEdit.Features.Video.ShotChanges;
 using Nikse.SubtitleEdit.Features.Video.SpeechToText;
 using Nikse.SubtitleEdit.Features.Video.SpeechToText.OpenAiCompatible;
+using Nikse.SubtitleEdit.Features.Video.VideoOcr;
 using Nikse.SubtitleEdit.Features.Video.TextToSpeech;
 using Nikse.SubtitleEdit.Features.Video.TextToSpeech.ReviewSpeech;
 using Nikse.SubtitleEdit.Features.Video.TransparentSubtitles;
@@ -6014,6 +6015,70 @@ private async Task ShowSpeechToTextWhisper()
         }
     }
 
+    [RelayCommand]
+    private async Task ShowVideoOcr()
+    {
+        if (Window == null)
+        {
+            return;
+        }
+
+        var ffmpegOk = await RequireFfmpegOk();
+        if (!ffmpegOk)
+        {
+            return;
+        }
+
+        var doContinue = await HasChangesContinue();
+        if (!doContinue)
+        {
+            return;
+        }
+
+        // Frame extraction needs a local file - for URL sources, ask for a file instead.
+        var videoFileName = !string.IsNullOrEmpty(_videoFileName) && File.Exists(_videoFileName) ? _videoFileName : string.Empty;
+        if (string.IsNullOrEmpty(videoFileName))
+        {
+            videoFileName = await _fileHelper.PickOpenVideoFile(Window!, Se.Language.General.OpenVideoFileTitle);
+            if (string.IsNullOrEmpty(videoFileName))
+            {
+                return;
+            }
+        }
+
+        var result = await ShowDialogAsync<VideoOcrWindow, VideoOcrViewModel>(vm => { vm.Initialize(videoFileName); });
+
+        if (result.OkPressed && result.ResultSubtitle.Paragraphs.Count > 0)
+        {
+            // Open the video before setting the subtitle state - VideoOpenFile persists
+            // the current subtitle file name to the recent-files list, and the OCR result
+            // has not been saved to disk yet.
+            if (_videoFileName != videoFileName)
+            {
+                await VideoOpenFile(videoFileName);
+            }
+
+            ResetSubtitle();
+
+            _subtitle = result.ResultSubtitle;
+
+            if (SelectedSubtitleFormat is AdvancedSubStationAlpha)
+            {
+                foreach (var p in _subtitle.Paragraphs)
+                {
+                    p.Text = AdvancedSubStationAlpha.FormatText(p.Text);
+                }
+            }
+
+            _subtitleFileName = Path.ChangeExtension(videoFileName, SelectedSubtitleFormat.Extension);
+            _converted = true;
+
+            SetSubtitles(_subtitle);
+            SelectAndScrollToRow(0);
+            ShowStatus(string.Format(Se.Language.Video.VideoOcr.LinesFoundX, _subtitle.Paragraphs.Count));
+        }
+    }
+
     [RelayCommand]
     private async Task SpeechToTextSelectedLines()
     {

src/ui/Features/Ocr/Download/PaddleOcrInstallHelper.cs

@@ -0,0 +1,109 @@
+using Avalonia.Controls;
+using Nikse.SubtitleEdit.Core.Common;
+using Nikse.SubtitleEdit.Features.Ocr.Engines;
+using Nikse.SubtitleEdit.Features.Shared;
+using Nikse.SubtitleEdit.Logic;
+using Nikse.SubtitleEdit.Logic.Config;
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Nikse.SubtitleEdit.Features.Ocr.Download;
+
+/// <summary>
+/// Prompts for and runs the Paddle OCR engine/models download when missing.
+/// Shared by the features that run Paddle OCR (image OCR, batch convert, video OCR).
+/// </summary>
+public static class PaddleOcrInstallHelper
+{
+    /// <summary>
+    /// Makes sure the Paddle OCR engine (standalone only) and models are installed,
+    /// prompting the user to download what is missing. Returns false if the user cancelled.
+    /// </summary>
+    public static async Task<bool> EnsureInstalled(Window window, IWindowService windowService, OcrEngineType engineType)
+    {
+        if (engineType == OcrEngineType.PaddleOcrStandalone)
+        {
+            if (Configuration.IsRunningOnWindows && !File.Exists(Path.Combine(Se.PaddleOcrFolder, "paddleocr.exe")))
+            {
+                var answer = await MessageBox.Show(
+                    window,
+                    "Download Paddle OCR?",
+                    $"{Environment.NewLine}\"Paddle OCR\" requires downloading Paddle OCR.{Environment.NewLine}{Environment.NewLine}Download and use Paddle OCR?",
+                    MessageBoxButtons.Cancel,
+                    MessageBoxIcon.Question,
+                    "CPU",
+                    "GPU CUDA 11",
+                    "GPU CUDA 12");
+
+                if (answer == MessageBoxResult.Cancel)
+                {
+                    return false;
+                }
+
+                var result = await windowService.ShowDialogAsync<DownloadPaddleOcrWindow, DownloadPaddleOcrViewModel>(window,
+                    vm =>
+                    {
+                        var engine = PaddleOcrDownloadType.EngineCpu;
+                        if (answer == MessageBoxResult.Custom2)
+                        {
+                            engine = PaddleOcrDownloadType.EngineGpu11;
+                        }
+                        else if (answer == MessageBoxResult.Custom3)
+                        {
+                            engine = PaddleOcrDownloadType.EngineGpu12;
+                        }
+
+                        vm.Initialize(engine);
+                    });
+
+                if (!result.OkPressed)
+                {
+                    return false;
+                }
+            }
+            else if (Configuration.IsRunningOnLinux && !File.Exists(Path.Combine(Se.PaddleOcrFolder, "paddleocr.bin")))
+            {
+                var answer = await MessageBox.Show(
+                    window,
+                    "Download Paddle OCR?",
+                    $"{Environment.NewLine}\"Paddle OCR\" requires downloading Paddle OCR.{Environment.NewLine}{Environment.NewLine}Download and use Paddle OCR?",
+                    MessageBoxButtons.Cancel,
+                    MessageBoxIcon.Question,
+                    "CPU",
+                    "GPU CUDA");
+
+                if (answer == MessageBoxResult.Cancel)
+                {
+                    return false;
+                }
+
+                var result = await windowService.ShowDialogAsync<DownloadPaddleOcrWindow, DownloadPaddleOcrViewModel>(window,
+                    vm =>
+                    {
+                        vm.Initialize(answer == MessageBoxResult.Custom1
+                            ? PaddleOcrDownloadType.EngineCpuLinux
+                            : PaddleOcrDownloadType.EngineGpuLinux);
+                    });
+
+                if (!result.OkPressed)
+                {
+                    return false;
+                }
+            }
+        }
+
+        if (!Directory.Exists(Se.PaddleOcrModelsFolder))
+        {
+            var result = await windowService.ShowDialogAsync<DownloadPaddleOcrWindow, DownloadPaddleOcrViewModel>(window,
+                vm => { vm.Initialize(PaddleOcrDownloadType.Models); });
+
+            if (!result.OkPressed)
+            {
+                return false;
+            }
+        }
+
+        return true;
+    }
+}