From f8b8a8a06dfeace668e8b15a1278478dbba960d2 Mon Sep 17 00:00:00 2001 From: niksedk Date: Wed, 8 Jul 2026 16:47:14 +0200 Subject: [PATCH] Video OCR: add llama.cpp engine + tidy scan-area hint Add a llama.cpp OCR engine to the burned-in/video OCR window, reusing the same curated OCR models, downloader, and local llama-server lifecycle as the normal OCR window. Includes model selection, download button, start/stop server button, and a language field, with the selected model/language persisted to settings. Rename the standalone Paddle entry to "Paddle OCR Standalone" to match the normal OCR window. Turn the "Drag on the video frame..." scan-area text into a tooltip on the video frame instead of an inline label that crowded/overlapped the scan-area buttons and coordinate text. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Video/VideoOcr/VideoOcrEngineItem.cs | 3 +- .../Video/VideoOcr/VideoOcrViewModel.cs | 180 ++++++++++++++++++ .../Features/Video/VideoOcr/VideoOcrWindow.cs | 28 ++- src/ui/Logic/Config/SeVideoOcr.cs | 4 + 4 files changed, 207 insertions(+), 8 deletions(-) diff --git a/src/ui/Features/Video/VideoOcr/VideoOcrEngineItem.cs b/src/ui/Features/Video/VideoOcr/VideoOcrEngineItem.cs index 65b3540840..8461531c23 100644 --- a/src/ui/Features/Video/VideoOcr/VideoOcrEngineItem.cs +++ b/src/ui/Features/Video/VideoOcr/VideoOcrEngineItem.cs @@ -28,11 +28,12 @@ public static List GetEngines() if (OperatingSystem.IsWindows() || OperatingSystem.IsLinux()) { - list.Add(new("Paddle OCR", OcrEngineType.PaddleOcrStandalone, "Local OCR engine (downloaded automatically) - fast and accurate")); + list.Add(new("Paddle OCR Standalone", OcrEngineType.PaddleOcrStandalone, "Local OCR engine (downloaded automatically) - fast and accurate")); } list.Add(new("Paddle OCR Python", OcrEngineType.PaddleOcrPython, "Local OCR engine via Python (requires \"pip install paddleocr\")")); list.Add(new("Ollama vision", OcrEngineType.Ollama, "Local vision model via Ollama - e.g. glm-ocr")); + list.Add(new("llama.cpp", OcrEngineType.LlamaCpp, "Local vision model via llama.cpp (downloaded automatically)")); list.Add(new("GLM API", OcrEngineType.Glm, "GLM vision model via Z.ai / bigmodel.cn API (requires API key)")); return list; diff --git a/src/ui/Features/Video/VideoOcr/VideoOcrViewModel.cs b/src/ui/Features/Video/VideoOcr/VideoOcrViewModel.cs index b14008d4bd..8b24439b6c 100644 --- a/src/ui/Features/Video/VideoOcr/VideoOcrViewModel.cs +++ b/src/ui/Features/Video/VideoOcr/VideoOcrViewModel.cs @@ -9,8 +9,10 @@ using Nikse.SubtitleEdit.Features.Ocr.Download; using Nikse.SubtitleEdit.Features.Ocr.Engines; using Nikse.SubtitleEdit.Features.Shared; +using Nikse.SubtitleEdit.Features.Translate; using Nikse.SubtitleEdit.Logic; using Nikse.SubtitleEdit.Logic.Config; +using Nikse.SubtitleEdit.Logic.LlamaCpp; using Nikse.SubtitleEdit.Logic.Media; using SkiaSharp; using System; @@ -33,6 +35,7 @@ public partial class VideoOcrViewModel : ObservableObject [ObservableProperty] private bool _isPaddleEngine; [ObservableProperty] private bool _isOllamaEngine; [ObservableProperty] private bool _isGlmEngine; + [ObservableProperty] private bool _isLlamaCppEngine; [ObservableProperty] private ObservableCollection _paddleLanguages; [ObservableProperty] private OcrLanguage2? _selectedPaddleLanguage; [ObservableProperty] private string _ollamaUrl; @@ -42,6 +45,10 @@ public partial class VideoOcrViewModel : ObservableObject [ObservableProperty] private string _glmModel; [ObservableProperty] private string _glmApiKey; [ObservableProperty] private string _glmLanguage; + [ObservableProperty] private ObservableCollection _llamaCppModels; + [ObservableProperty] private LlamaCppModelDisplay? _selectedLlamaCppModel; + [ObservableProperty] private string _llamaCppLanguage; + [ObservableProperty] private string _llamaCppServerButtonText; [ObservableProperty] private int _framesPerSecond; [ObservableProperty] private int _brightnessMinimum; [ObservableProperty] private int _textSimilarityPercent; @@ -99,6 +106,9 @@ public VideoOcrViewModel(IWindowService windowService) GlmModel = string.Empty; GlmApiKey = string.Empty; GlmLanguage = string.Empty; + LlamaCppModels = new ObservableCollection(); + LlamaCppLanguage = string.Empty; + LlamaCppServerButtonText = "Start server"; ProgressText = string.Empty; PreviewPositionText = string.Empty; ScanAreaText = string.Empty; @@ -191,6 +201,153 @@ partial void OnSelectedEngineChanged(VideoOcrEngineItem value) IsPaddleEngine = value.EngineType is OcrEngineType.PaddleOcrStandalone or OcrEngineType.PaddleOcrPython; IsOllamaEngine = value.EngineType == OcrEngineType.Ollama; IsGlmEngine = value.EngineType == OcrEngineType.Glm; + IsLlamaCppEngine = value.EngineType == OcrEngineType.LlamaCpp; + + if (IsLlamaCppEngine && LlamaCppModels.Count == 0) + { + var savedModelName = Path.GetFileName(Se.Settings.Video.VideoOcr.LlamaCppModel); + SelectedLlamaCppModel = LlamaCppDownloadHelper.PopulateModels(LlamaCppModels, LlamaCppServerManager.OcrModels, savedModelName); + } + } + + partial void OnSelectedLlamaCppModelChanged(LlamaCppModelDisplay? value) + { + if (value == null) + { + return; + } + + Se.Settings.Video.VideoOcr.LlamaCppModel = LlamaCppServerManager.GetModelPath(value.Model.FileName); + } + + private void UpdateLlamaCppServerButtonText() + { + LlamaCppServerButtonText = LlamaCppServerManager.IsServerRunning ? "Stop server" : "Start server"; + } + + [RelayCommand] + private async Task DownloadLlamaCpp() + { + if (Window == null) + { + return; + } + + var model = SelectedLlamaCppModel?.Model; + var forceModelDownload = false; + if (model != null && LlamaCppServerManager.IsModelInstalled(model)) + { + var answer = await MessageBox.Show( + Window, + Se.Language.General.Download, + string.Format(Se.Language.Translate.XIsAlreadyDownloadedReDownload, model.DisplayName), + MessageBoxButtons.YesNo, + MessageBoxIcon.Question); + + if (answer != MessageBoxResult.Yes) + { + return; + } + + forceModelDownload = true; + } + + var downloaded = await LlamaCppDownloadHelper.DownloadAsync(Window, _windowService, model, forceModelDownload: forceModelDownload); + if (downloaded != null) + { + var selectName = string.IsNullOrEmpty(downloaded) ? model?.FileName : downloaded; + SelectedLlamaCppModel = LlamaCppDownloadHelper.PopulateModels(LlamaCppModels, LlamaCppServerManager.OcrModels, selectName); + } + } + + [RelayCommand] + private async Task ToggleLlamaCppServer() + { + if (Window == null) + { + return; + } + + if (LlamaCppServerManager.IsServerRunning) + { + LlamaCppServerManager.StopServer(); + UpdateLlamaCppServerButtonText(); + return; + } + + await EnsureLlamaCppReady(); + UpdateLlamaCppServerButtonText(); + } + + private async Task EnsureLlamaCppReady() + { + if (Window == null) + { + return false; + } + + var model = SelectedLlamaCppModel?.Model; + if (model == null) + { + return false; + } + + var engineInstalled = LlamaCppServerManager.IsEngineInstalled(); + var modelInstalled = LlamaCppServerManager.IsModelInstalled(model); + if (!engineInstalled || !modelInstalled) + { + string message; + if (!engineInstalled && !modelInstalled) + { + message = "llama.cpp requires the llama-server engine and the selected OCR model to be downloaded. Download now?"; + } + else if (!engineInstalled) + { + message = "llama.cpp requires the llama-server engine to be downloaded. Download now?"; + } + else + { + message = "llama.cpp requires the selected OCR model to be downloaded. Download now?"; + } + + var answer = await MessageBox.Show( + Window, + Se.Language.General.Download, + message, + MessageBoxButtons.YesNo, + MessageBoxIcon.Question); + + if (answer != MessageBoxResult.Yes) + { + return false; + } + + await LlamaCppDownloadHelper.DownloadAsync(Window, _windowService, model); + if (!LlamaCppServerManager.IsEngineInstalled() || !LlamaCppServerManager.IsModelInstalled(model)) + { + return false; + } + } + + SelectedLlamaCppModel = LlamaCppDownloadHelper.PopulateModels(LlamaCppModels, LlamaCppServerManager.OcrModels, model.FileName); + + try + { + await LlamaCppServerManager.EnsureServerRunningAsync(model, _cancellationTokenSource.Token); + } + catch (Exception ex) + { + await MessageBox.Show( + Window, + Se.Language.General.Error, + ex.Message, + MessageBoxButtons.OK, + MessageBoxIcon.Error); + return false; + } + + UpdateLlamaCppServerButtonText(); + return true; } private void LoadPreview() @@ -581,6 +738,18 @@ await RunLlmOcr(ocrGroups, group => glmOcr.Ocr(group.RepresentativeFileName, GlmUrl, GlmModel, GlmLanguage, cancellationToken), () => glmOcr.Error, ReportOcrProgress, AddPreviewLine, cancellationToken); } + else if (engineType == OcrEngineType.LlamaCpp) + { + var llamaCppOcr = new LlamaCppOcr(Se.Settings.Ocr.LlamaCppOcrTimeoutMinutes); + var url = LlamaCppServerManager.ApiUrl; + var modelName = SelectedLlamaCppModel?.Model.FileName is { } fileName + ? Path.GetFileNameWithoutExtension(fileName) + : "glmocr"; + var prompt = Se.Settings.Ocr.LlamaCppOcrPrompt; + await RunLlmOcr(ocrGroups, group => OcrWithBitmap(group, bitmap => + llamaCppOcr.Ocr(bitmap, url, modelName, LlamaCppLanguage, prompt, cancellationToken)), + () => llamaCppOcr.Error, ReportOcrProgress, AddPreviewLine, cancellationToken); + } } private static async Task OcrWithBitmap(VideoOcrFrameGroup group, Func> ocr) @@ -643,6 +812,11 @@ await MessageBox.Show( return await PaddleOcrInstallHelper.EnsureInstalled(Window!, _windowService, engineType); } + if (engineType == OcrEngineType.LlamaCpp) + { + return await EnsureLlamaCppReady(); + } + return true; } @@ -673,6 +847,7 @@ private void LoadSettings() GlmModel = settings.GlmModel; GlmApiKey = settings.GlmApiKey; GlmLanguage = settings.GlmLanguage; + LlamaCppLanguage = string.IsNullOrEmpty(settings.LlamaCppLanguage) ? "English" : settings.LlamaCppLanguage; FramesPerSecond = Math.Clamp(settings.FramesPerSecond, 1, 30); BrightnessMinimum = Math.Clamp(settings.BrightnessMinimum, 0, 255); TextSimilarityPercent = Math.Clamp(settings.TextSimilarityPercent, 0, 100); @@ -693,6 +868,11 @@ private void SaveSettings() settings.GlmModel = GlmModel; settings.GlmApiKey = GlmApiKey; settings.GlmLanguage = GlmLanguage; + if (SelectedLlamaCppModel != null) + { + settings.LlamaCppModel = LlamaCppServerManager.GetModelPath(SelectedLlamaCppModel.Model.FileName); + } + settings.LlamaCppLanguage = LlamaCppLanguage; settings.FramesPerSecond = FramesPerSecond; settings.BrightnessMinimum = BrightnessMinimum; settings.TextSimilarityPercent = TextSimilarityPercent; diff --git a/src/ui/Features/Video/VideoOcr/VideoOcrWindow.cs b/src/ui/Features/Video/VideoOcr/VideoOcrWindow.cs index fab25691ab..7969abc144 100644 --- a/src/ui/Features/Video/VideoOcr/VideoOcrWindow.cs +++ b/src/ui/Features/Video/VideoOcr/VideoOcrWindow.cs @@ -100,6 +100,7 @@ private static Border MakePreviewView(VideoOcrViewModel vm) Background = Brushes.Black, Children = { image, cropSelector }, }; + ToolTip.SetTip(imageArea, Se.Language.Video.VideoOcr.ScanAreaInfo); var slider = new Slider { @@ -150,13 +151,6 @@ private static Border MakePreviewView(VideoOcrViewModel vm) UiUtil.MakeButton(Se.Language.Video.VideoOcr.BottomHalf, vm.SetScanAreaBottomHalfCommand), UiUtil.MakeButton(Se.Language.Video.VideoOcr.FullFrame, vm.SetScanAreaFullFrameCommand), scanAreaText, - new TextBlock - { - Text = Se.Language.Video.VideoOcr.ScanAreaInfo, - VerticalAlignment = VerticalAlignment.Center, - Opacity = 0.7, - Margin = new Thickness(10, 0, 0, 0), - }, }, }; @@ -224,6 +218,19 @@ private static Border MakeSettingsView(VideoOcrViewModel vm) glmPanel.Bind(StackPanel.IsVisibleProperty, new Binding(nameof(vm.IsGlmEngine)) { Source = vm }); panel.Children.Add(glmPanel); + // llama.cpp settings + var llamaCppPanel = new StackPanel { Orientation = Orientation.Vertical, Spacing = 4 }; + llamaCppPanel.Children.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.Model)); + llamaCppPanel.Children.Add(UiUtil.MakeComboBox(vm.LlamaCppModels, vm, nameof(vm.SelectedLlamaCppModel)).WithWidth(330)); + var llamaCppButtons = new StackPanel { Orientation = Orientation.Horizontal, Spacing = 5 }; + llamaCppButtons.Children.Add(UiUtil.MakeButton(vm.DownloadLlamaCppCommand, IconNames.Download, Se.Language.General.Download)); + llamaCppButtons.Children.Add(MakeLlamaCppServerButton(vm)); + llamaCppPanel.Children.Add(llamaCppButtons); + llamaCppPanel.Children.Add(UiUtil.MakeLabel(Se.Language.Video.VideoOcr.Language)); + llamaCppPanel.Children.Add(UiUtil.MakeTextBox(330, vm, nameof(vm.LlamaCppLanguage))); + llamaCppPanel.Bind(StackPanel.IsVisibleProperty, new Binding(nameof(vm.IsLlamaCppEngine)) { Source = vm }); + panel.Children.Add(llamaCppPanel); + // Scan settings panel.Children.Add(MakeHeader(Se.Language.Video.VideoOcr.Scan)); panel.Children.Add(MakeSettingRow( @@ -255,6 +262,13 @@ private static Border MakeSettingsView(VideoOcrViewModel vm) return UiUtil.MakeBorderForControl(scrollViewer); } + private static Button MakeLlamaCppServerButton(VideoOcrViewModel vm) + { + var button = UiUtil.MakeButton(string.Empty, vm.ToggleLlamaCppServerCommand); + button.Bind(Button.ContentProperty, new Binding(nameof(vm.LlamaCppServerButtonText))); + return button; + } + private static TextBlock MakeHeader(string text, bool isFirst = false) { return new TextBlock diff --git a/src/ui/Logic/Config/SeVideoOcr.cs b/src/ui/Logic/Config/SeVideoOcr.cs index 634bdabe8a..7757b2ef73 100644 --- a/src/ui/Logic/Config/SeVideoOcr.cs +++ b/src/ui/Logic/Config/SeVideoOcr.cs @@ -13,6 +13,8 @@ public class SeVideoOcr public string GlmModel { get; set; } public string GlmApiKey { get; set; } public string GlmLanguage { get; set; } + public string LlamaCppModel { get; set; } + public string LlamaCppLanguage { get; set; } public int FramesPerSecond { get; set; } public int ImageSimilarityPercent { get; set; } public int TextSimilarityPercent { get; set; } @@ -39,6 +41,8 @@ public SeVideoOcr() GlmModel = GlmOcr.DefaultModel; GlmApiKey = string.Empty; GlmLanguage = "English"; + LlamaCppModel = string.Empty; + LlamaCppLanguage = "English"; FramesPerSecond = 5; ImageSimilarityPercent = 92; TextSimilarityPercent = 80;