From 5af923afa931c7d30d04453436af9c9577b82e17 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 10 Sep 2026 19:00:52 +0200 Subject: [PATCH 1/3] Move the custom tokenizer into the expert settings behind the RAG preview --- .../Dialogs/ProviderDialog.razor | 53 +++++++++---------- .../Dialogs/ProviderDialog.razor.cs | 11 +++- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor index 80c9d6a82..f213ade7c 100644 --- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor @@ -175,33 +175,6 @@ UserAttributes="@SPELLCHECK_ATTRIBUTES" /> - @if (this.DataLLMProvider != LLMProviders.NONE) - { - - @T("For better token estimates, you can configure a custom tokenizer for this provider.") - - - - - @T("Choose File") - - - } - @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings")) @@ -273,6 +246,32 @@ @string.Format(T("The current model uses the {0}."), this.GetCurrentModelApiLabel()) + @if (this.ShowTokenizerSettings) + { + + @T("For better token estimates, you can configure a custom tokenizer for this provider.") + + + + + @T("Choose File") + + + } diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs index a436a9e07..e4831b9f4 100644 --- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs @@ -6,6 +6,7 @@ using AIStudio.Provider.HuggingFace; using AIStudio.Tools.Rust; using AIStudio.Settings; +using AIStudio.Settings.DataModel; using AIStudio.Tools.Services; using AIStudio.Tools.Validation; @@ -233,7 +234,9 @@ protected override async Task OnInitializedAsync() this.UsedInstanceNames = this.SettingsManager.GetAllProviders().Select(x => x.InstanceName.ToLowerInvariant()).ToList(); this.capabilityOverrides = this.DataCapabilityOverrides ?? new(); - this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters) || this.capabilityOverrides.HasOverrides; + this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters) + || this.capabilityOverrides.HasOverrides + || (this.ShowTokenizerSettings && !string.IsNullOrWhiteSpace(this.DataTokenizerPath)); // When editing, we need to load the data: if(this.IsEditing) @@ -583,6 +586,12 @@ this.DataHost is Host.LLAMA_CPP && /// private string ModelsOverviewURL => this.DataLLMProvider.GetModelsOverviewURL(this.HFInferenceProviderId); + /// + /// Whether the custom tokenizer is offered at all. It is an expert setting which only makes + /// sense while the RAG preview is enabled. + /// + private bool ShowTokenizerSettings => this.DataLLMProvider != LLMProviders.NONE && PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager); + private void UpdateModelSelectionAfterLoading() { if (this.DataLLMProvider is not LLMProviders.SELF_HOSTED || this.DataHost is not Host.LLAMA_CPP) From fa50a38f7156d998d920054f1b553c99d213bd25 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 10 Sep 2026 19:03:08 +0200 Subject: [PATCH 2/3] Lock the expert settings for enterprise-managed providers --- app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor | 5 ++++- app/MindWork AI Studio/Dialogs/ProviderDialog.razor | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor index 1299c3812..fe2d116ae 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor @@ -171,6 +171,7 @@ Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.Numbers" AdornmentColor="Color.Info" + Disabled="@this.IsEnterpriseConfiguration" Validation="@this.ValidateTokenLimit" HelperText="@T("Maximum number of tokens sent to the embedding model per chunk. The default is 8,192.")"/> @@ -191,6 +193,7 @@ Text="@this.dataFilePath" TextChanged="@this.OnDataFilePathChanged" Label="@T("Selected file path for the custom tokenizer")" + Disabled="@this.IsEnterpriseConfiguration" Validation="@this.providerValidation.ValidatingCustomTokenizer" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.AttachFile" @@ -201,7 +204,7 @@ ErrorText="@this.dataCustomTokenizerValidationIssue" OnClearButtonClick="@this.ClearPathTokenizer" /> - + @T("Choose File") diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor index f213ade7c..d4642b47d 100644 --- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor @@ -257,6 +257,7 @@ Text="@this.dataFilePath" TextChanged="@this.OnDataFilePathChanged" Label="@T("Selected file path for the custom tokenizer")" + Disabled="@this.IsEnterpriseConfiguration" Validation="@this.providerValidation.ValidatingCustomTokenizer" Adornment="Adornment.Start" AdornmentIcon="@Icons.Material.Filled.AttachFile" @@ -267,7 +268,7 @@ ErrorText="@this.dataCustomTokenizerValidationIssue" OnClearButtonClick="@this.ClearPathTokenizer" /> - + @T("Choose File") From b59e03e2ac1b1f397e7d4c35d000f058323941bf Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 10 Sep 2026 19:14:15 +0200 Subject: [PATCH 3/3] Add a drop zone to the custom tokenizer of both provider dialogs --- .../Assistants/I18N/allTexts.lua | 12 +++++ .../Dialogs/EmbeddingProviderDialog.razor | 47 ++++++++++--------- .../Dialogs/EmbeddingProviderDialog.razor.cs | 26 +++++++++- .../Dialogs/ProviderDialog.razor | 47 ++++++++++--------- .../Dialogs/ProviderDialog.razor.cs | 24 ++++++++++ .../plugin.lua | 12 +++++ .../plugin.lua | 12 +++++ 7 files changed, 137 insertions(+), 43 deletions(-) diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 7f71d85ee..583a52521 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -6505,9 +6505,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo -- Embedding batch size UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size" +-- You can also drag & drop the tokenizer file here. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here." + -- (Optional) API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key" +-- Please drop a tokenizer file in the JSON format. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format." + -- Failed to remove the API key from the operating system. The message was: {0}. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again." @@ -6955,9 +6961,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Use detecte -- Model UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Model" +-- You can also drag & drop the tokenizer file here. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here." + -- (Optional) API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API Key" +-- Please drop a tokenizer file in the JSON format. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format." + -- Failed to remove the API key from the operating system. The message was: {0}. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again." diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor index fe2d116ae..b60f4d64c 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor @@ -187,27 +187,32 @@ Disabled="@this.IsEnterpriseConfiguration" Validation="@this.ValidateEmbeddingBatchSize" HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/> - - - - @T("Choose File") - - + + + + + @T("Choose File") + + + + @T("You can also drag & drop the tokenizer file here.") + + } diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs index 81e201b52..bde93e45e 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs @@ -388,7 +388,31 @@ private Task ClearPathTokenizer(MouseEventArgs _) { return this.OnDataFilePathChanged(string.Empty); } - + + /// + /// Takes the first dropped path which can serve as a tokenizer. + /// + /// + /// A provider carries exactly one tokenizer, so a multi-selection cannot be honored as a whole. + /// Everything which is not a readable JSON file is skipped rather than handed to the runtime: + /// the validation would reject it anyway, and saying so right away names the actual mistake. + /// + /// The dropped paths. + private async Task OnTokenizerPathsDropped(List paths) + { + foreach (var path in paths) + { + if (!File.Exists(path) || !FileTypes.IsAllowedPath(path, FileTypes.JSON)) + continue; + + await this.OnDataFilePathChanged(path); + return; + } + + this.Logger.LogWarning("None of the {Count} dropped path(s) could be used as a tokenizer.", paths.Count); + await this.MessageBus.SendWarning(new(Icons.Material.Filled.Warning, T("Please drop a tokenizer file in the JSON format."))); + } + private async Task OnDataFilePathChanged(string filePath) { this.dataFilePath = filePath; diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor index d4642b47d..50c36e647 100644 --- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor @@ -251,27 +251,32 @@ @T("For better token estimates, you can configure a custom tokenizer for this provider.") - - - - @T("Choose File") - - + + + + + @T("Choose File") + + + + @T("You can also drag & drop the tokenizer file here.") + + } diff --git a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs index e4831b9f4..0eecec827 100644 --- a/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs @@ -425,6 +425,30 @@ private Task ClearPathTokenizer(MouseEventArgs _) return this.OnDataFilePathChanged(string.Empty); } + /// + /// Takes the first dropped path which can serve as a tokenizer. + /// + /// + /// A provider carries exactly one tokenizer, so a multi-selection cannot be honored as a whole. + /// Everything which is not a readable JSON file is skipped rather than handed to the runtime: + /// the validation would reject it anyway, and saying so right away names the actual mistake. + /// + /// The dropped paths. + private async Task OnTokenizerPathsDropped(List paths) + { + foreach (var path in paths) + { + if (!File.Exists(path) || !FileTypes.IsAllowedPath(path, FileTypes.JSON)) + continue; + + await this.OnDataFilePathChanged(path); + return; + } + + this.Logger.LogWarning("None of the {Count} dropped path(s) could be used as a tokenizer.", paths.Count); + await this.MessageBus.SendWarning(new(Icons.Material.Filled.Warning, T("Please drop a tokenizer file in the JSON format."))); + } + private async Task OnDataFilePathChanged(string filePath) { this.dataFilePath = filePath; diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 0cf69a8ea..5c97ddeea 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -6507,9 +6507,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo -- Embedding batch size UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Batch-Größe für Einbettungen" +-- You can also drag & drop the tokenizer file here. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "Sie können die Tokenizer-Datei auch per Drag-and-Drop hierher ziehen." + -- (Optional) API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API-Schlüssel" +-- Please drop a tokenizer file in the JSON format. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Bitte legen Sie hier eine Tokenizer-Datei im JSON-Format ab." + -- Failed to remove the API key from the operating system. The message was: {0}. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Der API-Schlüssel konnte nicht aus dem Betriebssystem entfernt werden. Die Meldung lautete: {0}. Bitte versuchen Sie es erneut." @@ -6957,9 +6963,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Erkanntes M -- Model UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Modell" +-- You can also drag & drop the tokenizer file here. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "Sie können die Tokenizer-Datei auch per Drag-and-Drop hierher ziehen." + -- (Optional) API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API-Schlüssel" +-- Please drop a tokenizer file in the JSON format. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Bitte legen Sie eine Tokenizer-Datei im JSON-Format ab." + -- Failed to remove the API key from the operating system. The message was: {0}. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Fehler beim Löschen des API-Schlüssels vom Betriebssystem. Die Nachricht war: {0}. Bitte versuchen Sie es erneut." diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 0fbcbe94d..f26294a41 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -6507,9 +6507,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo -- Embedding batch size UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size" +-- You can also drag & drop the tokenizer file here. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here." + -- (Optional) API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key" +-- Please drop a tokenizer file in the JSON format. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format." + -- Failed to remove the API key from the operating system. The message was: {0}. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again." @@ -6957,9 +6963,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Use detecte -- Model UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Model" +-- You can also drag & drop the tokenizer file here. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here." + -- (Optional) API Key UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API Key" +-- Please drop a tokenizer file in the JSON format. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format." + -- Failed to remove the API key from the operating system. The message was: {0}. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."