Skip to content

Moved the custom tokenizer into the expert settings and added a drop zone for it #956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SommerEngineering merged 3 commits into from
Sep 10, 2026
Merged

Moved the custom tokenizer into the expert settings and added a drop zone for it #956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5af923a
Move the custom tokenizer into the expert settings behind the RAG pre…
SommerEngineering Sep 10, 2026
fa50a38
Lock the expert settings for enterprise-managed providers
SommerEngineering Sep 10, 2026
b59e03e
Add a drop zone to the custom tokenizer of both provider dialogs
SommerEngineering Sep 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6505,9 +6505,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo
-- Embedding batch size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size"

-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."

-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key"

-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."

-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."

Expand Down Expand Up @@ -6955,9 +6961,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Use detecte
-- Model
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Model"

-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."

-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API Key"

-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."

-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."

Expand Down
48 changes: 28 additions & 20 deletions app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,7 @@
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.Numbers"
AdornmentColor="Color.Info"
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.ValidateTokenLimit"
HelperText="@T("Maximum number of tokens sent to the embedding model per chunk. The default is 8,192.")"/>
<MudNumericField
Expand All @@ -183,28 +184,35 @@
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
AdornmentColor="Color.Info"
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.ValidateEmbeddingBatchSize"
HelperText="@T("How many chunks are sent to the embedding provider at once. The default is 1.")"/>
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
T="string"
Text="@this.dataFilePath"
TextChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
Clearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@this.dataCustomTokenizerValidationIssue"
OnClearButtonClick="@this.ClearPathTokenizer"
/>
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
@T("Choose File")
</MudButton>
</MudStack>
<PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
T="string"
Text="@this.dataFilePath"
TextChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
Clearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@this.dataCustomTokenizerValidationIssue"
OnClearButtonClick="@this.ClearPathTokenizer"
/>
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@(this.IsEnterpriseConfiguration || this.isTokenizerFileDialogOpen)" OnClick="@this.OpenTokenizerFileDialog">
@T("Choose File")
</MudButton>
</MudStack>
<MudText Typo="Typo.body2">
@T("You can also drag & drop the tokenizer file here.")
</MudText>
</PathDropZone>
</MudCollapse>
</MudStack>
}
Expand Down
26 changes: 25 additions & 1 deletion app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,31 @@ private Task ClearPathTokenizer(MouseEventArgs _)
{
return this.OnDataFilePathChanged(string.Empty);
}


/// <summary>
/// Takes the first dropped path which can serve as a tokenizer.
/// </summary>
/// <remarks>
/// A provider carries exactly one tokenizer, so a multi-selection cannot be honored as a whole.
/// Everything which is not a readable JSON file is skipped rather than handed to the runtime:
/// the validation would reject it anyway, and saying so right away names the actual mistake.
/// </remarks>
/// <param name="paths">The dropped paths.</param>
private async Task OnTokenizerPathsDropped(List<string> paths)
{
foreach (var path in paths)
{
if (!File.Exists(path) || !FileTypes.IsAllowedPath(path, FileTypes.JSON))
continue;

await this.OnDataFilePathChanged(path);
return;
}

this.Logger.LogWarning("None of the {Count} dropped path(s) could be used as a tokenizer.", paths.Count);
await this.MessageBus.SendWarning(new(Icons.Material.Filled.Warning, T("Please drop a tokenizer file in the JSON format.")));
}

private async Task OnDataFilePathChanged(string filePath)
{
this.dataFilePath = filePath;
Expand Down
59 changes: 32 additions & 27 deletions app/MindWork AI Studio/Dialogs/ProviderDialog.razor
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,33 +175,6 @@
UserAttributes="@SPELLCHECK_ATTRIBUTES"
/>

@if (this.DataLLMProvider != LLMProviders.NONE)
{
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
</MudJustifiedText>
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
T="string"
Text="@this.dataFilePath"
TextChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
Clearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@this.dataCustomTokenizerValidationIssue"
OnClearButtonClick="@this.ClearPathTokenizer"
/>
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@this.isTokenizerFileDialogOpen" OnClick="@this.OpenTokenizerFileDialog">
@T("Choose File")
</MudButton>
</MudStack>
}

<MudStack>
<MudButton OnClick="@this.ToggleExpertSettings">
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
Expand Down Expand Up @@ -273,6 +246,38 @@
@string.Format(T("The current model uses the {0}."), this.GetCurrentModelApiLabel())
</MudJustifiedText>
<MudTextField T="string" Label=@T("Additional API parameters") Variant="Variant.Outlined" Lines="4" AutoGrow="true" MaxLines="10" HelperText=@T("""Add the parameters in proper JSON formatting, e.g., "temperature": 0.5. Remove trailing commas. The usual surrounding curly brackets {} must not be used, though.""") Placeholder="@GetPlaceholderExpertSettings" @bind-Value="@this.AdditionalJsonApiParameters" Immediate="true" Disabled="@this.IsEnterpriseConfiguration" Validation="@this.ValidateAdditionalJsonApiParameters" OnBlur="@this.OnInputChangeExpertSettings"/>
@if (this.ShowTokenizerSettings)
{
<MudJustifiedText Typo="Typo.body1" Class="mt-4 mb-3">
@T("For better token estimates, you can configure a custom tokenizer for this provider.")
</MudJustifiedText>
<PathDropZone IdPrefix="tokenizer" Disabled="@(() => this.IsEnterpriseConfiguration)" OnPathsDropped="@this.OnTokenizerPathsDropped">
<MudStack Row="@true" Spacing="3" Class="mb-3" StretchItems="StretchItems.None" AlignItems="AlignItems.Center">
<MudTextField
T="string"
Text="@this.dataFilePath"
TextChanged="@this.OnDataFilePathChanged"
Label="@T("Selected file path for the custom tokenizer")"
Disabled="@this.IsEnterpriseConfiguration"
Validation="@this.providerValidation.ValidatingCustomTokenizer"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
Clearable="@true"
Error="@(!string.IsNullOrWhiteSpace(this.dataCustomTokenizerValidationIssue))"
ErrorText="@this.dataCustomTokenizerValidationIssue"
OnClearButtonClick="@this.ClearPathTokenizer"
/>
<MudButton StartIcon="@Icons.Material.Filled.FolderOpen" Variant="Variant.Outlined" Color="Color.Primary" Disabled="@(this.IsEnterpriseConfiguration || this.isTokenizerFileDialogOpen)" OnClick="@this.OpenTokenizerFileDialog">
@T("Choose File")
</MudButton>
</MudStack>
<MudText Typo="Typo.body2">
@T("You can also drag & drop the tokenizer file here.")
</MudText>
</PathDropZone>
}
</MudCollapse>
</MudStack>
</MudForm>
Expand Down
35 changes: 34 additions & 1 deletion app/MindWork AI Studio/Dialogs/ProviderDialog.razor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using AIStudio.Provider.HuggingFace;
using AIStudio.Tools.Rust;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;

Expand Down Expand Up @@ -233,7 +234,9 @@ protected override async Task OnInitializedAsync()
this.UsedInstanceNames = this.SettingsManager.GetAllProviders().Select(x => x.InstanceName.ToLowerInvariant()).ToList();

this.capabilityOverrides = this.DataCapabilityOverrides ?? new();
this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters) || this.capabilityOverrides.HasOverrides;
this.showExpertSettings = !string.IsNullOrWhiteSpace(this.AdditionalJsonApiParameters)
|| this.capabilityOverrides.HasOverrides
|| (this.ShowTokenizerSettings && !string.IsNullOrWhiteSpace(this.DataTokenizerPath));

// When editing, we need to load the data:
if(this.IsEditing)
Expand Down Expand Up @@ -422,6 +425,30 @@ private Task ClearPathTokenizer(MouseEventArgs _)
return this.OnDataFilePathChanged(string.Empty);
}

/// <summary>
/// Takes the first dropped path which can serve as a tokenizer.
/// </summary>
/// <remarks>
/// A provider carries exactly one tokenizer, so a multi-selection cannot be honored as a whole.
/// Everything which is not a readable JSON file is skipped rather than handed to the runtime:
/// the validation would reject it anyway, and saying so right away names the actual mistake.
/// </remarks>
/// <param name="paths">The dropped paths.</param>
private async Task OnTokenizerPathsDropped(List<string> paths)
{
foreach (var path in paths)
{
if (!File.Exists(path) || !FileTypes.IsAllowedPath(path, FileTypes.JSON))
continue;

await this.OnDataFilePathChanged(path);
return;
}

this.Logger.LogWarning("None of the {Count} dropped path(s) could be used as a tokenizer.", paths.Count);
await this.MessageBus.SendWarning(new(Icons.Material.Filled.Warning, T("Please drop a tokenizer file in the JSON format.")));
}

private async Task OnDataFilePathChanged(string filePath)
{
this.dataFilePath = filePath;
Expand Down Expand Up @@ -583,6 +610,12 @@ this.DataHost is Host.LLAMA_CPP &&
/// </summary>
private string ModelsOverviewURL => this.DataLLMProvider.GetModelsOverviewURL(this.HFInferenceProviderId);

/// <summary>
/// Whether the custom tokenizer is offered at all. It is an expert setting which only makes
/// sense while the RAG preview is enabled.
/// </summary>
private bool ShowTokenizerSettings => this.DataLLMProvider != LLMProviders.NONE && PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager);

private void UpdateModelSelectionAfterLoading()
{
if (this.DataLLMProvider is not LLMProviders.SELF_HOSTED || this.DataHost is not Host.LLAMA_CPP)
Expand Down
12 changes: 12 additions & 0 deletions ...indWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6507,9 +6507,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo
-- Embedding batch size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Batch-Größe für Einbettungen"

-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "Sie können die Tokenizer-Datei auch per Drag-and-Drop hierher ziehen."

-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API-Schlüssel"

-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Bitte legen Sie hier eine Tokenizer-Datei im JSON-Format ab."

-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Der API-Schlüssel konnte nicht aus dem Betriebssystem entfernt werden. Die Meldung lautete: {0}. Bitte versuchen Sie es erneut."

Expand Down Expand Up @@ -6957,9 +6963,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Erkanntes M
-- Model
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Modell"

-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "Sie können die Tokenizer-Datei auch per Drag-and-Drop hierher ziehen."

-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API-Schlüssel"

-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Bitte legen Sie eine Tokenizer-Datei im JSON-Format ab."

-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Fehler beim Löschen des API-Schlüssels vom Betriebssystem. Die Nachricht war: {0}. Bitte versuchen Sie es erneut."

Expand Down
12 changes: 12 additions & 0 deletions ...indWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6507,9 +6507,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2189814010"] = "Mo
-- Embedding batch size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2209963239"] = "Embedding batch size"

-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."

-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(Optional) API Key"

-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."

-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."

Expand Down Expand Up @@ -6957,9 +6963,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2141072961"] = "Use detecte
-- Model
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2189814010"] = "Model"

-- You can also drag & drop the tokenizer file here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2282234384"] = "You can also drag & drop the tokenizer file here."

-- (Optional) API Key
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331453405"] = "(Optional) API Key"

-- Please drop a tokenizer file in the JSON format.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2331986401"] = "Please drop a tokenizer file in the JSON format."

-- Failed to remove the API key from the operating system. The message was: {0}. Please try again.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T2439094236"] = "Failed to remove the API key from the operating system. The message was: {0}. Please try again."

Expand Down