|
| 1 | +using System.Security.Cryptography; |
| 2 | +using System.Text; |
| 3 | +using System.Text.Json; |
| 4 | +using LLama; |
| 5 | +using LLama.Common; |
| 6 | +using LLama.Sampling; |
| 7 | + |
| 8 | +namespace NoteUI; |
| 9 | + |
| 10 | +public class AiManager |
| 11 | +{ |
| 12 | + // ── Data models ── |
| 13 | + |
| 14 | + public record ModelInfo(string Id, string Name); |
| 15 | + public record ChatMessage(string Role, string Content, DateTime Timestamp); |
| 16 | + |
| 17 | + public record LocalModel(string Name, string Repo, string FileName, string Size, bool IsPredefined = true) |
| 18 | + { |
| 19 | + public string DownloadUrl => $"https://huggingface.co/{Repo}/resolve/main/{FileName}"; |
| 20 | + public string LocalPath => Path.Combine(ModelsDir, FileName); |
| 21 | + public bool IsInstalled => File.Exists(LocalPath); |
| 22 | + } |
| 23 | + |
| 24 | + public class AiSettings |
| 25 | + { |
| 26 | + public float Temperature { get; set; } = 0.7f; |
| 27 | + public int MaxTokens { get; set; } = 2048; |
| 28 | + public int ContextSize { get; set; } = 2048; |
| 29 | + public int GpuLayers { get; set; } = 20; |
| 30 | + public string SystemPrompt { get; set; } = "Tu es un assistant utile et concis."; |
| 31 | + public string LastProviderId { get; set; } = ""; |
| 32 | + public string LastModelId { get; set; } = ""; |
| 33 | + public string LastLocalModelFileName { get; set; } = ""; |
| 34 | + } |
| 35 | + |
| 36 | + // ── Paths ── |
| 37 | + |
| 38 | + private static readonly string BaseDir = |
| 39 | + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "NoteUI"); |
| 40 | + public static readonly string ModelsDir = Path.Combine(BaseDir, "ai_models"); |
| 41 | + private static readonly string SettingsPath = Path.Combine(BaseDir, "ai_settings.json"); |
| 42 | + private static readonly string KeysPath = Path.Combine(BaseDir, "ai_keys.dat"); |
| 43 | + |
| 44 | + // ── State ── |
| 45 | + |
| 46 | + public AiSettings Settings { get; private set; } = new(); |
| 47 | + private Dictionary<string, string> _apiKeys = new(); |
| 48 | + |
| 49 | + // Local model state |
| 50 | + private LLamaWeights? _model; |
| 51 | + private ModelParams? _modelParams; |
| 52 | + private string? _loadedModelPath; |
| 53 | + private CancellationTokenSource? _inferenceCts; |
| 54 | + |
| 55 | + // ── Providers ── |
| 56 | + |
| 57 | + public static readonly ICloudAiProvider[] Providers = |
| 58 | + [ |
| 59 | + new OpenAiProvider(), |
| 60 | + new ClaudeProvider(), |
| 61 | + new GeminiProvider(), |
| 62 | + ]; |
| 63 | + |
| 64 | + // ── Predefined local models ── |
| 65 | + |
| 66 | + public static readonly LocalModel[] PredefinedModels = |
| 67 | + [ |
| 68 | + new("Gemma 2 2B", "bartowski/gemma-2-2b-it-GGUF", "gemma-2-2b-it-Q4_K_M.gguf", "~1.6 GB"), |
| 69 | + new("Phi-3 Mini 3.8B", "microsoft/Phi-3-mini-4k-instruct-gguf", "Phi-3-mini-4k-instruct-q4.gguf", "~2.4 GB"), |
| 70 | + new("Llama 3.2 3B", "bartowski/Llama-3.2-3B-Instruct-GGUF", "Llama-3.2-3B-Instruct-Q4_K_M.gguf", "~2 GB"), |
| 71 | + new("Mistral 7B", "TheBloke/Mistral-7B-Instruct-v0.2-GGUF", "mistral-7b-instruct-v0.2.Q4_K_M.gguf", "~4.4 GB"), |
| 72 | + new("Llama 3.1 8B", "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF", "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf", "~4.9 GB"), |
| 73 | + ]; |
| 74 | + |
| 75 | + // ── Init ── |
| 76 | + |
| 77 | + public void Load() |
| 78 | + { |
| 79 | + Directory.CreateDirectory(ModelsDir); |
| 80 | + LoadSettings(); |
| 81 | + LoadApiKeys(); |
| 82 | + } |
| 83 | + |
| 84 | + // ── Settings persistence ── |
| 85 | + |
| 86 | + private void LoadSettings() |
| 87 | + { |
| 88 | + try |
| 89 | + { |
| 90 | + if (File.Exists(SettingsPath)) |
| 91 | + { |
| 92 | + var json = File.ReadAllText(SettingsPath); |
| 93 | + Settings = JsonSerializer.Deserialize<AiSettings>(json) ?? new(); |
| 94 | + } |
| 95 | + } |
| 96 | + catch { Settings = new(); } |
| 97 | + } |
| 98 | + |
| 99 | + public void SaveSettings() |
| 100 | + { |
| 101 | + try |
| 102 | + { |
| 103 | + Directory.CreateDirectory(BaseDir); |
| 104 | + var json = JsonSerializer.Serialize(Settings, new JsonSerializerOptions { WriteIndented = true }); |
| 105 | + File.WriteAllText(SettingsPath, json); |
| 106 | + } |
| 107 | + catch { } |
| 108 | + } |
| 109 | + |
| 110 | + // ── API Keys (DPAPI) ── |
| 111 | + |
| 112 | + public string GetApiKey(string providerId) => |
| 113 | + _apiKeys.TryGetValue(providerId, out var key) ? key : ""; |
| 114 | + |
| 115 | + public void SetApiKey(string providerId, string key) |
| 116 | + { |
| 117 | + if (string.IsNullOrWhiteSpace(key)) |
| 118 | + _apiKeys.Remove(providerId); |
| 119 | + else |
| 120 | + _apiKeys[providerId] = key; |
| 121 | + SaveApiKeys(); |
| 122 | + } |
| 123 | + |
| 124 | + public bool HasApiKey(string providerId) => |
| 125 | + _apiKeys.TryGetValue(providerId, out var k) && !string.IsNullOrWhiteSpace(k); |
| 126 | + |
| 127 | + private void LoadApiKeys() |
| 128 | + { |
| 129 | + try |
| 130 | + { |
| 131 | + if (!File.Exists(KeysPath)) return; |
| 132 | + var encrypted = File.ReadAllBytes(KeysPath); |
| 133 | + var decrypted = ProtectedData.Unprotect(encrypted, null, DataProtectionScope.CurrentUser); |
| 134 | + var json = Encoding.UTF8.GetString(decrypted); |
| 135 | + _apiKeys = JsonSerializer.Deserialize<Dictionary<string, string>>(json) ?? new(); |
| 136 | + } |
| 137 | + catch { _apiKeys = new(); } |
| 138 | + } |
| 139 | + |
| 140 | + private void SaveApiKeys() |
| 141 | + { |
| 142 | + try |
| 143 | + { |
| 144 | + Directory.CreateDirectory(BaseDir); |
| 145 | + var json = JsonSerializer.Serialize(_apiKeys); |
| 146 | + var bytes = Encoding.UTF8.GetBytes(json); |
| 147 | + var encrypted = ProtectedData.Protect(bytes, null, DataProtectionScope.CurrentUser); |
| 148 | + File.WriteAllBytes(KeysPath, encrypted); |
| 149 | + } |
| 150 | + catch { } |
| 151 | + } |
| 152 | + |
| 153 | + // ── Provider helpers ── |
| 154 | + |
| 155 | + public ICloudAiProvider? GetProvider(string id) => |
| 156 | + Providers.FirstOrDefault(p => p.Id == id); |
| 157 | + |
| 158 | + // ── Local model management ── |
| 159 | + |
| 160 | + public List<LocalModel> GetInstalledModels() |
| 161 | + { |
| 162 | + if (!Directory.Exists(ModelsDir)) return []; |
| 163 | + var files = Directory.GetFiles(ModelsDir, "*.gguf"); |
| 164 | + var installed = new List<LocalModel>(); |
| 165 | + foreach (var file in files) |
| 166 | + { |
| 167 | + var fileName = Path.GetFileName(file); |
| 168 | + var predefined = PredefinedModels.FirstOrDefault(m => m.FileName == fileName); |
| 169 | + if (predefined != null) |
| 170 | + installed.Add(predefined); |
| 171 | + else |
| 172 | + { |
| 173 | + var sizeMb = new FileInfo(file).Length / (1024.0 * 1024.0); |
| 174 | + var sizeStr = sizeMb > 1024 ? $"~{sizeMb / 1024:F1} GB" : $"~{sizeMb:F0} MB"; |
| 175 | + installed.Add(new LocalModel(fileName, "", fileName, sizeStr, false)); |
| 176 | + } |
| 177 | + } |
| 178 | + return installed; |
| 179 | + } |
| 180 | + |
| 181 | + public Task DownloadModelAsync(LocalModel model, IProgress<(long downloaded, long? total)> progress, CancellationToken ct) |
| 182 | + => DownloadFileAsync(model.DownloadUrl, model.LocalPath, progress, ct); |
| 183 | + |
| 184 | + public Task DownloadFromUrlAsync(string url, string fileName, IProgress<(long downloaded, long? total)> progress, CancellationToken ct) |
| 185 | + => DownloadFileAsync(url, Path.Combine(ModelsDir, fileName), progress, ct); |
| 186 | + |
| 187 | + private static async Task DownloadFileAsync(string url, string destPath, IProgress<(long downloaded, long? total)> progress, CancellationToken ct) |
| 188 | + { |
| 189 | + Directory.CreateDirectory(Path.GetDirectoryName(destPath)!); |
| 190 | + var tempPath = destPath + ".tmp"; |
| 191 | + |
| 192 | + using var http = new HttpClient(); |
| 193 | + http.Timeout = TimeSpan.FromHours(2); |
| 194 | + using var response = await http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); |
| 195 | + response.EnsureSuccessStatusCode(); |
| 196 | + |
| 197 | + var totalBytes = response.Content.Headers.ContentLength; |
| 198 | + await using var stream = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); |
| 199 | + await using var fileStream = new FileStream(tempPath, FileMode.Create, FileAccess.Write, FileShare.None, 81920); |
| 200 | + |
| 201 | + var buffer = new byte[81920]; |
| 202 | + long totalRead = 0; |
| 203 | + int bytesRead; |
| 204 | + var lastReport = DateTime.UtcNow; |
| 205 | + while ((bytesRead = await stream.ReadAsync(buffer, ct).ConfigureAwait(false)) > 0) |
| 206 | + { |
| 207 | + await fileStream.WriteAsync(buffer.AsMemory(0, bytesRead), ct).ConfigureAwait(false); |
| 208 | + totalRead += bytesRead; |
| 209 | + |
| 210 | + var now = DateTime.UtcNow; |
| 211 | + if ((now - lastReport).TotalMilliseconds >= 50) |
| 212 | + { |
| 213 | + lastReport = now; |
| 214 | + progress.Report((totalRead, totalBytes)); |
| 215 | + } |
| 216 | + } |
| 217 | + |
| 218 | + // Final report to ensure 100% |
| 219 | + progress.Report((totalRead, totalBytes)); |
| 220 | + |
| 221 | + fileStream.Close(); |
| 222 | + if (File.Exists(destPath)) File.Delete(destPath); |
| 223 | + File.Move(tempPath, destPath); |
| 224 | + } |
| 225 | + |
| 226 | + public void DeleteModel(string fileName) |
| 227 | + { |
| 228 | + var path = Path.Combine(ModelsDir, fileName); |
| 229 | + if (_loadedModelPath == path) UnloadModel(); |
| 230 | + if (File.Exists(path)) File.Delete(path); |
| 231 | + } |
| 232 | + |
| 233 | + // ── Local inference ── |
| 234 | + |
| 235 | + public async Task LoadModelAsync(string fileName) |
| 236 | + { |
| 237 | + var path = Path.Combine(ModelsDir, fileName); |
| 238 | + if (_loadedModelPath == path && _model != null) return; |
| 239 | + |
| 240 | + UnloadModel(); |
| 241 | + |
| 242 | + await Task.Run(() => |
| 243 | + { |
| 244 | + var gpuLayers = Settings.GpuLayers; |
| 245 | + for (var attempt = 0; attempt < 2; attempt++) |
| 246 | + { |
| 247 | + try |
| 248 | + { |
| 249 | + var parameters = new ModelParams(path) |
| 250 | + { |
| 251 | + ContextSize = (uint)Settings.ContextSize, |
| 252 | + GpuLayerCount = attempt == 0 ? gpuLayers : 0, |
| 253 | + }; |
| 254 | + _model = LLamaWeights.LoadFromFile(parameters); |
| 255 | + _modelParams = parameters; |
| 256 | + _loadedModelPath = path; |
| 257 | + return; |
| 258 | + } |
| 259 | + catch when (attempt == 0 && gpuLayers > 0) |
| 260 | + { |
| 261 | + _model?.Dispose(); |
| 262 | + _model = null; |
| 263 | + } |
| 264 | + } |
| 265 | + }); |
| 266 | + } |
| 267 | + |
| 268 | + public async IAsyncEnumerable<string> ChatLocalAsync(string userMessage, List<ChatMessage> history, |
| 269 | + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default) |
| 270 | + { |
| 271 | + if (_model == null || _modelParams == null) yield break; |
| 272 | + |
| 273 | + var template = new LLamaTemplate(_model); |
| 274 | + template.Add("system", Settings.SystemPrompt); |
| 275 | + foreach (var msg in history) |
| 276 | + template.Add(msg.Role, msg.Content); |
| 277 | + template.Add("user", userMessage); |
| 278 | + |
| 279 | + var prompt = Encoding.UTF8.GetString(template.Apply().ToArray()); |
| 280 | + |
| 281 | + var inferenceParams = new InferenceParams |
| 282 | + { |
| 283 | + MaxTokens = Settings.MaxTokens, |
| 284 | + AntiPrompts = [ |
| 285 | + "<|eot_id|>", "<|start_header_id|>", |
| 286 | + "<|end|>", "<|assistant|>", "<|user|>", "<|system|>", |
| 287 | + "[/INST]", "</s>", |
| 288 | + "<|im_end|>", "<|endoftext|>", |
| 289 | + "<end_of_turn>", |
| 290 | + ], |
| 291 | + SamplingPipeline = new DefaultSamplingPipeline { Temperature = Settings.Temperature }, |
| 292 | + }; |
| 293 | + |
| 294 | + _inferenceCts = CancellationTokenSource.CreateLinkedTokenSource(ct); |
| 295 | + |
| 296 | + using var context = _model.CreateContext(_modelParams); |
| 297 | + var executor = new StatelessExecutor(_model, _modelParams); |
| 298 | + |
| 299 | + await foreach (var token in executor.InferAsync(prompt, inferenceParams, _inferenceCts.Token)) |
| 300 | + { |
| 301 | + yield return token; |
| 302 | + } |
| 303 | + } |
| 304 | + |
| 305 | + public void StopInference() => _inferenceCts?.Cancel(); |
| 306 | + |
| 307 | + public void UnloadModel() |
| 308 | + { |
| 309 | + _model?.Dispose(); |
| 310 | + _model = null; |
| 311 | + _modelParams = null; |
| 312 | + _loadedModelPath = null; |
| 313 | + } |
| 314 | + |
| 315 | + public bool IsModelLoaded => _model != null; |
| 316 | + public string? LoadedModelFileName => _loadedModelPath != null ? Path.GetFileName(_loadedModelPath) : null; |
| 317 | +} |
0 commit comments