Add vision (image understanding) web server samples for C#, JS, Rust

samples/README.md

@@ -8,8 +8,8 @@ Explore complete working examples that demonstrate how to use Foundry Local —
 
 | Language | Samples | Description |
 |----------|---------|-------------|
-| [**C#**](cs/) | 13 | .NET SDK samples including native chat, embeddings, audio transcription, tool calling, model management, web server, tutorials, and WinML EP verification. Uses WinML on Windows for hardware acceleration. |
-| [**JavaScript**](js/) | 15 | Node.js SDK samples including native chat, embeddings, audio transcription, Electron desktop app, Copilot SDK integration, LangChain, tool calling, web server, tutorials, and WinML EP verification. |
+| [**C#**](cs/) | 14 | .NET SDK samples including native chat, embeddings, audio transcription, tool calling, model management, web server, vision via Responses API, tutorials, and WinML EP verification. Uses WinML on Windows for hardware acceleration. |
+| [**JavaScript**](js/) | 16 | Node.js SDK samples including native chat, embeddings, audio transcription, Electron desktop app, Copilot SDK integration, LangChain, tool calling, web server, vision via Responses API, tutorials, and WinML EP verification. |
 | [**Python**](python/) | 14 | Python samples using the OpenAI-compatible API, including chat, embeddings, audio transcription, LangChain integration, tool calling, web server, Responses API, tutorials, and WinML EP verification. |
-| [**Rust**](rust/) | 11 | Rust SDK samples including native chat, embeddings, audio transcription, tool calling, web server, tutorials, and WinML EP verification. |
+| [**Rust**](rust/) | 12 | Rust SDK samples including native chat, embeddings, audio transcription, tool calling, web server, vision via Responses API, tutorials, and WinML EP verification. |
 | [**C++**](cpp/) | 1 | C++ sample for live audio transcription. |

samples/cs/README.md

@@ -15,6 +15,7 @@ Both packages provide the same APIs, so the same source code works on all platfo
 | [embeddings](embeddings/) | Generate single and batch text embeddings using the Foundry Local SDK. |
 | [audio-transcription-example](audio-transcription-example/) | Transcribe audio files using the Foundry Local SDK. |
 | [foundry-local-web-server](foundry-local-web-server/) | Set up a local OpenAI-compliant web server. |
+| [foundry-local-web-server-responses-vision](foundry-local-web-server-responses-vision/) | Stream a vision (image understanding) response from the local web server using the Responses API. |
 | [tool-calling-foundry-local-sdk](tool-calling-foundry-local-sdk/) | Use tool calling with native chat completions. |
 | [tool-calling-foundry-local-web-server](tool-calling-foundry-local-web-server/) | Use tool calling with the local web server. |
 | [model-management-example](model-management-example/) | Manage models, variant selection, and updates. |

samples/cs/foundry-local-web-server-responses-vision/FoundryLocalWebServerResponsesVision.csproj

@@ -0,0 +1,54 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <!-- Windows: target Windows SDK for WinML hardware acceleration -->
+  <PropertyGroup Condition="$([MSBuild]::IsOSPlatform('Windows'))">
+    <TargetFramework>net9.0-windows10.0.18362.0</TargetFramework>
+    <Platforms>ARM64;x64</Platforms>
+    <WindowsPackageType>None</WindowsPackageType>
+    <EnableCoreMrtTooling>false</EnableCoreMrtTooling>
+  </PropertyGroup>
+
+  <!-- Non-Windows: standard .NET -->
+  <PropertyGroup Condition="!$([MSBuild]::IsOSPlatform('Windows'))">
+    <TargetFramework>net9.0</TargetFramework>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(RuntimeIdentifier)'==''">
+    <RuntimeIdentifier>$(NETCoreSdkRuntimeIdentifier)</RuntimeIdentifier>
+  </PropertyGroup>
+
+  <!-- Windows: WinML for hardware acceleration -->
+  <ItemGroup Condition="$([MSBuild]::IsOSPlatform('Windows'))">
+    <PackageReference Include="Microsoft.AI.Foundry.Local.WinML" />
+  </ItemGroup>
+
+  <!-- Non-Windows: standard SDK -->
+  <ItemGroup Condition="!$([MSBuild]::IsOSPlatform('Windows'))">
+    <PackageReference Include="Microsoft.AI.Foundry.Local" />
+  </ItemGroup>
+
+  <!-- Linux GPU support -->
+  <ItemGroup Condition="'$(RuntimeIdentifier)' == 'linux-x64'">
+    <PackageReference Include="Microsoft.ML.OnnxRuntime.Gpu" />
+    <PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.Cuda" />
+  </ItemGroup>
+
+  <!-- Bundled test image is copied next to the binary so the default path resolves at runtime -->
+  <ItemGroup>
+    <None Update="test_image.jpg">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+  <!-- Shared utilities -->
+  <ItemGroup>
+    <Compile Include="../Shared/*.cs" />
+  </ItemGroup>
+
+</Project>

samples/cs/foundry-local-web-server-responses-vision/FoundryLocalWebServerResponsesVision.sln

@@ -0,0 +1,34 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.0.31903.59
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FoundryLocalWebServerResponsesVision", "FoundryLocalWebServerResponsesVision.csproj", "{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Debug|x64 = Debug|x64
+		Debug|x86 = Debug|x86
+		Release|Any CPU = Release|Any CPU
+		Release|x64 = Release|x64
+		Release|x86 = Release|x86
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Debug|Any CPU.ActiveCfg = Debug|ARM64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Debug|Any CPU.Build.0 = Debug|ARM64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Debug|x64.ActiveCfg = Debug|x64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Debug|x64.Build.0 = Debug|x64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Debug|x86.ActiveCfg = Debug|ARM64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Debug|x86.Build.0 = Debug|ARM64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Release|Any CPU.ActiveCfg = Release|ARM64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Release|Any CPU.Build.0 = Release|ARM64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Release|x64.ActiveCfg = Release|x64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Release|x64.Build.0 = Release|x64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Release|x86.ActiveCfg = Release|ARM64
+		{8B4D2C97-2B5D-4A4E-9D31-7C8A6E6F3F11}.Release|x86.Build.0 = Release|ARM64
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

samples/cs/foundry-local-web-server-responses-vision/Program.cs

@@ -0,0 +1,232 @@
+// <complete_code>
+// <imports>
+using System.Net.Http.Headers;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.AI.Foundry.Local;
+// </imports>
+
+const int DefaultMaxOutputTokens = 8192;
+
+if (args.Length < 1)
+{
+    Console.Error.WriteLine("Usage: dotnet run -- <model_alias_or_id> [image_path]");
+    Console.Error.WriteLine("         dotnet run -- --list-models");
+    Console.Error.WriteLine("  Example: dotnet run -- qwen3.5-0.8b");
+    Console.Error.WriteLine("  Example: dotnet run -- Qwen2.5-VL-7B-Instruct-generic-cpu");
+    return 1;
+}
+
+bool listModels = args[0] is "--list-models" or "-l";
+string? modelIdentifier = listModels ? null : args[0];
+string defaultImage = Path.Combine(AppContext.BaseDirectory, "test_image.jpg");
+string imagePath = !listModels && args.Length > 1 ? args[1] : defaultImage;
+
+// <init>
+var config = new Configuration
+{
+    AppName = "foundry_local_samples",
+    LogLevel = Microsoft.AI.Foundry.Local.LogLevel.Information,
+    Web = new Configuration.WebService
+    {
+        Urls = "http://127.0.0.1:52496"
+    }
+};
+
+await FoundryLocalManager.CreateAsync(config, Utils.GetAppLogger());
+var mgr = FoundryLocalManager.Instance;
+
+Console.WriteLine("\nDownloading execution providers:");
+var currentEp = "";
+await mgr.DownloadAndRegisterEpsAsync((epName, percent) =>
+{
+    if (epName != currentEp)
+    {
+        if (currentEp != "") Console.WriteLine();
+        currentEp = epName;
+    }
+    Console.Write($"\r  {epName.PadRight(30)}  {percent,6:F1}%");
+});
+if (currentEp != "") Console.WriteLine();
+// </init>
+
+var catalog = await mgr.GetCatalogAsync();
+
+if (listModels)
+{
+    var allModels = await catalog.ListModelsAsync();
+    var visionModels = allModels
+        .Where(m => (m.Info?.Task ?? "").Contains("vision", StringComparison.OrdinalIgnoreCase))
+        .OrderBy(m => m.Alias)
+        .ToList();
+
+    if (visionModels.Count == 0)
+    {
+        Console.WriteLine("\nNo vision models found in catalog.");
+        return 0;
+    }
+
+    var totalVariants = visionModels.Sum(m => m.Variants.Count);
+    Console.WriteLine($"\nVision models in catalog ({visionModels.Count} aliases, {totalVariants} variants):");
+    Console.WriteLine($"  {"ALIAS",-32}  {"INPUT MODALITIES",-20}  {"OUTPUT MODALITIES",-20}  {"TASK",-24}  CAPABILITIES");
+    foreach (var m in visionModels)
+    {
+        var task = m.Info?.Task ?? "";
+        var capabilities = m.Info?.Capabilities ?? "";
+        var inMod = m.Info?.InputModalities ?? "";
+        var outMod = m.Info?.OutputModalities ?? "";
+        Console.WriteLine($"  {m.Alias,-32}  {inMod,-20}  {outMod,-20}  {task,-24}  {capabilities}");
+
+        var variants = m.Variants
+            .OrderBy(v => v.Info?.Runtime?.DeviceType.ToString() ?? "")
+            .ThenBy(v => v.Info?.Runtime?.ExecutionProvider ?? "")
+            .ThenBy(v => v.Id)
+            .ToList();
+        if (variants.Count == 0) continue;
+
+        Console.WriteLine($"      {"VARIANT ID",-54}  {"DEVICE",-6}  {"EXECUTION PROVIDER",-32}  {"SIZE (MB)",10}  CACHED");
+        foreach (var v in variants)
+        {
+            var rt = v.Info?.Runtime;
+            var device = rt?.DeviceType.ToString() ?? "";
+            var ep = rt?.ExecutionProvider ?? "";
+            var size = v.Info?.FileSizeMb is int s ? s.ToString().PadLeft(10) : new string(' ', 10);
+            var cached = await v.IsCachedAsync() ? "yes" : "no";
+            Console.WriteLine($"      {v.Id,-54}  {device,-6}  {ep,-32}  {size}  {cached}");
+        }
+    }
+    return 0;
+}
+
+// <model_setup>
+var model = await catalog.GetModelAsync(modelIdentifier!);
+if (model is null)
+{
+    model = await catalog.GetModelVariantAsync(modelIdentifier!);
+}
+if (model is null)
+{
+    var available = (await catalog.ListModelsAsync()).Select(m => m.Alias);
+    Console.Error.WriteLine($"\nModel '{modelIdentifier}' not found in catalog (tried alias and variant id).");
+    Console.Error.WriteLine($"Available aliases: {string.Join(", ", available)}");
+    Console.Error.WriteLine("Run with --list-models to see variant ids.");
+    return 1;
+}
+
+if (!await model.IsCachedAsync())
+{
+    Console.WriteLine($"\nDownloading model {modelIdentifier}...");
+    await model.DownloadAsync(progress =>
+    {
+        Console.Write($"\rDownloading model: {progress:F2}%");
+        if (progress >= 100f) Console.WriteLine();
+    });
+    Console.WriteLine("Model downloaded");
+}
+
+Console.WriteLine("\nLoading model...");
+await model.LoadAsync();
+Console.WriteLine("Model loaded");
+// </model_setup>
+
+// <server_setup>
+Console.WriteLine("\nStarting web service...");
+await mgr.StartWebServiceAsync();
+var baseUrl = config.Web.Urls!.TrimEnd('/') + "/v1";
+Console.WriteLine($"Web service started on {baseUrl}");
+// </server_setup>
+
+// <inference>
+Console.WriteLine($"\nPreparing image: {imagePath}");
+var (imageB64, mediaType) = EncodeImage(imagePath);
+
+// The Foundry Local Responses API accepts an array of message items with input_text /
+// input_image content parts. The input_image part uses Foundry-specific `image_data` and
+// `media_type` fields (in place of OpenAI's `image_url`).
+var visionInput = new JsonArray
+{
+    new JsonObject
+    {
+        ["type"] = "message",
+        ["role"] = "user",
+        ["content"] = new JsonArray
+        {
+            new JsonObject { ["type"] = "input_text", ["text"] = "Describe this image." },
+            new JsonObject
+            {
+                ["type"] = "input_image",
+                ["image_data"] = imageB64,
+                ["media_type"] = mediaType,
+            },
+        },
+    },
+};
+
+var body = new JsonObject
+{
+    ["model"] = model.Id,
+    ["input"] = visionInput,
+    ["max_output_tokens"] = DefaultMaxOutputTokens,
+    ["stream"] = true,
+};
+
+using var http = new HttpClient { Timeout = Timeout.InfiniteTimeSpan };
+using var request = new HttpRequestMessage(HttpMethod.Post, $"{baseUrl}/responses")
+{
+    Content = new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json"),
+};
+request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/event-stream"));
+request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", "notneeded");
+
+Console.WriteLine("\nStreaming vision response...");
+using var response = await http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
+response.EnsureSuccessStatusCode();
+
+await using var stream = await response.Content.ReadAsStreamAsync();
+using var reader = new StreamReader(stream);
+
+Console.Write("[ASSISTANT]: ");
+while (await reader.ReadLineAsync() is string line)
+{
+    if (string.IsNullOrEmpty(line) || !line.StartsWith("data: ", StringComparison.Ordinal))
+        continue;
+    var data = line["data: ".Length..];
+    if (data == "[DONE]") break;
+    try
+    {
+        using var doc = JsonDocument.Parse(data);
+        var root = doc.RootElement;
+        if (root.TryGetProperty("type", out var t) &&
+            t.GetString() == "response.output_text.delta" &&
+            root.TryGetProperty("delta", out var d))
+        {
+            Console.Write(d.GetString());
+        }
+    }
+    catch (JsonException) { /* ignore non-JSON keepalives */ }
+}
+Console.WriteLine();
+// </inference>
+
+await mgr.StopWebServiceAsync();
+await model.UnloadAsync();
+return 0;
+
+static (string Base64, string MediaType) EncodeImage(string path)
+{
+    var mediaTypes = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
+    {
+        [".jpg"] = "image/jpeg",
+        [".jpeg"] = "image/jpeg",
+        [".png"] = "image/png",
+        [".gif"] = "image/gif",
+        [".bmp"] = "image/bmp",
+        [".webp"] = "image/webp",
+    };
+    var ext = Path.GetExtension(path);
+    var mediaType = mediaTypes.TryGetValue(ext, out var m) ? m : "image/jpeg";
+    var bytes = File.ReadAllBytes(path);
+    return (Convert.ToBase64String(bytes), mediaType);
+}
+// </complete_code>

samples/js/README.md

@@ -19,6 +19,7 @@ These samples demonstrate how to use the Foundry Local JavaScript SDK (`foundry-
 | [langchain-integration-example](langchain-integration-example/) | LangChain.js integration for building text generation chains. |
 | [tool-calling-foundry-local](tool-calling-foundry-local/) | Tool calling with custom function definitions and streaming responses. |
 | [web-server-example](web-server-example/) | Start a local OpenAI-compatible web server and call it with the OpenAI SDK. |
+| [web-server-responses-vision-example](web-server-responses-vision-example/) | Stream a vision (image understanding) response from the local web server using the Responses API. |
 | [tutorial-chat-assistant](tutorial-chat-assistant/) | Build an interactive multi-turn chat assistant (tutorial). |
 | [tutorial-document-summarizer](tutorial-document-summarizer/) | Summarize documents with AI (tutorial). |
 | [tutorial-tool-calling](tutorial-tool-calling/) | Create a tool-calling assistant (tutorial). |