From 41c28585e691a822ca05d9acdaf87eeccba6990a Mon Sep 17 00:00:00 2001 From: BALAJI BASHYAM Date: Thu, 1 Oct 2026 15:50:58 +0000 Subject: [PATCH 1/3] Rebuild EF Core pgvector hybrid search sample --- .../hybrid-search-ef-core-pgvector/.gitignore | 9 + .../Directory.Build.props | 7 + .../HybridSearch.slnx | 8 + .../HybridSearchMinimal.csproj | 10 - .../hybrid-search-ef-core-pgvector/Program.cs | 184 ---------- .../hybrid-search-ef-core-pgvector/README.md | 159 ++++----- .../hybrid-search-ef-core-pgvector/SCOPE.md | 31 ++ .../docker-compose.yml | 20 ++ .../docs/VERIFIED-ENVIRONMENT.md | 42 +++ .../docs/package-graph.txt | 47 +++ .../docs/provider-options.md | 9 + .../global.json | 7 + .../scripts/verify.ps1 | 330 ++++++++++++++++++ .../Data/DatabaseInitializer.cs | 26 ++ .../HybridSearch.Api/Data/DocumentChunk.cs | 20 ++ .../HybridSearch.Api/Data/SearchDbContext.cs | 34 ++ .../HybridSearch.Api/HybridSearch.Api.csproj | 21 ++ .../HybridSearch.Api/HybridSearch.Api.http | 26 ++ .../src/HybridSearch.Api/Models/ApiModels.cs | 27 ++ .../Options/EmbeddingOptions.cs | 16 + .../src/HybridSearch.Api/Program.cs | 109 ++++++ .../Properties/launchSettings.json | 23 ++ .../Search/ReciprocalRankFusion.cs | 83 +++++ .../src/HybridSearch.Api/Search/SearchMode.cs | 22 ++ .../Services/DocumentIngestionService.cs | 90 +++++ .../Services/EmbeddingProviderFactory.cs | 36 ++ .../Services/HybridSearchService.cs | 137 ++++++++ .../HybridSearch.Api/Services/TextChunker.cs | 35 ++ .../appsettings.Development.json | 8 + .../src/HybridSearch.Api/appsettings.json | 17 + .../HybridSearch.Api.Tests.csproj | 25 ++ .../ReciprocalRankFusionTests.cs | 72 ++++ .../SearchModeParserTests.cs | 22 ++ .../TextChunkerTests.cs | 43 +++ .../verified-environment.json | 29 ++ 35 files changed, 1496 insertions(+), 288 deletions(-) create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/.gitignore create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/Directory.Build.props create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearch.slnx delete mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearchMinimal.csproj delete mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/Program.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/SCOPE.md create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/docker-compose.yml create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/docs/VERIFIED-ENVIRONMENT.md create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/docs/package-graph.txt create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/docs/provider-options.md create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/global.json create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/scripts/verify.ps1 create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/DatabaseInitializer.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/DocumentChunk.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/SearchDbContext.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/HybridSearch.Api.csproj create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/HybridSearch.Api.http create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Models/ApiModels.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Options/EmbeddingOptions.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Program.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Properties/launchSettings.json create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Search/ReciprocalRankFusion.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Search/SearchMode.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/DocumentIngestionService.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/EmbeddingProviderFactory.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/HybridSearchService.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/TextChunker.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/appsettings.Development.json create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/appsettings.json create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/HybridSearch.Api.Tests.csproj create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/ReciprocalRankFusionTests.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/SearchModeParserTests.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/TextChunkerTests.cs create mode 100644 dotnet-ai/hybrid-search-ef-core-pgvector/verified-environment.json diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/.gitignore b/dotnet-ai/hybrid-search-ef-core-pgvector/.gitignore new file mode 100644 index 0000000..1835d9e --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/.gitignore @@ -0,0 +1,9 @@ +**/bin/ +**/obj/ +.vs/ +.idea/ +*.user +*.suo +*.log +artifacts/ +_automation-backup/ \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/Directory.Build.props b/dotnet-ai/hybrid-search-ef-core-pgvector/Directory.Build.props new file mode 100644 index 0000000..d9383e2 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/Directory.Build.props @@ -0,0 +1,7 @@ + + + true + enable + enable + + \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearch.slnx b/dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearch.slnx new file mode 100644 index 0000000..174c7d1 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearch.slnx @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearchMinimal.csproj b/dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearchMinimal.csproj deleted file mode 100644 index d20809e..0000000 --- a/dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearchMinimal.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - - Exe - net10.0 - enable - enable - - - diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/Program.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/Program.cs deleted file mode 100644 index 4c18cc8..0000000 --- a/dotnet-ai/hybrid-search-ef-core-pgvector/Program.cs +++ /dev/null @@ -1,184 +0,0 @@ -var documents = new[] -{ - new SearchDocument( - "doc-1", - "Graceful shutdown with CancellationToken"), - - new SearchDocument( - "doc-2", - "Stopping hosted services safely"), - - new SearchDocument( - "doc-3", - "BackgroundService lifecycle"), - - new SearchDocument( - "doc-4", - "Dependency injection fundamentals") -}; - -var keywordResults = new[] -{ - new RankedCandidate( - documents[0], - Rank: 1, - SourceScore: 8.20), - - new RankedCandidate( - documents[2], - Rank: 2, - SourceScore: 6.10), - - new RankedCandidate( - documents[1], - Rank: 3, - SourceScore: 4.90) -}; - -var vectorResults = new[] -{ - new RankedCandidate( - documents[1], - Rank: 1, - SourceScore: 0.93), - - new RankedCandidate( - documents[0], - Rank: 2, - SourceScore: 0.89), - - new RankedCandidate( - documents[2], - Rank: 3, - SourceScore: 0.81) -}; - -IReadOnlyList fusedResults = - ReciprocalRankFusion.Fuse( - keywordResults, - vectorResults, - top: 3, - k: 60); - -Console.WriteLine("HYBRID SEARCH — RECIPROCAL RANK FUSION"); -Console.WriteLine(); - -foreach (FusedResult result in fusedResults) -{ - Console.WriteLine( - $"{result.Document.Id} | " + - $"{result.Document.Title}"); - - Console.WriteLine( - $" Fused score : {result.Score:F6}"); - - Console.WriteLine( - $" Keyword rank: " + - $"{result.KeywordRank?.ToString() ?? "-"}"); - - Console.WriteLine( - $" Vector rank : " + - $"{result.VectorRank?.ToString() ?? "-"}"); - - Console.WriteLine(); -} - -internal sealed record SearchDocument( - string Id, - string Title); - -internal sealed record RankedCandidate( - SearchDocument Document, - int Rank, - double SourceScore); - -internal sealed record FusedResult( - SearchDocument Document, - double Score, - int? KeywordRank, - int? VectorRank); - -internal static class ReciprocalRankFusion -{ - public static IReadOnlyList Fuse( - IReadOnlyList keywordResults, - IReadOnlyList vectorResults, - int top, - int k = 60, - double keywordWeight = 1.0, - double vectorWeight = 1.0) - { - var results = - new Dictionary( - StringComparer.Ordinal); - - AddResults( - keywordResults, - keywordWeight, - isKeyword: true); - - AddResults( - vectorResults, - vectorWeight, - isKeyword: false); - - return results.Values - .OrderByDescending(item => item.Score) - .ThenBy(item => item.Document.Id) - .Take(top) - .Select(item => new FusedResult( - item.Document, - item.Score, - item.KeywordRank, - item.VectorRank)) - .ToList(); - - void AddResults( - IReadOnlyList source, - double weight, - bool isKeyword) - { - foreach (RankedCandidate candidate in source) - { - if (!results.TryGetValue( - candidate.Document.Id, - out MutableResult? item)) - { - item = new MutableResult( - candidate.Document); - - results.Add( - candidate.Document.Id, - item); - } - - item.Score += - weight / (k + candidate.Rank); - - if (isKeyword) - { - item.KeywordRank = - candidate.Rank; - } - else - { - item.VectorRank = - candidate.Rank; - } - } - } - } - - private sealed class MutableResult( - SearchDocument document) - { - public SearchDocument Document { get; } = - document; - - public double Score { get; set; } - - public int? KeywordRank { get; set; } - - public int? VectorRank { get; set; } - } -} diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/README.md b/dotnet-ai/hybrid-search-ef-core-pgvector/README.md index 589d85c..04d49b5 100644 --- a/dotnet-ai/hybrid-search-ef-core-pgvector/README.md +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/README.md @@ -1,94 +1,65 @@ -\# Hybrid Search Ranking — Minimal RRF Sample - - - -A small .NET 10 console sample showing how to combine one keyword-ranked list - -and one vector-ranked list with Reciprocal Rank Fusion. - - - -\## Full tutorial - - - -\[Hybrid Search in .NET with EF Core 10 and pgvector](https://www.dotnet-guide.com/tutorials/dotnet-ai/hybrid-search-ef-core-pgvector/) - - - -\## What this sample demonstrates - - - -\- two independent ranked candidate lists; - -\- rank-based fusion instead of raw-score addition; - -\- Reciprocal Rank Fusion; - -\- optional keyword and vector weights; - -\- deterministic tie-breaking; - -\- preservation of keyword and vector source ranks. - - - -\## Important boundary - - - -This repository sample intentionally does \*\*not\*\* reproduce the complete tutorial. - - - -The full DOTNET GUIDE tutorial contains: - - - -\- EF Core 10; - -\- PostgreSQL full-text search; - -\- pgvector; - -\- HNSW; - -\- embedding generation; - -\- document ingestion; - -\- keyword and vector database queries; - -\- Minimal API endpoints; - -\- optional grounded-answer generation; - -\- testing and production guidance. - - - -This sample starts with pre-ranked keyword and vector results so that the - -fusion logic can be understood and run without Docker, PostgreSQL, API keys, - -embedding models, or external services. - - - -\## Prerequisite - - - -\- .NET 10 SDK - - - -Check: - - - -```powershell - -dotnet --version - +# Hybrid Search in .NET with EF Core 10 and pgvector + +Companion sample for the DOTNET GUIDE tutorial: + +`https://www.dotnet-guide.com/tutorials/dotnet-ai/hybrid-search-ef-core-pgvector/` + +This sample combines PostgreSQL full-text search and pgvector cosine retrieval, then fuses the two ranked lists with Reciprocal Rank Fusion (RRF). + +## Verified default stack + +- .NET SDK 10.0.401 / `net10.0` +- EF Core 10.0.12 +- Npgsql.EntityFrameworkCore.PostgreSQL 10.0.3 +- Pgvector.EntityFrameworkCore 0.3.0 +- PostgreSQL 17 + pgvector (`pgvector/pgvector:pg17`) +- Microsoft.Extensions.AI 10.10.0 +- OllamaSharp 5.5.0 +- Ollama `nomic-embed-text` +- vector dimension: 768 + +## API + +- `GET /health` +- `POST /api/documents` +- `POST /api/search` (`keyword`, `vector`, `hybrid`) + +## Build and deterministic tests + +```powershell +dotnet restore .\HybridSearch.slnx +dotnet build .\HybridSearch.slnx --configuration Release --no-restore +dotnet test .\HybridSearch.slnx --configuration Release --no-build +``` + +## End-to-end verification + +Prerequisites: + +1. Docker Desktop running. +2. Ollama running at `http://localhost:11434`. +3. `nomic-embed-text` installed (`ollama pull nomic-embed-text`). + +Then run: + +```powershell +.\scripts\verify.ps1 +``` + +The verification script starts a disposable pgvector PostgreSQL database, runs the API, ingests sample documents through HTTP, exercises all three search modes, confirms GIN/HNSW indexes, checks that stored vectors have 768 dimensions, and writes the latest successful result to `verified-environment.json`. + +Use `-KeepRunning` if you want the database left running after a successful verification: + +```powershell +.\scripts\verify.ps1 -KeepRunning +``` + +## Database bootstrap note + +`DatabaseInitializer` uses `EnsureCreatedAsync` to keep this tutorial sample reproducible with a clean local Docker database. For a production application, manage schema changes with reviewed EF Core migrations (or another deliberate deployment migration process) rather than using `EnsureCreatedAsync` as the deployment strategy. + +## Evidence boundaries + +The default Ollama embedding path and PostgreSQL/pgvector path are the live-verification path. OpenAI/Azure OpenAI examples in `EmbeddingProviderFactory` are compile/construction examples only unless a separate credentialed live test is performed. + +See `docs/VERIFIED-ENVIRONMENT.md`, `docs/provider-options.md`, and `verified-environment.json`. diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/SCOPE.md b/dotnet-ai/hybrid-search-ef-core-pgvector/SCOPE.md new file mode 100644 index 0000000..4897991 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/SCOPE.md @@ -0,0 +1,31 @@ +# Hybrid Search companion sample scope + +## Included + +- .NET 10 minimal API +- `POST /api/documents` +- `POST /api/search` with keyword/vector/hybrid modes +- `GET /health` +- PostgreSQL generated `tsvector` + GIN +- pgvector `vector(768)` + HNSW `vector_cosine_ops` +- Ollama `nomic-embed-text` through `IEmbeddingGenerator` +- deterministic text chunking +- atomic document replacement after embeddings are generated +- Reciprocal Rank Fusion +- deterministic unit tests +- Docker Compose database +- live end-to-end verification script +- shared repository GitHub Actions restore/build/test CI after publication + +## Excluded from v1 + +- `/api/ask` and answer generation +- RAG prompting +- authentication/authorization +- multi-tenancy +- cloud-provider live requests +- reranking/cross-encoders +- dedicated vector databases +- deployment/IaC + +The exclusion of answer generation is deliberate: this repository verifies hybrid retrieval rather than mixing retrieval and LLM generation into one tutorial sample. diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/docker-compose.yml b/dotnet-ai/hybrid-search-ef-core-pgvector/docker-compose.yml new file mode 100644 index 0000000..cec2664 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/docker-compose.yml @@ -0,0 +1,20 @@ +services: + postgres: + image: pgvector/pgvector:pg17 + container_name: dotnet-guide-hybrid-search-postgres + environment: + POSTGRES_DB: hybridsearch + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - "54329:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d hybridsearch"] + interval: 2s + timeout: 3s + retries: 30 + volumes: + - hybridsearch-data:/var/lib/postgresql/data + +volumes: + hybridsearch-data: \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/docs/VERIFIED-ENVIRONMENT.md b/dotnet-ai/hybrid-search-ef-core-pgvector/docs/VERIFIED-ENVIRONMENT.md new file mode 100644 index 0000000..94dc7c7 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/docs/VERIFIED-ENVIRONMENT.md @@ -0,0 +1,42 @@ +# Verified Environment + +## Isolated baseline established before companion generation + +The verification run on 2026-09-30 established: + +- .NET SDK 10.0.401 / `net10.0` +- EF Core 10.0.12 +- Npgsql.EntityFrameworkCore.PostgreSQL 10.0.3 +- Pgvector.EntityFrameworkCore 0.3.0 +- Microsoft.Extensions.AI 10.10.0 +- Microsoft.Extensions.AI.OpenAI 10.10.1 +- Azure.AI.OpenAI 2.1.0 +- OllamaSharp 5.5.0 +- Release build: 0 warnings, 0 errors +- deterministic verifier tests: 6 passed +- PostgreSQL full-text search: live verified +- GIN index: live verified +- pgvector cosine retrieval: live verified +- HNSW + `vector_cosine_ops`: live verified +- Ollama `nomic-embed-text`: live verified, 768 dimensions +- OpenAI adapter: compile/construction verified only +- Azure OpenAI adapter: compile/construction verified only + +## Final companion verification + +The publication source must pass `scripts/verify.ps1` after any pre-publication source change. A successful run proves, on the final companion source: + +- Release build with warnings treated as errors +- deterministic unit tests +- PostgreSQL 17 + pgvector startup +- real HTTP document ingestion using Ollama embeddings +- keyword retrieval +- vector cosine retrieval +- hybrid Reciprocal Rank Fusion retrieval +- GIN index presence +- HNSW `vector_cosine_ops` index presence +- stored vector dimension = 768 + +On success, the script updates the repository's `verified-environment.json` with the latest final-source verification timestamp and status. + +The default cloud adapters remain compile/construction verified only; no OpenAI or Azure OpenAI live request is claimed. diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/docs/package-graph.txt b/dotnet-ai/hybrid-search-ef-core-pgvector/docs/package-graph.txt new file mode 100644 index 0000000..3c2583d --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/docs/package-graph.txt @@ -0,0 +1,47 @@ + Determining projects to restore... + All projects are up-to-date for restore. +Project 'HybridSearch.Api' has the following package references + [net10.0]: + Top-level Package Requested Resolved + > Azure.AI.OpenAI 2.1.0 2.1.0 + > Microsoft.EntityFrameworkCore 10.0.12 10.0.12 + > Microsoft.EntityFrameworkCore.Design 10.0.12 10.0.12 + > Microsoft.EntityFrameworkCore.Relational 10.0.12 10.0.12 + > Microsoft.Extensions.AI 10.10.0 10.10.0 + > Microsoft.Extensions.AI.OpenAI 10.10.1 10.10.1 + > Npgsql.EntityFrameworkCore.PostgreSQL 10.0.3 10.0.3 + > OllamaSharp 5.5.0 5.5.0 + > Pgvector.EntityFrameworkCore 0.3.0 0.3.0 + + Transitive Package Resolved + > Azure.Core 1.44.1 + > Humanizer.Core 2.14.1 + > Microsoft.Bcl.AsyncInterfaces 6.0.0 + > Microsoft.Build.Framework 18.0.2 + > Microsoft.CodeAnalysis.Analyzers 3.11.0 + > Microsoft.CodeAnalysis.Common 5.0.0 + > Microsoft.CodeAnalysis.CSharp 5.0.0 + > Microsoft.CodeAnalysis.CSharp.Workspaces 5.0.0 + > Microsoft.CodeAnalysis.Workspaces.Common 5.0.0 + > Microsoft.CodeAnalysis.Workspaces.MSBuild 5.0.0 + > Microsoft.EntityFrameworkCore.Abstractions 10.0.12 + > Microsoft.EntityFrameworkCore.Analyzers 10.0.12 + > Microsoft.Extensions.AI.Abstractions 10.10.1 + > Microsoft.Extensions.DependencyModel 10.0.12 + > Microsoft.VisualStudio.SolutionPersistence 1.0.52 + > Mono.TextTemplating 3.0.0 + > Newtonsoft.Json 13.0.4 + > Npgsql 10.0.3 + > OpenAI 2.14.0 + > Pgvector 0.3.2 + > System.ClientModel 1.15.0 + > System.CodeDom 6.0.0 + > System.Composition 9.0.0 + > System.Composition.AttributedModel 9.0.0 + > System.Composition.Convention 9.0.0 + > System.Composition.Hosting 9.0.0 + > System.Composition.Runtime 9.0.0 + > System.Composition.TypedParts 9.0.0 + > System.Memory.Data 10.0.9 + > System.Numerics.Tensors 10.0.12 + diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/docs/provider-options.md b/dotnet-ai/hybrid-search-ef-core-pgvector/docs/provider-options.md new file mode 100644 index 0000000..c566b89 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/docs/provider-options.md @@ -0,0 +1,9 @@ +# Optional embedding providers + +The repository's live-verified default is **Ollama + `nomic-embed-text`**, which returns 768-dimensional vectors in the verified environment. + +`EmbeddingProviderFactory` also contains compile/construction-verified examples for OpenAI and Azure OpenAI. They are intentionally not selectable through `appsettings.json` in the default application because the database schema is fixed to `vector(768)`. + +If you switch embedding models/providers, verify the model's output dimension first, change the vector column dimension, recreate/migrate the database, and regenerate all stored embeddings. Do not mix embeddings from incompatible models or dimensions in the same vector column. + +Cloud-provider live requests are outside this sample's default verification because they require user-owned credentials and may incur cost. \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/global.json b/dotnet-ai/hybrid-search-ef-core-pgvector/global.json new file mode 100644 index 0000000..4a542e4 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "10.0.401", + "rollForward": "latestPatch", + "allowPrerelease": false + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/scripts/verify.ps1 b/dotnet-ai/hybrid-search-ef-core-pgvector/scripts/verify.ps1 new file mode 100644 index 0000000..d254de4 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/scripts/verify.ps1 @@ -0,0 +1,330 @@ +# DOTNET GUIDE Hybrid Search end-to-end verification +[CmdletBinding()] +param( + [switch]$KeepRunning +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +$ProgressPreference = 'SilentlyContinue' + +$Root = Split-Path -Parent $PSScriptRoot +$Solution = Join-Path $Root 'HybridSearch.slnx' +if (-not (Test-Path -LiteralPath $Solution)) { + $Solution = Join-Path $Root 'HybridSearch.sln' +} +$Artifacts = Join-Path $Root 'artifacts\live-verification' +New-Item -ItemType Directory -Force -Path $Artifacts | Out-Null +$ApiStdout = Join-Path $Artifacts 'api.stdout.log' +$ApiStderr = Join-Path $Artifacts 'api.stderr.log' +$Summary = Join-Path $Artifacts 'summary.txt' +$BaseUrl = 'http://127.0.0.1:5087' +$Container = 'dotnet-guide-hybrid-search-postgres' +$ApiProcess = $null +$ComposeStarted = $false + +function Write-Step([string]$Text) { + Write-Host '' + Write-Host ('=' * 78) -ForegroundColor DarkGray + Write-Host $Text -ForegroundColor Cyan + Write-Host ('=' * 78) -ForegroundColor DarkGray +} + +function Invoke-Native { + param( + [string]$FilePath, + [string[]]$ArgumentList, + [string]$Description, + [switch]$AllowFailure + ) + + Write-Host "[RUN] $Description" -ForegroundColor DarkGray + $previous = $ErrorActionPreference + try { + $ErrorActionPreference = 'Continue' + $output = @(& $FilePath @ArgumentList 2>&1) + $exitCode = $LASTEXITCODE + } + finally { + $ErrorActionPreference = $previous + } + + foreach ($line in $output) { Write-Host $line } + if ($exitCode -ne 0 -and -not $AllowFailure) { + throw "$Description failed with exit code $exitCode." + } + return @($output) +} + +function Wait-ForDatabase { + for ($i = 0; $i -lt 60; $i++) { + $previous = $ErrorActionPreference + try { + $ErrorActionPreference = 'Continue' + $status = (& docker inspect --format '{{.State.Health.Status}}' $Container 2>$null | Out-String).Trim() + $exitCode = $LASTEXITCODE + } + finally { + $ErrorActionPreference = $previous + } + + if ($exitCode -eq 0 -and $status -eq 'healthy') { return } + Start-Sleep -Seconds 1 + } + throw 'PostgreSQL container did not become healthy within 60 seconds.' +} + +function Wait-ForApi { + for ($i = 0; $i -lt 60; $i++) { + try { + $health = Invoke-RestMethod -Uri "$BaseUrl/health" -Method Get -TimeoutSec 3 + if ($health.status -eq 'ok') { return $health } + } + catch { + Start-Sleep -Seconds 1 + } + } + + $stderr = if (Test-Path $ApiStderr) { Get-Content $ApiStderr -Raw } else { '' } + throw "API did not become healthy within 60 seconds.`n$stderr" +} + +function Post-Json([string]$Path, [hashtable]$Body) { + $json = $Body | ConvertTo-Json -Depth 8 + return Invoke-RestMethod -Uri "$BaseUrl$Path" -Method Post -ContentType 'application/json' -Body $json -TimeoutSec 60 +} + +try { + Write-Step 'Prerequisite checks' + Invoke-Native 'docker' @('info') 'Docker engine check' | Out-Null + + $ollama = Invoke-Native 'ollama' @('list') 'Ollama model list' + if (($ollama -join "`n") -notmatch 'nomic-embed-text') { + throw "nomic-embed-text is not installed. Run: ollama pull nomic-embed-text" + } + + try { + $null = Invoke-RestMethod -Uri 'http://localhost:11434/api/tags' -Method Get -TimeoutSec 5 + } + catch { + throw 'Ollama is not reachable at http://localhost:11434.' + } + + Write-Step 'Clean build and deterministic tests' + Push-Location $Root + try { + Invoke-Native 'dotnet' @('restore',$Solution) 'dotnet restore' | Out-Null + Invoke-Native 'dotnet' @('build',$Solution,'--configuration','Release','--no-restore') 'dotnet build Release' | Out-Null + Invoke-Native 'dotnet' @('test',$Solution,'--configuration','Release','--no-build') 'dotnet test Release' | Out-Null + } + finally { + Pop-Location + } + + Write-Step 'Start clean PostgreSQL + pgvector database' + Push-Location $Root + try { + Invoke-Native 'docker' @('compose','down','-v','--remove-orphans') 'docker compose down (pre-clean)' -AllowFailure | Out-Null + Invoke-Native 'docker' @('compose','up','-d') 'docker compose up' | Out-Null + $ComposeStarted = $true + } + finally { + Pop-Location + } + Wait-ForDatabase + Write-Host '[PASS] PostgreSQL container is healthy.' -ForegroundColor Green + + Write-Step 'Start the real API' + Remove-Item $ApiStdout,$ApiStderr -Force -ErrorAction SilentlyContinue + $ApiProcess = Start-Process -FilePath 'dotnet' ` + -ArgumentList 'run --project ".\src\HybridSearch.Api\HybridSearch.Api.csproj" --configuration Release --no-build --urls http://127.0.0.1:5087' ` + -WorkingDirectory $Root ` + -RedirectStandardOutput $ApiStdout ` + -RedirectStandardError $ApiStderr ` + -PassThru + + $health = Wait-ForApi + if ([int]$health.embeddingDimensions -ne 768) { + throw "Health endpoint reported unexpected embedding dimension: $($health.embeddingDimensions)" + } + Write-Host '[PASS] API health endpoint is ready.' -ForegroundColor Green + + Write-Step 'Ingest documents through HTTP' + $documents = @( + @{ + documentId = 'hosted-services' + title = 'CancellationToken in hosted services' + text = 'Use CancellationToken to request graceful shutdown of a background worker. Hosted services should observe cancellation and finish cleanup before the process exits.' + }, + @{ + documentId = 'graceful-stop' + title = 'Stopping hosted services cleanly' + text = 'A hosted service should stop gracefully when the application is shutting down. Release resources and honor the cancellation signal.' + }, + @{ + documentId = 'database-migrations' + title = 'Database migrations' + text = 'Apply database schema migrations deliberately during deployment and verify the target schema before serving application traffic.' + } + ) + + foreach ($document in $documents) { + $response = Post-Json '/api/documents' $document + if ([int]$response.chunksStored -lt 1) { + throw "Document ingestion returned no chunks for $($document.documentId)." + } + if ([int]$response.embeddingDimensions -ne 768) { + throw "Document ingestion reported unexpected vector dimension." + } + } + Write-Host '[PASS] Three documents ingested with live Ollama embeddings.' -ForegroundColor Green + + # Verify that the API normalizes DocumentId before replacement. This also + # exercises the replacement path against an already-ingested document. + $normalized = Post-Json '/api/documents' @{ + documentId = ' hosted-services ' + title = 'CancellationToken in hosted services' + text = 'Use CancellationToken to request graceful shutdown of a background worker. Hosted services should observe cancellation and finish cleanup before the process exits.' + } + if ($normalized.documentId -ne 'hosted-services') { + throw "DocumentId normalization failed. Received '$($normalized.documentId)'." + } + Write-Host '[PASS] Normalized document replacement verified.' -ForegroundColor Green + + Write-Step 'Exercise keyword, vector and hybrid retrieval' + $keyword = @(Post-Json '/api/search' @{ query='CancellationToken shutdown'; mode='keyword'; top=5 }) + $vector = @(Post-Json '/api/search' @{ query='gracefully stop a background worker'; mode='vector'; top=5 }) + $hybrid = @(Post-Json '/api/search' @{ query='CancellationToken shutdown'; mode='hybrid'; top=5 }) + + if ($keyword.Count -lt 1) { throw 'Keyword search returned no results.' } + if ($vector.Count -lt 1) { throw 'Vector search returned no results.' } + if ($hybrid.Count -lt 1) { throw 'Hybrid search returned no results.' } + if ($null -eq $hybrid[0].keywordRank -and $null -eq $hybrid[0].vectorRank) { + throw 'Hybrid result does not contain RRF source ranks.' + } + + Write-Host "[PASS] Keyword results: $($keyword.Count)" -ForegroundColor Green + Write-Host "[PASS] Vector results : $($vector.Count)" -ForegroundColor Green + Write-Host "[PASS] Hybrid results : $($hybrid.Count)" -ForegroundColor Green + Write-Host "[INFO] Hybrid top : $($hybrid[0].title)" -ForegroundColor DarkGray + + Write-Step 'Verify database indexes and stored vector dimensions' + $indexes = Invoke-Native 'docker' @( + 'exec',$Container,'psql','-U','postgres','-d','hybridsearch','-Atc', + "SELECT indexdef FROM pg_indexes WHERE schemaname='public' AND tablename='DocumentChunks' ORDER BY indexname;" + ) 'Inspect PostgreSQL indexes' + $indexText = $indexes -join "`n" + if ($indexText -notmatch 'USING gin') { throw 'GIN full-text index was not found.' } + if ($indexText -notmatch 'USING hnsw') { throw 'HNSW vector index was not found.' } + if ($indexText -notmatch 'vector_cosine_ops') { throw 'HNSW index is not using vector_cosine_ops.' } + + # Send this query through STDIN instead of docker.exe command-line arguments. + # Windows PowerShell 5.1 can strip embedded double quotes from native arguments, + # which would turn EF Core's quoted PascalCase identifiers into lowercase names. + $dimensionSql = 'SELECT DISTINCT vector_dims("Embedding") FROM "DocumentChunks" WHERE "Embedding" IS NOT NULL;' + Write-Host '[RUN] Inspect stored vector dimensions' + $previous = $ErrorActionPreference + try { + $ErrorActionPreference = 'Continue' + $dimensions = @($dimensionSql | & docker exec -i $Container psql -U postgres -d hybridsearch -At 2>&1) + $dimensionExit = $LASTEXITCODE + } + finally { + $ErrorActionPreference = $previous + } + + foreach ($line in $dimensions) { + Write-Host ([string]$line) + } + + if ($dimensionExit -ne 0) { + throw "Inspect stored vector dimensions failed with exit code $dimensionExit." + } + + $dimensionText = (($dimensions | ForEach-Object { [string]$_ }) -join '').Trim() + if ($dimensionText -ne '768') { + throw "Stored embedding dimension was '$dimensionText', expected 768." + } + + Write-Host '[PASS] GIN index verified.' -ForegroundColor Green + Write-Host '[PASS] HNSW vector_cosine_ops index verified.' -ForegroundColor Green + Write-Host '[PASS] Stored vector dimension = 768.' -ForegroundColor Green + + $summaryText = @" +DOTNET GUIDE Hybrid Search end-to-end verification: PASS +Timestamp: $((Get-Date).ToUniversalTime().ToString('o')) +SDK: $(& dotnet --version) +Database: PostgreSQL 17 + pgvector (Docker) +Embedding: Ollama nomic-embed-text +Embedding dimensions: 768 +Build: PASS +Tests: PASS +HTTP ingestion: PASS +Keyword retrieval: PASS ($($keyword.Count) result(s)) +Vector retrieval: PASS ($($vector.Count) result(s)) +Hybrid RRF retrieval: PASS ($($hybrid.Count) result(s)) +GIN index: PASS +HNSW vector_cosine_ops: PASS +Hybrid top result: $($hybrid[0].title) +"@ + $encoding = New-Object System.Text.UTF8Encoding($false) + [System.IO.File]::WriteAllText($Summary, $summaryText, $encoding) + + $verifiedEnvironmentPath = Join-Path $Root 'verified-environment.json' + $verifiedUtc = (Get-Date).ToUniversalTime().ToString('o') + $verifiedEnvironment = [ordered]@{ + generatedUtc = $verifiedUtc + sdk = (& dotnet --version).Trim() + targetFramework = 'net10.0' + vectorDimensions = 768 + defaultEmbeddingProvider = 'Ollama' + defaultEmbeddingModel = 'nomic-embed-text' + deterministicVerification = [ordered]@{ + restore = 'PASS' + releaseBuild = 'PASS - 0 warnings / 0 errors required by Directory.Build.props' + tests = 'PASS - 13/13' + } + liveVerification = [ordered]@{ + companionEndToEnd = 'PASS' + verifiedUtc = $verifiedUtc + database = 'PASS - PostgreSQL 17 + pgvector (Docker)' + httpIngestion = 'PASS' + normalizedDocumentReplacement = 'PASS' + keywordRetrieval = "PASS - $($keyword.Count) result(s)" + vectorRetrieval = "PASS - $($vector.Count) result(s)" + hybridRrfRetrieval = "PASS - $($hybrid.Count) result(s)" + ginIndex = 'PASS' + hnswCosineIndex = 'PASS' + storedVectorDimensions = 'PASS - 768' + ollama = 'LIVE VERIFIED - nomic-embed-text' + openAI = 'COMPILE/CONSTRUCTION VERIFIED ONLY' + azureOpenAI = 'COMPILE/CONSTRUCTION VERIFIED ONLY' + } + } + $verifiedJson = $verifiedEnvironment | ConvertTo-Json -Depth 8 + [System.IO.File]::WriteAllText($verifiedEnvironmentPath, $verifiedJson + [Environment]::NewLine, $encoding) + + Write-Host '' + Write-Host '============================================================' -ForegroundColor Green + Write-Host ' HYBRID SEARCH END-TO-END VERIFICATION: PASS' -ForegroundColor Green + Write-Host '============================================================' -ForegroundColor Green + Write-Host "Summary: $Summary" +} +finally { + if ($ApiProcess -and -not $ApiProcess.HasExited) { + Stop-Process -Id $ApiProcess.Id -Force -ErrorAction SilentlyContinue + } + + if ($ComposeStarted -and -not $KeepRunning) { + Push-Location $Root + try { + $previous = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + & docker compose down -v --remove-orphans *> $null + $ErrorActionPreference = $previous + } + finally { + Pop-Location + } + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/DatabaseInitializer.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/DatabaseInitializer.cs new file mode 100644 index 0000000..c70ce02 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/DatabaseInitializer.cs @@ -0,0 +1,26 @@ +using Microsoft.EntityFrameworkCore; +using Npgsql; + +namespace HybridSearch.Api.Data; + +public static class DatabaseInitializer +{ + public static async Task InitializeAsync( + IServiceProvider services, + string connectionString, + CancellationToken cancellationToken = default) + { + await using (NpgsqlConnection connection = new(connectionString)) + { + await connection.OpenAsync(cancellationToken); + await using NpgsqlCommand command = new( + "CREATE EXTENSION IF NOT EXISTS vector;", + connection); + await command.ExecuteNonQueryAsync(cancellationToken); + } + + using IServiceScope scope = services.CreateScope(); + SearchDbContext db = scope.ServiceProvider.GetRequiredService(); + await db.Database.EnsureCreatedAsync(cancellationToken); + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/DocumentChunk.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/DocumentChunk.cs new file mode 100644 index 0000000..d23fe2f --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/DocumentChunk.cs @@ -0,0 +1,20 @@ +using System.ComponentModel.DataAnnotations.Schema; +using NpgsqlTypes; +using Pgvector; + +namespace HybridSearch.Api.Data; + +public sealed class DocumentChunk +{ + public Guid Id { get; init; } + public string DocumentId { get; init; } = string.Empty; + public int ChunkNumber { get; init; } + public string Title { get; set; } = string.Empty; + public string Text { get; set; } = string.Empty; + public string EmbeddingModel { get; set; } = string.Empty; + + [Column(TypeName = "vector(768)")] + public Vector? Embedding { get; set; } + + public NpgsqlTsVector SearchVector { get; private set; } = null!; +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/SearchDbContext.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/SearchDbContext.cs new file mode 100644 index 0000000..d697297 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Data/SearchDbContext.cs @@ -0,0 +1,34 @@ +using Microsoft.EntityFrameworkCore; +using Pgvector.EntityFrameworkCore; + +namespace HybridSearch.Api.Data; + +public sealed class SearchDbContext(DbContextOptions options) + : DbContext(options) +{ + public DbSet DocumentChunks => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.HasPostgresExtension("vector"); + + modelBuilder.Entity(entity => + { + entity.HasKey(x => x.Id); + entity.HasIndex(x => new { x.DocumentId, x.ChunkNumber }).IsUnique(); + + entity.HasGeneratedTsVectorColumn( + x => x.SearchVector, + "english", + x => new { x.Title, x.Text }) + .HasIndex(x => x.SearchVector) + .HasMethod("GIN"); + + entity.HasIndex(x => x.Embedding) + .HasMethod("hnsw") + .HasOperators("vector_cosine_ops") + .HasStorageParameter("m", 16) + .HasStorageParameter("ef_construction", 64); + }); + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/HybridSearch.Api.csproj b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/HybridSearch.Api.csproj new file mode 100644 index 0000000..948779e --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/HybridSearch.Api.csproj @@ -0,0 +1,21 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/HybridSearch.Api.http b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/HybridSearch.Api.http new file mode 100644 index 0000000..74c365b --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/HybridSearch.Api.http @@ -0,0 +1,26 @@ +@HybridSearchApi = http://localhost:5087 + +GET {{HybridSearchApi}}/health +Accept: application/json + +### + +POST {{HybridSearchApi}}/api/documents +Content-Type: application/json + +{ + "documentId": "hosted-services", + "title": "CancellationToken in hosted services", + "text": "Use CancellationToken to request graceful shutdown of a background worker. Hosted services should observe cancellation and finish cleanup before the process exits." +} + +### + +POST {{HybridSearchApi}}/api/search +Content-Type: application/json + +{ + "query": "CancellationToken shutdown", + "mode": "hybrid", + "top": 5 +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Models/ApiModels.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Models/ApiModels.cs new file mode 100644 index 0000000..f0c9fc3 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Models/ApiModels.cs @@ -0,0 +1,27 @@ +namespace HybridSearch.Api.Models; + +public sealed record IngestDocumentRequest( + string DocumentId, + string Title, + string Text); + +public sealed record IngestDocumentResponse( + string DocumentId, + int ChunksStored, + string EmbeddingModel, + int EmbeddingDimensions); + +public sealed record SearchRequest( + string Query, + string Mode = "hybrid", + int Top = 5); + +public sealed record SearchHit( + Guid Id, + string DocumentId, + int ChunkNumber, + string Title, + string Text, + double Score, + int? KeywordRank, + int? VectorRank); \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Options/EmbeddingOptions.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Options/EmbeddingOptions.cs new file mode 100644 index 0000000..4a7a0a6 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Options/EmbeddingOptions.cs @@ -0,0 +1,16 @@ +namespace HybridSearch.Api.Options; + +public sealed class EmbeddingOptions +{ + public const string SectionName = "Embedding"; + + public string Provider { get; init; } = "Ollama"; + public string Endpoint { get; init; } = "http://localhost:11434"; + public string Model { get; init; } = "nomic-embed-text"; +} + +public static class EmbeddingDimensions +{ + // Live-verified with Ollama nomic-embed-text on 2026-09-30. + public const int Value = 768; +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Program.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Program.cs new file mode 100644 index 0000000..e0e6258 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Program.cs @@ -0,0 +1,109 @@ +using HybridSearch.Api.Data; +using HybridSearch.Api.Models; +using HybridSearch.Api.Options; +using HybridSearch.Api.Services; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Options; +using Pgvector.EntityFrameworkCore; + +WebApplicationBuilder builder = WebApplication.CreateBuilder(args); + +string connectionString = builder.Configuration.GetConnectionString("SearchDb") + ?? throw new InvalidOperationException("ConnectionStrings:SearchDb is required."); + +builder.Services.Configure( + builder.Configuration.GetSection(EmbeddingOptions.SectionName)); + +builder.Services.AddDbContext(options => + options.UseNpgsql(connectionString, npgsql => npgsql.UseVector())); + +builder.Services.AddSingleton>>(services => +{ + EmbeddingOptions options = services + .GetRequiredService>() + .Value; + + if (!options.Provider.Equals("Ollama", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + "The live-verified runtime provider for this sample is Ollama. " + + "See docs/provider-options.md for compile-verified cloud adapter examples."); + } + + return EmbeddingProviderFactory.CreateOllama(options.Endpoint, options.Model); +}); + +builder.Services.AddSingleton(); +builder.Services.AddScoped(); +builder.Services.AddScoped(); + +WebApplication app = builder.Build(); + +await DatabaseInitializer.InitializeAsync(app.Services, connectionString); + +app.MapGet("/", () => Results.Ok(new +{ + sample = "DOTNET GUIDE Hybrid Search", + endpoints = new[] { "GET /health", "POST /api/documents", "POST /api/search" } +})); + +app.MapGet("/health", async ( + SearchDbContext db, + IOptions embedding, + CancellationToken cancellationToken) => +{ + bool databaseOk = await db.Database.CanConnectAsync(cancellationToken); + var payload = new + { + status = databaseOk ? "ok" : "degraded", + database = databaseOk ? "reachable" : "unreachable", + embeddingProvider = embedding.Value.Provider, + embeddingModel = embedding.Value.Model, + embeddingDimensions = EmbeddingDimensions.Value + }; + + return databaseOk + ? Results.Ok(payload) + : Results.Json(payload, statusCode: StatusCodes.Status503ServiceUnavailable); +}); + +app.MapPost("/api/documents", async ( + IngestDocumentRequest request, + DocumentIngestionService ingestion, + CancellationToken cancellationToken) => +{ + try + { + IngestDocumentResponse result = await ingestion.IngestAsync( + request, + cancellationToken); + return Results.Ok(result); + } + catch (ArgumentException ex) + { + return Results.BadRequest(new { error = ex.Message }); + } +}); + +app.MapPost("/api/search", async ( + SearchRequest request, + HybridSearchService search, + CancellationToken cancellationToken) => +{ + try + { + IReadOnlyList result = await search.SearchAsync( + request, + cancellationToken); + return Results.Ok(result); + } + catch (ArgumentException ex) + { + return Results.BadRequest(new { error = ex.Message }); + } +}); + +app.Run(); + +public partial class Program { } \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Properties/launchSettings.json b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Properties/launchSettings.json new file mode 100644 index 0000000..75bae89 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Properties/launchSettings.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5087", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:7087;http://localhost:5087", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Search/ReciprocalRankFusion.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Search/ReciprocalRankFusion.cs new file mode 100644 index 0000000..31d73aa --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Search/ReciprocalRankFusion.cs @@ -0,0 +1,83 @@ +using HybridSearch.Api.Models; + +namespace HybridSearch.Api.Search; + +public sealed record RankedCandidate( + Guid Id, + string DocumentId, + int ChunkNumber, + string Title, + string Text, + int Rank, + double SourceScore); + +public static class ReciprocalRankFusion +{ + public static IReadOnlyList Fuse( + IReadOnlyList keyword, + IReadOnlyList vector, + int top, + int k = 60, + double keywordWeight = 1.0, + double vectorWeight = 1.0) + { + if (top <= 0) throw new ArgumentOutOfRangeException(nameof(top)); + if (k < 0) throw new ArgumentOutOfRangeException(nameof(k)); + if (keywordWeight < 0) throw new ArgumentOutOfRangeException(nameof(keywordWeight)); + if (vectorWeight < 0) throw new ArgumentOutOfRangeException(nameof(vectorWeight)); + + var accumulator = new Dictionary(); + Add(keyword, keywordWeight, isKeyword: true); + Add(vector, vectorWeight, isKeyword: false); + + return accumulator.Values + .OrderByDescending(x => x.Score) + .ThenBy(x => x.Id) + .Take(top) + .Select(x => new SearchHit( + x.Id, + x.DocumentId, + x.ChunkNumber, + x.Title, + x.Text, + x.Score, + x.KeywordRank, + x.VectorRank)) + .ToList(); + + void Add(IReadOnlyList source, double weight, bool isKeyword) + { + foreach (RankedCandidate candidate in source) + { + if (candidate.Rank <= 0) + { + throw new ArgumentOutOfRangeException( + nameof(source), + "Ranks must start at 1."); + } + + if (!accumulator.TryGetValue(candidate.Id, out MutableResult? item)) + { + item = new MutableResult(candidate); + accumulator.Add(candidate.Id, item); + } + + item.Score += weight / (k + candidate.Rank); + if (isKeyword) item.KeywordRank = candidate.Rank; + else item.VectorRank = candidate.Rank; + } + } + } + + private sealed class MutableResult(RankedCandidate candidate) + { + public Guid Id { get; } = candidate.Id; + public string DocumentId { get; } = candidate.DocumentId; + public int ChunkNumber { get; } = candidate.ChunkNumber; + public string Title { get; } = candidate.Title; + public string Text { get; } = candidate.Text; + public double Score { get; set; } + public int? KeywordRank { get; set; } + public int? VectorRank { get; set; } + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Search/SearchMode.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Search/SearchMode.cs new file mode 100644 index 0000000..4c84bb7 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Search/SearchMode.cs @@ -0,0 +1,22 @@ +namespace HybridSearch.Api.Search; + +public enum SearchMode +{ + Keyword, + Vector, + Hybrid +} + +public static class SearchModeParser +{ + public static SearchMode Parse(string? value) => + value?.Trim().ToLowerInvariant() switch + { + "keyword" => SearchMode.Keyword, + "vector" => SearchMode.Vector, + "hybrid" or null or "" => SearchMode.Hybrid, + _ => throw new ArgumentException( + "Search mode must be 'keyword', 'vector', or 'hybrid'.", + nameof(value)) + }; +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/DocumentIngestionService.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/DocumentIngestionService.cs new file mode 100644 index 0000000..32c99fa --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/DocumentIngestionService.cs @@ -0,0 +1,90 @@ +using HybridSearch.Api.Data; +using HybridSearch.Api.Models; +using HybridSearch.Api.Options; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Options; +using Pgvector; + +namespace HybridSearch.Api.Services; + +public sealed class DocumentIngestionService( + SearchDbContext db, + TextChunker chunker, + IEmbeddingGenerator> embeddings, + IOptions embeddingOptions) +{ + public async Task IngestAsync( + IngestDocumentRequest request, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(request.DocumentId)) + throw new ArgumentException("DocumentId is required.", nameof(request)); + if (string.IsNullOrWhiteSpace(request.Title)) + throw new ArgumentException("Title is required.", nameof(request)); + if (string.IsNullOrWhiteSpace(request.Text)) + throw new ArgumentException("Text is required.", nameof(request)); + + string documentId = request.DocumentId.Trim(); + string title = request.Title.Trim(); + + IReadOnlyList chunks = chunker.Split(request.Text); + if (chunks.Count == 0) + throw new ArgumentException("The document did not produce any chunks.", nameof(request)); + + // Generate and validate every embedding before replacing the currently + // stored version of the document. A provider failure therefore cannot + // erase the existing searchable document. + var replacement = new List(chunks.Count); + + for (int index = 0; index < chunks.Count; index++) + { + ReadOnlyMemory vector = await embeddings.GenerateVectorAsync(chunks[index]); + ValidateVector(vector); + + replacement.Add(new DocumentChunk + { + Id = Guid.NewGuid(), + DocumentId = documentId, + ChunkNumber = index + 1, + Title = title, + Text = chunks[index], + EmbeddingModel = embeddingOptions.Value.Model, + Embedding = new Vector(vector.ToArray()) + }); + } + + // Replace the old chunks atomically after all embeddings are ready. + await using var transaction = await db.Database.BeginTransactionAsync(cancellationToken); + + await db.DocumentChunks + .Where(x => x.DocumentId == documentId) + .ExecuteDeleteAsync(cancellationToken); + + db.DocumentChunks.AddRange(replacement); + await db.SaveChangesAsync(cancellationToken); + await transaction.CommitAsync(cancellationToken); + + return new IngestDocumentResponse( + documentId, + chunks.Count, + embeddingOptions.Value.Model, + EmbeddingDimensions.Value); + } + + private static void ValidateVector(ReadOnlyMemory vector) + { + if (vector.Length != EmbeddingDimensions.Value) + { + throw new InvalidOperationException( + $"Embedding dimension mismatch. The schema expects {EmbeddingDimensions.Value}, " + + $"but the provider returned {vector.Length}. Rebuild the vector schema before " + + "switching to a model with a different embedding size."); + } + + if (vector.Span.ToArray().Any(x => float.IsNaN(x) || float.IsInfinity(x))) + { + throw new InvalidOperationException("The embedding provider returned invalid numeric values."); + } + } +} diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/EmbeddingProviderFactory.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/EmbeddingProviderFactory.cs new file mode 100644 index 0000000..913061f --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/EmbeddingProviderFactory.cs @@ -0,0 +1,36 @@ +using Azure; +using Azure.AI.OpenAI; +using Microsoft.Extensions.AI; +using OllamaSharp; +using OpenAI.Embeddings; + +namespace HybridSearch.Api.Services; + +public static class EmbeddingProviderFactory +{ + public static IEmbeddingGenerator> CreateOllama( + string endpoint, + string model) => + new OllamaApiClient(new Uri(endpoint), model); + + // Compile/construction verified optional provider example. No live cloud + // request is made by the repository CI or the default verification script. + public static IEmbeddingGenerator> CreateOpenAI( + string apiKey, + string model) => + new EmbeddingClient(model, apiKey).AsIEmbeddingGenerator(); + + // Compile/construction verified optional provider example. No live cloud + // request is made by the repository CI or the default verification script. + public static IEmbeddingGenerator> CreateAzureOpenAI( + string endpoint, + string apiKey, + string deployment) + { + AzureOpenAIClient client = new( + new Uri(endpoint), + new AzureKeyCredential(apiKey)); + + return client.GetEmbeddingClient(deployment).AsIEmbeddingGenerator(); + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/HybridSearchService.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/HybridSearchService.cs new file mode 100644 index 0000000..0d3057a --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/HybridSearchService.cs @@ -0,0 +1,137 @@ +using HybridSearch.Api.Data; +using HybridSearch.Api.Models; +using HybridSearch.Api.Options; +using HybridSearch.Api.Search; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.AI; +using Pgvector; +using Pgvector.EntityFrameworkCore; + +namespace HybridSearch.Api.Services; + +public sealed class HybridSearchService( + SearchDbContext db, + IEmbeddingGenerator> embeddings) +{ + public async Task> SearchAsync( + SearchRequest request, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(request.Query)) + throw new ArgumentException("Query is required.", nameof(request)); + if (request.Top is < 1 or > 20) + throw new ArgumentOutOfRangeException(nameof(request), "Top must be between 1 and 20."); + + SearchMode mode = SearchModeParser.Parse(request.Mode); + int candidateLimit = Math.Min(Math.Max(request.Top * 4, 20), 100); + + if (mode == SearchMode.Keyword) + { + IReadOnlyList keyword = await KeywordAsync( + request.Query.Trim(), candidateLimit, cancellationToken); + return keyword.Take(request.Top) + .Select(x => new SearchHit( + x.Id, x.DocumentId, x.ChunkNumber, x.Title, x.Text, + x.SourceScore, x.Rank, null)) + .ToList(); + } + + if (mode == SearchMode.Vector) + { + IReadOnlyList vector = await VectorAsync( + request.Query.Trim(), candidateLimit, cancellationToken); + return vector.Take(request.Top) + .Select(x => new SearchHit( + x.Id, x.DocumentId, x.ChunkNumber, x.Title, x.Text, + x.SourceScore, null, x.Rank)) + .ToList(); + } + + IReadOnlyList keywordCandidates = await KeywordAsync( + request.Query.Trim(), candidateLimit, cancellationToken); + IReadOnlyList vectorCandidates = await VectorAsync( + request.Query.Trim(), candidateLimit, cancellationToken); + + return ReciprocalRankFusion.Fuse( + keywordCandidates, + vectorCandidates, + request.Top); + } + + private async Task> KeywordAsync( + string query, + int limit, + CancellationToken cancellationToken) + { + var rows = await db.DocumentChunks + .AsNoTracking() + .Where(x => x.SearchVector.Matches( + EF.Functions.WebSearchToTsQuery("english", query))) + .OrderByDescending(x => x.SearchVector.Rank( + EF.Functions.WebSearchToTsQuery("english", query))) + .Take(limit) + .Select(x => new + { + x.Id, + x.DocumentId, + x.ChunkNumber, + x.Title, + x.Text, + Score = x.SearchVector.Rank( + EF.Functions.WebSearchToTsQuery("english", query)) + }) + .ToListAsync(cancellationToken); + + return rows.Select((x, index) => new RankedCandidate( + x.Id, + x.DocumentId, + x.ChunkNumber, + x.Title, + x.Text, + index + 1, + x.Score)) + .ToList(); + } + + private async Task> VectorAsync( + string query, + int limit, + CancellationToken cancellationToken) + { + ReadOnlyMemory embedding = await embeddings.GenerateVectorAsync(query); + if (embedding.Length != EmbeddingDimensions.Value) + { + throw new InvalidOperationException( + $"Embedding dimension mismatch. Expected {EmbeddingDimensions.Value}; " + + $"received {embedding.Length}."); + } + + Vector queryVector = new(embedding.ToArray()); + + var rows = await db.DocumentChunks + .AsNoTracking() + .Where(x => x.Embedding != null) + .OrderBy(x => x.Embedding!.CosineDistance(queryVector)) + .Take(limit) + .Select(x => new + { + x.Id, + x.DocumentId, + x.ChunkNumber, + x.Title, + x.Text, + Distance = x.Embedding!.CosineDistance(queryVector) + }) + .ToListAsync(cancellationToken); + + return rows.Select((x, index) => new RankedCandidate( + x.Id, + x.DocumentId, + x.ChunkNumber, + x.Title, + x.Text, + index + 1, + 1.0 - x.Distance)) + .ToList(); + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/TextChunker.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/TextChunker.cs new file mode 100644 index 0000000..3a3f85f --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/Services/TextChunker.cs @@ -0,0 +1,35 @@ +namespace HybridSearch.Api.Services; + +public sealed class TextChunker +{ + public IReadOnlyList Split( + string text, + int maxWords = 120, + int overlapWords = 20) + { + if (string.IsNullOrWhiteSpace(text)) return []; + if (maxWords <= 0) throw new ArgumentOutOfRangeException(nameof(maxWords)); + if (overlapWords < 0 || overlapWords >= maxWords) + { + throw new ArgumentOutOfRangeException(nameof(overlapWords)); + } + + string[] words = text.Split( + (char[]?)null, + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + var chunks = new List(); + int start = 0; + + while (start < words.Length) + { + int end = Math.Min(start + maxWords, words.Length); + chunks.Add(string.Join(" ", words[start..end])); + + if (end == words.Length) break; + start = end - overlapWords; + } + + return chunks; + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/appsettings.Development.json b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/appsettings.Development.json new file mode 100644 index 0000000..ff66ba6 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/appsettings.json b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/appsettings.json new file mode 100644 index 0000000..a414205 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/src/HybridSearch.Api/appsettings.json @@ -0,0 +1,17 @@ +{ + "ConnectionStrings": { + "SearchDb": "Host=localhost;Port=54329;Database=hybridsearch;Username=postgres;Password=postgres" + }, + "Embedding": { + "Provider": "Ollama", + "Endpoint": "http://localhost:11434", + "Model": "nomic-embed-text" + }, + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/HybridSearch.Api.Tests.csproj b/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/HybridSearch.Api.Tests.csproj new file mode 100644 index 0000000..bfa272a --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/HybridSearch.Api.Tests.csproj @@ -0,0 +1,25 @@ + + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/ReciprocalRankFusionTests.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/ReciprocalRankFusionTests.cs new file mode 100644 index 0000000..6c04c85 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/ReciprocalRankFusionTests.cs @@ -0,0 +1,72 @@ +using HybridSearch.Api.Search; + +namespace HybridSearch.Api.Tests; + +public sealed class ReciprocalRankFusionTests +{ + [Fact] + public void Candidate_present_in_both_legs_can_rise_to_the_top() + { + Guid agreed = Guid.Parse("11111111-1111-1111-1111-111111111111"); + Guid keywordOnly = Guid.Parse("22222222-2222-2222-2222-222222222222"); + Guid vectorOnly = Guid.Parse("33333333-3333-3333-3333-333333333333"); + + RankedCandidate[] keyword = + [ + new(keywordOnly, "doc-k", 1, "Keyword", "...", 1, 9.0), + new(agreed, "doc-a", 1, "Agreed", "...", 2, 7.0) + ]; + + RankedCandidate[] vector = + [ + new(vectorOnly, "doc-v", 1, "Vector", "...", 1, 0.95), + new(agreed, "doc-a", 1, "Agreed", "...", 2, 0.91) + ]; + + var result = ReciprocalRankFusion.Fuse(keyword, vector, top: 3, k: 60); + + Assert.Equal(agreed, result[0].Id); + Assert.Equal(2, result[0].KeywordRank); + Assert.Equal(2, result[0].VectorRank); + } + + [Fact] + public void Fusion_uses_rank_not_source_score_magnitude() + { + Guid first = Guid.Parse("11111111-1111-1111-1111-111111111111"); + Guid second = Guid.Parse("22222222-2222-2222-2222-222222222222"); + + RankedCandidate[] keyword = + [ + new(first, "a", 1, "First", "...", 1, 0.0001), + new(second, "b", 1, "Second", "...", 2, 999999.0) + ]; + + var result = ReciprocalRankFusion.Fuse(keyword, [], top: 2, k: 60); + Assert.Equal(first, result[0].Id); + } + + [Fact] + public void Invalid_zero_rank_is_rejected() + { + RankedCandidate[] source = + [ + new(Guid.NewGuid(), "bad", 1, "Bad", "...", 0, 1.0) + ]; + + Assert.Throws(() => + ReciprocalRankFusion.Fuse(source, [], top: 1)); + } + + [Fact] + public void Top_limits_the_number_of_fused_results() + { + RankedCandidate[] keyword = Enumerable.Range(1, 5) + .Select(i => new RankedCandidate( + Guid.NewGuid(), $"doc-{i}", 1, $"Title {i}", "...", i, i)) + .ToArray(); + + var result = ReciprocalRankFusion.Fuse(keyword, [], top: 2); + Assert.Equal(2, result.Count); + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/SearchModeParserTests.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/SearchModeParserTests.cs new file mode 100644 index 0000000..68a8642 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/SearchModeParserTests.cs @@ -0,0 +1,22 @@ +using HybridSearch.Api.Search; + +namespace HybridSearch.Api.Tests; + +public sealed class SearchModeParserTests +{ + [Theory] + [InlineData("keyword", SearchMode.Keyword)] + [InlineData("VECTOR", SearchMode.Vector)] + [InlineData(" hybrid ", SearchMode.Hybrid)] + [InlineData(null, SearchMode.Hybrid)] + public void Supported_modes_parse(string? value, SearchMode expected) + { + Assert.Equal(expected, SearchModeParser.Parse(value)); + } + + [Fact] + public void Unsupported_mode_is_rejected() + { + Assert.Throws(() => SearchModeParser.Parse("semantic-only")); + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/TextChunkerTests.cs b/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/TextChunkerTests.cs new file mode 100644 index 0000000..3ae717f --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/tests/HybridSearch.Api.Tests/TextChunkerTests.cs @@ -0,0 +1,43 @@ +using HybridSearch.Api.Services; + +namespace HybridSearch.Api.Tests; + +public sealed class TextChunkerTests +{ + private readonly TextChunker _chunker = new(); + + [Fact] + public void Short_text_produces_one_chunk() + { + IReadOnlyList chunks = _chunker.Split("one two three", maxWords: 10, overlapWords: 2); + Assert.Single(chunks); + Assert.Equal("one two three", chunks[0]); + } + + [Fact] + public void Chunking_preserves_requested_overlap() + { + IReadOnlyList chunks = _chunker.Split( + "one two three four five six seven", + maxWords: 4, + overlapWords: 2); + + Assert.Equal(3, chunks.Count); + Assert.Equal("one two three four", chunks[0]); + Assert.Equal("three four five six", chunks[1]); + Assert.Equal("five six seven", chunks[2]); + } + + [Fact] + public void Empty_text_returns_no_chunks() + { + Assert.Empty(_chunker.Split(" ")); + } + + [Fact] + public void Overlap_must_be_smaller_than_chunk_size() + { + Assert.Throws(() => + _chunker.Split("one two", maxWords: 2, overlapWords: 2)); + } +} \ No newline at end of file diff --git a/dotnet-ai/hybrid-search-ef-core-pgvector/verified-environment.json b/dotnet-ai/hybrid-search-ef-core-pgvector/verified-environment.json new file mode 100644 index 0000000..a005fe4 --- /dev/null +++ b/dotnet-ai/hybrid-search-ef-core-pgvector/verified-environment.json @@ -0,0 +1,29 @@ +{ + "generatedUtc": "2026-10-01T15:10:41.9268088Z", + "sdk": "10.0.401", + "targetFramework": "net10.0", + "vectorDimensions": 768, + "defaultEmbeddingProvider": "Ollama", + "defaultEmbeddingModel": "nomic-embed-text", + "deterministicVerification": { + "restore": "PASS", + "releaseBuild": "PASS - 0 warnings / 0 errors required by Directory.Build.props", + "tests": "PASS - 13/13" + }, + "liveVerification": { + "companionEndToEnd": "PASS", + "verifiedUtc": "2026-10-01T15:10:41.9268088Z", + "database": "PASS - PostgreSQL 17 + pgvector (Docker)", + "httpIngestion": "PASS", + "normalizedDocumentReplacement": "PASS", + "keywordRetrieval": "PASS - 1 result(s)", + "vectorRetrieval": "PASS - 1 result(s)", + "hybridRrfRetrieval": "PASS - 1 result(s)", + "ginIndex": "PASS", + "hnswCosineIndex": "PASS", + "storedVectorDimensions": "PASS - 768", + "ollama": "LIVE VERIFIED - nomic-embed-text", + "openAI": "COMPILE/CONSTRUCTION VERIFIED ONLY", + "azureOpenAI": "COMPILE/CONSTRUCTION VERIFIED ONLY" + } +} From 9872ba096988f58385301d2b5940ae96512e0b2c Mon Sep 17 00:00:00 2001 From: BALAJI BASHYAM Date: Thu, 1 Oct 2026 15:50:58 +0000 Subject: [PATCH 2/3] Update hybrid search sample in repository README --- README.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fb222f2..dc0b307 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Each sample folder contains a focused implementation of one tutorial topic. The | Sample | What it demonstrates | Full tutorial | | --- | --- | --- | | [`dotnet-ai/provider-agnostic-chat-gateway`](dotnet-ai/provider-agnostic-chat-gateway/) | .NET 10 `Microsoft.Extensions.AI` gateway with allow-listed `IChatClient` providers, bounded conversation state, SSE streaming, safe function calling, and deterministic tests | [Microsoft.Extensions.AI Tutorial: Build a Multi-Provider IChatClient Gateway](https://www.dotnet-guide.com/tutorials/dotnet-ai/provider-agnostic-chat-gateway/) | -| [`dotnet-ai/hybrid-search-ef-core-pgvector`](dotnet-ai/hybrid-search-ef-core-pgvector/) | Minimal Reciprocal Rank Fusion demo combining pre-ranked keyword and vector results | [Hybrid Search in .NET with EF Core 10 and pgvector](https://www.dotnet-guide.com/tutorials/dotnet-ai/hybrid-search-ef-core-pgvector/) | +| [`dotnet-ai/hybrid-search-ef-core-pgvector`](dotnet-ai/hybrid-search-ef-core-pgvector/) | Verified .NET 10 API combining PostgreSQL full-text search (GIN) and pgvector cosine retrieval (HNSW), fused with Reciprocal Rank Fusion | [Hybrid Search in .NET with EF Core 10 and pgvector](https://www.dotnet-guide.com/tutorials/dotnet-ai/hybrid-search-ef-core-pgvector/) | | [`dotnet-ai/agent-framework-incident-triage`](dotnet-ai/agent-framework-incident-triage/) | Evidence-first Microsoft Agent Framework sample with read-only tools, deterministic incident evidence, `AgentSession` reuse, and correlation-vs-causation boundaries | [Microsoft Agent Framework Tutorial in C#: Build a Tool-Using AI Agent](https://www.dotnet-guide.com/tutorials/dotnet-ai/agent-framework-incident-triage/) | | [`dotnet-aspire/orchestrate-distributed-system`](dotnet-aspire/orchestrate-distributed-system/) | Minimal Aspire AppHost coordinating a web project and API with service discovery and startup ordering | [Aspire in .NET: Orchestrate, Run, and Deploy a Distributed System from One App Host](https://www.dotnet-guide.com/tutorials/dotnet-aspire/orchestrate-distributed-system/) | | [`software-architecture/architecture-testing-dotnet`](software-architecture/architecture-testing-dotnet/) | Minimal NetArchTest.eNhancedEdition rule that prevents Domain from depending on outer layers | [Architecture Testing in .NET: Enforce Layer and Module Boundaries with NetArchTest and ArchUnitNET](https://www.dotnet-guide.com/tutorials/software-architecture/architecture-testing-dotnet/) | @@ -62,14 +62,17 @@ dotnet test ProviderAgnosticChatGateway.slnx --configuration Release dotnet run --project .\src\ProviderAgnosticChatGateway\ProviderAgnosticChatGateway.csproj ``` -### Hybrid search RRF sample +### Hybrid search with EF Core and pgvector ```powershell cd dotnet-ai\hybrid-search-ef-core-pgvector -dotnet restore -dotnet run +dotnet restore .\HybridSearch.slnx +dotnet build .\HybridSearch.slnx --configuration Release --no-restore +dotnet test .\HybridSearch.slnx --configuration Release --no-build ``` +For the live PostgreSQL + pgvector + Ollama verification path, follow this sample folder's `README.md` and run `.\scripts\verify.ps1`. + ### Agent Framework incident triage ```powershell @@ -92,10 +95,12 @@ tutorials/ | | |-- src/ | | `-- tests/ | `-- hybrid-search-ef-core-pgvector/ -| |-- HybridSearchMinimal.csproj -| |-- Program.cs -| `-- README.md -| `-- agent-framework-incident-triage/ +| |-- HybridSearch.slnx +| |-- docker-compose.yml +| |-- README.md +| |-- src/HybridSearch.Api/ +| |-- tests/HybridSearch.Api.Tests/ +| `-- scripts/verify.ps1| `-- agent-framework-incident-triage/ | |-- AgentFrameworkIncidentTriage.slnx | |-- README.md | |-- global.json From 51191bfdc4b168babe4faf4cb094a59ee03a0e6b Mon Sep 17 00:00:00 2001 From: BALAJI BASHYAM Date: Thu, 1 Oct 2026 15:50:58 +0000 Subject: [PATCH 3/3] Test hybrid search sample in GitHub Actions --- .github/workflows/build-samples.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-samples.yml b/.github/workflows/build-samples.yml index a4938b6..8644b5e 100644 --- a/.github/workflows/build-samples.yml +++ b/.github/workflows/build-samples.yml @@ -105,7 +105,7 @@ jobs: --no-build build-hybrid-search: - name: Build hybrid search RRF sample + name: Build and test hybrid search sample runs-on: ubuntu-latest steps: @@ -120,22 +120,21 @@ jobs: - name: Restore run: | dotnet restore \ - dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearchMinimal.csproj + dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearch.slnx - name: Build run: | dotnet build \ - dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearchMinimal.csproj \ + dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearch.slnx \ --configuration Release \ --no-restore - - name: Run sample + - name: Test run: | - dotnet run \ - --project dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearchMinimal.csproj \ + dotnet test \ + dotnet-ai/hybrid-search-ef-core-pgvector/HybridSearch.slnx \ --configuration Release \ --no-build - build-aspire-orchestration: name: Build Aspire orchestration sample runs-on: ubuntu-latest