From 8b5a57940d37c2c792de447cf5227d63fc3728e2 Mon Sep 17 00:00:00 2001 From: Nikhiladiga Date: Sat, 4 Jul 2026 23:53:21 +0530 Subject: [PATCH 1/5] docs: add guide on the limitations of Postgres LIKE/ILIKE at scale --- docs-site/content/.vuepress/config.js | 1 + .../guide/postgres-like-ilike-scale.md | 139 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 docs-site/content/guide/postgres-like-ilike-scale.md diff --git a/docs-site/content/.vuepress/config.js b/docs-site/content/.vuepress/config.js index e4434520..0c2d26ce 100644 --- a/docs-site/content/.vuepress/config.js +++ b/docs-site/content/.vuepress/config.js @@ -311,6 +311,7 @@ let config = { ['/guide/migrating-from-algolia', 'Migrating from Algolia'], ['/guide/testcontainers', 'Running Tests with Testcontainers'], ['/guide/personalized-search-join', 'Personalized Search with JOINs'], + ['/guide/postgres-like-ilike-scale', 'Postgres LIKE/ILIKE at Scale'] ], }, diff --git a/docs-site/content/guide/postgres-like-ilike-scale.md b/docs-site/content/guide/postgres-like-ilike-scale.md new file mode 100644 index 00000000..153e1af7 --- /dev/null +++ b/docs-site/content/guide/postgres-like-ilike-scale.md @@ -0,0 +1,139 @@ +# Why LIKE and ILIKE Fail at Scale: The Hidden Costs of Postgres Pattern Matching + +If you are working with Postgres, you have almost certainly come across the `LIKE` and `ILIKE` keywords in some of your queries. When building a new application, adding a quick `WHERE title ILIKE '%search_term%'` is often the fastest way to implement a basic search feature. + +It works perfectly fine, until it doesn't. + +As your database grows from thousands of rows to millions, and as your users begin to expect a fast, Google-like search experience, relying on basic pattern matching starts to show its cracks. In this article, we'll explore why `LIKE` and `ILIKE` break down at scale and what you can do about it. + +## The Performance Bottleneck: Full Table Scans + +The most immediate issue you'll encounter with `LIKE` and `ILIKE` is performance degradation. + +By default, Postgres uses [B-tree indexes](https://www.postgresql.org/docs/current/btree.html), which are fantastic for exact matches (`=`, `>`, `<`). B-trees are optimized for balance and retrieval efficiency, making them suitable for systems that require frequent data insertions and retrievals. + +However, if you use a wildcard at the *beginning* of your search string (e.g., `ILIKE '%query%'`), Postgres cannot use a standard B-tree index. This is because B-trees store their data in a sorted manner, and a leading wildcard breaks that sorted order. As a result, Postgres must perform a **Sequential Scan** (a full table scan) to find the matching rows. + +```sql +-- This query will ignore standard B-tree indexes and scan every row +SELECT * FROM products WHERE description ILIKE '%laptop%'; +``` + +When you have 10,000 rows, a sequential scan might take a few milliseconds. But when you hit 10 million rows, that same query could take seconds. If you have concurrent users running searches, those slow queries will quickly exhaust your database connections and CPU, bringing your application to a halt. + +:::tip +While you can use `pg_trgm` to create GIN indexes that support leading wildcards, they come with their own trade-offs: massive index sizes, slower write speeds, and they still don't solve the other problems listed below. +::: + +### The Experiment: Searching 5 Million Rows + +Let's look at a concrete example. We can easily generate a dummy dataset of 5 million product records in Postgres: + +```sql +CREATE TABLE products ( + id SERIAL PRIMARY KEY, + name VARCHAR(255), + description TEXT +); + +-- Insert 5 million rows of dummy data +INSERT INTO products (name, description) +SELECT + 'Product ' || i, + 'This is a standard description for product ' || i || '. It has some generic text.' +FROM generate_series(1, 5000000) AS i; + +-- Inject our target keyword near the end of the table +UPDATE products SET description = 'The ultimate gaming laptop with high specs' WHERE id = 4900000; +``` + +Now, let's run a typical `ILIKE` search and analyze its performance: + +```sql +EXPLAIN ANALYZE SELECT * FROM products WHERE description ILIKE '%gaming laptop%'; +``` + +Here is the output you'll typically see on a standard database instance: + +```text +Gather (cost=1000.00..108052.24 rows=500 width=96) (actual time=1514.831..1547.060 rows=1 loops=1) + Workers Planned: 2 + Workers Launched: 2 + -> Parallel Seq Scan on products (cost=0.00..107002.24 rows=208 width=96) (actual time=1513.529..1523.547 rows=0 loops=3) +" Filter: (description ~~* '%gaming laptop%'::text)" + Rows Removed by Filter: 1666666 +Planning Time: 0.671 ms +JIT: + Functions: 6 + Options: Inlining false, Optimization false, Expressions true, Deforming true + Timing: Generation 1.141 ms, Inlining 0.000 ms, Optimization 0.951 ms, Emission 12.196 ms, Total 14.288 ms +Execution Time: 1547.978 ms +``` + +In this output, **Seq Scan** means Postgres looked at your query and realized it couldn't use any indexes because of the leading wildcard `%`. Therefore, it had to read the table sequentially, literally checking every row one by one from start to finish. The "Parallel" part just means Postgres threw multiple CPU threads at the problem to try and speed it up. Now imagine concurrent users running this query. This will immediately spike your CPU usage, causing other critical queries to slow down or time out completely, creating a severe performance bottleneck. + +### What if we add an Index? + +A common follow-up question is: *"Why not just add an index to the description field?"* + +If we were to create a standard B-tree index on our dummy table: + +```sql +CREATE INDEX idx_products_description ON products(description); +``` + +And run the exact same `ILIKE '%gaming laptop%'` query, Postgres would **still** perform a Sequential Scan and take the same time to execute the query. There are two main reasons for this: + +1. **Leading Wildcards Bypass Indexes:** As mentioned earlier, B-trees store data sorted from left to right. Because our search starts with a wildcard (`%`), the index cannot be used to look up the value. +2. **B-trees are Case-Sensitive:** Even if we removed the leading wildcard and searched for `ILIKE 'gaming laptop%'`, a standard B-tree index is case-sensitive. It cannot be used with the case-insensitive `ILIKE` operator. You would need to create a special functional index (e.g., `LOWER(description)`) just to support it. + +Between the leading wildcards and case-insensitivity, Postgres is often forced to bypass your indexes and fall back to brute force. As your dataset grows in size, this brute-force approach becomes completely unsustainable, leading to sluggish search experiences for your users and overloaded database servers. + +However, **"Scale"** is not just data volume, it's also user expectations. As an application grows, users expect a modern, consumer-grade search experience. If search is fast but returns zero results like "iphne" or ranks irrelevant results first, the search has still failed at scale from a product perspective. + +The Postgres workarounds for these features make the performance even worse. If you try to fix the typo problem using modules like `fuzzystrmatch` or `pg_trgm` on millions of rows, the performance hits are even more catastrophic than a simple sequential scan. + +### The Search Engine Contrast + +If we take this exact same 5 million row dataset and index it in a purpose-built search engine like Typesense, the performance difference is staggering. + +When you issue a search request to Typesense: + +```json +{ + "q": "gaming laptop", + "query_by": "description" +} +``` + +The response time drops from ~1.5 seconds to 5 milliseconds! + +```json +{ + "found": 1, + "out_of": 5000000, + "search_time_ms": 5, + "hits": [ + { + "document": { + "id": "4900000", + "name": "Product 4900000", + "description": "The ultimate gaming laptop with high specs" + } + // Highlights and relevance metadata omitted for brevity + } + ] +} +``` + +Because Typesense is purpose-built for search, it stores data in highly optimized, memory-mapped data structures that don't need to scan every record. The result is instant, search-as-you-type performance that Postgres `ILIKE` simply cannot match at scale. + +Typesense also solves additional limitations of Postgres `LIKE`/`ILIKE` searches: + +* **Intelligent Relevance Ranking** - Results are automatically ranked by relevance. You can configure weights so that a match in a product `name` ranks higher than a match in its `description`. +* **Instant Typo Tolerance** - Typo tolerance is active by default. A search for "iphne" will instantly find "iphone" in under 10ms with zero extra configuration. +* **Automatic Accent and Character Normalization** - Accents, case variations, and special characters are normalized automatically, meaning "café" and "cafe" match seamlessly. + +## Conclusion + +Postgres `LIKE` and `ILIKE` pattern matching is a simple and effective solution for prototypes or applications with small datasets. However, as your database scales into the millions of rows, relational databases hit a hard wall in both raw scan times and search relevance UX. Offloading search workloads to a purpose-built search engine like Typesense becomes essential to guarantee sub-millisecond query performance, typo tolerance, and consumer-grade search results. From 991a7e79b6b11e2b8038ed2c117cda59b77d19ae Mon Sep 17 00:00:00 2001 From: Nikhiladiga Date: Mon, 6 Jul 2026 10:37:29 +0530 Subject: [PATCH 2/5] docs: refine explanation of pg_trgm limitations --- docs-site/content/guide/postgres-like-ilike-scale.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-site/content/guide/postgres-like-ilike-scale.md b/docs-site/content/guide/postgres-like-ilike-scale.md index 153e1af7..5597eb43 100644 --- a/docs-site/content/guide/postgres-like-ilike-scale.md +++ b/docs-site/content/guide/postgres-like-ilike-scale.md @@ -22,7 +22,7 @@ SELECT * FROM products WHERE description ILIKE '%laptop%'; When you have 10,000 rows, a sequential scan might take a few milliseconds. But when you hit 10 million rows, that same query could take seconds. If you have concurrent users running searches, those slow queries will quickly exhaust your database connections and CPU, bringing your application to a halt. :::tip -While you can use `pg_trgm` to create GIN indexes that support leading wildcards, they come with their own trade-offs: massive index sizes, slower write speeds, and they still don't solve the other problems listed below. +While you can use `pg_trgm` to create GIN indexes that support leading wildcards, they come with their own trade-offs such as massive index sizes and slower write speeds. They still fail to address the core user-experience limitations of database pattern matching such as relevance ranking, typo tolerance, and accent sensitivity. ::: ### The Experiment: Searching 5 Million Rows From 534d1c971241f904056f7e4273d975debe2986d5 Mon Sep 17 00:00:00 2001 From: Nikhiladiga Date: Thu, 9 Jul 2026 17:46:13 +0530 Subject: [PATCH 3/5] docs: add guide on migrating from PostgreSQL to Typesense for full-text search and remove duplicate content. --- docs-site/content/.vuepress/config.js | 2 +- .../guide/postgres-like-ilike-scale.md | 139 -------- .../guide/postgres-typesense-migration-fts.md | 313 ++++++++++++++++++ 3 files changed, 314 insertions(+), 140 deletions(-) delete mode 100644 docs-site/content/guide/postgres-like-ilike-scale.md create mode 100644 docs-site/content/guide/postgres-typesense-migration-fts.md diff --git a/docs-site/content/.vuepress/config.js b/docs-site/content/.vuepress/config.js index 0c2d26ce..0e77407e 100644 --- a/docs-site/content/.vuepress/config.js +++ b/docs-site/content/.vuepress/config.js @@ -311,7 +311,7 @@ let config = { ['/guide/migrating-from-algolia', 'Migrating from Algolia'], ['/guide/testcontainers', 'Running Tests with Testcontainers'], ['/guide/personalized-search-join', 'Personalized Search with JOINs'], - ['/guide/postgres-like-ilike-scale', 'Postgres LIKE/ILIKE at Scale'] + ['/guide/postgres-typesense-migration-fts', 'Migrating from Postgres for Full-Text Search'] ], }, diff --git a/docs-site/content/guide/postgres-like-ilike-scale.md b/docs-site/content/guide/postgres-like-ilike-scale.md deleted file mode 100644 index 5597eb43..00000000 --- a/docs-site/content/guide/postgres-like-ilike-scale.md +++ /dev/null @@ -1,139 +0,0 @@ -# Why LIKE and ILIKE Fail at Scale: The Hidden Costs of Postgres Pattern Matching - -If you are working with Postgres, you have almost certainly come across the `LIKE` and `ILIKE` keywords in some of your queries. When building a new application, adding a quick `WHERE title ILIKE '%search_term%'` is often the fastest way to implement a basic search feature. - -It works perfectly fine, until it doesn't. - -As your database grows from thousands of rows to millions, and as your users begin to expect a fast, Google-like search experience, relying on basic pattern matching starts to show its cracks. In this article, we'll explore why `LIKE` and `ILIKE` break down at scale and what you can do about it. - -## The Performance Bottleneck: Full Table Scans - -The most immediate issue you'll encounter with `LIKE` and `ILIKE` is performance degradation. - -By default, Postgres uses [B-tree indexes](https://www.postgresql.org/docs/current/btree.html), which are fantastic for exact matches (`=`, `>`, `<`). B-trees are optimized for balance and retrieval efficiency, making them suitable for systems that require frequent data insertions and retrievals. - -However, if you use a wildcard at the *beginning* of your search string (e.g., `ILIKE '%query%'`), Postgres cannot use a standard B-tree index. This is because B-trees store their data in a sorted manner, and a leading wildcard breaks that sorted order. As a result, Postgres must perform a **Sequential Scan** (a full table scan) to find the matching rows. - -```sql --- This query will ignore standard B-tree indexes and scan every row -SELECT * FROM products WHERE description ILIKE '%laptop%'; -``` - -When you have 10,000 rows, a sequential scan might take a few milliseconds. But when you hit 10 million rows, that same query could take seconds. If you have concurrent users running searches, those slow queries will quickly exhaust your database connections and CPU, bringing your application to a halt. - -:::tip -While you can use `pg_trgm` to create GIN indexes that support leading wildcards, they come with their own trade-offs such as massive index sizes and slower write speeds. They still fail to address the core user-experience limitations of database pattern matching such as relevance ranking, typo tolerance, and accent sensitivity. -::: - -### The Experiment: Searching 5 Million Rows - -Let's look at a concrete example. We can easily generate a dummy dataset of 5 million product records in Postgres: - -```sql -CREATE TABLE products ( - id SERIAL PRIMARY KEY, - name VARCHAR(255), - description TEXT -); - --- Insert 5 million rows of dummy data -INSERT INTO products (name, description) -SELECT - 'Product ' || i, - 'This is a standard description for product ' || i || '. It has some generic text.' -FROM generate_series(1, 5000000) AS i; - --- Inject our target keyword near the end of the table -UPDATE products SET description = 'The ultimate gaming laptop with high specs' WHERE id = 4900000; -``` - -Now, let's run a typical `ILIKE` search and analyze its performance: - -```sql -EXPLAIN ANALYZE SELECT * FROM products WHERE description ILIKE '%gaming laptop%'; -``` - -Here is the output you'll typically see on a standard database instance: - -```text -Gather (cost=1000.00..108052.24 rows=500 width=96) (actual time=1514.831..1547.060 rows=1 loops=1) - Workers Planned: 2 - Workers Launched: 2 - -> Parallel Seq Scan on products (cost=0.00..107002.24 rows=208 width=96) (actual time=1513.529..1523.547 rows=0 loops=3) -" Filter: (description ~~* '%gaming laptop%'::text)" - Rows Removed by Filter: 1666666 -Planning Time: 0.671 ms -JIT: - Functions: 6 - Options: Inlining false, Optimization false, Expressions true, Deforming true - Timing: Generation 1.141 ms, Inlining 0.000 ms, Optimization 0.951 ms, Emission 12.196 ms, Total 14.288 ms -Execution Time: 1547.978 ms -``` - -In this output, **Seq Scan** means Postgres looked at your query and realized it couldn't use any indexes because of the leading wildcard `%`. Therefore, it had to read the table sequentially, literally checking every row one by one from start to finish. The "Parallel" part just means Postgres threw multiple CPU threads at the problem to try and speed it up. Now imagine concurrent users running this query. This will immediately spike your CPU usage, causing other critical queries to slow down or time out completely, creating a severe performance bottleneck. - -### What if we add an Index? - -A common follow-up question is: *"Why not just add an index to the description field?"* - -If we were to create a standard B-tree index on our dummy table: - -```sql -CREATE INDEX idx_products_description ON products(description); -``` - -And run the exact same `ILIKE '%gaming laptop%'` query, Postgres would **still** perform a Sequential Scan and take the same time to execute the query. There are two main reasons for this: - -1. **Leading Wildcards Bypass Indexes:** As mentioned earlier, B-trees store data sorted from left to right. Because our search starts with a wildcard (`%`), the index cannot be used to look up the value. -2. **B-trees are Case-Sensitive:** Even if we removed the leading wildcard and searched for `ILIKE 'gaming laptop%'`, a standard B-tree index is case-sensitive. It cannot be used with the case-insensitive `ILIKE` operator. You would need to create a special functional index (e.g., `LOWER(description)`) just to support it. - -Between the leading wildcards and case-insensitivity, Postgres is often forced to bypass your indexes and fall back to brute force. As your dataset grows in size, this brute-force approach becomes completely unsustainable, leading to sluggish search experiences for your users and overloaded database servers. - -However, **"Scale"** is not just data volume, it's also user expectations. As an application grows, users expect a modern, consumer-grade search experience. If search is fast but returns zero results like "iphne" or ranks irrelevant results first, the search has still failed at scale from a product perspective. - -The Postgres workarounds for these features make the performance even worse. If you try to fix the typo problem using modules like `fuzzystrmatch` or `pg_trgm` on millions of rows, the performance hits are even more catastrophic than a simple sequential scan. - -### The Search Engine Contrast - -If we take this exact same 5 million row dataset and index it in a purpose-built search engine like Typesense, the performance difference is staggering. - -When you issue a search request to Typesense: - -```json -{ - "q": "gaming laptop", - "query_by": "description" -} -``` - -The response time drops from ~1.5 seconds to 5 milliseconds! - -```json -{ - "found": 1, - "out_of": 5000000, - "search_time_ms": 5, - "hits": [ - { - "document": { - "id": "4900000", - "name": "Product 4900000", - "description": "The ultimate gaming laptop with high specs" - } - // Highlights and relevance metadata omitted for brevity - } - ] -} -``` - -Because Typesense is purpose-built for search, it stores data in highly optimized, memory-mapped data structures that don't need to scan every record. The result is instant, search-as-you-type performance that Postgres `ILIKE` simply cannot match at scale. - -Typesense also solves additional limitations of Postgres `LIKE`/`ILIKE` searches: - -* **Intelligent Relevance Ranking** - Results are automatically ranked by relevance. You can configure weights so that a match in a product `name` ranks higher than a match in its `description`. -* **Instant Typo Tolerance** - Typo tolerance is active by default. A search for "iphne" will instantly find "iphone" in under 10ms with zero extra configuration. -* **Automatic Accent and Character Normalization** - Accents, case variations, and special characters are normalized automatically, meaning "café" and "cafe" match seamlessly. - -## Conclusion - -Postgres `LIKE` and `ILIKE` pattern matching is a simple and effective solution for prototypes or applications with small datasets. However, as your database scales into the millions of rows, relational databases hit a hard wall in both raw scan times and search relevance UX. Offloading search workloads to a purpose-built search engine like Typesense becomes essential to guarantee sub-millisecond query performance, typo tolerance, and consumer-grade search results. diff --git a/docs-site/content/guide/postgres-typesense-migration-fts.md b/docs-site/content/guide/postgres-typesense-migration-fts.md new file mode 100644 index 00000000..c85a5d90 --- /dev/null +++ b/docs-site/content/guide/postgres-typesense-migration-fts.md @@ -0,0 +1,313 @@ +# Migrating from PostgreSQL to Typesense for Full-Text Search + +When adding search functionality to an application backed by PostgreSQL, developers naturally start with the tools built into the database. PostgreSQL is an exceptional database that excels at transactional consistency, relational queries, and JOINs. It also provides highly capable native tools for moderate full-text search requirements. For many applications, particularly those where search is a secondary feature, PostgreSQL is more than sufficient. + +However, as search becomes a central feature of an application, user expectations increase. Modern search experiences require high query throughput, instant autocomplete, typo tolerance, relevance tuning, faceting, and dynamic merchandising. While it is possible to build these features using PostgreSQL, doing so introduces significant operational complexity. + +This article explores the journey of implementing search in PostgreSQL. We will start with basic pattern matching and typo tolerance using `pg_trgm`, explore PostgreSQL's native Full Text Search (FTS) capabilities, and finally examine the operational challenges of extending these features at scale and how Typesense addresses them. + +--- + +## Performance of LIKE and ILIKE + +The most immediate bottleneck with `LIKE` and `ILIKE` queries is search performance on large tables. + +By default, PostgreSQL uses [B-tree indexes](https://www.postgresql.org/docs/current/btree.html) for primary and secondary indexes. B-trees are highly optimized for exact matches (`=`) and range comparisons (`>`, `<`). They can also optimize prefix searches (e.g., `LIKE 'laptop%'`) and can handle case-insensitive lookups if configured as expression indexes (e.g., `CREATE INDEX ON products (LOWER(description))`). + +However, if you place a wildcard at the beginning of your search string (for example, `ILIKE '%laptop%'`), a standard B-tree index cannot be used. B-tree indexes rely on the sorted order of keys from left to right. A leading wildcard bypasses this sorted structure, forcing PostgreSQL to fall back on a **Sequential Scan** (a full table scan). + +```sql +-- This query bypasses B-tree indexes and scans every row +SELECT * FROM products WHERE description ILIKE '%laptop%'; +``` + +--- + +## Test dataset with 5 million rows + +To see the impact of full table scans, let's create a dataset of 5 million product records in PostgreSQL: + +```sql +CREATE TABLE products ( + id SERIAL PRIMARY KEY, + name VARCHAR(255), + description TEXT +); + +-- Insert 5 million rows of test data +INSERT INTO products (name, description) +SELECT + 'Product ' || i, + 'This is a standard description for product ' || i || '. It has some generic text.' +FROM generate_series(1, 5000000) AS i; + +-- Insert a target keyword near the end of the table +UPDATE products SET description = 'The ultimate gaming laptop with high specs' WHERE id = 4900000; +``` + +Running a standard wildcard search on this table highlights the performance cost: + +```sql +EXPLAIN ANALYZE SELECT * FROM products WHERE description ILIKE '%gaming laptop%'; +``` + +This query produces an execution plan similar to this: + +```text +Gather (cost=1000.00..108052.24 rows=500 width=96) (actual time=1514.831..1547.060 rows=1 loops=1) + Workers Planned: 2 + Workers Launched: 2 + -> Parallel Seq Scan on products (cost=0.00..107002.24 rows=208 width=96) (actual time=1513.529..1523.547 rows=0 loops=3) + Filter: (description ~~* '%gaming laptop%'::text) + Rows Removed by Filter: 1666666 +Planning Time: 0.671 ms +Execution Time: 1547.978 ms +``` + +Because of the leading wildcard `%`, PostgreSQL performs a sequential scan across every row to evaluate the filter expression. While parallel workers mitigate this, the query still takes over 1.5 seconds. Under concurrent search traffic, sequential scans quickly consume CPU resources, leading to query queues and timeouts. + +--- + +## Typo tolerance with pg_trgm + +To handle spelling mistakes and optimize wildcard searches, PostgreSQL includes the `pg_trgm` extension. + +### How trigrams work +A trigram is a group of three consecutive characters extracted from a string. When generating trigrams, PostgreSQL prefixes the string with two spaces and suffixes it with one space. + +You can inspect the trigrams generated for any string using `show_trgm()`: + +```sql +SELECT show_trgm('apple'); +-- Output: {" a"," ap","app","le ","ppl","ple"} +``` + +To calculate similarity between two strings, PostgreSQL uses the **Jaccard similarity coefficient** i.e, the size of the intersection of their trigrams divided by the size of their union. + +Comparing `"apple"` and `"aple"`: +* Trigrams of `"apple"`: `{" a"," ap","app","le ","ppl","ple"}` (6 trigrams) +* Trigrams of `"aple"`: `{" a"," ap","apl","ple","le "}` (5 trigrams) +* Intersection: `{" a"," ap","le ","ple"}` (4 trigrams) +* Union: `{" a"," ap","apl","app","le ","ppl","ple"}` (7 trigrams) +* **Similarity score**: `4 / 7 ≈ 0.57` + +### Querying with pg_trgm +With the extension enabled (`CREATE EXTENSION pg_trgm;`), you can measure string similarity and run similarity-based queries: + +```sql +-- Compute raw similarity +SELECT similarity('gaming laptop', 'gaming leptop'); -- Returns ~0.65 + +-- Filter using the % similarity operator +SELECT * FROM products +WHERE name % 'gaming leptop' +ORDER BY similarity(name, 'gaming leptop') DESC; +``` + +The `%` operator evaluates to `true` if the Jaccard similarity exceeds the threshold configured by `pg_trgm.similarity_threshold` (default is `0.3`). You can adjust this threshold to control search sensitivity: + +```sql +SET pg_trgm.similarity_threshold = 0.4; +``` + +--- + +## Trigram indexes + +Running similarity queries without a dedicated index still forces a sequential scan, calculating trigrams for every row. PostgreSQL supports two index types to optimize trigram matches. +- **GIN** [Generalized Inverted Index](https://www.postgresql.org/docs/current/gin.html). +- **GiST** [Generalized Search Tree](https://www.postgresql.org/docs/current/gist.html). + +```sql +-- Create a GIN trigram index +CREATE INDEX idx_products_name_trgm_gin ON products USING gin (name gin_trgm_ops); + +-- Create a GiST trigram index +CREATE INDEX idx_products_name_trgm_gist ON products USING gist (name gist_trgm_ops); +``` + +### Trade-offs between GIN and GiST + +Choosing between GIN and GiST depends on your read-to-write ratio and disk space: + +| Feature | GIN Index | GiST Index | +| :--- | :--- | :--- | +| **Search Speed** | Fast (resolves trigrams directly via an inverted index) | Slower (requires traversing multiple nodes) | +| **Write/Update Overhead** | High (updating the inverted index is write-intensive) | Low (faster inserts and updates) | +| **Index Size on Disk** | Large (often 2–3x larger than GiST) | Small (compact structure) | +| **Primary Use Case** | Read-heavy search with infrequent writes | Write-heavy tables or limited disk space | + +Since our `products` table is read-heavy, let's add a **GIN** index and observe the impact on our wildcard query performance. + +```sql +CREATE INDEX idx_products_description_trgm +ON products +USING gin (description gin_trgm_ops); +``` + +We can now rerun our original query with `EXPLAIN ANALYZE` to see the new execution plan. + +```sql +EXPLAIN ANALYZE +SELECT * FROM products WHERE description ILIKE '%gaming laptop%'; +``` + +After adding a GIN index, PostgreSQL uses a Bitmap Index Scan instead of a Sequential Scan, reducing execution time significantly for this workload. + +```plaintext +Bitmap Heap Scan on products (cost=1859.51..3744.23 rows=500 width=96) (actual time=0.823..0.832 rows=1 loops=1) + Recheck Cond: (description ~~* '%gaming laptop%'::text) + Heap Blocks: exact=1 + -> Bitmap Index Scan on idx_products_description_trgm (cost=0.00..1859.38 rows=500 width=0) (actual time=0.775..0.775 rows=1 loops=1) + Index Cond: (description ~~* '%gaming laptop%'::text) +Planning Time: 2.563 ms +Execution Time: 1.078 ms +``` + +--- + +## PostgreSQL Full Text Search (FTS) + +While `pg_trgm` provides typo tolerance and speeds up wildcard search queries, PostgreSQL also includes a mature, native Full Text Search engine designed for linguistic processing. + +FTS works by parsing text into a `tsvector` (a sorted list of distinct lexical tokens, or "lexemes") and querying it using a `tsquery`. + +### A realistic FTS query + +```sql +-- Add a GIN index for FTS +CREATE INDEX idx_products_description_fts +ON products +USING gin (to_tsvector('english', description)); + +-- Search using websearch_to_tsquery for a Google-like query syntax +SELECT + id, + name, + ts_rank(to_tsvector('english', description), websearch_to_tsquery('english', '"gaming laptop" or desktop -refurbished')) AS rank, + ts_headline('english', description, websearch_to_tsquery('english', '"gaming laptop" or desktop -refurbished')) AS snippet +FROM products +WHERE to_tsvector('english', description) @@ websearch_to_tsquery('english', '"gaming laptop" or desktop -refurbished') +ORDER BY rank DESC +LIMIT 10; +``` + +### Strengths of PostgreSQL FTS + +PostgreSQL's FTS engine is extremely capable out of the box: +* **Stemming and Stop Words**: It automatically removes common words (like "the" or "and") and normalizes words to their root forms (e.g., "running" and "ran" become "run"). +* **Ranking**: The `ts_rank` function provides sophisticated document scoring based on lexical frequency. +* **Dictionaries and Field Weighting**: You can configure custom dictionaries and assign relative weights (A, B, C, D) to different columns (like boosting `name` over `description`). +* **Highlighting**: The `ts_headline` function dynamically generates text snippets highlighting the matched query terms. + +### Limitations at scale + +Despite its strengths, extending PostgreSQL FTS to support modern search UX features introduces significant operational complexity: +* **Typo tolerance is not native**: While FTS handles stemming, it doesn't handle misspellings. Combining `pg_trgm` with FTS requires complex query orchestration. +* **Autocomplete requires additional work**: Implementing instant search-as-you-type requires generating prefix queries, managing multiple index types, and tuning for latency. +* **Synonyms require dictionaries**: Managing synonyms and curations requires manually maintaining and reloading PostgreSQL dictionary files on the database server. +* **Relevance tuning becomes manual**: Continuously tweaking `ts_rank` weights or combining text relevance with business metrics (like popularity or rating) involves writing increasingly complex SQL. +* **Faceting requires implementation**: Extracting dynamic aggregations (like category counts or price ranges) requires complex `GROUP BY` queries that can degrade performance on large result sets. +* **Index size and write degradation**: GIN indexes map every generated lexeme and trigram back to the row IDs. Larger indexes place greater pressure on the buffer cache and may reduce cache efficiency. Furthermore, PostgreSQL updates search indexes synchronously as part of the transaction, which can introduce write latency on high-throughput systems. + +--- + +## Migrating to Typesense + +To overcome the operational complexity of building advanced search features in PostgreSQL, you can migrate your search workloads directly to Typesense. + +### How Typesense structures search data + +Dedicated search engines optimize for a different workload than relational databases: +* **In-Memory Indexes**: Typesense stores search indexes in memory (mapped to disk for persistence), avoiding slow disk operations during query execution. +* **Asynchronous Indexing**: Unlike PostgreSQL, which updates search indexes as part of the transaction, Typesense indexes documents independently after synchronization. +* **Adaptive Radix Tree (ART)**: Typesense utilizes an Adaptive Radix Tree to index prefixes. This structure provides efficient prefix lookups with complexity proportional to the search key length, returning results as the user types without generating high CPU load. ART can also handle the fuzzy/typo-tolerance + prefix case natively in one structure, where Postgres needs two separate indexes/extensions bolted together to get similar coverage. + +### Keeping data in sync + +When offloading search workloads, PostgreSQL remains your primary database and source of truth. You do not need to duplicate your entire database into Typesense. Instead, you only index the specific fields required for search and filtering (like `name` and `description`), along with a primary key to link back to PostgreSQL. + +To keep Typesense in sync with PostgreSQL, you have two common approaches: +* **Batch Synchronization**: Run a periodic script that queries recently updated rows in PostgreSQL and pushes those changes to Typesense in batches. +* **Change Data Capture (CDC)**: For real-time synchronization, use a tool like [Debezium](https://debezium.io/). Debezium reads the PostgreSQL write-ahead logs (WAL) and streams every insert, update, or delete event to Typesense asynchronously. + +### Search capabilities comparison + +Instead of writing complex SQL to compose search features, Typesense provides them natively. + +#### Typo tolerance +Typesense automatically handles misspellings based on word length without needing to configure similarity thresholds. +```text +iphnoe + +↓ + +iphone +``` + +#### Prefix search +You do not need to construct wildcard queries. Typesense instantly returns useful results for partial words. +```text +iph + +↓ + +iphone +``` + +#### Search-as-you-type +Relevant matches are returned instantly at each keystroke. +```text +User types: +g +ga +gam +gami +gaming +``` + +#### Field weighting +Boosting matches in a title over a description requires a simple configuration rather than complex text ranking functions. +```text +Boost `title` over `description` +``` + +#### Faceting +Extracting dynamic filters and counts is built-in and requires no `GROUP BY` logic. +```text +Brand +Category +Price +Rating +``` + +#### Synonyms +Instead of editing database dictionary files, you can define synonyms dynamically via the API or a dashboard. +```text +tv + +↓ + +television +``` + +#### Curations and Merchandising +You can pin products or boost specific brands dynamically. +```text +Pin "MacBook Pro" for the query "laptop" +Boost brand "Apple" +``` + +#### Highlighting +Typesense returns highlighted text snippets natively in the response payload without requiring functions like `ts_headline()` to compose them. + +#### Search analytics +Query analytics and popular search tracking are built into the search platform rather than implemented separately in your backend. + +--- + +## Conclusion + +PostgreSQL provides excellent search capabilities for many applications, especially when search is not the primary workload. Features like Full Text Search and `pg_trgm` allow developers to build capable search experiences without introducing another system. + +As search becomes a central feature of an application—with requirements like typo tolerance, autocomplete, relevance tuning, faceting, custom ranking, and high query throughput—the operational complexity of extending PostgreSQL increases. At that point, dedicated search engines such as Typesense become attractive because they are purpose-built for those workloads while allowing PostgreSQL to remain the source of truth. From 46f44dc1351034bea6cd94a7f6578f6274e2ab5f Mon Sep 17 00:00:00 2001 From: Nikhiladiga Date: Thu, 9 Jul 2026 21:46:51 +0530 Subject: [PATCH 4/5] docs: rewrite postgres to typesense migration guide with simplified architecture and implementation comparison --- .../guide/postgres-typesense-migration-fts.md | 376 ++++++++---------- 1 file changed, 171 insertions(+), 205 deletions(-) diff --git a/docs-site/content/guide/postgres-typesense-migration-fts.md b/docs-site/content/guide/postgres-typesense-migration-fts.md index c85a5d90..3272c1b6 100644 --- a/docs-site/content/guide/postgres-typesense-migration-fts.md +++ b/docs-site/content/guide/postgres-typesense-migration-fts.md @@ -1,313 +1,279 @@ # Migrating from PostgreSQL to Typesense for Full-Text Search -When adding search functionality to an application backed by PostgreSQL, developers naturally start with the tools built into the database. PostgreSQL is an exceptional database that excels at transactional consistency, relational queries, and JOINs. It also provides highly capable native tools for moderate full-text search requirements. For many applications, particularly those where search is a secondary feature, PostgreSQL is more than sufficient. +When adding search functionality to an application backed by PostgreSQL, it's natural to start with the tools built into the database. PostgreSQL is widely used for transactional applications and provides native support for full-text search. For many applications, particularly those where search is a secondary feature, PostgreSQL is sufficient. -However, as search becomes a central feature of an application, user expectations increase. Modern search experiences require high query throughput, instant autocomplete, typo tolerance, relevance tuning, faceting, and dynamic merchandising. While it is possible to build these features using PostgreSQL, doing so introduces significant operational complexity. +However, modern search experiences require instant autocomplete, typo tolerance, relevance tuning, faceting, and dynamic merchandising. While it is possible to build these features using PostgreSQL, doing so introduces additional implementation and operational work. -This article explores the journey of implementing search in PostgreSQL. We will start with basic pattern matching and typo tolerance using `pg_trgm`, explore PostgreSQL's native Full Text Search (FTS) capabilities, and finally examine the operational challenges of extending these features at scale and how Typesense addresses them. +This guide compares how common search capabilities are implemented in PostgreSQL versus Typesense. ---- - -## Performance of LIKE and ILIKE +## Test dataset -The most immediate bottleneck with `LIKE` and `ILIKE` queries is search performance on large tables. +All PostgreSQL execution plans and performance metrics in this guide are based on a test dataset containing ~5 million product records. The equivalent Typesense examples assume these records are indexed with `id`, `name`, and `description` fields. -By default, PostgreSQL uses [B-tree indexes](https://www.postgresql.org/docs/current/btree.html) for primary and secondary indexes. B-trees are highly optimized for exact matches (`=`) and range comparisons (`>`, `<`). They can also optimize prefix searches (e.g., `LIKE 'laptop%'`) and can handle case-insensitive lookups if configured as expression indexes (e.g., `CREATE INDEX ON products (LOWER(description))`). +--- -However, if you place a wildcard at the beginning of your search string (for example, `ILIKE '%laptop%'`), a standard B-tree index cannot be used. B-tree indexes rely on the sorted order of keys from left to right. A leading wildcard bypasses this sorted structure, forcing PostgreSQL to fall back on a **Sequential Scan** (a full table scan). +## Implementation Comparison -```sql --- This query bypasses B-tree indexes and scans every row -SELECT * FROM products WHERE description ILIKE '%laptop%'; -``` +| Capability | PostgreSQL | Typesense | +| :--- | :--- | :--- | +| **Basic substring search** | `ILIKE` | `q` | +| **Typo tolerance** | `pg_trgm` + GIN index | Built-in | +| **Full text search** | `tsvector` + `tsquery` | Built-in | +| **Ranking** | `ts_rank` | Built-in | +| **Highlighting** | `ts_headline` | Built-in | +| **Prefix search** | Multiple approaches | Built-in | +| **Faceting** | `GROUP BY` | Built-in | --- -## Test dataset with 5 million rows - -To see the impact of full table scans, let's create a dataset of 5 million product records in PostgreSQL: +## Architecture overview -```sql -CREATE TABLE products ( - id SERIAL PRIMARY KEY, - name VARCHAR(255), - description TEXT -); - --- Insert 5 million rows of test data -INSERT INTO products (name, description) -SELECT - 'Product ' || i, - 'This is a standard description for product ' || i || '. It has some generic text.' -FROM generate_series(1, 5000000) AS i; - --- Insert a target keyword near the end of the table -UPDATE products SET description = 'The ultimate gaming laptop with high specs' WHERE id = 4900000; -``` +When offloading search workloads, PostgreSQL remains your primary database and source of truth. You do not duplicate your entire database into Typesense. Instead, you only index the specific fields required for search and filtering. -Running a standard wildcard search on this table highlights the performance cost: +### Query flow +Search requests are sent directly to Typesense to ensure low latency. Typesense returns the matching document IDs (and any fields needed for rendering the UI), and the application can fetch the full relational data from PostgreSQL if necessary. -```sql -EXPLAIN ANALYZE SELECT * FROM products WHERE description ILIKE '%gaming laptop%'; +```plaintext + Search Request + │ + Typesense + │ + Product IDs + │ + PostgreSQL ``` -This query produces an execution plan similar to this: +### Keeping data synchronized +To keep Typesense in sync with PostgreSQL, developers typically use Change Data Capture (CDC) tools like [Debezium](https://debezium.io/). Debezium reads the PostgreSQL write-ahead logs (WAL) and streams every insert, update, or delete event to Typesense asynchronously. -```text -Gather (cost=1000.00..108052.24 rows=500 width=96) (actual time=1514.831..1547.060 rows=1 loops=1) - Workers Planned: 2 - Workers Launched: 2 - -> Parallel Seq Scan on products (cost=0.00..107002.24 rows=208 width=96) (actual time=1513.529..1523.547 rows=0 loops=3) - Filter: (description ~~* '%gaming laptop%'::text) - Rows Removed by Filter: 1666666 -Planning Time: 0.671 ms -Execution Time: 1547.978 ms +```plaintext +PostgreSQL + │ +Debezium + │ +Typesense ``` -Because of the leading wildcard `%`, PostgreSQL performs a sequential scan across every row to evaluate the filter expression. While parallel workers mitigate this, the query still takes over 1.5 seconds. Under concurrent search traffic, sequential scans quickly consume CPU resources, leading to query queues and timeouts. - --- -## Typo tolerance with pg_trgm +## Basic substring search -To handle spelling mistakes and optimize wildcard searches, PostgreSQL includes the `pg_trgm` extension. +### PostgreSQL -### How trigrams work -A trigram is a group of three consecutive characters extracted from a string. When generating trigrams, PostgreSQL prefixes the string with two spaces and suffixes it with one space. - -You can inspect the trigrams generated for any string using `show_trgm()`: +The simplest way to search in PostgreSQL is using the `ILIKE` operator for case-insensitive pattern matching. ```sql -SELECT show_trgm('apple'); --- Output: {" a"," ap","app","le ","ppl","ple"} +SELECT * FROM products WHERE description ILIKE '%laptop%'; ``` -To calculate similarity between two strings, PostgreSQL uses the **Jaccard similarity coefficient** i.e, the size of the intersection of their trigrams divided by the size of their union. - -Comparing `"apple"` and `"aple"`: -* Trigrams of `"apple"`: `{" a"," ap","app","le ","ppl","ple"}` (6 trigrams) -* Trigrams of `"aple"`: `{" a"," ap","apl","ple","le "}` (5 trigrams) -* Intersection: `{" a"," ap","le ","ple"}` (4 trigrams) -* Union: `{" a"," ap","apl","app","le ","ppl","ple"}` (7 trigrams) -* **Similarity score**: `4 / 7 ≈ 0.57` - -### Querying with pg_trgm -With the extension enabled (`CREATE EXTENSION pg_trgm;`), you can measure string similarity and run similarity-based queries: +**Implementation considerations:** +If you place a wildcard at the beginning of your search string (`%laptop`), standard B-tree indexes cannot be used. PostgreSQL performs a sequential scan across every row to evaluate the filter expression. While parallel workers mitigate this, the query quickly consumes CPU resources under concurrent search traffic. For example, queries against a 5M products table: ```sql --- Compute raw similarity -SELECT similarity('gaming laptop', 'gaming leptop'); -- Returns ~0.65 +EXPLAIN ANALYZE SELECT * FROM products WHERE description ILIKE '%gaming laptop%'; +``` --- Filter using the % similarity operator -SELECT * FROM products -WHERE name % 'gaming leptop' -ORDER BY similarity(name, 'gaming leptop') DESC; +```plaintext +Gather (cost=1000.00..108052.24 rows=500 width=96) (actual time=1514.831..1547.060 rows=1 loops=1) + -> Parallel Seq Scan on products (cost=0.00..107002.24 rows=208 width=96) (actual time=1513.529..1523.547 rows=0 loops=3) + Filter: (description ~~* '%gaming laptop%'::text) + Rows Removed by Filter: 1666666 +Execution Time: 1547.978 ms ``` -The `%` operator evaluates to `true` if the Jaccard similarity exceeds the threshold configured by `pg_trgm.similarity_threshold` (default is `0.3`). You can adjust this threshold to control search sensitivity: +### Equivalent implementation in Typesense -```sql -SET pg_trgm.similarity_threshold = 0.4; +Typesense handles basic substring search natively through the `q` parameter. All search queries complete in under 5 milliseconds. + +```json +{ + "q": "laptop", + "query_by": "description" +} ``` --- -## Trigram indexes +## Typo tolerance -Running similarity queries without a dedicated index still forces a sequential scan, calculating trigrams for every row. PostgreSQL supports two index types to optimize trigram matches. -- **GIN** [Generalized Inverted Index](https://www.postgresql.org/docs/current/gin.html). -- **GiST** [Generalized Search Tree](https://www.postgresql.org/docs/current/gist.html). - -```sql --- Create a GIN trigram index -CREATE INDEX idx_products_name_trgm_gin ON products USING gin (name gin_trgm_ops); +### PostgreSQL --- Create a GiST trigram index -CREATE INDEX idx_products_name_trgm_gist ON products USING gist (name gist_trgm_ops); -``` +To handle spelling mistakes and optimize wildcard searches, we enable the `pg_trgm` extension. A trigram is a group of three consecutive characters extracted from a string. -### Trade-offs between GIN and GiST +You enable the extension, configure the similarity threshold, create a Generalized Inverted Index (GIN), and write a similarity query: -Choosing between GIN and GiST depends on your read-to-write ratio and disk space: +```sql +CREATE EXTENSION pg_trgm; -| Feature | GIN Index | GiST Index | -| :--- | :--- | :--- | -| **Search Speed** | Fast (resolves trigrams directly via an inverted index) | Slower (requires traversing multiple nodes) | -| **Write/Update Overhead** | High (updating the inverted index is write-intensive) | Low (faster inserts and updates) | -| **Index Size on Disk** | Large (often 2–3x larger than GiST) | Small (compact structure) | -| **Primary Use Case** | Read-heavy search with infrequent writes | Write-heavy tables or limited disk space | +SET pg_trgm.similarity_threshold = 0.3; -Since our `products` table is read-heavy, let's add a **GIN** index and observe the impact on our wildcard query performance. +CREATE INDEX idx_products_description_trgm ON products USING gin (description gin_trgm_ops); -```sql -CREATE INDEX idx_products_description_trgm -ON products -USING gin (description gin_trgm_ops); +SELECT * FROM products +WHERE description % 'gaming leptop' +ORDER BY similarity(description, 'gaming leptop') DESC; ``` -We can now rerun our original query with `EXPLAIN ANALYZE` to see the new execution plan. +**Implementation considerations:** +After adding a GIN index, PostgreSQL uses a Bitmap Index Scan instead of a Sequential Scan, reducing execution time significantly. However, GIN indexes map every generated trigram back to the row IDs. Larger indexes place greater pressure on the buffer cache and may reduce cache efficiency. Furthermore, PostgreSQL updates search indexes synchronously as part of the transaction, which can impact write performance. -```sql -EXPLAIN ANALYZE -SELECT * FROM products WHERE description ILIKE '%gaming laptop%'; -``` +### Equivalent implementation in Typesense -After adding a GIN index, PostgreSQL uses a Bitmap Index Scan instead of a Sequential Scan, reducing execution time significantly for this workload. +Typo tolerance is enabled by default in Typesense. The engine automatically adjusts the number of allowed typos based on the length of the search word, leveraging its in-memory Adaptive Radix Tree (ART) to efficiently traverse and identify fuzzy matches without requiring a separate trigram index. -```plaintext -Bitmap Heap Scan on products (cost=1859.51..3744.23 rows=500 width=96) (actual time=0.823..0.832 rows=1 loops=1) - Recheck Cond: (description ~~* '%gaming laptop%'::text) - Heap Blocks: exact=1 - -> Bitmap Index Scan on idx_products_description_trgm (cost=0.00..1859.38 rows=500 width=0) (actual time=0.775..0.775 rows=1 loops=1) - Index Cond: (description ~~* '%gaming laptop%'::text) -Planning Time: 2.563 ms -Execution Time: 1.078 ms +```json +{ + "q": "gaming leptop", + "query_by": "description" +} ``` --- -## PostgreSQL Full Text Search (FTS) +## Full text search and Ranking -While `pg_trgm` provides typo tolerance and speeds up wildcard search queries, PostgreSQL also includes a mature, native Full Text Search engine designed for linguistic processing. +### PostgreSQL -FTS works by parsing text into a `tsvector` (a sorted list of distinct lexical tokens, or "lexemes") and querying it using a `tsquery`. +PostgreSQL provides a native Full Text Search engine designed for linguistic processing. It parses text into a `tsvector` (a sorted list of distinct lexical tokens) and queries it using a `tsquery`. -### A realistic FTS query +To implement full text search with ranking and highlighting, you create a specialized index, generate a query, rank the results using `ts_rank`, and extract snippets using `ts_headline`: ```sql --- Add a GIN index for FTS CREATE INDEX idx_products_description_fts ON products USING gin (to_tsvector('english', description)); --- Search using websearch_to_tsquery for a Google-like query syntax SELECT id, name, - ts_rank(to_tsvector('english', description), websearch_to_tsquery('english', '"gaming laptop" or desktop -refurbished')) AS rank, - ts_headline('english', description, websearch_to_tsquery('english', '"gaming laptop" or desktop -refurbished')) AS snippet + ts_rank( + to_tsvector('english', description), + websearch_to_tsquery('english', '"gaming laptop" or desktop -refurbished') + ) AS rank, + ts_headline( + 'english', + description, + websearch_to_tsquery('english', '"gaming laptop" or desktop -refurbished') + ) AS snippet FROM products WHERE to_tsvector('english', description) @@ websearch_to_tsquery('english', '"gaming laptop" or desktop -refurbished') ORDER BY rank DESC LIMIT 10; ``` -### Strengths of PostgreSQL FTS +**Implementation considerations:** +`ts_rank` calculates scores purely based on lexical statistics. If you need to factor in business metrics (like product popularity or rating), you must implement custom scoring formulas directly in your SQL. Furthermore, because FTS relies on exact dictionary matching, it does not handle typos natively. To achieve typo-tolerant full text search, you must maintain both a GIN trigram index and a `tsvector` index, and write application logic that first corrects misspelled words using trigrams before passing them to the FTS engine. -PostgreSQL's FTS engine is extremely capable out of the box: -* **Stemming and Stop Words**: It automatically removes common words (like "the" or "and") and normalizes words to their root forms (e.g., "running" and "ran" become "run"). -* **Ranking**: The `ts_rank` function provides sophisticated document scoring based on lexical frequency. -* **Dictionaries and Field Weighting**: You can configure custom dictionaries and assign relative weights (A, B, C, D) to different columns (like boosting `name` over `description`). -* **Highlighting**: The `ts_headline` function dynamically generates text snippets highlighting the matched query terms. +### Equivalent implementation in Typesense -### Limitations at scale +Full text search, ranking, and highlighting are built into the core engine. You define the fields to search, and Typesense returns highlighted text snippets natively in the response payload. -Despite its strengths, extending PostgreSQL FTS to support modern search UX features introduces significant operational complexity: -* **Typo tolerance is not native**: While FTS handles stemming, it doesn't handle misspellings. Combining `pg_trgm` with FTS requires complex query orchestration. -* **Autocomplete requires additional work**: Implementing instant search-as-you-type requires generating prefix queries, managing multiple index types, and tuning for latency. -* **Synonyms require dictionaries**: Managing synonyms and curations requires manually maintaining and reloading PostgreSQL dictionary files on the database server. -* **Relevance tuning becomes manual**: Continuously tweaking `ts_rank` weights or combining text relevance with business metrics (like popularity or rating) involves writing increasingly complex SQL. -* **Faceting requires implementation**: Extracting dynamic aggregations (like category counts or price ranges) requires complex `GROUP BY` queries that can degrade performance on large result sets. -* **Index size and write degradation**: GIN indexes map every generated lexeme and trigram back to the row IDs. Larger indexes place greater pressure on the buffer cache and may reduce cache efficiency. Furthermore, PostgreSQL updates search indexes synchronously as part of the transaction, which can introduce write latency on high-throughput systems. +```json +{ + "q": "gaming laptop", + "query_by": "name,description", + "query_by_weights": "2,1" +} +``` + +Field weighting (`query_by_weights`) allows you to boost matches in a title over a description using a simple configuration parameter. --- -## Migrating to Typesense +## Autocomplete and Prefix search -To overcome the operational complexity of building advanced search features in PostgreSQL, you can migrate your search workloads directly to Typesense. +### PostgreSQL -### How Typesense structures search data +Implementing search-as-you-type (autocomplete) in PostgreSQL requires generating prefix queries and often managing multiple index types to handle both prefix matching and typo tolerance simultaneously. -Dedicated search engines optimize for a different workload than relational databases: -* **In-Memory Indexes**: Typesense stores search indexes in memory (mapped to disk for persistence), avoiding slow disk operations during query execution. -* **Asynchronous Indexing**: Unlike PostgreSQL, which updates search indexes as part of the transaction, Typesense indexes documents independently after synchronization. -* **Adaptive Radix Tree (ART)**: Typesense utilizes an Adaptive Radix Tree to index prefixes. This structure provides efficient prefix lookups with complexity proportional to the search key length, returning results as the user types without generating high CPU load. ART can also handle the fuzzy/typo-tolerance + prefix case natively in one structure, where Postgres needs two separate indexes/extensions bolted together to get similar coverage. +Developers typically use `tsquery` with prefix matching (e.g., `gaming:*`) or `LIKE 'gaming%'` combined with trigrams, requiring careful query construction to balance latency and accuracy. -### Keeping data in sync +### Equivalent implementation in Typesense -When offloading search workloads, PostgreSQL remains your primary database and source of truth. You do not need to duplicate your entire database into Typesense. Instead, you only index the specific fields required for search and filtering (like `name` and `description`), along with a primary key to link back to PostgreSQL. +Typesense utilizes an Adaptive Radix Tree (ART) to index prefixes. This structure provides efficient prefix lookups with complexity proportional to the search key length. Relevant matches are returned instantly at each keystroke. -To keep Typesense in sync with PostgreSQL, you have two common approaches: -* **Batch Synchronization**: Run a periodic script that queries recently updated rows in PostgreSQL and pushes those changes to Typesense in batches. -* **Change Data Capture (CDC)**: For real-time synchronization, use a tool like [Debezium](https://debezium.io/). Debezium reads the PostgreSQL write-ahead logs (WAL) and streams every insert, update, or delete event to Typesense asynchronously. +```json +{ + "q": "gam", + "query_by": "name", + "prefix": true +} +``` -### Search capabilities comparison +--- -Instead of writing complex SQL to compose search features, Typesense provides them natively. +## Faceting -#### Typo tolerance -Typesense automatically handles misspellings based on word length without needing to configure similarity thresholds. -```text -iphnoe +### PostgreSQL -↓ +Extracting dynamic aggregations, such as category counts or price ranges for a search interface, requires computing aggregates across the result set using `GROUP BY`. -iphone +```sql +SELECT + category, + COUNT(*) as count +FROM products +WHERE to_tsvector('english', description) @@ websearch_to_tsquery('english', 'laptop') +GROUP BY category; ``` -#### Prefix search -You do not need to construct wildcard queries. Typesense instantly returns useful results for partial words. -```text -iph +**Implementation considerations:** +As the search logic becomes more complex (incorporating trigrams, `ts_rank`, and complex `WHERE` clauses), calculating dynamic `GROUP BY` aggregates alongside the search results requires executing multiple expensive queries or complex Common Table Expressions (CTEs), which can degrade performance on large datasets. -↓ +### Equivalent implementation in Typesense -iphone -``` +Extracting dynamic filters and counts requires no `GROUP BY` logic. You simply specify the `facet_by` parameter. -#### Search-as-you-type -Relevant matches are returned instantly at each keystroke. -```text -User types: -g -ga -gam -gami -gaming +```json +{ + "q": "laptop", + "query_by": "description", + "facet_by": "brand,category,price" +} ``` -#### Field weighting -Boosting matches in a title over a description requires a simple configuration rather than complex text ranking functions. -```text -Boost `title` over `description` -``` +Typesense automatically returns the facet counts for "Apple", "Electronics", and price brackets alongside the search results in a single request. -#### Faceting -Extracting dynamic filters and counts is built-in and requires no `GROUP BY` logic. -```text -Brand -Category -Price -Rating -``` +--- -#### Synonyms -Instead of editing database dictionary files, you can define synonyms dynamically via the API or a dashboard. -```text -tv +## Synonyms and Merchandising -↓ +### PostgreSQL -television -``` +Managing synonyms (e.g., matching "tv" to "television") requires maintaining and reloading PostgreSQL dictionary files on the database server. Merchandising tasks, such as pinning a specific product to the top of the results for a specific query, require building custom application-level logic to inject those records into the SQL results. -#### Curations and Merchandising -You can pin products or boost specific brands dynamically. -```text -Pin "MacBook Pro" for the query "laptop" -Boost brand "Apple" -``` +### Equivalent implementation in Typesense + +Synonyms and curations (merchandising) can be defined dynamically via the API or a dashboard without touching the underlying search schema. -#### Highlighting -Typesense returns highlighted text snippets natively in the response payload without requiring functions like `ts_headline()` to compose them. +```json +// Create a synonym +{ + "synonyms": ["tv", "television"] +} -#### Search analytics -Query analytics and popular search tracking are built into the search platform rather than implemented separately in your backend. +// Create a curation (override) +{ + "rule": { + "match": "exact", + "query": "laptop" + }, + "includes": [ + {"id": "macbook-pro-14", "position": 1} + ] +} +``` --- -## Conclusion +## When to use what -PostgreSQL provides excellent search capabilities for many applications, especially when search is not the primary workload. Features like Full Text Search and `pg_trgm` allow developers to build capable search experiences without introducing another system. +The decision to scale search beyond PostgreSQL is rarely about raw query latency, but rather about the architectural overhead of query concurrency, write frequency, and search complexity. -As search becomes a central feature of an application—with requirements like typo tolerance, autocomplete, relevance tuning, faceting, custom ranking, and high query throughput—the operational complexity of extending PostgreSQL increases. At that point, dedicated search engines such as Typesense become attractive because they are purpose-built for those workloads while allowing PostgreSQL to remain the source of truth. +| Use PostgreSQL when... | Use Typesense when... | +| :--- | :--- | +| Search is secondary | Search is a core feature | +| Small feature set | Autocomplete | +| Simple ranking | Typo tolerance | +| Few search requests | High query throughput | +| One database is preferred | Dedicated search infrastructure | From 532b12d7d1719bbcbe8d2ee54df25e93c395ec5e Mon Sep 17 00:00:00 2001 From: Nikhiladiga Date: Thu, 9 Jul 2026 21:54:34 +0530 Subject: [PATCH 5/5] docs: refine Postgres to Typesense migration guide with updated architecture descriptions and enhanced comparison table --- .../guide/postgres-typesense-migration-fts.md | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/docs-site/content/guide/postgres-typesense-migration-fts.md b/docs-site/content/guide/postgres-typesense-migration-fts.md index 3272c1b6..3e7bb3d5 100644 --- a/docs-site/content/guide/postgres-typesense-migration-fts.md +++ b/docs-site/content/guide/postgres-typesense-migration-fts.md @@ -28,7 +28,7 @@ All PostgreSQL execution plans and performance metrics in this guide are based o ## Architecture overview -When offloading search workloads, PostgreSQL remains your primary database and source of truth. You do not duplicate your entire database into Typesense. Instead, you only index the specific fields required for search and filtering. +When offloading search workloads, PostgreSQL remains your primary database and source of truth. In many deployments, only the fields required for search and filtering are indexed into Typesense. ### Query flow Search requests are sent directly to Typesense to ensure low latency. Typesense returns the matching document IDs (and any fields needed for rendering the UI), and the application can fetch the full relational data from PostgreSQL if necessary. @@ -83,7 +83,7 @@ Execution Time: 1547.978 ms ### Equivalent implementation in Typesense -Typesense handles basic substring search natively through the `q` parameter. All search queries complete in under 5 milliseconds. +Typesense handles basic substring search natively through the `q` parameter. Search requests are served directly from the search index without requiring SQL pattern matching. ```json { @@ -119,7 +119,7 @@ After adding a GIN index, PostgreSQL uses a Bitmap Index Scan instead of a Seque ### Equivalent implementation in Typesense -Typo tolerance is enabled by default in Typesense. The engine automatically adjusts the number of allowed typos based on the length of the search word, leveraging its in-memory Adaptive Radix Tree (ART) to efficiently traverse and identify fuzzy matches without requiring a separate trigram index. +Typo tolerance is enabled by default in Typesense. The engine uses an Adaptive Radix Tree (ART) to efficiently perform prefix and fuzzy lookups ```json { @@ -188,6 +188,19 @@ Implementing search-as-you-type (autocomplete) in PostgreSQL requires generating Developers typically use `tsquery` with prefix matching (e.g., `gaming:*`) or `LIKE 'gaming%'` combined with trigrams, requiring careful query construction to balance latency and accuracy. +```sql +SELECT * +FROM products +WHERE name ILIKE 'gam%'; +``` + +```sql +SELECT * +FROM products +WHERE to_tsvector('english', name) + @@ to_tsquery('english', 'gam:*'); +``` + ### Equivalent implementation in Typesense Typesense utilizes an Adaptive Radix Tree (ART) to index prefixes. This structure provides efficient prefix lookups with complexity proportional to the search key length. Relevant matches are returned instantly at each keystroke. @@ -270,10 +283,12 @@ Synonyms and curations (merchandising) can be defined dynamically via the API or The decision to scale search beyond PostgreSQL is rarely about raw query latency, but rather about the architectural overhead of query concurrency, write frequency, and search complexity. -| Use PostgreSQL when... | Use Typesense when... | +| PostgreSQL is often sufficient when... | Consider Typesense when... | | :--- | :--- | -| Search is secondary | Search is a core feature | -| Small feature set | Autocomplete | -| Simple ranking | Typo tolerance | -| Few search requests | High query throughput | -| One database is preferred | Dedicated search infrastructure | +| Search is a secondary feature of the application | Search is a primary part of the user experience | +| Basic substring search or Full Text Search meets your requirements | You need autocomplete and search-as-you-type | +| PostgreSQL's built-in ranking is sufficient | You need typo tolerance, synonyms, and relevance tuning | +| You prefer to keep search within your primary database | You want to offload search workloads to a dedicated search engine | +| Search traffic is relatively modest | You expect high search concurrency or search-intensive workloads | + +The two systems are often used together rather than as replacements for one another. PostgreSQL continues to manage transactional data, while Typesense indexes the fields required for search, providing a dedicated engine optimized for low-latency, feature-rich search experiences.