feat: add logical delete/forget API

README.md

@@ -60,6 +60,8 @@ ctx.add(
     external_id="conversation-2026-03-01#turn-1",
 )
 print(ctx.get(external_id="conversation-2026-03-01#turn-1"))
+ctx.delete(external_id="conversation-2026-03-01#turn-1")
+assert ctx.get(external_id="conversation-2026-03-01#turn-1") is None
 
 from PIL import Image
 image = Image.new("RGB", (2, 2), color="teal")
@@ -149,6 +151,13 @@ metrics = ctx.compact()
 print(f"Compaction removed {metrics['fragments_removed']} fragments")
 ```
 
+`delete()` and its alias `forget()` write a versioned tombstone for the target
+record and return `False` if the id is already absent. Default `list()`,
+`get()`, and `search()` calls hide tombstoned records, but this is logical
+forgetting rather than guaranteed physical erasure: older dataset versions and
+underlying files may still contain the original payload until retention or
+physical cleanup policies remove them.
+
 ### Rust
 
 ```rust

crates/lance-context-core/src/record.rs

@@ -1,5 +1,7 @@
 use chrono::{DateTime, Utc};
 
+use crate::serde::CONTENT_TYPE_TOMBSTONE;
+
 /// Structured metadata captured alongside each context entry.
 #[derive(Debug, Clone, Default)]
 pub struct StateMetadata {
@@ -26,6 +28,13 @@ pub struct ContextRecord {
     pub embedding: Option<Vec<f32>>,
 }
 
+impl ContextRecord {
+    #[must_use]
+    pub fn is_tombstone(&self) -> bool {
+        self.content_type == CONTENT_TYPE_TOMBSTONE
+    }
+}
+
 /// Result returned from a vector similarity search.
 #[derive(Debug, Clone)]
 pub struct SearchResult {

crates/lance-context-core/src/serde.rs

@@ -5,6 +5,7 @@ use serde::{Deserialize, Serialize};
 
 pub const CONTENT_TYPE_TEXT: &str = "text/plain";
 pub const CONTENT_TYPE_ARROW_STREAM: &str = "application/vnd.apache.arrow.stream";
+pub const CONTENT_TYPE_TOMBSTONE: &str = "application/vnd.lance-context.tombstone";
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct SerializedContent {

crates/lance-context-core/src/store.rs

@@ -32,6 +32,7 @@ use tracing::{error, info, warn};
 use uuid::Uuid;
 
 use crate::record::{ContextRecord, SearchResult, StateMetadata};
+use crate::serde::CONTENT_TYPE_TOMBSTONE;
 
 /// Embedding length used for the semantic index column.
 const DEFAULT_EMBEDDING_DIM: i32 = 1536;
@@ -204,6 +205,13 @@ impl ContextStore {
         }
 
         self.validate_unique_ids(entries).await?;
+        self.write_entries(entries).await
+    }
+
+    async fn write_entries(&mut self, entries: &[ContextRecord]) -> LanceResult<u64> {
+        if entries.is_empty() {
+            return Ok(self.dataset.manifest.version);
+        }
 
         // Group entries by (bot_id, session_id)
         let mut groups: HashMap<(Option<String>, Option<String>), Vec<ContextRecord>> =
@@ -252,10 +260,56 @@ impl ContextStore {
         Ok(self.dataset.manifest.version)
     }
 
+    /// Logically forget a record by internal storage id.
+    ///
+    /// This writes a tombstone with the same primary key, preserving prior
+    /// dataset versions while hiding the record from default reads.
+    pub async fn delete_by_id(&mut self, id: &str) -> LanceResult<bool> {
+        let Some(record) = self.get_by_id(id).await? else {
+            return Ok(false);
+        };
+        self.write_tombstone_for(record).await?;
+        Ok(true)
+    }
+
+    /// Logically forget a record by caller-supplied external id.
+    pub async fn delete_by_external_id(&mut self, external_id: &str) -> LanceResult<bool> {
+        let Some(record) = self.get_by_external_id(external_id).await? else {
+            return Ok(false);
+        };
+        self.write_tombstone_for(record).await?;
+        Ok(true)
+    }
+
+    async fn write_tombstone_for(&mut self, record: ContextRecord) -> LanceResult<u64> {
+        let tombstone = ContextRecord {
+            id: record.id,
+            external_id: record.external_id,
+            run_id: record.run_id,
+            bot_id: record.bot_id,
+            session_id: record.session_id,
+            created_at: Utc::now(),
+            role: record.role,
+            state_metadata: None,
+            content_type: CONTENT_TYPE_TOMBSTONE.to_string(),
+            text_payload: None,
+            binary_payload: None,
+            embedding: None,
+        };
+        self.write_entries(std::slice::from_ref(&tombstone)).await
+    }
+
     async fn validate_unique_ids(&self, entries: &[ContextRecord]) -> LanceResult<()> {
         let mut ids = HashSet::new();
         let mut external_ids = HashSet::new();
         for entry in entries {
+            if entry.is_tombstone() {
+                return Err(ArrowError::InvalidArgumentError(format!(
+                    "content_type '{}' is reserved for internal tombstones",
+                    CONTENT_TYPE_TOMBSTONE
+                ))
+                .into());
+            }
             if !ids.insert(entry.id.as_str()) {
                 return Err(ArrowError::InvalidArgumentError(format!(
                     "duplicate id '{}' in batch",
@@ -333,7 +387,11 @@ impl ContextStore {
         let mut stream = scanner.try_into_stream().await?;
         let mut results = Vec::new();
         while let Some(batch) = stream.try_next().await? {
-            results.extend(batch_to_records(&batch)?);
+            results.extend(
+                batch_to_records(&batch)?
+                    .into_iter()
+                    .filter(|record| !record.is_tombstone()),
+            );
         }
 
         if let Some(offset) = offset {
@@ -1347,6 +1405,133 @@ mod tests {
         });
     }
 
+    #[test]
+    fn add_rejects_reserved_tombstone_content_type() {
+        let dir = TempDir::new().unwrap();
+        let uri = dir.path().to_string_lossy().to_string();
+        let runtime = tokio::runtime::Runtime::new().unwrap();
+        runtime.block_on(async {
+            let mut store = ContextStore::open(&uri).await.unwrap();
+            let mut record = text_record("a", 0.0);
+            record.content_type = CONTENT_TYPE_TOMBSTONE.to_string();
+
+            let err = store.add(&[record]).await.unwrap_err();
+            let message = err.to_string();
+            assert!(
+                message.contains("reserved") && message.contains("tombstone"),
+                "unexpected error message: {message}"
+            );
+        });
+    }
+
+    #[test]
+    fn delete_by_external_id_hides_record_from_default_reads() {
+        let dir = TempDir::new().unwrap();
+        let uri = dir.path().to_string_lossy().to_string();
+        let runtime = tokio::runtime::Runtime::new().unwrap();
+        runtime.block_on(async {
+            let mut store = ContextStore::open(&uri).await.unwrap();
+            let mut first = text_record("a", 0.0);
+            first.external_id = Some("doc-123#chunk-1".to_string());
+            let second = text_record("b", 2.0);
+            store.add(&[first.clone(), second.clone()]).await.unwrap();
+
+            assert!(store
+                .delete_by_external_id("doc-123#chunk-1")
+                .await
+                .unwrap());
+
+            assert!(store
+                .get_by_external_id("doc-123#chunk-1")
+                .await
+                .unwrap()
+                .is_none());
+            assert!(store.get_by_id(&first.id).await.unwrap().is_none());
+
+            let records = store.list(None, None).await.unwrap();
+            assert_eq!(records.len(), 1);
+            assert_eq!(records[0].id, second.id);
+
+            let query = make_embedding(0.0);
+            let hits = store.search(&query, Some(10)).await.unwrap();
+            assert_eq!(hits.len(), 1);
+            assert_eq!(hits[0].record.id, second.id);
+        });
+    }
+
+    #[test]
+    fn delete_by_id_hides_record_from_default_reads() {
+        let dir = TempDir::new().unwrap();
+        let uri = dir.path().to_string_lossy().to_string();
+        let runtime = tokio::runtime::Runtime::new().unwrap();
+        runtime.block_on(async {
+            let mut store = ContextStore::open(&uri).await.unwrap();
+            let mut first = text_record("a", 0.0);
+            first.external_id = Some("doc-123#chunk-1".to_string());
+            let second = text_record("b", 2.0);
+            store.add(&[first.clone(), second.clone()]).await.unwrap();
+
+            assert!(store.delete_by_id(&first.id).await.unwrap());
+
+            assert!(store.get_by_id(&first.id).await.unwrap().is_none());
+            assert!(store
+                .get_by_external_id("doc-123#chunk-1")
+                .await
+                .unwrap()
+                .is_none());
+
+            let records = store.list(None, None).await.unwrap();
+            assert_eq!(records.len(), 1);
+            assert_eq!(records[0].id, second.id);
+
+            let query = make_embedding(0.0);
+            let hits = store.search(&query, Some(10)).await.unwrap();
+            assert_eq!(hits.len(), 1);
+            assert_eq!(hits[0].record.id, second.id);
+        });
+    }
+
+    #[test]
+    fn delete_missing_id_is_noop() {
+        let dir = TempDir::new().unwrap();
+        let uri = dir.path().to_string_lossy().to_string();
+        let runtime = tokio::runtime::Runtime::new().unwrap();
+        runtime.block_on(async {
+            let mut store = ContextStore::open(&uri).await.unwrap();
+            assert!(!store.delete_by_id("missing").await.unwrap());
+            assert!(!store.delete_by_external_id("missing").await.unwrap());
+        });
+    }
+
+    #[test]
+    fn external_id_can_be_reused_after_delete() {
+        let dir = TempDir::new().unwrap();
+        let uri = dir.path().to_string_lossy().to_string();
+        let runtime = tokio::runtime::Runtime::new().unwrap();
+        runtime.block_on(async {
+            let mut store = ContextStore::open(&uri).await.unwrap();
+            let mut first = text_record("a", 0.0);
+            first.external_id = Some("doc-123#chunk-1".to_string());
+            store.add(std::slice::from_ref(&first)).await.unwrap();
+            assert!(store
+                .delete_by_external_id("doc-123#chunk-1")
+                .await
+                .unwrap());
+
+            let mut replacement = text_record("b", 1.0);
+            replacement.external_id = first.external_id.clone();
+            store.add(std::slice::from_ref(&replacement)).await.unwrap();
+
+            let by_external_id = store
+                .get_by_external_id("doc-123#chunk-1")
+                .await
+                .unwrap()
+                .unwrap();
+            assert_eq!(by_external_id.id, replacement.id);
+            assert_eq!(store.list(None, None).await.unwrap().len(), 1);
+        });
+    }
+
     #[test]
     fn test_region_id_derivation_explicit() {
         let bot_id = Some("bot-123".to_string());

python/python/lance_context/api.py

@@ -451,6 +451,23 @@ def get(
             return None
         return _normalize_record(result)
 
+    def delete(self, *, id: str | None = None, external_id: str | None = None) -> bool:
+        """Logically forget one entry by internal id or caller-supplied external id.
+
+        Returns True when an entry was found and tombstoned, False when the
+        identifier is already absent. This is a versioned logical delete:
+        default reads hide the entry, but older dataset versions and underlying
+        storage files may still contain the original payload until retention or
+        physical cleanup policies remove them.
+        """
+        if (id is None) == (external_id is None):
+            raise ValueError("Specify exactly one of id or external_id")
+        return bool(self._inner.delete(id, external_id))
+
+    def forget(self, *, id: str | None = None, external_id: str | None = None) -> bool:
+        """Alias for :meth:`delete`."""
+        return self.delete(id=id, external_id=external_id)
+
     def compact(
         self,
         *,

python/python/tests/test_context.py

@@ -29,3 +29,16 @@ def test_context_snapshot_and_fork():
     assert fork.branch() == "branch-a"
     assert fork.entries() == ctx.entries()
     assert fork.version() == ctx.version()
+
+
+def test_context_delete_and_forget_by_external_id(tmp_path):
+    ctx = lc.Context.create(str(tmp_path / "context.lance"))
+    ctx.add("user", "stale", external_id="doc-1#chunk-1")
+
+    entry = ctx.get(external_id="doc-1#chunk-1")
+    assert entry is not None
+
+    assert ctx.delete(external_id="doc-1#chunk-1") is True
+    assert ctx.get(external_id="doc-1#chunk-1") is None
+    assert ctx.get(id=entry["id"]) is None
+    assert ctx.forget(external_id="doc-1#chunk-1") is False