Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions src/kb/storage/objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

Both raw files and cached parse artifacts (Unstructured element JSON) live here.
Keying convention:
raw/<domain>/<content_hash>/<filename>
raw/<domain>/<content_hash>
parse/<content_hash>/elements.json
"""

Expand All @@ -24,6 +24,13 @@ def _content_hash(blob: bytes) -> str:
return hashlib.sha256(blob).hexdigest()


def _safe_key_segment(segment: str) -> str:
"""Reject path-traversal-shaped segments before building object keys."""
if not segment or segment in {".", ".."} or "/" in segment or "\\" in segment:
raise ValueError(f"unsafe object-store key segment: {segment!r}")
return segment


# ─── Backend interface ────────────────────────────────────────────────────
class _Backend:
async def put(self, key: str, blob: bytes, mime: str | None = None) -> None: ...
Expand Down Expand Up @@ -125,7 +132,7 @@ def _get_backend() -> _Backend:
async def put_raw_file(*, domain: str, filename: str, blob: bytes) -> tuple[str, str]:
"""Store a raw upload; returns (object_key, content_hash). Idempotent by content_hash."""
h = _content_hash(blob)
key = f"raw/{domain}/{h}/{filename}"
key = f"raw/{_safe_key_segment(domain)}/{h}"
backend = _get_backend()
if not await backend.exists(key):
await backend.put(key, blob)
Expand Down
9 changes: 7 additions & 2 deletions src/kb/vector/pgvector_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@
from kb.vector.embed import embed_dense

# Whitelist filter columns — anything not in here is silently dropped to keep SQL safe.
_ALLOWED_FILTER_COLS = {"project", "file_id", "entity_id", "parent_chunk", "domain"}
#
# `parent_id` is the public query-scoping key used by callers; `parent_chunk`
# is the backing column in `chunks`. Support both so scope filters behave the
# same across vector backends.
_ALLOWED_FILTER_COLS = {"project", "file_id", "entity_id", "parent_id", "parent_chunk", "domain"}


def _vec_literal(v: list[float]) -> str:
Expand Down Expand Up @@ -140,7 +144,8 @@ async def hybrid_search(
for k, v in filters.items():
if v is None or k not in _ALLOWED_FILTER_COLS:
continue
where.append(f"{k} = :flt_{k}")
col = "parent_chunk" if k == "parent_id" else k
where.append(f"{col} = :flt_{k}")
params[f"flt_{k}"] = v
wsql = " AND ".join(where)

Expand Down
40 changes: 40 additions & 0 deletions tests/test_objects.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
"""Raw object store keying and idempotency."""

from __future__ import annotations

import asyncio
import hashlib

import pytest

from kb.storage import objects


class FakeBackend:
def __init__(self) -> None:
self.keys: list[str] = []
self._exists: set[str] = set()

async def exists(self, key: str) -> bool:
return key in self._exists

async def put(self, key: str, blob: bytes, mime: str | None = None) -> None:
self.keys.append(key)
self._exists.add(key)


def test_put_raw_file_is_idempotent_by_content_hash(monkeypatch) -> None:
backend = FakeBackend()
monkeypatch.setattr(objects, "_backend", backend)

first = asyncio.run(objects.put_raw_file(domain="sec", filename="a.pdf", blob=b"same"))
second = asyncio.run(objects.put_raw_file(domain="sec", filename="b.pdf", blob=b"same"))

expected_key = f"raw/sec/{hashlib.sha256(b'same').hexdigest()}"
assert first == second
assert backend.keys == [expected_key]


def test_put_raw_file_rejects_unsafe_domain_segment() -> None:
with pytest.raises(ValueError):
asyncio.run(objects.put_raw_file(domain="../evil", filename="a.pdf", blob=b"same"))
Loading