diff --git a/.github/workflows/build-and-publish.yaml b/.github/workflows/build-and-publish.yaml index a93aa53..3976002 100644 --- a/.github/workflows/build-and-publish.yaml +++ b/.github/workflows/build-and-publish.yaml @@ -3,10 +3,10 @@ name: Build and Publish to PyPI on: push: tags: - - 'v*' + - "v[0-9]+.[0-9]+.[0-9]+" jobs: - build: + test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -19,9 +19,35 @@ jobs: - name: Install Poetry run: pipx install poetry - - name: Set version from tag - if: startsWith(github.ref, 'refs/tags/v') - run: poetry version "${GITHUB_REF_NAME#v}" + - name: Check tag matches pyproject version + run: | + PYPROJECT_VERSION="$(poetry version -s)" + TAG_VERSION="${GITHUB_REF_NAME#v}" + if [ "$PYPROJECT_VERSION" != "$TAG_VERSION" ]; then + echo "Tag v$TAG_VERSION does not match pyproject version $PYPROJECT_VERSION" >&2 + exit 1 + fi + + - name: Install dependencies + run: poetry install --all-extras + + - name: Run tests + run: poetry run pytest + + build: + runs-on: ubuntu-latest + needs: + - test + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install Poetry + run: pipx install poetry - name: Build distributions run: poetry build @@ -35,7 +61,7 @@ jobs: publish-to-test-pypi: environment: name: testpypi - url: https://pypi.org/project/epoch8-smartfeed/ + url: https://test.pypi.org/project/epoch8-smartfeed/ permissions: id-token: write runs-on: ubuntu-latest @@ -54,6 +80,7 @@ jobs: publish-to-pypi: environment: name: pypi + url: https://pypi.org/project/epoch8-smartfeed/ permissions: id-token: write runs-on: ubuntu-latest diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index c5996e3..0e0b1a7 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -5,7 +5,10 @@ on: paths: - ".github/workflows/lint.yaml" - "smartfeed/**" + - "tests/**" - "pyproject.toml" + - "poetry.lock" + pull_request: jobs: lint: @@ -13,8 +16,6 @@ jobs: strategy: matrix: python-version: - # - "3.8" - # - "3.9" - "3.10" steps: @@ -37,10 +38,16 @@ jobs: run: | poetry install --all-extras - - name: Lint with mypy + - name: Lint with ruff run: | - poetry run mypy smartfeed + pip install ruff==0.15.20 + ruff check smartfeed tests + + - name: Type-check with pyright + run: | + pip install pyright==1.1.409 + pyright --pythonpath "$(which python)" smartfeed tests - name: Lint with black run: | - poetry run black --check smartfeed + poetry run black --check smartfeed tests diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 14b0d53..f9b8b51 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -5,7 +5,10 @@ on: paths: - ".github/workflows/tests.yaml" - "smartfeed/**" + - "tests/**" - "pyproject.toml" + - "poetry.lock" + pull_request: jobs: test: @@ -13,18 +16,12 @@ jobs: strategy: matrix: python-version: - # - "3.8" - "3.9" - "3.10" + - "3.11" - "3.12" - "3.13" - services: - redis: - image: redis - ports: - - 6379:6379 - steps: - uses: actions/checkout@v3 diff --git a/.gitignore b/.gitignore index 95f8006..4d10ba0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ __pycache__/ local_settings.py db.sqlite3 media - +frontend/ ### Linux ### *~ @@ -210,4 +210,4 @@ ENV/ /site # Poetry Lock -*.lock \ No newline at end of file +*.lock diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..6a4b1c0 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,161 @@ +# SmartFeed Architecture + +## Overview + +SmartFeed builds a paginated feed from multiple async data sources using a tree config. + +``` +FeedManager.get_feed(session_id, limit, cursor) + -> executor.run(root_node, ctx, limit, cursor) + -> tree of nodes execute in parallel + -> stamp smartfeed_position on results + -> return FeedResult +``` + +## Node Types + +Three types of nodes: + +``` +SubFeed (leaf) -- calls async method from methods_dict +Wrapper (pipeline) -- fetch -> dedup -> rerank -> cache -> paginate +Mixer (coordinator) -- runs children in parallel, assembles results +``` + +## Execution Flow + +``` + FeedManager + | + executor.run() + | + +---------+---------+ + | | + node.execute() node.build_mix_plan() + (SubFeed,Wrapper) (Mixers) + | + asyncio.gather(children) + | + plan.assemble() +``` + +### executor.py + +Module-level functions (no class): + +- `run(node, ctx, limit, cursor)` -- dispatches on `isinstance(node, MixerNode)`: mixers build a MixPlan, leaf/pipeline nodes execute() +- `_execute_mix(plan, ctx, cursor)` -- runs MixPlan children in parallel via `asyncio.gather` + +### Wrapper Pipeline + +``` +fetch child (session_size or limit) + -> dedup (priority arbitration, seen-set from Redis) + -> rerank (call methods_dict callable) + -> cache (write to Redis with TTL) + -> paginate (slice by page from cursor) +``` + +Two paths: +- **Cached** (`self.cache` set): warm = read cache + paginate. Cold = full pipeline. +- **Passthrough** (no cache): fetch + dedup (with Redis seen-set) + rerank per page. + +### MixPlan + +Mixers return `MixPlan(children, assemble)`: + +```python +MixChild(node_id="mix_0", node=subfeed_a, demand=8) # ask for 8 items +MixChild(node_id="mix_1", node=subfeed_b, demand=12) # ask for 12 items +``` + +Executor runs all children via `asyncio.gather`, passes results to `assemble(buffers, child_cursors)`. + +## Key Models + +### Config (Pydantic v2) + +``` +FeedConfig(version, feed: FeedNode) +FeedNode = Union[SubFeed, Wrapper, MergerPercentage, MergerPositional, ...] +``` + +Discriminated union on `type` field. Forward refs rebuilt at import time. + +### Runtime + +``` +ExecutionContext(session_id, methods_dict, redis) +FeedResult(data: list, next_page: dict, has_next_page: bool) +``` + +### Output + +Items are plain dicts; each carries a `_smartfeed_debug_info` dict bundle +(source, smartfeed_position, rerank_position when reranked -- all 0-based). + +## Redis State + +``` +sf:{session_id}:{cache_key or node_id}:{config_hash} -- cached session batch (Redis LIST of orjson items) +sf:{session_id}:{cache_key or node_id}:{config_hash}:meta -- metadata (gen, child_cursor, child_has_next) +sf:{session_id}:{cache_key or node_id}:{config_hash}:coldlock -- cold-build lock (SETNX, ttl 10s) +sf:{session_id}:{node_id}:{config_hash}:seen -- session-scoped dedup seen-set (Redis SET) +sf:{session_id}:{cache_key}:{child_hash}:{segment} -- shared base segment (blob, per continuation window) +sf:{session_id}:{cache_key}:{child_hash}:{segment}:meta -- shared segment meta (child cursor / has_next) +sf:{session_id}:{cache_key}:{child_hash}:{segment}:lock -- shared segment cold-build lock (SETNX, ttl 10s) +``` + +Warm page reads are one MULTI/EXEC pipeline: GET meta + LLEN + LRANGE of the page +window + EXPIRE on data/meta/seen -- an atomic snapshot that transfers only the +requested window, never the whole batch. + +TTL = inactivity timeout. Refreshed on every access via pipeline EXPIRE. +Cursor for a cached wrapper is `{node_id: {offset, gen}}` (absolute offset). + +## Dedup + +Two dedup paths, unified `_dedup(data, seen=None)` method. Both fetch exactly the +outstanding deficit and refill until the target is filled or the child is exhausted +(no over-fetch, so no item loss; no early give-up, so no short pages), and both carry +a session-scoped Redis seen-set so dedup holds across the whole scroll: + +1. **Cached path** (`_fetch_and_dedup` / `_cold_build`): seen-set carried across cold + rebuilds, reset on a fresh scroll (empty cursor). +2. **Passthrough path** (`_passthrough`): seen-set persisted across pages; SADD after each page. + +Priority arbitration: higher `dedup_priority` wins even when the higher-priority copy is +not first-seen; equal priority = first-seen. + +## Config Hash + +`BaseNode.config_hash()` = md5(json.dumps(model_dump(), sort_keys=True, default=str))[:8], +memoized per node instance (config is immutable after validation). + +Used in Redis keys. Any config change = different hash = fresh cache. + +## Generation ID + +Cached wrapper stamps `gen` (random nonce) in cursor and `:meta`. Stale gen = rebuild from scratch. + +## Shared Cache + +Two wrappers with same `cache_key` share base data. Each applies its own rerank. RedisLock prevents duplicate fetches. + +## File Structure + +``` +smartfeed/ + models/ + base.py -- BaseNode, MixerNode, FeedResult, config_hash + subfeed.py -- SubFeed (leaf) + wrapper.py -- Wrapper (cache + dedup + rerank pipeline) + mixers.py -- Percentage, Positional, Append, Distribute, Gradient + __init__.py -- FeedNode union, FeedConfig, exports + execution/ + executor.py -- run(), _execute_mix() (module-level functions) + context.py -- ExecutionContext + plans.py -- MixPlan, MixChild + redis_lock.py -- RedisLock (SETNX async context manager) + manager.py -- FeedManager (public entry point) +``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 094e455..c41b80e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,61 @@ -# 0.2.0 (2025-11-25) +# Changelog + +## 1.0.0 (2026-07-07) + +SmartFeed v2: complete rewrite. Changes below are relative to the last published release, **0.2.0**. (The pre-rewrite internal working tree contained additional intermediate modules — never part of a published release — that are not listed here.) See [MIGRATION.md](MIGRATION.md) for step-by-step upgrade instructions. + +### New Features + +- **Wrapper** with optional pipeline stages: cache, dedup, rerank +- **Session-scoped dedup contract**: within one continuous scroll a user sees each item at most once, no matter how far they scroll; an empty cursor starts a fresh scroll +- **Cross-page dedup** via Redis seen-set (SET with TTL) +- **Deficit-based refill**: dedup returns full pages without discarding unique items (both cached and passthrough paths) +- **Rerank callable** registered in `methods_dict`, referenced by `method_name` in config +- **Shared cache** via `cache_key` for A/B testing with different rerankers +- **Generation ID** for stale cursor detection +- **TTL touch** (inactivity timeout) on every cache access +- **`raise_error`** on WrapperRerank: fail-hard or fail-soft mode +- **`_smartfeed_debug_info`** stamped automatically: source, positions, per-wrapper data (plain dicts, all positions 0-based) +- **`config_hash`** for automatic cache invalidation on config changes +- **Config validation at construction**: duplicate `subfeed_id`/`node_id` anywhere in the tree are rejected (they share one cursor/Redis namespace); gradient `step`/`size_to_step` must be > 0 +- **Input validation**: `get_feed` rejects non-positive `limit`, empty `session_id`, non-dict `cursor`; malformed per-node cursor contents (tampered `offset`/`gen`/`page`) raise `ValueError` naming the node +- **Session batch stored as a Redis LIST**: warm page reads transfer and parse only the requested window (single atomic pipeline: meta + LLEN + LRANGE + TTL refresh) +- **PEP 561 `py.typed` marker**: annotations are visible to downstream type checkers + +### Breaking Changes + +- Models moved from `smartfeed.schemas` to `smartfeed.models` (`FeedManager` keeps its name) +- `FeedManager.get_data(user_id, limit, next_page, **params)` -> `FeedManager.get_feed(session_id, limit, cursor)` +- `user_id` -> `session_id` in ExecutionContext +- Cursors are plain dicts, not `FeedResultNextPage` Pydantic models +- Config: `merger_view_session` -> unified `wrapper` node with optional `cache` / `dedup` / `rerank` stages (`session_live_time` -> `cache.session_ttl`) +- Config: `merger_id` -> `node_id` on merger and wrapper nodes (subfeed `subfeed_id` is unchanged) +- Config: merger-level `shuffle` removed; only `SubFeed.shuffle` remains +- Config: `MergerPositional.start` / `end` / `step` removed; `positions` are now page-relative (were absolute across pages) +- Config: top-level `version` is no longer required or read (unknown top-level keys are ignored, so configs carrying it still parse) +- `FeedResultClient`, `FeedResultNextPage` removed (use `FeedResult` and plain-dict cursors) +- Pydantic v2 required (v1 support removed) + +### Removed (from the 0.2.0 public surface) + +- `smartfeed.schemas` module (models now live in `smartfeed.models`) +- `MergerViewSession` node (replaced by `Wrapper(cache=..., dedup=...)`) +- merger-level `shuffle`; `MergerPositional.start` / `end` / `step` +- `FeedResultClient`, `FeedResultNextPage` +- `examples/` + +### Stats (v2 vs the pre-rewrite internal working tree) + +| Metric | pre-rewrite | v2 | +|--------|-------------|----| +| Python files | 24 | 11 | +| Lines of code | 2838 | ~1650 | +| Test files | 26 | 43 | + +## 0.2.0 (2025-11-25) * Bump dependency versions -# 0.1.0 (2024-12-26) +## 0.1.0 (2024-12-26) * Initial build and publish to PyPI diff --git a/LICENSE b/LICENSE index 90cc1c9..f95ad24 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2023 Epoch8 + Copyright 2023-2026 Epoch8 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile b/Makefile index 7e9caf0..aba1647 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,11 @@ lint: - mypy smartfeed - black --check smartfeed + ruff check smartfeed tests + pyright smartfeed tests + black --check smartfeed tests format: - black --verbose smartfeed tests + black smartfeed tests isort smartfeed tests test: - pytest -s -vv -k "not test_merger_view_session" - -test_cache: - pytest -s -vv -k "test_merger_view_session" + pytest diff --git a/README.md b/README.md index fe96da4..b599c2e 100644 --- a/README.md +++ b/README.md @@ -1,117 +1,367 @@ # SmartFeed -Python-package для формирования ленты (Feed) из клиентских данных с заданной конфигурацией. +Feed orchestrator. Builds one paginated feed from many independent async data sources, described by a declarative JSON/dict config tree. You register your own async fetch functions; SmartFeed handles parallel fetch, cross-source dedup, optional rerank, per-session caching, and cursor-based pagination. -## Содержание: +This README is written for engineers evaluating how it works and how much integration effort it takes. For internal architecture and control flow, see `ARCHITECTURE.md`. -- [Использование](#использование) - - [Установка](#установка) - - [Формирование конфигурации](#формирование-конфигурации) - - [Требования к клиентскому методу](#требования-к-клиентскому-методу) - - [Запуск](#запуск) +- [Install](#install) +- [Quick start](#quick-start) +- [How it works](#how-it-works) +- [Public API](#public-api) +- [Callable contracts you implement](#callable-contracts-you-implement) +- [Config reference](#config-reference) +- [Wrapper pipeline semantics](#wrapper-pipeline-semantics) +- [Dedup and dedup_priority](#dedup-and-dedup_priority) +- [Redis keys and state](#redis-keys-and-state) +- [Error handling and resilience](#error-handling-and-resilience) +- [`_smartfeed_debug_info`](#_smartfeed_debug_info) +- [Integration checklist and footguns](#integration-checklist-and-footguns) +- [Testing](#testing) +- [Integration effort: honest read](#integration-effort-honest-read) -## Использование - -### Установка +## Install ``` -poetry add git+ssh://git@github.com:epoch8/looky-timeline.git +pip install epoch8-smartfeed ``` -### Формирование конфигурации +Requires: Python 3.9+, **pydantic v2**, orjson. An async Redis client (`redis.asyncio.Redis`) is required only if any Wrapper uses `cache` or cross-page `dedup`; without it those features silently fall back to uncached behavior. -Конфигурация каждого фида должна быть словарем следующего вида: -``` -"version": "1", -"view_session": True or False, -"session_size": 800, # по умолчанию 100 -"session_live_time": 500, # по умолчанию 300 -"feed": { - "merger_id": "merger_pos", - "type": "merger_positional", - "positions": [1, 3, 15], - "start": 17, - "end": 200, - "step": 2, - "positional": { - "subfeed_id": "sf_positional", - "type": "subfeed", - "method_name": "ads", - "subfeed_params": { - "limit_to_return": 10, +Note the distribution name: an unrelated 2015 project owns the PyPI name `smartfeed` and installs the **same** top-level `smartfeed` package — install `epoch8-smartfeed`, and do not co-install the other one. + +## Quick start + +```python +from redis.asyncio import Redis + +from smartfeed.manager import FeedManager +from smartfeed.models.base import FeedResult + +config = { + "feed": { + "type": "wrapper", + "node_id": "main", + "cache": {"session_size": 300, "session_ttl": 300}, + "dedup": {"dedup_key": "id"}, + "rerank": {"method_name": "my_rerank"}, + # NOTE: cache_key is a field on the wrapper, NOT inside "cache". + # "cache_key": "shared_pool", + "data": { + "type": "merger_percentage", + "node_id": "mix", + "items": [ + {"percentage": 40, "data": {"type": "subfeed", "subfeed_id": "source_a", "method_name": "source_a"}}, + {"percentage": 60, "data": {"type": "subfeed", "subfeed_id": "source_b", "method_name": "source_b"}}, + ], }, }, - "default": { - "merger_id": "merger_percent", - "type": "merger_percentage", - "shuffle": False, - "items": [ - { - "percentage": 40, - "data": { - "subfeed_id": "sf_1_default_merger_of_main", - "type": "subfeed", - "method_name": "followings", - }, - }, - { - "percentage": 60, - "data": { - "subfeed_id": "sf_2_default_merger_of_main", - "type": "subfeed", - "method_name": "ads", - "raise_error": True or False (default = True), - }, - }, - ], - }, -}, +} + +# A subfeed fetch function. **kwargs is REQUIRED (see Callable contracts). +async def source_a(user_id, limit, next_page, **kwargs): + # ... fetch your data ... + return FeedResult(data=[{"id": 1}, {"id": 2}], next_page={"page": 2}, has_next_page=True) + +async def source_b(user_id, limit, next_page, **kwargs): + return FeedResult(data=[{"id": 3}], next_page={}, has_next_page=False) + +# A rerank function. MUST return exactly len(items) items (reorder only). +async def my_rerank(items, session_id): + return sorted(items, key=lambda x: x.get("score", 0), reverse=True) + +redis = Redis() # or None: cache/dedup then degrade to per-page passthrough + +manager = FeedManager( + config=config, + methods_dict={"source_a": source_a, "source_b": source_b, "my_rerank": my_rerank}, + redis_client=redis, # needed for cache/dedup persistence +) + +result = await manager.get_feed(session_id="user_123", limit=20, cursor=None) +# result.data -> list of item dicts, each stamped with _smartfeed_debug_info +# result.next_page -> opaque cursor; pass back verbatim as `cursor` next call +# result.has_next_page -> bool ``` -### Требования к клиентскому методу +Config is validated when `FeedManager` is constructed. A malformed config raises `pydantic.ValidationError` at construction, not on the first `get_feed`. `subfeed_id` and `node_id` share one namespace (cursor keys, Redis keys) and must be unique across the whole tree — duplicates raise at construction. + +## How it works + +A feed is a tree of nodes described in config. There are three node kinds (seven concrete `type` values): + +- **SubFeed** (`subfeed`) — a leaf. Calls one of your async functions from `methods_dict` and adapts its result into SmartFeed's shape, tagging each item with its source. +- **Wrapper** (`wrapper`) — a pipeline over one child. Optional stages applied in order: dedup, rerank, cache, paginate. Each stage is independent. +- **Mixer** — a combiner over several children, run concurrently and merged by a rule. Five concrete types: `merger_percentage`, `merger_append`, `merger_positional`, `merger_percentage_gradient`, `merger_distribute`. "Mixer" is a documentation umbrella, not a class. + +The executor walks the tree. For each node it either calls `execute()` (SubFeed, Wrapper) or builds a mix plan and runs the children concurrently via `asyncio.gather`, then assembles them. Nodes nest arbitrarily. Each node paginates independently: the returned `next_page` cursor is a dict keyed per node, and you round-trip it verbatim. -Клиентский метод для получения данных должен обязательно включать в себя следующие параметры: -- **user_id: Any** - ID объекта, на который ориентируемся при получении данных субфида. -- **limit: int** - Количество возвращаемых данных. -- **next_page: FeedResultNextPageInside** - Объект курсора пагинации, формируется на стороне клиента после обработки данных. +"Concurrent" here means `asyncio` concurrency on one event loop (sibling sources are awaited together, so page latency is close to the slowest source, not the sum). It is not multi-core. -Возвращаемый тип данных: **FeedResultClient**. +## Public API -### Запуск -Для получения ленты с помощью SmartFeed нужно выполнить следующий код: +The entire public surface is one class with two methods. +```python +FeedManager(config: dict, methods_dict: dict, redis_client: Optional[redis.asyncio.Redis] = None) ``` -from smartfeed.manager import FeedManager -from smartfeed.schemas import FeedResult, FeedResultNextPage, FeedResultNextPageInside +Parses and validates `config` into a `FeedConfig` immediately (pydantic v2). `methods_dict` maps names to your async fetch/rerank callables. `redis_client` must be an async `redis.asyncio.Redis`; if `None`, every Wrapper with `cache`/`dedup` degrades to uncached passthrough. Holds no per-request state. -from client.services import ClientService +```python +async get_feed(session_id: str, limit: int, cursor: Optional[dict] = None) -> FeedResult +``` +`cursor=None` is treated as the first page (`{}`). Builds a fresh execution context per call and delegates to the executor, then stamps each dict item's final 0-based page position. Returns `FeedResult(data, next_page, has_next_page)`. `next_page` is opaque; store it and pass it back as `cursor`. -config = {} # получаем конфигурацию фида -methods_dict = { - "method_1": ClientService().method_1, - "method_2": ClientService().method_2, - # и т.д. -} -# для конфигурации view_session = False, -# Redis передавать небязательно -redis_client = redis.Redis() +Inputs are validated: a non-positive or non-int `limit` and an empty `session_id` raise `ValueError`; a non-dict `cursor` raises `TypeError`. Cursor **contents** are treated as untrusted (they round-trip through clients): a malformed per-node entry (non-dict slice, negative or non-int `offset`, non-str `gen`, non-positive-int `page`) raises `ValueError` naming the node, regardless of any `raise_error` setting. -feed_manager = FeedManager( - config=config, - methods_dict=methods_dict, - redis_client=redis_client, -) +There is **no request-parameters channel**: only `session_id`, `limit`, and the cursor are threaded to sources. Per-request context must be baked into `subfeed_params` when you build the config, or captured in closures when you build `methods_dict` for that request. + +`FeedResult` (`smartfeed.models.base`): `data: List[Any]`, `next_page: dict`, `has_next_page: bool`. Items are usually dicts; non-dict items are passed through untouched by stamping/dedup (hence the deliberately loose element type). + +## Callable contracts you implement + +### SubFeed fetch function + +```python +async def fetch(user_id: str, limit: int, next_page: dict, **kwargs) -> FeedResult: ... +``` + +- **`**kwargs` is mandatory.** The executor injects `ctx=` into every call. A function without `**kwargs` raises `TypeError` on every invocation. (And if that source has `raise_error=False`, the `TypeError` is swallowed into an empty result, so a signature bug looks exactly like "this source had no items today".) +- `user_id` is the feed `session_id` (not necessarily an account id). +- `limit` is this leaf's computed demand for the page. Under a mixer it is a fraction of the page; under a dedup wrapper it is the outstanding refill deficit. +- `next_page` is this subfeed's own cursor slice (`cursor.get(subfeed_id, {})`), opaque. Echo back whatever you need on the next page. +- Return a `FeedResult` (or any object exposing `.data` / `.next_page` / `.has_next_page`; there is no `isinstance` check). +- On success each dict item is stamped with `_smartfeed_debug_info.source = subfeed_id`. Non-dict items are passed through untouched. + +### Rerank function + +```python +async def rerank(items: list, session_id: str) -> list: ... +``` + +- Must return a list of **exactly `len(items)`** items. Reordering or replacing is allowed; adding or dropping is not — a length mismatch always raises `ValueError`, even with `raise_error=False`. +- `items` are already fetched and deduped, in pre-rerank order. +- Exceptions thrown by your function are governed by `rerank.raise_error` (see [Error handling](#error-handling-and-resilience)). + +## Config reference + +All defaults below are byte-exact from the code. `FeedConfig` is the top-level shape: + +| Field | Type | Default | Meaning | +|---|---|---|---| +| `feed` | `FeedNode` | required | Root node. Discriminated union on `type` over the 7 node types. | + +Unknown top-level keys (e.g. a legacy `version` tag) are ignored. + +### SubFeed — `type: "subfeed"` + +| Field | Type | Default | Meaning | +|---|---|---|---| +| `subfeed_id` | `str` | required | Logical name. Stamped as `_smartfeed_debug_info.source`, is the cursor namespace key, and the identity dedup priority resolves against. | +| `method_name` | `str` | required | Key into `methods_dict`. Looked up **before** the try block, so a missing/typo'd name raises `KeyError` even with `raise_error=False`. | +| `subfeed_params` | `dict` | `{}` | Static kwargs merged into the fetch call. A key colliding with `user_id`/`limit`/`next_page`/`ctx` raises `TypeError` at call time (not validated at parse time). | +| `raise_error` | `bool` | `True` | `True`: exceptions propagate. `False`: swallow, return an empty page. On the swallow path `next_page` is `{subfeed_id: }` with `has_next_page=False`, so the source retries the same position next page without clobbering sibling cursors on merge. | +| `shuffle` | `bool` | `False` | Shuffle this source's page in place before stamping. Per-page only. | +| `dedup_priority` | `int` | `0` | See [dedup_priority](#dedup-and-dedup_priority). | + +### Wrapper — `type: "wrapper"` + +Wraps exactly one child with optional stages. `cache`, `dedup`, `rerank` are all independent and optional (all 8 on/off combinations parse). + +| Field | Type | Default | Meaning | +|---|---|---|---| +| `node_id` | `str` | required | Namespaces this wrapper's cursor, its cache keys (when `cache_key` unset), and its debug sub-dict. | +| `data` | node | required | The single child node. | +| `cache` | `WrapperCache?` | `None` | Enables per-session caching. Requires a live `redis_client` too, else passthrough. | +| `dedup` | `WrapperDedup?` | `None` | Enables duplicate removal with overfetch/refill. | +| `rerank` | `WrapperRerank?` | `None` | Enables a rerank pass (after fetch+dedup). | +| `cache_key` | `str?` | `None` | **Sibling of `cache`, not inside it.** Enables a shared base cache across wrappers with the same key (see [Wrapper pipeline](#wrapper-pipeline-semantics)). No-op unless `cache` is also set. | +| `dedup_priority` | `int` | `0` | See below. | + +`WrapperCache`: + +| Field | Type | Default | Meaning | +|---|---|---|---| +| `session_size` | `int` | `300` | Items built and cached per session (cold build). Pages are sliced from this. | +| `session_ttl` | `int` | `300` | Redis TTL (s) on the data + `:meta` keys. Refreshed on every warm read — an inactivity timeout, not a fixed expiry. | + +`WrapperDedup`: + +| Field | Type | Default | Meaning | +|---|---|---|---| +| `dedup_key` | `str` | required | Item field for identity. Compared as `str(item[key])`, so `1` and `"1"` collide. | +| `missing_key_policy` | `"error" \| "keep" \| "drop"` | `"error"` | When an item lacks `dedup_key`: `error` raises, `keep` passes it through un-deduped, `drop` discards it. | +| `state_ttl` | `int` | `300` | TTL (s) on the Redis seen-set (session-scoped dedup state, both cache and passthrough paths). Refreshed on every warm read, like the cache TTL. | + +(The pre-release `overfetch_factor` / `max_refill_loops` knobs are gone: the dedup fetch pulls exactly the outstanding deficit and refills until the page is full or the child is exhausted. Unknown keys in the config are ignored, so configs still carrying them parse.) + +`WrapperRerank`: + +| Field | Type | Default | Meaning | +|---|---|---|---| +| `method_name` | `str` | required | Key into `methods_dict`. `KeyError` if missing. | +| `raise_error` | `bool` | `True` | Governs only exceptions thrown by your rerank function. Does **not** govern the length-mismatch check, which always raises. | + +### Mixers + +**MergerPercentage** — `type: "merger_percentage"`. Splits the page by fixed percentages, then concatenates a solid block per branch in config order (it does not interleave). + +```json +{"type": "merger_percentage", "node_id": "mix", + "items": [{"percentage": 40, "data": {…}}, {"percentage": 60, "data": {…}}]} +``` +| Field | Type | Default | Meaning | +|---|---|---|---| +| `node_id` | `str` | required | Prefix for internal child ids. | +| `items` | `list[{percentage:int, data:node}]` | required | Per-branch demand = `limit * percentage // 100`. Rounding remainder is topped up (largest-remainder) **only when the percentages sum to exactly 100**. `percentage` has no range validator; sum is not enforced. | +| `dedup_priority` | `int` | `0` | | + +There is no `[:limit]` trim: percentages summing over 100 overflow the page, under 100 underfill. + +**MergerAppend** — `type: "merger_append"`. Equal-share split, concatenated in order, then trimmed to `limit`. + +```json +{"type": "merger_append", "node_id": "append", "items": [{…}, {…}]} +``` +| Field | Type | Default | Meaning | +|---|---|---|---| +| `node_id` | `str` | required | | +| `items` | `list[node]` | required | Each child gets `limit // n`; the remainder goes one-each to the first children. Empty `items` yields an empty page (no error). | +| `dedup_priority` | `int` | `0` | | + +**MergerPositional** — `type: "merger_positional"`. Places `positional` items at fixed slots, `default` items everywhere else. + +```json +{"type": "merger_positional", "node_id": "pos", + "positions": [1, 3, 5, 7], "positional": {…}, "default": {…}} +``` +| Field | Type | Default | Meaning | +|---|---|---|---| +| `node_id` | `str` | required | | +| `positions` | `list[int]` | `[]` | 1-indexed slots, relative to **each page's** `1..limit` window (they repeat every page; they are not absolute feed positions). | +| `positional` | node | required | Fills the pinned slots. Demand = count of in-range positions. | +| `default` | node | required | Fills the rest. Demand = `limit - pos_count`, fixed regardless of actual returns. | +| `dedup_priority` | `int` | `0` | | + +If a source runs dry the other backfills; if the positional source is exhausted the page can underfill (default demand is fixed). + +**MergerPercentageGradient** — `type: "merger_percentage_gradient"`. A two-child percentage merger whose ratio shifts by `step` points every `size_to_step` cumulative items, tracked across pages via its own cursor counter. + +```json +{"type": "merger_percentage_gradient", "node_id": "grad", + "item_from": {"percentage": 80, "data": {…}}, "item_to": {"percentage": 20, "data": {…}}, + "step": 10, "size_to_step": 30} +``` +| Field | Type | Default | Meaning | +|---|---|---|---| +| `node_id` | `str` | required | Also the merged-cursor key holding `{"page": n}`. | +| `item_from` | `{percentage:int, data:node}` | required | Starting branch. Its share falls toward 0. | +| `item_to` | `{percentage:int, data:node}` | required | Ending branch. Its share rises toward 100, then clamps. | +| `step` | `int` | required | Percentage points shifted per segment. Must be > 0 (validated at parse). | +| `size_to_step` | `int` | required | Items per shift. Must be > 0 (validated at parse). | +| `dedup_priority` | `int` | `0` | | + +**MergerAppendDistribute** — `type: "merger_distribute"` (note the literal). Round-robin by a key so no two adjacent items share it. + +```json +{"type": "merger_distribute", "node_id": "diverse", + "distribution_key": "operator_id", "sorting_key": "score", "sorting_desc": true, "items": [{…}]} +``` +| Field | Type | Default | Meaning | +|---|---|---|---| +| `node_id` | `str` | required | | +| `items` | `list[node]` | required | **Every child is asked for the full `limit`** (up to `N * limit` fetched), then pooled and trimmed to `limit`. Higher backend load than items shown. | +| `distribution_key` | `str` | required | Group key. `entry[key]` with no missing-key policy — `KeyError` if an item lacks it. | +| `sorting_key` | `str?` | `None` | If set, the whole pool is sorted by this key before grouping. `KeyError` if missing. | +| `sorting_desc` | `bool` | `False` | Sort direction. Ignored if `sorting_key` is `None`. | +| `dedup_priority` | `int` | `0` | | + +## Wrapper pipeline semantics + +A Wrapper has two paths, chosen at runtime: + +- **Cached** (`cache` set **and** `redis_client` present). First request for a session is a cold build: fetch `session_size` items, dedup, rerank, store the batch in Redis, serve page 1. Later pages slice out of the stored batch with no refetch. When the batch is exhausted the pipeline continues from where the child left off (it does not restart the source). Concurrent first requests for a session are serialized by a cold-build lock, so the child is fetched once. Rerank runs once per cold build over the whole batch, not per page. +- **Passthrough** (no `cache`, or no Redis). The pipeline reruns every page. + +**Dedup contract (session-scoped).** Within one continuous forward scroll, a user sees each item at most once — no matter how far they scroll, and across the internal rebuild boundaries the cursor crosses. Dedup is scoped to the session, not across sessions: re-sending an empty cursor (or page 1) starts a **fresh** scroll and may repeat items already shown in a previous scroll. Both paths honor this via a Redis seen-set (`:seen`): the passthrough path persists shown ids across pages; the cached path carries the seen-set across cold rebuilds and resets it when a fresh scroll begins (empty cursor). Neither path over-fetches, so no unique item is silently skipped, and refill fills each page to `limit` unless the source is genuinely exhausted. + +**Shared base cache (`cache_key`).** Two wrappers with different `node_id` but the same `cache_key` share one fetched+deduped base dataset (keyed by `cache_key` + the child's config hash + the continuation position), each applying its own rerank on top. Each page-window is its own shared segment, so a shared wrapper paginates past `session_size` normally. A distributed lock ensures only one caller does each cold fetch. Caveat: whichever wrapper wins the cold-build race bakes in **its** `session_size` and dedup settings for all sharers; a sibling with different settings reads that same pool. The shared base dedups within each segment; for strict cross-scroll uniqueness prefer a non-shared cached wrapper. Use `cache_key` for A/B testing two rerankers over one item pool; do not rely on per-sharer dedup/size differences. + +## Dedup and dedup_priority + +Every node has `dedup_priority: int = 0`. When a Wrapper's dedup sees the same key from two sources, the item whose source subtree has the higher effective priority wins; equal priority means first-seen wins (order follows the mixer's child list). Priority propagates down: a node with a non-zero `dedup_priority` overrides its whole subtree unless a descendant sets its own non-zero value. It is config-only and never written onto items. + +## Redis keys and state + +All SmartFeed state lives in Redis under `sf:{session_id}:...`. `config_hash` is the first 8 hex chars of an md5 over the node's sorted JSON dump, so any config change moves the key and old data ages out. + +``` +sf:{session_id}:{cache_key or node_id}:{config_hash} cached session batch (Redis LIST of orjson items) +sf:{session_id}:{cache_key or node_id}:{config_hash}:meta gen nonce, child cursor, child has_next +sf:{session_id}:{cache_key or node_id}:{config_hash}:coldlock cold-build lock (ttl 10s) +sf:{session_id}:{node_id}:{config_hash}:seen dedup seen-set (session-scoped; cache + passthrough) +sf:{session_id}:{cache_key}:{child_config_hash}:{segment} shared base segment (when cache_key set) +sf:{session_id}:{cache_key}:{child_config_hash}:{segment}:meta shared base segment meta +sf:{session_id}:{cache_key}:{child_config_hash}:{segment}:lock shared segment cold-build lock (ttl 10s) +``` + +(`{segment}` is `"0"` for page 1 and a short hash of the continuation cursor for later windows.) + +TTL (`session_ttl`) is refreshed on every warm read, so an active user's cache does not expire mid-scroll. The `:seen` set has its own `state_ttl`. Each cold build stamps a random `gen` nonce into the cursor and `:meta`; if a returning cursor's `gen` no longer matches (cache expired or rebuilt), SmartFeed rebuilds cleanly from page 1 rather than resuming into an inconsistent state. + +Note the `next_page` cursor shape is config-dependent: a cached wrapper hides the child cursor behind `{node_id: {offset, gen}}` (an absolute offset, so a client may change `limit` between pages without skipping or repeating items), an uncached wrapper exposes the raw child cursor, and the gradient injects `{node_id: {page}}`. Adding or removing a `cache` block changes the cursor your clients round-trip. + +## Error handling and resilience + +Error handling is **opt-in per node**. By default (`raise_error=True` everywhere) one failing source or rerank fails the entire `get_feed` call, including sibling branches that already succeeded — `asyncio.gather` runs without `return_exceptions=True`, and `FeedManager` has no try/except of its own. For fail-soft behavior, set `raise_error=False` on the specific SubFeed/rerank nodes. + +Five sharp edges an integrator will hit: + +1. **Missing `method_name`** is looked up before the try block, so it raises `KeyError` regardless of `raise_error`. +2. **Rerank length mismatch** always raises `ValueError`, regardless of `rerank.raise_error`. +3. **A subfeed function without `**kwargs`** raises `TypeError` on every call (the injected `ctx` has nowhere to go). With `raise_error=False` this is swallowed into an empty page — a signature bug that reads as "no items". +4. **`subfeed_params` key collisions** (with `user_id`/`limit`/`next_page`/`ctx`) raise `TypeError` at call time, with the same silent-swallow caveat under `raise_error=False`. +5. **Malformed cursor contents** raise `ValueError` regardless of `raise_error` — cursors are validated as untrusted client input (see [Public API](#public-api)). Catch it at your API boundary and treat it as a bad request, not a server error. + +Silent behavior downgrades (no exception): `cache` without a `redis_client`; `cache_key` without `cache`; divergent shared-`cache_key` settings. These ship without error and only show up as degraded cache/dedup metrics. + +## `_smartfeed_debug_info` + +Every dict item carries a `_smartfeed_debug_info` bundle — a plain dict; there is no model class behind it. The core writes these (all positions **0-based**): + +- `source` — the producing `subfeed_id`. +- `smartfeed_position` — final 0-based page position (set by `FeedManager`). Wrappers also stamp a pre-rerank position under their `node_id`. +- `rerank_position` — 0-based post-rerank index under the wrapper's `node_id`, only when rerank is configured. + +Any other fields (`strategy`, `rrf_score`, `feature_score`, `feature_position`, ...) are **conventions only**; the core never writes them — your fetch/rerank callables may attach whatever extra keys they like. The pipeline operates on plain dicts throughout. + +## Integration checklist and footguns + +What you must provide: + +- One async fetch function per `method_name`, each accepting `**kwargs` and returning a `FeedResult`. +- Any rerank functions, each returning exactly `len(items)` items. +- The `methods_dict` (a plain dict; no registration API). +- The config tree. pydantic checks types and shapes only; **semantic correctness is yours** — matching `dedup_key`s across siblings, percentages summing to 100, `cache_key` placed on the Wrapper, distribution/sorting keys present on items. +- An async `redis.asyncio.Redis` if you want caching or cross-page dedup persistence. +- Explicit `raise_error=False` on each node you want to fail soft (no global switch). +- Keep cached items orjson-serializable (`str/int/float/bool/None/list/dict`). A `Decimal` that works in passthrough throws once caching serializes the batch. + +Footguns, condensed: + +- `**kwargs` is load-bearing on every subfeed function (absorbs injected `ctx`). +- `cache_key` goes on the Wrapper, not inside `cache`, and is a no-op without `cache`. +- `cache` without Redis, and shared `cache_key` with divergent settings, degrade silently. +- Percentage mixers do not interleave and do not enforce sum==100; positional slots are per-page; distribute over-fetches and has no missing-key policy. +- Cursor shape depends on whether a `cache` block is present. + +## Testing + +- `fakeredis.aioredis.FakeRedis()` is the standard fixture; the whole integration is testable without a real Redis. +- `pytest` + `pytest-asyncio` (`@pytest.mark.asyncio`). Both are pinned dev deps. +- `tests/conftest.py` has a minimal `METHODS` dict (items / empty / error) as a starting point. +- There is no built-in validator for your subfeed signatures. Add a smoke test that runs each configured method through `get_feed` once before deploy — specifically to catch the `**kwargs`/`ctx` and `subfeed_params`-collision classes, which surface only at call time. + +``` +pytest tests/ -v +``` -user_id = "sjjdj?" # любой тип данных -limit = 100 -next_page = FeedResultNextPage( - data={ - "subfeed_id": FeedResultNextPageInside(page=1, after=None), - } -) -data: FeedResult = await feed_manager.get_data( - user_id=user_id, - limit=limit, - next_page=next_page, -) -``` \ No newline at end of file diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..84c2ce1 --- /dev/null +++ b/TODO.md @@ -0,0 +1,121 @@ +# TODO — next version + +Planning notes for the next batch of changes. Current error handling works but is +far from ideal; this file scopes what "ideal" looks like and the open questions. + +> Compat reminder (from CLAUDE.md): runtime is **Python 3.9+**. Anything below that +> relies on 3.11-only APIs (e.g. `Exception.add_note`) needs a fallback. + +--- + +## Theme: error handling overhaul + +### Current state (for reference) + +- **Fail-hard is the default** (`SubFeed.raise_error = True`). On failure the exception + is re-raised at `smartfeed/models/subfeed.py:47`, then propagates through the mixer's + `asyncio.gather(...)` at `smartfeed/execution/executor.py:34` (no `return_exceptions`), + so it bubbles out of `get_feed` raw. +- **Fail-soft is opt-in** (`raise_error=False`). The subfeed swallows the exception and + returns `FeedResult(data=[], next_page={subfeed_id: }, has_next_page=False)` + (`subfeed.py:47`). It is **completely silent** — no log, no debug stamp (empty data = + nothing to stamp), so the caller cannot tell "source failed" from "source was empty". +- **Backfill of a dead source's slots** only happens via the wrapper refill loop + (`wrapper.py:251` passthrough-with-dedup, `wrapper.py:663` cached cold build). A bare + mixer or a non-dedup passthrough does **not** redistribute — the dead child's demand + share just yields nothing and the page comes back short. +- The package currently has **no logging and no `print`** anywhere — there is no logger + to hook into yet. + +--- + +### 1. Fail-hard: propagate a clear, understandable error + +**Problem.** When a hard failure crashes the whole feed, the exception that surfaces is +the raw error from the user's method. It carries no smartfeed-level context: *which* +subfeed (`subfeed_id`), *where* in the config tree, on *which* page/cursor. Debugging a +crash means guessing which source blew up. + +**Desired.** The propagated error names the failing node and preserves the original +traceback — e.g. "SubFeed 'trending_posts' raised while fetching page N" chained to the +original exception, so `raise ... from e` keeps the root cause visible. + +**Ideas / anchors.** +- Catch at `subfeed.py:47` in the `raise_error=True` branch; attach context, then re-raise. +- Preserve the original: either a dedicated `SubFeedError`/`SmartFeedError` that chains via + `from e`, or annotate in place. **`Exception.add_note` is 3.11+** — needs a 3.9/3.10 + fallback (wrap in a custom exception, or prepend context to a re-raised copy). +- Consider the `gather` semantics: with the default, only the *first* exception is raised + and sibling failures are discarded. Decide whether to keep first-wins or switch to + `return_exceptions=True` and aggregate (e.g. surface an ExceptionGroup / a combined + message) — `executor.py:34`. + +**Open questions.** +- Introduce a real exception hierarchy (`SmartFeedError` base) or keep raising the + original type with added context? +- Should the error object carry structured fields (node_id, cursor slice, page) for + programmatic handling, not just a message? + +--- + +### 2. Fail-soft: observability + self-heal without a dedup node + +Two sub-goals. + +**2a. Make soft failures observable (not silent).** + +Today a soft failure vanishes. We want the failure recorded somewhere the integrator can +see. Options to weigh: +- Emit via the stdlib `logging` module (`logging.getLogger("smartfeed")`) — library-correct, + but there's no logger anywhere yet, so this is a from-scratch decision on logger name / + levels / what we log. +- Surface it in the result: a feed-level `errors` channel (per-source failure list) on + `FeedResult`, or a debug entry. Note: item-level `_smartfeed_debug_info` won't work for a + failed source because `data=[]` — there's no item to stamp. Needs a feed-level home. +- Optional user callback / hook (`on_source_error=...`) so the integrator decides. +- (Explicitly *not* `print` — a published library must not print.) + +**Open questions.** +- Where does a soft error live: logs only, `FeedResult` field, callback, or several? +- Does adding an `errors` field to `FeedResult` break the documented "byte-exact" result + shape? If so, gate it / version it. + +**2b. Self-heal (backfill dead-source positions) even with no dedup node.** *(ideal / exploratory — no chosen design yet)* + +Today, surviving siblings only backfill a dead source's slots when a dedup wrapper (or a +cached cold build) sits above, because only the wrapper runs a deficit-refill loop. Goal: +the page fills from healthy sources even when there's no dedup wrapper in the tree. + +**Tensions / constraints to respect.** +- **Do not reintroduce over-fetch.** Over-fetching demand as a buffer was deliberately + removed (commit `d799de8` "overfetch killed bc YAGNI"; pinned by + `tests/test_bug_overfetch_item_loss.py`). Any backfill design must not silently discard + fetched items or re-open that bug. +- The current mixer model fans out **once** via `gather` then `assemble` — there is no + re-fetch round at the mixer level today. + +**Candidate directions (pick / prototype later).** +- Generalize the wrapper's deficit-refill loop **down into mixers / the executor**, so any + mixer can re-request the outstanding deficit from its surviving children (mirrors + `wrapper.py:663`, without requiring dedup). Decide where it lives: `executor.run` for + mixers, or a new `MixerNode` capability. +- Decide proportion behavior when a source dies: do the survivors' percentages + **renormalize** to cover the gap, or just fill in configured order? +- Bound it like the wrapper does (`_REFILL_SCAN_FACTOR`) so a permanently-dead or + duplicate-only source can't spin the loop forever. + +**Open questions.** +- Should self-heal be always-on, or opt-in per mixer (a `backfill=True` flag)? +- Interaction with soft-fail's unadvanced self-scoped cursor (`subfeed.py:52`): a source + that fails every round must not stall the loop — needs a "give up on this child" signal. + +--- + +## Cross-cutting (don't forget when implementing) + +- Add `tests/test_bug_*.py` pins for each fixed behavior (repo convention — don't fold into + existing files). +- Update `README.md` "Error philosophy" + the `ARCHITECTURE.md` internals if behavior/keys + move; add a `CHANGELOG.md` entry. +- Keep public annotations precise (`py.typed`); run `make lint` (ruff + pyright + black). +- Honor 3.9+ compat for any exception-notes / logging approach. diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 8657196..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '3.8' - -services: - sfredis: - image: redis - container_name: sfredis - command: [ 'redis-server', '--port', '6379' ] - ports: - - '6379:6379' diff --git a/pyproject.toml b/pyproject.toml index 108a4c7..ee6a9e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "epoch8-smartfeed" -version = "0.2.0" +version = "0.3.0" description = "" authors = [ "Epoch8 Team " @@ -12,15 +12,17 @@ packages = [ [tool.poetry.dependencies] python = ">=3.9" -pydantic = ">=1.10.7" +orjson = ">=3.9.0" +pydantic = ">=2.0" redis = ">=4.5.5" [tool.poetry.group.dev.dependencies] -isort = "^5.12.0" black = "^23.3.0" +fakeredis = "^2.34.1" +isort = "^5.12.0" mypy = "^1.3.0" -pytest = "^7.3.1" -pytest-asyncio = "^0.21.0" +pytest = ">=8.2" +pytest-asyncio = ">=1.0" types-redis = "^4.5.5.2" [tool.black] @@ -43,6 +45,9 @@ strict_optional = true disallow_any_expr = false python_version = "3.10" +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["F811"] + [tool.pytest.ini_options] testpaths = ["tests"] python_files = ["tests.py", "test_*.py", "*_test.py"] diff --git a/smartfeed/__init__.py b/smartfeed/__init__.py index e69de29..c6a7df9 100644 --- a/smartfeed/__init__.py +++ b/smartfeed/__init__.py @@ -0,0 +1,39 @@ +from .manager import FeedManager +from .models import ( + BaseNode, + FeedConfig, + FeedNode, + FeedResult, + MergerAppend, + MergerAppendDistribute, + MergerPercentage, + MergerPercentageGradient, + MergerPercentageItem, + MergerPositional, + MixerNode, + SubFeed, + Wrapper, + WrapperCache, + WrapperDedup, + WrapperRerank, +) + +__all__ = [ + "BaseNode", + "FeedConfig", + "FeedManager", + "FeedNode", + "FeedResult", + "MergerAppend", + "MergerAppendDistribute", + "MergerPercentage", + "MergerPercentageGradient", + "MergerPercentageItem", + "MergerPositional", + "MixerNode", + "SubFeed", + "Wrapper", + "WrapperCache", + "WrapperDedup", + "WrapperRerank", +] diff --git a/smartfeed/examples/example_client.py b/smartfeed/examples/example_client.py deleted file mode 100644 index 9a421ff..0000000 --- a/smartfeed/examples/example_client.py +++ /dev/null @@ -1,171 +0,0 @@ -import base64 -import json -from typing import Optional, Union - -from pydantic import BaseModel, Field, validator - -from smartfeed.schemas import FeedResultClient, FeedResultNextPage, FeedResultNextPageInside - - -class TestClientRequest(BaseModel): - """ - Пример модели клиентского входящего запроса. - """ - - profile_id: str = Field(...) - limit: int = Field(...) - next_page: Union[str, FeedResultNextPage] = Field( - base64.urlsafe_b64encode(json.dumps({"data": {}}).encode()).decode() - ) - - class Config: - validate_all = True - - @validator("next_page") - def validate_next_page(cls, value: Union[str, FeedResultNextPage]) -> Union[str, FeedResultNextPage]: - if isinstance(value, str): - return FeedResultNextPage.parse_obj(json.loads(base64.urlsafe_b64decode(value))) - return value - - -class ClientMixerClass: - """ - Пример клиентского класса ClientMixer. - """ - - @staticmethod - async def example_method( - user_id: str, - limit: int, - next_page: FeedResultNextPageInside, - limit_to_return: Optional[int] = None, - ) -> FeedResultClient: - """ - Пример клиентского метода. - - :param user_id: ID профиля. - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param limit_to_return: ограничить кол-во результата. - :return: массив букв "profile_id" в количестве "limit" штук. - """ - - data = [f"{user_id}_{i}" for i in range(1, 1000)] - - from_index = (data.index(next_page.after) + 1) if next_page.after else 0 - to_index = from_index + limit - - result_data = data[from_index:to_index] - - if isinstance(limit_to_return, int) and limit_to_return > 0: - result_data = result_data[:limit_to_return] - - next_page.after = result_data[-1] if result_data else None - next_page.page += 1 - - result = FeedResultClient(data=result_data, next_page=next_page, has_next_page=True) - return result - - @staticmethod - async def empty_method( - user_id: str, # pylint: disable=W0613 - limit: int, # pylint: disable=W0613 - next_page: FeedResultNextPageInside, - limit_to_return: Optional[int] = None, # pylint: disable=W0613 - ) -> FeedResultClient: - """ - Пример клиентского метода, возвращающего пустые данные. - - :param user_id: ID профиля. - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param limit_to_return: ограничить кол-во результата. - :return: массив букв "profile_id" в количестве "limit" штук. - """ - - next_page.after = None - next_page.page += 1 - - result = FeedResultClient(data=[], next_page=next_page, has_next_page=False) - return result - - @staticmethod - async def error_method( - user_id: str, # pylint: disable=W0613 - limit: int, # pylint: disable=W0613 - next_page: FeedResultNextPageInside, - limit_to_return: Optional[int] = None, # pylint: disable=W0613 - ) -> FeedResultClient: - """ - Пример клиентского метода, возвращающего пустые данные. - - :param user_id: ID профиля. - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param limit_to_return: ограничить кол-во результата. - :return: массив букв "profile_id" в количестве "limit" штук. - """ - - next_page.after = None - next_page.page = int(10 / 0) - - result = FeedResultClient(data=[], next_page=next_page, has_next_page=False) - return result - - @staticmethod - async def doubles_method( - user_id: str, # pylint: disable=W0613 - limit: int, # pylint: disable=W0613 - next_page: FeedResultNextPageInside, - limit_to_return: Optional[int] = None, # pylint: disable=W0613 - ) -> FeedResultClient: - """ - Пример клиентского метода, возвращающего данные с дублями. - - :param user_id: ID профиля. - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param limit_to_return: ограничить кол-во результата. - :return: массив целых чисел, равный [i for i in range(1, 11)] после удаления дублей. - """ - - data = [1, 2, 3, 4, 3, 2, 5, 6, 4, 4, 7, 8, 9, 10, 9, 9, 9] - - next_page.after = None - next_page.page += 1 - - result = FeedResultClient(data=data, next_page=next_page, has_next_page=False) - return result - - @staticmethod - async def keys_method( - user_id: str, - limit: int, - next_page: FeedResultNextPageInside, - limit_to_return: Optional[int] = None, - ) -> FeedResultClient: - """ - Пример клиентского метода. - - :param user_id: ID профиля. - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param limit_to_return: ограничить кол-во результата. - :return: массив букв "profile_id" в количестве "limit" штук. - """ - - data = [{"user_id": f"{user_id}_{i%10}", "value": i} for i in range(1, 1000)] - - from_index = (data.index(next_page.after) + 1) if next_page.after else 0 - to_index = from_index + limit - - result_data = data[from_index:to_index] - - if isinstance(limit_to_return, int) and limit_to_return > 0: - result_data = result_data[:limit_to_return] - - next_page.after = result_data[-1] if result_data else None - next_page.page += 1 - - result = FeedResultClient(data=result_data, next_page=next_page, has_next_page=True) - return result diff --git a/smartfeed/execution/context.py b/smartfeed/execution/context.py new file mode 100644 index 0000000..a87d6e4 --- /dev/null +++ b/smartfeed/execution/context.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional + +from redis.asyncio import Redis as AsyncRedis + + +@dataclass +class ExecutionContext: + session_id: str + methods_dict: Dict[str, Callable[..., Any]] + redis: Optional[AsyncRedis[Any]] = None diff --git a/smartfeed/execution/executor.py b/smartfeed/execution/executor.py new file mode 100644 index 0000000..8eb555e --- /dev/null +++ b/smartfeed/execution/executor.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import asyncio +from typing import Any, Dict, List + +from smartfeed.models.base import BaseNode, FeedResult, MixerNode + +from .context import ExecutionContext +from .plans import MixPlan + + +async def run(node: BaseNode, ctx: ExecutionContext, limit: int, cursor: Dict[str, Any]) -> FeedResult: + """Execute a feed node. Mixers fan out via a MixPlan; leaf/pipeline nodes execute directly.""" + if isinstance(node, MixerNode): + plan = node.build_mix_plan(ctx=ctx, limit=limit, cursor=cursor) + return await _execute_mix(plan, ctx, cursor) + + return await node.execute( + methods_dict=ctx.methods_dict, + session_id=ctx.session_id, + limit=limit, + cursor=cursor, + ctx=ctx, + ) + + +async def _execute_mix(plan: MixPlan, ctx: ExecutionContext, cursor: Dict[str, Any]) -> FeedResult: + """Execute a MixPlan: run children in parallel, assemble results.""" + if not plan.children: + merged, merged_cursor = plan.assemble({}, {}) + return FeedResult(data=merged, next_page=merged_cursor, has_next_page=False) + + tasks = [run(c.node, ctx, c.demand, cursor) for c in plan.children] + results = await asyncio.gather(*tasks) + + buffers: Dict[str, List[Any]] = {} + child_cursors: Dict[str, Dict[str, Any]] = {} + any_has_next = False + for child, result in zip(plan.children, results): + buffers[child.node_id] = result.data + child_cursors[child.node_id] = result.next_page + any_has_next = any_has_next or result.has_next_page + + merged, merged_cursor = plan.assemble(buffers, child_cursors) + return FeedResult(data=merged, next_page=merged_cursor, has_next_page=any_has_next) diff --git a/smartfeed/execution/plans.py b/smartfeed/execution/plans.py new file mode 100644 index 0000000..fab4cd3 --- /dev/null +++ b/smartfeed/execution/plans.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple + +if TYPE_CHECKING: + from smartfeed.models.base import BaseNode + + +@dataclass +class MixChild: + node_id: str + node: BaseNode + demand: int + + +@dataclass +class MixPlan: + children: List[MixChild] + # (buffers: {child_id: items}, cursors: {child_id: cursor}) -> (merged items, merged cursor) + assemble: Callable[[Dict[str, List[Any]], Dict[str, Dict[str, Any]]], Tuple[List[Any], Dict[str, Any]]] diff --git a/smartfeed/execution/redis_lock.py b/smartfeed/execution/redis_lock.py new file mode 100644 index 0000000..6bfbb54 --- /dev/null +++ b/smartfeed/execution/redis_lock.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import secrets +from typing import Any + +from redis.asyncio import Redis as AsyncRedis +from redis.exceptions import WatchError + + +class RedisLock: + """Distributed lock via SETNX with unique token. + + async with RedisLock(redis, "key") as acquired: + if acquired: + ... # we hold the lock + else: + ... # someone else holds it + """ + + def __init__(self, redis: "AsyncRedis[Any]", key: str, ttl: int = 10): + self._redis = redis + self._key = key + self._ttl = ttl + self._token = secrets.token_hex(8) + self._owned = False + + async def __aenter__(self) -> bool: + self._owned = bool(await self._redis.set(self._key, self._token, nx=True, ex=self._ttl)) + return self._owned + + async def __aexit__(self, *exc: object) -> None: + if not self._owned: + return + # Atomic compare-and-delete: WATCH the key so that if it changes between our + # read and the DELETE (our TTL expired and someone else acquired), the EXEC + # aborts instead of deleting the new holder's lock. + async with self._redis.pipeline(transaction=True) as pipe: + try: + await pipe.watch(self._key) + val = await self._redis.get(self._key) + if val is None: + return # lock expired; nothing to release + # Redis clients configured with decode_responses=True return str, not bytes. + token = val.decode() if isinstance(val, bytes) else val + if token != self._token: + return # lock expired and re-acquired by someone else; not ours + pipe.multi() + pipe.delete(self._key) + await pipe.execute() + except WatchError: + pass # key changed under us -> it is no longer ours to delete diff --git a/smartfeed/manager.py b/smartfeed/manager.py index e91bbe9..a385776 100644 --- a/smartfeed/manager.py +++ b/smartfeed/manager.py @@ -1,46 +1,64 @@ -from typing import Any, Dict, Optional, Union +from __future__ import annotations + +from typing import Any, Callable, Dict, Optional -import redis from redis.asyncio import Redis as AsyncRedis -from .schemas import FeedConfig, FeedResult, FeedResultNextPage +from .execution import executor as _executor +from .execution.context import ExecutionContext +from .models import FeedConfig +from .models.base import FeedResult class FeedManager: - """ - Класс FeedManager. - """ - - def __init__(self, config: Dict, methods_dict: Dict, redis_client: Optional[Union[redis.Redis, AsyncRedis]] = None): - """ - Инициализация класса FeedManager. - - :param config: конфигурация. - :param methods_dict: словарь с используемыми методами. - :param redis_client: объект клиента Redis (для конфигурации с view_session = True). - """ - - self.feed_config = FeedConfig.parse_obj(config) + def __init__( + self, + config: Dict[str, Any], + methods_dict: Dict[str, Callable[..., Any]], + redis_client: Optional[AsyncRedis[Any]] = None, + ) -> None: + self.config = FeedConfig.model_validate(config) self.methods_dict = methods_dict self.redis_client = redis_client - async def get_data(self, user_id: Any, limit: int, next_page: FeedResultNextPage, **params: Any) -> FeedResult: - """ - Метод для получения данных согласно конфигурации. - - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: лимит на выдачу данных. - :param next_page: курсор для пагинации в формате SmartFeedResultNextPage. - :param params: любые внешние параметры, передаваемые в исполняемую функцию на клиентской стороне. - :return: результат получения данных согласно конфигурации фида. - """ - - result = await self.feed_config.feed.get_data( + async def get_feed( + self, + session_id: str, + limit: int, + cursor: Optional[Dict[str, Any]] = None, + ) -> FeedResult: + # Boundary guards: these inputs come from the integrator's request handler + # (and the cursor round-trips through untrusted clients); fail loudly here + # instead of slicing/keying garbage deeper in the pipeline. + if not isinstance(session_id, str) or not session_id: + raise ValueError(f"session_id must be a non-empty string, got {session_id!r}") + if not isinstance(limit, int) or isinstance(limit, bool) or limit <= 0: + raise ValueError(f"limit must be a positive int, got {limit!r}") + cursor = self._validated_cursor(cursor) + ctx = ExecutionContext( + session_id=session_id, methods_dict=self.methods_dict, - user_id=user_id, - limit=limit, - next_page=next_page, - redis_client=self.redis_client, - **params, + redis=self.redis_client, ) + result = await _executor.run(self.config.feed, ctx, limit, cursor) + for i, item in enumerate(result.data): + if isinstance(item, dict): + item.setdefault("_smartfeed_debug_info", {})["smartfeed_position"] = i return result + + @staticmethod + def _validated_cursor(cursor: object) -> Dict[str, Any]: + """Runtime gate for the declared Optional[dict] contract. + + Untyped callers can pass anything, so the check runs on an `object` view: + checking the already-narrowed parameter would make the raise provably dead + code to type analysis (pyright reportUnreachable). + """ + if cursor is None: + return {} + if not isinstance(cursor, dict): + raise TypeError( + f"cursor must be the dict from the previous FeedResult.next_page (or None), " + f"got {type(cursor).__name__}" + ) + return cursor diff --git a/smartfeed/models/__init__.py b/smartfeed/models/__init__.py new file mode 100644 index 0000000..84c519f --- /dev/null +++ b/smartfeed/models/__init__.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +from typing import Annotated, Dict, Union + +from pydantic import BaseModel, Field, model_validator + +from .base import BaseNode, FeedResult, MixerNode +from .mixers import ( + MergerAppend, + MergerAppendDistribute, + MergerPercentage, + MergerPercentageGradient, + MergerPercentageItem, + MergerPositional, +) +from .subfeed import SubFeed +from .wrapper import Wrapper, WrapperCache, WrapperDedup, WrapperRerank + +# --------------------------------------------------------------------------- +# FeedNode: discriminated union of all node types +# --------------------------------------------------------------------------- + +FeedNode = Annotated[ + Union[ + SubFeed, + Wrapper, + MergerAppend, + MergerPercentage, + MergerPercentageGradient, + MergerPositional, + MergerAppendDistribute, + ], + Field(discriminator="type"), +] + +# --------------------------------------------------------------------------- +# FeedConfig: top-level config model +# --------------------------------------------------------------------------- + + +def _walk_ids(node: BaseNode, seen: Dict[str, str]) -> None: + """Collect every subfeed_id/node_id in the tree, raising on a duplicate. + + Note the BaseNode check comes BEFORE the generic has-a-.data check: a Wrapper is + both a node AND has .data, and must be visited itself, not skipped over. + """ + node_id = getattr(node, "subfeed_id", None) or getattr(node, "node_id", None) + if node_id is not None: + kind = type(node).__name__ + if node_id in seen: + raise ValueError( + f"duplicate id '{node_id}' ({seen[node_id]} and {kind}): subfeed_id and node_id share " + f"one namespace and must be unique across the config tree -- duplicates silently " + f"collide in cursors and Redis keys" + ) + seen[node_id] = kind + for attr in ("items", "data", "positional", "default", "item_from", "item_to"): + child = getattr(node, attr, None) + if child is None or child is node: + continue + children = child if isinstance(child, list) else [child] + for item in children: + if isinstance(item, BaseNode): + _walk_ids(item, seen) + else: + # MergerPercentageItem and friends wrap their node in .data + inner = getattr(item, "data", None) + if isinstance(inner, BaseNode): + _walk_ids(inner, seen) + + +class FeedConfig(BaseModel): + # Top-level keys other than `feed` (e.g. a legacy `version` tag) are ignored. + feed: FeedNode + + @model_validator(mode="after") + def _validate_unique_ids(self) -> "FeedConfig": + seen: Dict[str, str] = {} + _walk_ids(self.feed, seen) + return self + + +# Rebuild forward refs so that nested FeedNode fields resolve correctly +_types_ns = {"FeedNode": FeedNode} +for _model in ( + SubFeed, + Wrapper, + WrapperCache, + MergerAppend, + MergerPercentage, + MergerPercentageItem, + MergerPositional, + MergerPercentageGradient, + MergerAppendDistribute, + FeedConfig, +): + _model.model_rebuild(force=True, _types_namespace=_types_ns) + +__all__ = [ + "BaseNode", + "MixerNode", + "FeedResult", + "SubFeed", + "MergerPercentage", + "MergerPercentageItem", + "MergerAppend", + "MergerPositional", + "MergerPercentageGradient", + "MergerAppendDistribute", + "Wrapper", + "WrapperCache", + "WrapperRerank", + "WrapperDedup", + "FeedNode", + "FeedConfig", +] diff --git a/smartfeed/models/base.py b/smartfeed/models/base.py new file mode 100644 index 0000000..69b3cf5 --- /dev/null +++ b/smartfeed/models/base.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import hashlib +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from pydantic import BaseModel, PrivateAttr + +if TYPE_CHECKING: + from smartfeed.execution.context import ExecutionContext + from smartfeed.execution.plans import MixPlan + + +class BaseNode(BaseModel): + dedup_priority: int = 0 + + # Config is immutable once FeedManager validates it, so the hash is computed once + # per node instance instead of on every Redis key construction. + _config_hash: Optional[str] = PrivateAttr(default=None) + + def config_hash(self) -> str: + if self._config_hash is None: + # json.dumps with sort_keys for deterministic hashing + raw = json.dumps(self.model_dump(), sort_keys=True, default=str) + self._config_hash = hashlib.md5(raw.encode()).hexdigest()[:8] + return self._config_hash + + async def execute( + self, + methods_dict: Dict[str, Any], + session_id: str, + limit: int, + cursor: Dict[str, Any], + ctx: Optional[ExecutionContext] = None, + **params: Any, + ) -> FeedResult: + """Leaf/pipeline nodes (SubFeed, Wrapper) override this; mixers build a MixPlan instead.""" + raise NotImplementedError(f"{type(self).__name__} does not execute directly") + + +class MixerNode(BaseNode): + """Base for combiner nodes. The executor dispatches on this type: mixers return a + MixPlan of children to run concurrently instead of executing directly.""" + + def build_mix_plan(self, *, ctx: ExecutionContext, limit: int, cursor: Dict[str, Any]) -> MixPlan: + raise NotImplementedError + + +class FeedResult(BaseModel): + # Items are usually dicts; non-dict items are passed through untouched by + # stamping/dedup, so the element type is deliberately Any. + data: List[Any] + next_page: Dict[str, Any] + has_next_page: bool + + +def coerce_feed_node(value: Any) -> Any: + """Coerce a dict to the appropriate FeedNode model instance. + + Uses a lazy import of FeedNode to avoid circular imports. + Non-dict values are returned unchanged. + """ + if not isinstance(value, dict): + return value + # Lazy import to break circular dependency + from pydantic import TypeAdapter + + from smartfeed.models import FeedNode + + return TypeAdapter(FeedNode).validate_python(value) diff --git a/smartfeed/models/mixers.py b/smartfeed/models/mixers.py new file mode 100644 index 0000000..1c25ac6 --- /dev/null +++ b/smartfeed/models/mixers.py @@ -0,0 +1,429 @@ +from __future__ import annotations + +from collections import defaultdict, deque +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple + +from pydantic import BaseModel, Field, model_validator + +from ..execution.plans import MixChild, MixPlan +from .base import MixerNode, coerce_feed_node + +if TYPE_CHECKING: + from ..execution.context import ExecutionContext + + +def _merge_cursor(child_cursors: Dict[str, Dict[str, Any]]) -> Dict[str, Any]: + merged: Dict[str, Any] = {} + for c in child_cursors.values(): + merged.update(c) + return merged + + +# --------------------------------------------------------------------------- +# MergerPercentageItem +# --------------------------------------------------------------------------- + + +class MergerPercentageItem(BaseModel): + percentage: int + data: Any # BaseNode subclass + + @model_validator(mode="before") + @classmethod + def _coerce_data(cls, values: Any) -> Any: + if isinstance(values, dict) and isinstance(values.get("data"), dict): + values["data"] = coerce_feed_node(values["data"]) + return values + + +# --------------------------------------------------------------------------- +# MergerPercentage +# --------------------------------------------------------------------------- + + +class MergerPercentage(MixerNode): + type: Literal["merger_percentage"] = "merger_percentage" + node_id: str + items: List[MergerPercentageItem] + dedup_priority: int = 0 + + def build_mix_plan( + self, + *, + ctx: ExecutionContext, + limit: int, + cursor: Dict[str, Any], + ) -> MixPlan: + total_pct = sum(item.percentage for item in self.items) + + # Compute per-child demand with remainder distribution + demands: List[int] = [] + remainders: List[Tuple[int, int]] = [] + for idx, item in enumerate(self.items): + raw = limit * item.percentage + child_limit = raw // 100 + demands.append(max(0, child_limit)) + remainders.append((raw % 100, idx)) + + if total_pct == 100: + missing = max(0, limit - sum(demands)) + if missing > 0: + for _rem, idx in sorted(remainders, key=lambda x: (-x[0], x[1])): + if missing <= 0: + break + demands[idx] += 1 + missing -= 1 + + children = [ + MixChild( + node_id=f"{self.node_id}_{idx}", + node=item.data, + demand=demands[idx], + ) + for idx, item in enumerate(self.items) + ] + + def assemble( + buffers: Dict[str, List[Any]], + child_cursors: Dict[str, Dict[str, Any]], + ) -> Tuple[List[Any], Dict[str, Any]]: + # Simple concatenation: demand already ensures correct proportions + merged_data: List[Any] = [] + for child in children: + merged_data.extend(buffers.get(child.node_id, [])) + return merged_data, _merge_cursor(child_cursors) + + return MixPlan(children=children, assemble=assemble) + + +# --------------------------------------------------------------------------- +# MergerAppend +# --------------------------------------------------------------------------- + + +class MergerAppend(MixerNode): + type: Literal["merger_append"] = "merger_append" + node_id: str + items: List[Any] # list of BaseNode subclasses + dedup_priority: int = 0 + + @model_validator(mode="before") + @classmethod + def _coerce_items(cls, values: Any) -> Any: + if isinstance(values, dict): + raw_items = values.get("items") + if isinstance(raw_items, list): + values["items"] = [coerce_feed_node(item) if isinstance(item, dict) else item for item in raw_items] + return values + + def build_mix_plan( + self, + *, + ctx: ExecutionContext, + limit: int, + cursor: Dict[str, Any], + ) -> MixPlan: + # Each child gets equal demand share; leftover goes to first children + n = len(self.items) + if n == 0: + + def assemble_empty( + buffers: Dict[str, List[Any]], child_cursors: Dict[str, Dict[str, Any]] + ) -> Tuple[List[Any], Dict[str, Any]]: + return [], _merge_cursor(child_cursors) + + return MixPlan(children=[], assemble=assemble_empty) + + base_demand = limit // n + extra = limit - base_demand * n + + children = [ + MixChild( + node_id=f"{self.node_id}_{idx}", + node=item, + demand=base_demand + (1 if idx < extra else 0), + ) + for idx, item in enumerate(self.items) + ] + + def assemble( + buffers: Dict[str, List[Any]], + child_cursors: Dict[str, Dict[str, Any]], + ) -> Tuple[List[Any], Dict[str, Any]]: + merged_data: List[Any] = [] + for child in children: + merged_data.extend(buffers.get(child.node_id, [])) + # Trim to limit in case of overfill + return merged_data[:limit], _merge_cursor(child_cursors) + + return MixPlan(children=children, assemble=assemble) + + +# --------------------------------------------------------------------------- +# MergerPositional +# --------------------------------------------------------------------------- + + +class MergerPositional(MixerNode): + type: Literal["merger_positional"] = "merger_positional" + node_id: str + positions: List[int] = [] + positional: Any # BaseNode subclass + default: Any # BaseNode subclass + dedup_priority: int = 0 + + @model_validator(mode="before") + @classmethod + def _coerce_children(cls, values: Any) -> Any: + if isinstance(values, dict): + for field in ("positional", "default"): + if isinstance(values.get(field), dict): + values[field] = coerce_feed_node(values[field]) + return values + + def build_mix_plan( + self, + *, + ctx: ExecutionContext, + limit: int, + cursor: Dict[str, Any], + ) -> MixPlan: + # positions are 1-indexed; determine how many positional slots fall within [1..limit] + pos_slots = [p for p in self.positions if 1 <= p <= limit] + pos_count = len(pos_slots) + default_count = limit - pos_count + + positional_child = MixChild( + node_id=f"{self.node_id}_positional", + node=self.positional, + demand=pos_count, + ) + default_child = MixChild( + node_id=f"{self.node_id}_default", + node=self.default, + demand=default_count, + ) + + children = [positional_child, default_child] + + def assemble( + buffers: Dict[str, List[Any]], + child_cursors: Dict[str, Dict[str, Any]], + ) -> Tuple[List[Any], Dict[str, Any]]: + pos_items = deque(buffers.get(positional_child.node_id, [])) + def_items = deque(buffers.get(default_child.node_id, [])) + + result: List[Any] = [] + pos_set = set(pos_slots) + for position in range(1, limit + 1): + if position in pos_set and pos_items: + result.append(pos_items.popleft()) + elif def_items: + result.append(def_items.popleft()) + elif pos_items: + result.append(pos_items.popleft()) + + return result, _merge_cursor(child_cursors) + + return MixPlan(children=children, assemble=assemble) + + +# --------------------------------------------------------------------------- +# MergerPercentageGradient +# --------------------------------------------------------------------------- + + +class MergerPercentageGradient(MixerNode): + """Percentage-based merger that shifts the ratio over pages.""" + + type: Literal["merger_percentage_gradient"] = "merger_percentage_gradient" + node_id: str + item_from: MergerPercentageItem + item_to: MergerPercentageItem + step: int = Field(gt=0) # a zero/negative shift cannot progress (use merger_percentage for a fixed split) + size_to_step: int = Field(gt=0) # segment length; 0 would make the gradient walk undefined + dedup_priority: int = 0 + + def _percentages_at(self, k: int) -> Tuple[int, int]: + """Percentage split at 1-indexed gradient segment k. + + Closed form of "shift by `step` once per segment while `to` < 100, clamping + to (0, 100) on overshoot" -- so demand computation never has to replay the + shift history from segment 1. + """ + pct_from = self.item_from.percentage + pct_to = self.item_to.percentage + if k <= 1 or pct_to >= 100: + return pct_from, pct_to + # The walk freezes at the first shift where `to` reaches 100 (guard stops + # shifting) or an overshoot clamps the pair to (0, 100). + freeze = min( + -((pct_to - 100) // self.step), # ceil((100 - pct_to) / step): `to` reaches 100 + pct_from // self.step + 1, # first shift where `from` would pass 0 + ) + shifts = min(k - 1, freeze) + raw_from = pct_from - self.step * shifts + raw_to = pct_to + self.step * shifts + if raw_to > 100 or raw_from < 0: + return 0, 100 + return raw_from, raw_to + + def _calculate_demands(self, page: int, limit: int) -> Tuple[int, int, List[Dict[str, int]]]: + """Demands and interleave segments for THIS page's window only. + + Iterates just the gradient segments overlapping [limit*(page-1), limit*page) + -- at most limit/size_to_step + 2 of them -- instead of replaying the whole + scroll history, so per-request work is bounded by the page size no matter + how deep (or how tampered) `page` is. + """ + page_start = limit * (page - 1) + page_end = limit * page + limit_from = 0 + limit_to = 0 + segments: List[Dict[str, int]] = [] + + first_seg = page_start // self.size_to_step + 1 + last_seg = -(-page_end // self.size_to_step) # ceil + for k in range(first_seg, last_seg + 1): + lo = max(page_start, (k - 1) * self.size_to_step) + hi = min(k * self.size_to_step, page_end) + iter_limit = hi - lo + if iter_limit <= 0: + continue + pct_from, _ = self._percentages_at(k) + from_take = iter_limit * pct_from // 100 + to_take = iter_limit - from_take + limit_from += from_take + limit_to += to_take + segments.append({"limit": iter_limit, "from_take": from_take, "to_take": to_take}) + + return limit_from, limit_to, segments + + def build_mix_plan( + self, + *, + ctx: ExecutionContext, + limit: int, + cursor: Dict[str, Any], + ) -> MixPlan: + node_cursor = cursor.get(self.node_id, {}) + if not isinstance(node_cursor, dict): + raise ValueError( + f"MergerPercentageGradient '{self.node_id}': cursor entry must be a dict, " + f"got {type(node_cursor).__name__}" + ) + page = node_cursor.get("page", 1) + if not isinstance(page, int) or isinstance(page, bool) or page < 1: + raise ValueError( + f"MergerPercentageGradient '{self.node_id}': cursor 'page' must be a positive int, got {page!r}" + ) + limit_from, limit_to, segments = self._calculate_demands(page, limit) + + from_child = MixChild( + node_id=f"{self.node_id}_from", + node=self.item_from.data, + demand=max(0, limit_from), + ) + to_child = MixChild( + node_id=f"{self.node_id}_to", + node=self.item_to.data, + demand=max(0, limit_to), + ) + children = [from_child, to_child] + + def assemble( + buffers: Dict[str, List[Any]], + child_cursors: Dict[str, Dict[str, Any]], + ) -> Tuple[List[Any], Dict[str, Any]]: + from_data = list(buffers.get(from_child.node_id, [])) + to_data = list(buffers.get(to_child.node_id, [])) + result: List[Any] = [] + fi = ti = 0 + for seg in segments: + ft = int(seg["from_take"]) + tt = int(seg["to_take"]) + result.extend(from_data[fi : fi + ft]) + result.extend(to_data[ti : ti + tt]) + fi += ft + ti += tt + merged_cur = _merge_cursor(child_cursors) + merged_cur[self.node_id] = {"page": page + 1} + return result, merged_cur + + return MixPlan(children=children, assemble=assemble) + + +# --------------------------------------------------------------------------- +# MergerAppendDistribute +# --------------------------------------------------------------------------- + + +class MergerAppendDistribute(MixerNode): + """Append merger that round-robins items by a distribution key.""" + + type: Literal["merger_distribute"] = "merger_distribute" + node_id: str + items: List[Any] # list of BaseNode subclasses + distribution_key: str + sorting_key: Optional[str] = None + sorting_desc: bool = False + dedup_priority: int = 0 + + @model_validator(mode="before") + @classmethod + def _coerce_items(cls, values: Any) -> Any: + if isinstance(values, dict): + raw_items = values.get("items") + if isinstance(raw_items, list): + values["items"] = [coerce_feed_node(item) if isinstance(item, dict) else item for item in raw_items] + return values + + def _uniform_distribute(self, data: List[Any]) -> List[Any]: + if self.sorting_key: + data = sorted(data, key=lambda x: x[self.sorting_key], reverse=self.sorting_desc) + + grouped_entries: Dict[Any, deque[Any]] = defaultdict(deque) + for entry in data: + grouped_entries[entry[self.distribution_key]].append(entry) + + result: List[Any] = [] + prev_key = None + while any(grouped_entries.values()): + for key in list(grouped_entries.keys()): + if grouped_entries[key]: + if key != prev_key or len(grouped_entries) == 1: + result.append(grouped_entries[key].popleft()) + prev_key = key + if not grouped_entries[key]: + del grouped_entries[key] + else: + del grouped_entries[key] + return result + + def build_mix_plan( + self, + *, + ctx: ExecutionContext, + limit: int, + cursor: Dict[str, Any], + ) -> MixPlan: + children = [ + MixChild( + node_id=f"{self.node_id}_{idx}", + node=item, + demand=limit, + ) + for idx, item in enumerate(self.items) + ] + + def assemble( + buffers: Dict[str, List[Any]], + child_cursors: Dict[str, Dict[str, Any]], + ) -> Tuple[List[Any], Dict[str, Any]]: + all_items: List[Any] = [] + for child in children: + all_items.extend(buffers.get(child.node_id, [])) + distributed = self._uniform_distribute(all_items) + return distributed[:limit], _merge_cursor(child_cursors) + + return MixPlan(children=children, assemble=assemble) diff --git a/smartfeed/models/subfeed.py b/smartfeed/models/subfeed.py new file mode 100644 index 0000000..5f5ceb2 --- /dev/null +++ b/smartfeed/models/subfeed.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +from random import shuffle +from typing import TYPE_CHECKING, Any, Dict, Literal, Optional + +from .base import BaseNode, FeedResult + +if TYPE_CHECKING: + from smartfeed.execution.context import ExecutionContext + + +class SubFeed(BaseNode): + type: Literal["subfeed"] = "subfeed" + subfeed_id: str + method_name: str + subfeed_params: Dict[str, Any] = {} + raise_error: bool = True + shuffle: bool = False + + async def execute( + self, + methods_dict: Dict[str, Any], + session_id: str, + limit: int, + cursor: Dict[str, Any], + ctx: Optional[ExecutionContext] = None, + **params: Any, + ) -> FeedResult: + method = methods_dict[self.method_name] + subfeed_cursor = cursor.get(self.subfeed_id, {}) + # Like the missing-method KeyError, this raises regardless of raise_error: + # a malformed cursor is client tampering, not a source failure to fail soft on. + if not isinstance(subfeed_cursor, dict): + raise ValueError( + f"SubFeed '{self.subfeed_id}': cursor entry must be a dict, got {type(subfeed_cursor).__name__}" + ) + + try: + result: FeedResult = await method( + user_id=session_id, + limit=limit, + next_page=subfeed_cursor, + ctx=ctx, + **params, + **self.subfeed_params, + ) + except Exception: + if self.raise_error: + raise + # Scope the failure cursor to THIS subfeed (its own, unadvanced position) + # so it retries next page without clobbering sibling cursors on merge. + return FeedResult(data=[], next_page={self.subfeed_id: subfeed_cursor}, has_next_page=False) + + if self.shuffle: + shuffle(result.data) + + # Stamp source on every dict item (merge, don't overwrite) + for item in result.data: + if isinstance(item, dict): + item.setdefault("_smartfeed_debug_info", {})["source"] = self.subfeed_id + + return FeedResult( + data=result.data, + next_page={self.subfeed_id: result.next_page}, + has_next_page=result.has_next_page, + ) diff --git a/smartfeed/models/wrapper.py b/smartfeed/models/wrapper.py new file mode 100644 index 0000000..e61e5ad --- /dev/null +++ b/smartfeed/models/wrapper.py @@ -0,0 +1,780 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import secrets +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Set, Tuple + +import orjson +from pydantic import BaseModel, PrivateAttr, model_validator + +from smartfeed.execution import executor as _executor + +from .base import BaseNode, FeedResult, coerce_feed_node + +if TYPE_CHECKING: + from smartfeed.execution.context import ExecutionContext + +# Safety bound for the dedup refill loop: stop scanning after this many fetched items +# per `target` when the child keeps yielding duplicates (guards against a misbehaving +# source that returns has_next=True forever). Well above any realistic duplicate rate. +_REFILL_SCAN_FACTOR = 50 + +# Cold-build lock: how long a builder may hold the lock, and how a loser waits it out. +# The loser's poll budget covers the FULL lock TTL -- giving up earlier and rebuilding +# without the lock would double-fetch the child and write a competing generation. +_COLD_LOCK_TTL = 10 +_LOCK_POLL_INTERVAL = 0.1 +_LOCK_POLL_ROUNDS = int(_COLD_LOCK_TTL / _LOCK_POLL_INTERVAL) + + +class WrapperCache(BaseModel): + session_size: int = 300 + session_ttl: int = 300 + + +class WrapperRerank(BaseModel): + method_name: str + raise_error: bool = True # True = crash if rerank fails, False = keep original order + + +class WrapperDedup(BaseModel): + dedup_key: str + missing_key_policy: Literal["error", "keep", "drop"] = "error" + state_ttl: int = 300 # TTL for Redis seen-set (seconds) + + +class Wrapper(BaseNode): + type: Literal["wrapper"] = "wrapper" + node_id: str + cache: Optional[WrapperCache] = None + rerank: Optional[WrapperRerank] = None + dedup: Optional[WrapperDedup] = None + data: Any # BaseNode subclass; typed as Any to support discriminated union deserialization + cache_key: Optional[str] = None + + # Config is immutable after validation, so the priority map is computed once. + _priorities: Optional[Dict[str, int]] = PrivateAttr(default=None) + + @model_validator(mode="before") + @classmethod + def _coerce_data(cls, values: Any) -> Any: + if isinstance(values, dict) and isinstance(values.get("data"), dict): + values["data"] = coerce_feed_node(values["data"]) + return values + + async def execute( + self, + methods_dict: Dict[str, Any], + session_id: str, + limit: int, + cursor: Dict[str, Any], + ctx: Optional[ExecutionContext] = None, + **params: Any, + ) -> FeedResult: + """Entry point. Routes to cached or passthrough path.""" + if self.cache is None or ctx is None or ctx.redis is None: + return await self._passthrough(methods_dict, session_id, limit, cursor, ctx) + + return await self._execute_with_cache(methods_dict, session_id, limit, cursor, ctx) + + # -- dedup ----------------------------------------------------------------- + + def _resolve_dedup_priorities(self) -> Dict[str, int]: + """Build {subfeed_id: effective_priority} by walking the config tree. + + The config never changes after validation, so the walk runs once per + wrapper instance instead of once per dedup round.""" + if self._priorities is None: + result: Dict[str, int] = {} + self._collect_priorities(self.data, override_priority=0, out=result) + self._priorities = result + return self._priorities + + @staticmethod + def _collect_priorities(node: BaseNode, override_priority: int, out: Dict[str, int]) -> None: + """Recursive walk: propagate override_priority down, write SubFeed priorities to out.""" + from .subfeed import SubFeed + + # Determine the effective priority for this subtree. + # A node with dedup_priority != 0 overrides all children. + effective = node.dedup_priority if node.dedup_priority != 0 else override_priority + + if isinstance(node, SubFeed): + out[node.subfeed_id] = effective + return + + # Walk children: look in known child-bearing attributes. + # The BaseNode check comes BEFORE the .data unwrap: a Wrapper is the one + # node that ALSO has .data, and it must be visited itself so its own + # dedup_priority participates in propagation. + for attr in ("items", "data", "positional", "default", "item_from", "item_to"): + child = getattr(node, attr, None) + if child is None: + continue + children = child if isinstance(child, list) else [child] + for item in children: + if isinstance(item, BaseNode): + Wrapper._collect_priorities(item, effective, out) + elif isinstance(item, BaseModel): + # MergerPercentageItem and friends wrap their node in .data + inner = getattr(item, "data", None) + if isinstance(inner, BaseNode): + Wrapper._collect_priorities(inner, effective, out) + + def _dedup(self, data: List[Any], seen: Optional[Set[str]] = None) -> List[Any]: + """Remove duplicates by dedup_key. Higher dedup_priority wins, equal = first-seen wins.""" + if not self.dedup: + return data + + priorities = self._resolve_dedup_priorities() + key_field = self.dedup.dedup_key + policy = self.dedup.missing_key_policy + if seen is None: + seen = set() + + # {key_val: (priority, index_in_result)} for in-batch priority arbitration + batch: Dict[str, Tuple[int, int]] = {} + result: List[Any] = [] + + for item in data: + if not isinstance(item, dict): + result.append(item) + continue + + if key_field not in item: + if policy == "error": + raise KeyError(f"Dedup key '{key_field}' missing from item: {item}") + elif policy == "keep": + result.append(item) + elif policy == "drop": + pass + continue + + key_val = str(item[key_field]) + + # Cross-page / cross-rebuild: already shown earlier in this scroll. + # `key_val in batch` means it is a *within-batch* duplicate, which must + # fall through to priority arbitration below rather than be skipped here. + if key_val in seen and key_val not in batch: + continue + + # In-batch priority arbitration + source = (item.get("_smartfeed_debug_info") or {}).get("source") + item_priority = priorities.get(source, 0) if source else 0 + + if key_val not in batch: + idx = len(result) + batch[key_val] = (item_priority, idx) + result.append(item) + else: + existing_priority, existing_idx = batch[key_val] + if item_priority > existing_priority: + result[existing_idx] = item + batch[key_val] = (item_priority, existing_idx) + + seen.add(key_val) + + return result + + # -- rerank ---------------------------------------------------------------- + + async def _apply_rerank(self, data: List[Any], methods_dict: Dict[str, Any], session_id: str) -> List[Any]: + """Call rerank callable from methods_dict. Validates output length.""" + if not self.rerank: + return data + rerank_fn = methods_dict[self.rerank.method_name] + original_len = len(data) + try: + reranked: List[Any] = await rerank_fn(data, session_id) + except Exception: + if self.rerank.raise_error: + raise + # Fail-soft: keep original order + return data + if len(reranked) != original_len: + raise ValueError( + f"Rerank '{self.rerank.method_name}' must return exactly " f"{original_len} items, got {len(reranked)}" + ) + return reranked + + # -- debug stamping -------------------------------------------------------- + + def _stamp_pre_rerank(self, data: List[Any]) -> None: + """Stamp smartfeed_position on each item (position before rerank).""" + for i, item in enumerate(data): + if isinstance(item, dict): + item.setdefault("_smartfeed_debug_info", {})[self.node_id] = { + "smartfeed_position": i, + } + + def _stamp_post_rerank(self, data: List[Any]) -> None: + """Stamp rerank_position on each item (position after rerank).""" + for i, item in enumerate(data): + if isinstance(item, dict): + item.setdefault("_smartfeed_debug_info", {}).setdefault(self.node_id, {})["rerank_position"] = i + + # -- passthrough (no cache) ----------------------------------------------- + + async def _passthrough( + self, + methods_dict: Dict[str, Any], + session_id: str, + limit: int, + cursor: Dict[str, Any], + ctx: Optional[ExecutionContext], + ) -> FeedResult: + """No-cache path: fetch -> dedup (refill to a full page) -> rerank -> return.""" + assert ctx is not None, "wrapper execution requires ctx (the executor always provides it)" + if self.dedup: + # Session-scoped seen-set, mirroring the cached path: an empty cursor is a + # fresh scroll, so reset the set (re-requesting page 1 returns page 1); + # otherwise load it so shown ids persist across pages. + if not cursor: + await self._reset_seen_set(ctx, session_id) + seen_keys: Set[str] = set() + else: + seen_keys = await self._load_seen_set(ctx, session_id) + + current_cursor = dict(cursor) + data: List[Any] = [] + has_next = True + scanned = 0 + scan_cap = max(limit, 1) * _REFILL_SCAN_FACTOR + + # Passthrough has no buffer, so it must never over-fetch: pull exactly the + # outstanding deficit each round and refill until the page is full or the + # child is exhausted. Nothing is discarded, so nothing is lost. Keep scanning + # through duplicate runs (don't stop on a single all-duplicate window), bounded + # by scan_cap so a duplicate-only source can't spin forever. + while len(data) < limit and has_next and scanned < scan_cap: + need = limit - len(data) + result = await _executor.run(self.data, ctx, need, current_cursor) + current_cursor = result.next_page + has_next = result.has_next_page + if not result.data: + break # child yielded nothing -> stop instead of spinning + scanned += len(result.data) + data.extend(self._dedup(result.data, seen_keys)) + + # Persist seen-set to Redis for the next page. + new_keys: Set[str] = set() + key_field = self.dedup.dedup_key + for item in data: + if isinstance(item, dict) and key_field in item: + new_keys.add(str(item[key_field])) + await self._save_seen_set(ctx, session_id, new_keys) + else: + result = await _executor.run(self.data, ctx, limit, cursor) + data = result.data + current_cursor = result.next_page + has_next = result.has_next_page + + self._stamp_pre_rerank(data) + if self.rerank: + data = await self._apply_rerank(data, methods_dict, session_id) + self._stamp_post_rerank(data) + return FeedResult( + data=data, + next_page=current_cursor, + has_next_page=has_next, + ) + + def _seen_set_key(self, session_id: str) -> str: + """Redis key for cross-page dedup seen-set.""" + return f"sf:{session_id}:{self.node_id}:{self.config_hash()}:seen" + + async def _load_seen_set(self, ctx: Optional[ExecutionContext], session_id: str) -> Set[str]: + """Load seen keys from Redis SET for cross-page dedup.""" + if not ctx or not ctx.redis: + return set() + key = self._seen_set_key(session_id) + members = await ctx.redis.smembers(key) + return {m.decode() if isinstance(m, bytes) else m for m in members} if members else set() + + async def _save_seen_set(self, ctx: Optional[ExecutionContext], session_id: str, new_keys: Set[str]) -> None: + """Append new keys to Redis seen-set and refresh TTL (one round-trip).""" + if not ctx or not ctx.redis or not new_keys: + return + key = self._seen_set_key(session_id) + assert self.dedup is not None, "seen-set only written when dedup configured" + pipe = ctx.redis.pipeline() + pipe.sadd(key, *new_keys) + pipe.expire(key, self.dedup.state_ttl) + await pipe.execute() + + async def _reset_seen_set(self, ctx: Optional[ExecutionContext], session_id: str) -> None: + """Clear the seen-set so a fresh scroll (empty cursor) starts from page 1.""" + if not ctx or not ctx.redis: + return + await ctx.redis.delete(self._seen_set_key(session_id)) + + # -- cache helpers --------------------------------------------------------- + # + # The session batch is stored as a Redis LIST of individually-encoded items, so a + # warm page read transfers and parses only the requested window (LRANGE), never + # the whole batch. Shared-base segments stay single blobs: they are always read + # whole (each sharing wrapper reranks the full segment). + + def _base_key(self, session_id: str) -> str: + """Redis key prefix for this wrapper's cache data.""" + key_part = self.cache_key or self.node_id + return f"sf:{session_id}:{key_part}:{self.config_hash()}" + + async def _read_meta(self, ctx: ExecutionContext, session_id: str) -> Optional[Dict[str, Any]]: + """Read cache metadata (gen, child_cursor, child_has_next) from Redis.""" + assert ctx.redis is not None, "cached path requires redis" + base = self._base_key(session_id) + raw = await ctx.redis.get(f"{base}:meta") + if raw is None: + return None + meta: Dict[str, Any] = orjson.loads(raw) + return meta + + async def _write_cache( + self, + ctx: ExecutionContext, + session_id: str, + data: List[Any], + gen: str, + child_cursor: Dict[str, Any], + child_has_next: bool = False, + ) -> None: + """Replace session data (as a LIST of encoded items) and metadata atomically.""" + assert self.cache is not None, "cached path; execute() routes cache=None to passthrough" + assert ctx.redis is not None, "cached path requires redis" + base = self._base_key(session_id) + ttl = self.cache.session_ttl + + try: + encoded = [orjson.dumps(item) for item in data] + meta = orjson.dumps({"gen": gen, "child_cursor": child_cursor, "child_has_next": child_has_next}) + except TypeError as e: + raise TypeError( + f"Wrapper '{self.node_id}': cache is enabled, so items and child cursors must be " + f"orjson-serializable (str/int/float/bool/None/list/dict): {e}" + ) from e + pipe = ctx.redis.pipeline() + pipe.delete(base) + if encoded: + pipe.rpush(base, *encoded) + pipe.expire(base, ttl) + pipe.set(f"{base}:meta", meta, ex=ttl) + await pipe.execute() + + async def _read_snapshot_and_touch( + self, ctx: ExecutionContext, session_id: str, offset: int, limit: int + ) -> Tuple[Optional[Dict[str, Any]], List[Any], int]: + """Read meta + one page window as ONE atomic snapshot and refresh TTLs. + + A single MULTI/EXEC round-trip: reading meta and data as separate commands + would let a concurrent rebuild land in between, pairing old-gen meta with the + replacement batch (torn read) -- `_write_cache` swaps both keys in one + transaction, so reads go through one transaction too. Only the requested + window is transferred and parsed (LRANGE), not the whole session batch. The + TTL refresh rides along: data/meta get session_ttl and the dedup seen-set gets + state_ttl, so dedup state cannot expire mid-scroll while the cache it guards + is being kept alive. + + Returns (meta, page_items, total_batch_len); meta is None on miss. + """ + assert self.cache is not None, "cached path; execute() routes cache=None to passthrough" + assert ctx.redis is not None, "cached path requires redis" + base = self._base_key(session_id) + ttl = self.cache.session_ttl + pipe = ctx.redis.pipeline() + pipe.get(f"{base}:meta") + pipe.llen(base) + pipe.lrange(base, offset, offset + limit - 1) + pipe.expire(base, ttl) + pipe.expire(f"{base}:meta", ttl) + if self.dedup: + pipe.expire(self._seen_set_key(session_id), self.dedup.state_ttl) + results = await pipe.execute() + raw_meta, total, raw_page = results[0], results[1], results[2] + meta: Optional[Dict[str, Any]] = orjson.loads(raw_meta) if raw_meta is not None else None + page = [orjson.loads(x) for x in raw_page] + return meta, page, int(total) + + async def _serve_new_generation( + self, ctx: ExecutionContext, session_id: str, limit: int, stale_gen: Optional[str] + ) -> Optional[FeedResult]: + """Serve page 1 of the current batch, but only if it is a NEW generation. + + Used by cold-build waiters: the outgoing batch's keys are never deleted before + a rebuild, so a batch still carrying `stale_gen` is the one being replaced -- + serving it would re-show items and hand back an instantly-stale cursor. + """ + meta, page, total = await self._read_snapshot_and_touch(ctx, session_id, 0, limit) + if meta is None or meta.get("gen") == stale_gen: + return None + child_has_next = meta.get("child_has_next", bool(meta.get("child_cursor"))) + return self._page_result(page, 0, limit, total, meta.get("gen", ""), child_has_next) + + # -- pagination ------------------------------------------------------------ + + def _page_result( + self, + page_data: List[Any], + offset: int, + limit: int, + total: int, + gen: str, + child_has_next: bool = False, + ) -> FeedResult: + """Build a page response from an already-sliced window of the session batch. + + The cursor carries an absolute offset (not a page number), so pagination stays + correct even if the client varies `limit` between requests. + """ + end = offset + limit + has_next = end < total or child_has_next + + next_cursor = { + self.node_id: { + "offset": end, + "gen": gen, + } + } + + return FeedResult( + data=page_data, + next_page=next_cursor, + has_next_page=has_next, + ) + + # -- main cached flow ------------------------------------------------------ + + async def _execute_with_cache( + self, + methods_dict: Dict[str, Any], + session_id: str, + limit: int, + cursor: Dict[str, Any], + ctx: ExecutionContext, + ) -> FeedResult: + """Cached path: warm hit -> paginate, stale/miss -> cold build.""" + # Cursors round-trip through untrusted clients: reject malformed contents + # loudly instead of crashing mid-slice or serving a wrong page. + my_cursor = cursor.get(self.node_id, {}) + if not isinstance(my_cursor, dict): + raise ValueError( + f"Wrapper '{self.node_id}': cursor entry must be a dict, " + f"got {type(my_cursor).__name__} (pass back next_page verbatim)" + ) + cursor_gen = my_cursor.get("gen") + if cursor_gen is not None and not isinstance(cursor_gen, str): + raise ValueError(f"Wrapper '{self.node_id}': cursor 'gen' must be a string, got {cursor_gen!r}") + cursor_offset = my_cursor.get("offset", 0) + if not isinstance(cursor_offset, int) or isinstance(cursor_offset, bool) or cursor_offset < 0: + raise ValueError( + f"Wrapper '{self.node_id}': cursor 'offset' must be a non-negative int, got {cursor_offset!r}" + ) + + # Warm path: try to read existing cache + if cursor_gen: + meta, page, total = await self._read_snapshot_and_touch(ctx, session_id, cursor_offset, limit) + if meta and meta.get("gen") == cursor_gen: + # Serve from cache while this offset is still within the batch. + if cursor_offset < total: + child_has_next = meta.get("child_has_next", bool(meta.get("child_cursor"))) + return self._page_result(page, cursor_offset, limit, total, cursor_gen, child_has_next) + # Cache exhausted: rebuild with continuation cursor + return await self._cold_build_locked( + methods_dict, + session_id, + limit, + ctx, + child_cursor=meta.get("child_cursor", {}), + stale_gen=cursor_gen, + ) + # Stale gen or evicted cache -> fresh build replacing whatever batch exists now. + return await self._cold_build_locked( + methods_dict, + session_id, + limit, + ctx, + child_cursor={}, + stale_gen=meta.get("gen") if meta else None, + ) + + # Fresh scroll (no gen): rebuild, replacing the current batch if one exists. + # Its gen is the baseline a lock waiter must NOT accept as the new build. + meta = await self._read_meta(ctx, session_id) + return await self._cold_build_locked( + methods_dict, + session_id, + limit, + ctx, + child_cursor={}, + stale_gen=meta.get("gen") if meta else None, + ) + + async def _cold_build_locked( + self, + methods_dict: Dict[str, Any], + session_id: str, + limit: int, + ctx: ExecutionContext, + child_cursor: Dict[str, Any], + stale_gen: Optional[str] = None, + ) -> FeedResult: + """Cold build guarded by a lock so concurrent rebuilds fetch the child ONCE: + one caller builds and writes the cache, the rest wait for it and serve the + new batch from offset 0. + + `stale_gen` is the generation being replaced (None if none existed). The old + keys are not deleted before a rebuild, so a waiter must ignore metas still + carrying `stale_gen`: accepting the first meta it sees would re-serve the + outgoing batch and hand back a cursor that is stale the moment the builder + finishes. Waiters poll for the full lock TTL (the builder can hold it no + longer); if no new generation appears by then the builder is dead, so the + second round retries the lock, and only after that do we build unlocked as + a last resort. + """ + from smartfeed.execution.redis_lock import RedisLock + + assert ctx.redis is not None, "cached path requires redis" + lock_key = f"{self._base_key(session_id)}:coldlock" + for _ in range(2): + async with RedisLock(ctx.redis, lock_key, ttl=_COLD_LOCK_TTL) as acquired: + if acquired: + # Another caller may have finished this same rebuild while we + # queued for the lock -- serve its batch instead of re-fetching. + meta = await self._read_meta(ctx, session_id) + if meta and meta.get("gen") != stale_gen: + served = await self._serve_new_generation(ctx, session_id, limit, stale_gen) + if served is not None: + return served + return await self._cold_build(methods_dict, session_id, limit, ctx, child_cursor) + # Another coroutine is building -- wait for it to publish a NEW generation. + for _ in range(_LOCK_POLL_ROUNDS): + await asyncio.sleep(_LOCK_POLL_INTERVAL) + meta = await self._read_meta(ctx, session_id) + if meta and meta.get("gen") != stale_gen: + served = await self._serve_new_generation(ctx, session_id, limit, stale_gen) + if served is not None: + return served + # Two rounds without acquiring or seeing a new generation: build unlocked so + # the request still completes. + return await self._cold_build(methods_dict, session_id, limit, ctx, child_cursor) + + # -- shared base cache helpers ---------------------------------------------- + + @staticmethod + def _cursor_segment(child_cursor: Dict[str, Any]) -> str: + """Stable short tag for a continuation position, so each page-window of the + shared base is its own segment (page 1 = "0").""" + if not child_cursor: + return "0" + raw = json.dumps(child_cursor, sort_keys=True, default=str) + return hashlib.md5(raw.encode()).hexdigest()[:8] + + def _base_shared_key(self, session_id: str, child_cursor: Dict[str, Any]) -> str: + """Key for a shared-base segment (deduped, NOT reranked), keyed by cache_key + and the continuation position so continuations don't collide with page 1.""" + seg = self._cursor_segment(child_cursor) + return f"sf:{session_id}:{self.cache_key}:{self.data.config_hash()}:{seg}" + + async def _read_shared_base( + self, ctx: ExecutionContext, session_id: str, child_cursor: Dict[str, Any] + ) -> Optional[List[Any]]: + """Read a shared-base segment from Redis. Returns None on miss.""" + assert ctx.redis is not None, "cached path requires redis" + key = self._base_shared_key(session_id, child_cursor) + raw = await ctx.redis.get(key) + if raw is None: + return None + segment: List[Any] = orjson.loads(raw) + return segment + + async def _write_shared_base( + self, + ctx: ExecutionContext, + session_id: str, + child_cursor: Dict[str, Any], + data: List[Any], + child_cursor_out: Dict[str, Any], + child_has_next: bool = False, + ) -> None: + """Write a shared-base segment and its meta to Redis with TTL.""" + assert self.cache is not None, "cached path; execute() routes cache=None to passthrough" + assert ctx.redis is not None, "cached path requires redis" + key = self._base_shared_key(session_id, child_cursor) + ttl = self.cache.session_ttl + try: + payload = orjson.dumps(data) + meta = orjson.dumps({"child_cursor": child_cursor_out, "child_has_next": child_has_next}) + except TypeError as e: + raise TypeError( + f"Wrapper '{self.node_id}': cache is enabled, so items and child cursors must be " + f"orjson-serializable (str/int/float/bool/None/list/dict): {e}" + ) from e + pipe = ctx.redis.pipeline() + pipe.set(key, payload, ex=ttl) + pipe.set(f"{key}:meta", meta, ex=ttl) + await pipe.execute() + + async def _read_shared_base_meta( + self, ctx: ExecutionContext, session_id: str, child_cursor: Dict[str, Any] + ) -> Optional[Dict[str, Any]]: + """Read a shared-base segment's metadata (child_cursor, child_has_next).""" + assert ctx.redis is not None, "cached path requires redis" + key = self._base_shared_key(session_id, child_cursor) + raw = await ctx.redis.get(f"{key}:meta") + if raw is None: + return None + meta: Dict[str, Any] = orjson.loads(raw) + return meta + + async def _fetch_and_dedup( + self, + ctx: ExecutionContext, + target: int, + child_cursor: Dict[str, Any], + seen: Optional[Set[str]] = None, + ) -> Tuple[List[Any], Dict[str, Any], bool]: + """Collect up to `target` deduped items from the child. + + Fetches exactly the outstanding deficit each round and keeps EVERY survivor + (never truncates, never advances the child cursor past an unconsumed unique), + so no item is lost. Refills until `target` unique items are collected or the + child is exhausted, so pages fill under dedup attrition instead of coming up + short. `seen`, if given, holds keys already shown earlier in this scroll (so + they are not repeated across a rebuild boundary); `_dedup` mutates it in place + with the keys emitted here. + + Returns (data, child_cursor, child_has_next). + """ + cursor = dict(child_cursor) + data: List[Any] = [] + # One seen-set across ALL refill rounds: a fresh set per round would re-admit + # round-1 ids, letting duplicates survive within one build. + if seen is None: + seen = set() + has_next = True + scanned = 0 + scan_cap = max(target, 1) * _REFILL_SCAN_FACTOR + + # Keep scanning through duplicate runs until `target` unique items are + # collected or the child is exhausted; bounded by scan_cap so a + # duplicate-only source can't spin forever. + while len(data) < target and has_next and scanned < scan_cap: + need = target - len(data) + result = await _executor.run(self.data, ctx, need, cursor) + cursor = result.next_page + has_next = result.has_next_page + if not result.data: + break # child yielded nothing -> stop instead of spinning + scanned += len(result.data) + new = self._dedup(result.data, seen) if self.dedup else result.data + data.extend(new) + + return data, cursor, has_next + + async def _cold_build( + self, + methods_dict: Dict[str, Any], + session_id: str, + limit: int, + ctx: ExecutionContext, + child_cursor: Dict[str, Any], + ) -> FeedResult: + """Build full session: fetch -> dedup -> rerank -> write cache -> paginate page 1.""" + assert self.cache is not None, "cached path; execute() routes cache=None to passthrough" + session_size = self.cache.session_size + + if self.cache_key is not None: + # Shared cache path: base data (fetch + dedup) is shared across wrappers + # with the same cache_key; rerank is applied per-wrapper after. Each + # continuation position is a distinct shared segment (see _base_shared_key). + base_data = await self._build_shared_base(methods_dict, session_id, ctx, child_cursor) + shared_meta = await self._read_shared_base_meta(ctx, session_id, child_cursor) + child_cursor_out = shared_meta.get("child_cursor", {}) if shared_meta else {} + child_has_next = shared_meta.get("child_has_next", False) if shared_meta else False + # Per-wrapper: stamp pre-rerank, apply rerank, stamp post-rerank + data = list(base_data) + self._stamp_pre_rerank(data) + data = await self._apply_rerank(data, methods_dict, session_id) + if self.rerank: + self._stamp_post_rerank(data) + else: + # Standard path. Session-scoped seen-set: reset on a fresh scroll (empty + # cursor) so re-requesting page 1 returns page 1, and carry it across + # rebuilds so a shown id never repeats later in the same scroll. + seen: Optional[Set[str]] = None + preloaded: Set[str] = set() + if self.dedup: + if not child_cursor: + await self._reset_seen_set(ctx, session_id) + seen = set() + else: + seen = await self._load_seen_set(ctx, session_id) + preloaded = set(seen) + data, child_cursor_out, child_has_next = await self._fetch_and_dedup( + ctx, session_size, child_cursor, seen=seen + ) + if self.dedup: + assert seen is not None + # SADD only the delta: everything in `preloaded` is already in Redis. + await self._save_seen_set(ctx, session_id, seen - preloaded) + self._stamp_pre_rerank(data) + data = await self._apply_rerank(data, methods_dict, session_id) + if self.rerank: + self._stamp_post_rerank(data) + + gen = secrets.token_hex(4) + await self._write_cache(ctx, session_id, data, gen, child_cursor_out, child_has_next) + return self._page_result(data[:limit], 0, limit, len(data), gen, child_has_next) + + async def _build_shared_base( + self, + methods_dict: Dict[str, Any], + session_id: str, + ctx: ExecutionContext, + child_cursor: Dict[str, Any], + ) -> List[Any]: + """Fetch and dedup the shared base data, using a distributed lock to ensure + only one caller fetches from child on a cold build. Others wait and read.""" + from smartfeed.execution.redis_lock import RedisLock + + assert self.cache is not None, "cached path; execute() routes cache=None to passthrough" + assert ctx.redis is not None, "cached path requires redis" + session_size = self.cache.session_size + lock_key = f"{self._base_shared_key(session_id, child_cursor)}:lock" + + # Fast path: this segment already cached + existing = await self._read_shared_base(ctx, session_id, child_cursor) + if existing is not None: + return existing + + # Segments are only ever created (never rebuilt in place), so "segment exists" + # is the completion signal. Waiters poll for the full lock TTL; if the segment + # never appears the builder is dead, so the second round retries the lock, and + # only after that do we fetch unlocked as a last resort. + for _ in range(2): + async with RedisLock(ctx.redis, lock_key, ttl=_COLD_LOCK_TTL) as acquired: + if acquired: + # Double-check after acquiring lock + existing = await self._read_shared_base(ctx, session_id, child_cursor) + if existing is not None: + return existing + + base_data, child_cursor_out, child_has_next = await self._fetch_and_dedup( + ctx, session_size, child_cursor + ) + await self._write_shared_base( + ctx, session_id, child_cursor, base_data, child_cursor_out, child_has_next + ) + return base_data + # Another coroutine holds the lock -- poll until the segment appears. + for _ in range(_LOCK_POLL_ROUNDS): + await asyncio.sleep(_LOCK_POLL_INTERVAL) + existing = await self._read_shared_base(ctx, session_id, child_cursor) + if existing is not None: + return existing + # Two rounds without acquiring or seeing the segment: fetch unlocked so the + # request still completes. + data, _, _ = await self._fetch_and_dedup(ctx, session_size, child_cursor) + return data diff --git a/smartfeed/examples/__init__.py b/smartfeed/py.typed similarity index 100% rename from smartfeed/examples/__init__.py rename to smartfeed/py.typed diff --git a/smartfeed/schemas.py b/smartfeed/schemas.py deleted file mode 100644 index 45df221..0000000 --- a/smartfeed/schemas.py +++ /dev/null @@ -1,1132 +0,0 @@ -import inspect -import json -import logging -from abc import ABC, abstractmethod -from collections import defaultdict, deque -from random import shuffle -from typing import Annotated, Any, Callable, Dict, List, Literal, Optional, Union, no_type_check - -import redis -from pydantic import BaseModel, Field, root_validator -from redis.asyncio import Redis as AsyncRedis -from redis.asyncio import RedisCluster as AsyncRedisCluster - -FeedTypes = Annotated[ - Union[ - "MergerAppend", - "MergerAppendDistribute", - "MergerPositional", - "MergerPercentage", - "MergerPercentageGradient", - "MergerViewSession", - "SubFeed", - ], - Field(discriminator="type"), -] - - -class FeedResultNextPageInside(BaseModel): - """ - Модель данных курсора пагинации конкретной позиции. - - Attributes: - page порядковый номер страницы. - after данные для пагинации клиентского метода. - """ - - page: int = 1 - after: Any = None - - -class FeedResultNextPage(BaseModel): - """ - Модель курсора пагинации. - - Attributes: - data словарь вида "ключ: данные по пагинации", где ключ - subfeed_id или merger_id. - """ - - data: Dict[str, FeedResultNextPageInside] - - -class FeedResult(BaseModel): - """ - Модель результата метода get_data() любой позиции / целого фида. - - Attributes: - data список данных, возвращенных мерджером / субфидом. - next_page курсор пагинации. - has_next_page флаг наличия следующей страницы данных. - """ - - data: List - next_page: FeedResultNextPage - has_next_page: bool - - -class FeedResultClient(BaseModel): - """ - Модель результата клиентского метода субфида. - - Attributes: - data список данных, возвращенных мерджером / субфидом. - next_page курсор пагинации клиентского метода. - has_next_page флаг наличия следующей страницы данных. - """ - - data: List - next_page: FeedResultNextPageInside - has_next_page: bool - - -class BaseFeedConfigModel(ABC, BaseModel): - """ - Абстрактный класс для мерджера / субфида конфигурации. - """ - - @abstractmethod - async def get_data( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - limit: int, - next_page: FeedResultNextPage, - redis_client: Optional[Union[redis.Redis, AsyncRedis]] = None, - **params: Any, - ) -> FeedResult: - """ - Метод для получения данных. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param redis_client: объект клиента Redis (для конфигурации с view_session мерджером). - :param params: параметры для метода. - :return: список данных. - """ - - -class MergerViewSession(BaseFeedConfigModel): - """ - Модель мерджера с кэшированием. - - Attributes: - merger_id уникальный ID мерджера. - type тип объекта - всегда "merger_view_session". - view_session флаг использования механизма расчета всего фида сразу и сохранения в кэш. - session_size размер кэшируемого фида (limit получения данных для сохранения в кэш). - session_live_time срок хранения в кэше для кэшируемого фида (в секундах). - data мерджер или субфид. - deduplicate флаг дедупликации (удаления дублей из сессии). - dedup_key название ключа или атрибута, по которому логика дедпликации найдет дубли. - shuffle флаг для перемешивания полученных данных мерджера. - """ - - merger_id: str - type: Literal["merger_view_session"] - session_size: int - session_live_time: int - data: FeedTypes - deduplicate: bool = False - dedup_key: str = None # type: ignore - shuffle: bool = False - - def _get_dedup_key_or_attr(self, item: Any) -> str: - """ - Метод для получения ключа объекта кешируемой сессии. - - Если указанное в конфиге сессии название ключа имеет значение None, - в качестве ключа вернется сам объект. - Если название ключа не None, и для одного из объектов ни найден ни ключ, ни атрибут, - метод выбросит AssertionError. - - :param item: объект, для которого нужен ключ. - :return: ключ объекта. - """ - - if not self.dedup_key: - return item - - try: - dedup_value = item.get(self.dedup_key) - except AttributeError: - dedup_value = getattr(item, self.dedup_key, None) - - assert dedup_value is not None, f"Deduplication failed: entity {item} has no key or attr {self.dedup_key}" - return dedup_value - - def _dedup_data(self, data: List[Any]) -> List[Any]: - """ - Метод для удаления дублей в списке data с сохранением последовательности. - - :param data: список, в котором нужно удалить дубли. - :return: результат удаления дублей. - """ - - deduplicated_data = {self._get_dedup_key_or_attr(item): item for item in data} - return list(deduplicated_data.values()) - - async def _set_cache( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - redis_client: redis.Redis, - cache_key: str, - **params: Any, - ) -> List[Any]: - """ - Метод для кэширования данных Merger View Session. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param redis_client: объект клиента Redis. - :param cache_key: ключ для кэширования. - :param params: любые внешние параметры, передаваемые в исполняемую функцию на клиентской стороне. - :return: обработанные данные, которые были записаны в кэш. - """ - - result = await self.data.get_data( - methods_dict=methods_dict, - user_id=user_id, - limit=self.session_size, - next_page=FeedResultNextPage(data={}), - **params, - ) - - data = result.data - if self.deduplicate: - data = self._dedup_data(data) - redis_client.set(name=cache_key, value=json.dumps(data), ex=self.session_live_time) - return data - - async def _set_cache_async( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - redis_client: AsyncRedis, - cache_key: str, - **params: Any, - ) -> List[Any]: - """ - Метод для кэширования данных Merger View Session. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param redis_client: объект клиента Redis. - :param cache_key: ключ для кэширования. - :param params: любые внешние параметры, передаваемые в исполняемую функцию на клиентской стороне. - :return: обработанные данные, которые были записаны в кэш. - """ - - result = await self.data.get_data( - methods_dict=methods_dict, - user_id=user_id, - limit=self.session_size, - next_page=FeedResultNextPage(data={}), - **params, - ) - - data = result.data - if self.deduplicate: - data = self._dedup_data(data) - await redis_client.set(cache_key, json.dumps(data)) - await redis_client.expire(cache_key, self.session_live_time) - return data - - async def _get_cache( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - limit: int, - next_page: FeedResultNextPage, - redis_client: redis.Redis, - **params: Any, - ) -> FeedResult: - """ - Метод для получения данных Merger View Session из кэша Redis. - При отсутствии данных в кэше - получить и сохранить. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: лимит на выдачу данных. - :param next_page: курсор для пагинации в формате SmartFeedResultNextPage. - :param redis_client: объект клиента Redis. - :param params: любые внешние параметры, передаваемые в исполняемую функцию на клиентской стороне. - :return: результат получения данных согласно конфигурации фида. - """ - - # Формируем ключ для кэширования данных мерджера. - if session_cache_key := params.get("custom_view_session_key", None): - cache_key = f"{self.merger_id}_{user_id}_{session_cache_key}" - else: - cache_key = f"{self.merger_id}_{user_id}" - - logging.info("MergerViewSession cache request for %s", cache_key) - # Если кэш не найден или передан пустой курсор пагинации на мерджер, обновляем данные и записываем в кэш. - if not redis_client.exists(cache_key) or self.merger_id not in next_page.data: - logging.info("Cache miss or new session - generating fresh data for %s", cache_key) - # Получаем свежие данные и используем их напрямую (избегаем чтение из кэша) - session_data = await self._set_cache( - methods_dict=methods_dict, user_id=user_id, redis_client=redis_client, cache_key=cache_key, **params - ) - else: - logging.info("Cache exists - attempting read from Redis for %s", cache_key) - # Читаем из кэша только если он уже существовал - cached_data = redis_client.get(name=cache_key) - if cached_data is None: - # Fallback: если кэш пропал, получаем свежие данные - logging.info( - "Redis returned None for %s - falling back to fresh data (cluster replication issue)", cache_key - ) - session_data = await self._set_cache( - methods_dict=methods_dict, user_id=user_id, redis_client=redis_client, cache_key=cache_key, **params - ) - else: - logging.info("Successfully read cached data for %s", cache_key) - session_data = json.loads(cached_data) - page = next_page.data[self.merger_id].page if self.merger_id in next_page.data else 1 - result = FeedResult( - data=session_data[(page - 1) * limit :][:limit], - next_page=FeedResultNextPage(data={self.merger_id: FeedResultNextPageInside(page=page + 1, after=None)}), - has_next_page=bool(len(session_data) > limit * page), - ) - return result - - async def _get_cache_async( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - limit: int, - next_page: FeedResultNextPage, - redis_client: AsyncRedis, - **params: Any, - ) -> FeedResult: - """ - Метод для получения данных Merger View Session из кэша Redis. - При отсутствии данных в кэше - получить и сохранить. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: лимит на выдачу данных. - :param next_page: курсор для пагинации в формате SmartFeedResultNextPage. - :param redis_client: объект клиента Redis. - :param params: любые внешние параметры, передаваемые в исполняемую функцию на клиентской стороне. - :return: результат получения данных согласно конфигурации фида. - """ - - # Формируем ключ для кэширования данных мерджера. - if session_cache_key := params.get("custom_view_session_key", None): - cache_key = f"{self.merger_id}_{user_id}_{session_cache_key}" - else: - cache_key = f"{self.merger_id}_{user_id}" - - # Если кэш не найден или передан пустой курсор пагинации на мерджер, обновляем данные и записываем в кэш. - if not await redis_client.exists(cache_key) or self.merger_id not in next_page.data: - # Получаем свежие данные и используем их напрямую (избегаем чтение из кэша) - session_data = await self._set_cache_async( - methods_dict=methods_dict, user_id=user_id, redis_client=redis_client, cache_key=cache_key, **params - ) - else: - # Читаем из кэша только если он уже существовал - cached_data = await redis_client.get(cache_key) - if cached_data is None: - # Fallback: если кэш пропал, получаем свежие данные - logging.info( - "Redis returned None for %s - falling back to fresh data (cluster replication issue)", cache_key - ) - session_data = await self._set_cache_async( - methods_dict=methods_dict, user_id=user_id, redis_client=redis_client, cache_key=cache_key, **params - ) - else: - logging.info("Successfully read cached data for %s", cache_key) - session_data = json.loads(cached_data) - page = next_page.data[self.merger_id].page if self.merger_id in next_page.data else 1 - result = FeedResult( - data=session_data[(page - 1) * limit :][:limit], - next_page=FeedResultNextPage(data={self.merger_id: FeedResultNextPageInside(page=page + 1, after=None)}), - has_next_page=bool(len(session_data) > limit * page), - ) - return result - - async def get_data( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - limit: int, - next_page: FeedResultNextPage, - redis_client: Optional[Union[redis.Redis, AsyncRedis]] = None, - **params: Any, - ) -> FeedResult: - """ - Метод для получения данных методом append. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param redis_client: объект клиента Redis (для конфигурации с view_session мерджером). - :param params: для метода класса. - :return: список данных методом append. - """ - - # Проверяем наличие клиента Redis в конфигурации фида. - if not redis_client: - raise ValueError("Redis client must be provided if using Merger View Session") - - # Формируем результат view session мерджера. - if isinstance(redis_client, (AsyncRedis, AsyncRedisCluster)): - result = await self._get_cache_async( - methods_dict=methods_dict, - user_id=user_id, - limit=limit, - next_page=next_page, - redis_client=redis_client, - **params, - ) - else: - result = await self._get_cache( - methods_dict=methods_dict, - user_id=user_id, - limit=limit, - next_page=next_page, - redis_client=redis_client, - **params, - ) - - # Если в конфигурации указано "смешать" данные. - if self.shuffle: - shuffle(result.data) - - return result - - -class MergerAppend(BaseFeedConfigModel): - """ - Модель append мерджера. - - Attributes: - merger_id уникальный ID мерджера. - type тип объекта - всегда "merger_append". - items позиции мерджера. - shuffle флаг для перемешивания полученных данных мерджера. - """ - - merger_id: str - type: Literal["merger_append"] - items: List[FeedTypes] - shuffle: bool = False - - async def get_data( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - limit: int, - next_page: FeedResultNextPage, - redis_client: Optional[Union[redis.Redis, AsyncRedis]] = None, - **params: Any, - ) -> FeedResult: - """ - Метод для получения данных методом append. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param redis_client: объект клиента Redis (для конфигурации с view_session мерджером). - :param params: для метода класса. - :return: список данных методом append. - """ - - # Формируем результат append мерджера. - result = FeedResult(data=[], next_page=FeedResultNextPage(data={}), has_next_page=False) - - result_limit = limit - for item in self.items: - # Получаем данные из позиции мерджера. - item_result = await item.get_data( - methods_dict=methods_dict, - user_id=user_id, - limit=result_limit, - next_page=next_page, - redis_client=redis_client, - **params, - ) - - # Добавляем данные позиции к общему результату процентного мерджера. - result.data.extend(item_result.data) - - # Обновляем result_limit - result_limit -= len(item_result.data) - - # Если has_next_page = False, то проверяем has_next_page у позиции и, если необходимо, обновляем. - if not result.has_next_page and item_result.has_next_page: - result.has_next_page = True - - # Обновляем next_page. - result.next_page.data.update(item_result.next_page.data) - - # Если полученных данных хватает, то прерываем итерацию и возвращаем результат. - if result_limit <= 0: - break - - # Если в конфигурации указано "смешать" данные. - if self.shuffle: - shuffle(result.data) - - return result - - -class MergerPositional(BaseFeedConfigModel): - """ - Модель позиционного мерджера. - - Attributes: - merger_id уникальный ID мерджера. - type тип объекта - всегда "merger_positional". - positions позиции для вставки из мерджера / субфида "positional" [обязателен, если нет start, end, step]. - start начальная позиция [обязателен, если нет positions]. - end завершающая позиция [обязателен, если нет positions]. - step шаг позиций между "start" и "end" [обязателен, если нет positions]. - positional мерджер / субфид из которого берутся позиционные данные. - default мерджер / субфид из которого берутся остальные данные. - """ - - merger_id: str - type: Literal["merger_positional"] - positions: List[int] = [] - start: Optional[int] = None - end: Optional[int] = None - step: Optional[int] = None - positional: FeedTypes - default: FeedTypes - - @root_validator(skip_on_failure=True) - def validate_merger_positional(cls, values: Dict[str, Any]) -> Dict[str, Any]: - if not values["positions"] and not all((values["start"], values["end"], values["step"])): - raise ValueError('Either "positions" or "start", "end", and "step" must be provided') - if values["start"] and values["positions"]: - if isinstance(values["start"], int) and values["start"] <= max(values["positions"]): - raise ValueError('"start" must be bigger than maximum value of "positions"') - if isinstance(values["start"], int) and isinstance(values["end"], int): - if values["end"] <= values["start"]: - raise ValueError('"end" must be bigger than "start"') - return values - - async def get_data( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - limit: int, - next_page: FeedResultNextPage, - redis_client: Optional[Union[redis.Redis, AsyncRedis]] = None, - **params: Any, - ) -> FeedResult: - """ - Метод для получения данных в позиционном соотношении из данных позиций. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param redis_client: объект клиента Redis (для конфигурации с view_session мерджером). - :param params: для метода класса. - :return: список данных в процентном соотношении. - """ - - # Получаем данные "default". - default_res = await self.default.get_data( - methods_dict=methods_dict, - user_id=user_id, - limit=limit, - next_page=next_page, - redis_client=redis_client, - **params, - ) - - # Формируем результат позиционного мерджера. - result = FeedResult( - data=default_res.data, - next_page=FeedResultNextPage( - data={ - self.merger_id: FeedResultNextPageInside( - page=next_page.data[self.merger_id].page if self.merger_id in next_page.data else 1, - after=next_page.data[self.merger_id].after if self.merger_id in next_page.data else None, - ) - }, - ), - has_next_page=default_res.has_next_page, - ) - - # Получаем список позиций с учетом текущей страницы. - positional_has_next_page = True - page_positions = [] - available_positions = range( - (result.next_page.data[self.merger_id].page - 1) * limit, - (result.next_page.data[self.merger_id].page * limit) + 1, - ) - for position in self.positions: - if position in available_positions: - page_positions.append(available_positions.index(position)) - - # Если конечная позиция текущей страницы больше или равна MAX позиции в конфигурации, то has_next_page = False - if max(available_positions) >= max(self.positions, default=0): - positional_has_next_page = False - - if self.start is not None and self.end is not None and self.step is not None: - # Если конечная позиция текущей страницы больше или равна конечной шаговой позиции, то has_next_page = False - positional_has_next_page = not max(available_positions) >= self.end - - for position in range(self.start, self.end, self.step): - if position in available_positions: - page_positions.append(available_positions.index(position)) - - # Получаем данные "positional". - pos_res = await self.positional.get_data( - methods_dict=methods_dict, - user_id=user_id, - limit=len(page_positions), - next_page=next_page, - redis_client=redis_client, - **params, - ) - - # Если has_next_page = False, то проверяем has_next_page у позиции и, если необходимо, обновляем. - if not result.has_next_page and all([positional_has_next_page, pos_res.has_next_page]): - result.has_next_page = True - - # Обновляем next_page. - result.next_page.data.update(default_res.next_page.data) - result.next_page.data.update(pos_res.next_page.data) - - # Формируем общие данные позиционного мерджера. - for i, post in enumerate(pos_res.data): - result.data = result.data[: page_positions[i] - 1] + [post] + result.data[page_positions[i] - 1 :] - - # Проверка на возврат данных в количестве не более limit. - if len(result.data) > limit: - result.data = result.data[:limit] - - # Обновляем страницу для курсора пагинации мерджера. - result.next_page.data[self.merger_id].page += 1 - - return result - - -class MergerPercentageItem(BaseModel): - """ - Модель позиции процентного мерджера. - - Attributes: - percentage процент позиции в мерджере. - data мерджер / субфид. - """ - - percentage: int - data: FeedTypes - - -class MergerPercentage(BaseFeedConfigModel): - """ - Модель процентного мерджера. - - Attributes: - merger_id уникальный ID мерджера. - type тип объекта - всегда "merger_percentage". - shuffle флаг для перемешивания полученных данных мерджера. - items позиции мерджера. - """ - - merger_id: str - type: Literal["merger_percentage"] - items: List[MergerPercentageItem] - shuffle: bool = False - - @staticmethod - async def _merge_items_data(items_data: List[List]) -> List: - """ - Метод для получения максимально равномерно распределенных данных позиций процентного мерджера. - - :param items_data: список со списками данных из каждой позиции. - :return: максимально равномерно распределенные данные позиций процентного мерджера. - """ - - # Формируем возвращаемый результат и список курсоров для списка каждой позиции. - result: List = [] - cursor: List[Dict] = [] - - # Получаем длину самого маленького списка и формируем курсор для каждого списка. - min_length = min(len(item_data) for item_data in items_data) or 1 - for item_data in items_data: - cursor.append( - { - "items": item_data, - "current": 0, - "size": round(len(item_data) / min_length), - } - ) - - # Получаем общий размер всех элементов всех списков и пока не получаем результат такого же размера - # производим операции по распределению элементов. - full_length = sum(len(item_data) for item_data in items_data) - while len(result) < full_length: - for item_cursor in cursor: - items = item_cursor["items"] - start = item_cursor["current"] - end = start + item_cursor["size"] if start + item_cursor["size"] < len(items) else len(items) - result.extend(items[start:end]) - item_cursor["current"] = end - - return result - - async def get_data( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - limit: int, - next_page: FeedResultNextPage, - redis_client: Optional[Union[redis.Redis, AsyncRedis]] = None, - **params: Any, - ) -> FeedResult: - """ - Метод для получения данных в процентном соотношении из данных позиций. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param redis_client: объект клиента Redis (для конфигурации с view_session мерджером). - :param params: для метода класса. - :return: список данных в процентном соотношении. - """ - - # Формируем результат процентного мерджера. - result = FeedResult(data=[], next_page=FeedResultNextPage(data={}), has_next_page=False) - - items_data: List = [] - for item in self.items: - # Получаем данные из позиций процентного мерджера. - item_result = await item.data.get_data( - methods_dict=methods_dict, - user_id=user_id, - limit=limit * item.percentage // 100, - next_page=next_page, - redis_client=redis_client, - **params, - ) - - # Добавляем данные позиции в список данных позиций. - items_data.append(item_result.data) - - # Если has_next_page = False, то проверяем has_next_page у позиции и, если необходимо, обновляем. - if not result.has_next_page and item_result.has_next_page: - result.has_next_page = True - - # Обновляем next_page. - result.next_page.data.update(item_result.next_page.data) - - # Добавляем данные позиции к общему результату процентного мерджера. - result.data = await self._merge_items_data(items_data=items_data) - - # Если в конфигурации указано "смешать" данные. - if self.shuffle: - shuffle(result.data) - - return result - - -class MergerPercentageGradient(BaseFeedConfigModel): - """ - Модель процентного мерджера с градиентном. - - Attributes: - merger_id уникальный ID мерджера. - type тип объекта - всегда "merger_percentage_gradient". - item_from мерджер / субфид из которого начинается "перетекание" градиента. - item_to мерджер / субфид в который "перетекает" градиент. - step изменение в % соотношения из item_from в item_to. - size_to_step шаг для применения изменений % соотношения (например, через каждые 30 позиций). - shuffle флаг для перемешивания полученных данных мерджера. - """ - - merger_id: str - type: Literal["merger_percentage_gradient"] - item_from: MergerPercentageItem - item_to: MergerPercentageItem - step: int - size_to_step: int - shuffle: bool = False - - @root_validator(skip_on_failure=True) - def validate_merger_percentage_gradient(cls, values: Dict[str, Any]) -> Dict[str, Any]: - if values["step"] < 1 or values["step"] > 100: - raise ValueError('"step" must be in range from 1 to 100') - if values["size_to_step"] < 1: - raise ValueError('"size_to_step" must be bigger than 1') - return values - - async def _calculate_limits_and_percents(self, page: int, limit: int) -> Dict: - """ - Метод для получения списка лимитов данных с процентным соотношением позиций item_from & item_to, - учитывая градиентное изменение соотношений. - - :param page: порядковый номер страницы. - :param limit: общий лимит данных для страницы. - :return: список лимитов данных с процентным соотношением позиций item_from & item_to. - """ - - result: Dict = { - "limit_from": 0, - "limit_to": 0, - "percentages": [], - } - - percentage_from = self.item_from.percentage - percentage_to = self.item_to.percentage - start_position = limit * (page - 1) - first_iter = True - - for i in range(self.size_to_step, limit * page + self.size_to_step, self.size_to_step): - # При первой итерации и percentage_to >= 100 не меняем соотношение % между позициями. - if not first_iter and percentage_to < 100: - # Меняем процентное соотношение позиций на "шаг", указанный в конфигурации. - percentage_from -= self.step - percentage_to += self.step - - # Если процентное соотношение вышло за 100+, то устанавливаем предельные значения. - if percentage_to > 100 or percentage_from < 0: - percentage_from = 0 - percentage_to = 100 - - # Если индекс итерации по величине больше стартовой позиции согласно переданной странице, - # то начинаем обработку. - if i > start_position: - # Рассчитываем лимит получения данных для конкретной итерации. - iter_limit = (limit * page - start_position) if i > limit * page else (i - start_position) - start_position = i - - # Формируем результат для каждой итерации и добавляем в возвращаемый список, но если процентное - # соотношение у последней итерации 0 - 100, то добавляем лимит к ней. - if result["percentages"] and result["percentages"][-1]["to"] >= 100: - result["limit_to"] += iter_limit - result["percentages"][-1]["limit"] += iter_limit - else: - result["limit_from"] += iter_limit * percentage_from // 100 - result["limit_to"] += iter_limit * percentage_to // 100 - iter_result = {"limit": iter_limit, "from": percentage_from, "to": percentage_to} - result["percentages"].append(iter_result) - - # Если первая итерация цикла - if first_iter: - first_iter = False - - return result - - async def get_data( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - limit: int, - next_page: FeedResultNextPage, - redis_client: Optional[Union[redis.Redis, AsyncRedis]] = None, - **params: Any, - ) -> FeedResult: - """ - Метод для получения данных в процентном соотношении с градиентом из данных позиций. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param redis_client: объект клиента Redis (для конфигурации с view_session мерджером). - :param params: для метода класса. - :return: список данных в процентном соотношении. - """ - - # Формируем результат процентного мерджера с градиентом. - result = FeedResult( - data=[], - next_page=FeedResultNextPage( - data={ - self.merger_id: FeedResultNextPageInside( - page=next_page.data[self.merger_id].page if self.merger_id in next_page.data else 1, - after=next_page.data[self.merger_id].after if self.merger_id in next_page.data else None, - ) - }, - ), - has_next_page=False, - ) - - # Получаем список лимитов данных и соотношений согласно странице и градиенту. - limits_and_percents = await self._calculate_limits_and_percents( - page=result.next_page.data[self.merger_id].page, - limit=limit, - ) - - # Получаем данные из позиций в процентном соотношений. - item_from = await self.item_from.data.get_data( - methods_dict=methods_dict, - user_id=user_id, - limit=limits_and_percents["limit_from"], - next_page=next_page, - redis_client=redis_client, - **params, - ) - item_to = await self.item_to.data.get_data( - methods_dict=methods_dict, - user_id=user_id, - limit=limits_and_percents["limit_to"], - next_page=next_page, - redis_client=redis_client, - **params, - ) - - from_start_index = 0 - to_start_index = 0 - for lp_data in limits_and_percents["percentages"]: - # Высчитываем лимиты для каждой позиции исходя из процентного соотношения. - from_end_index = (lp_data["limit"] * lp_data["from"] // 100) + from_start_index - to_end_index = (lp_data["limit"] * lp_data["to"] // 100) + to_start_index - - # Добавляем данные позиции к общему результату процентного мерджера с градиентом. - result.data.extend(item_from.data[from_start_index:from_end_index]) - result.data.extend(item_to.data[to_start_index:to_end_index]) - - # Обновляем стартовые индексы. - from_start_index = from_end_index - to_start_index = to_end_index - - # Обновляем next_page. - result.next_page.data.update(item_from.next_page.data) - result.next_page.data.update(item_to.next_page.data) - - # Если has_next_page = False, то проверяем has_next_page у позиций и, если необходимо, обновляем. - if any([item_from.has_next_page, item_to.has_next_page]): - result.has_next_page = True - - # Если в конфигурации указано "смешать" данные. - if self.shuffle: - shuffle(result.data) - - # Обновляем страницу для курсора пагинации мерджера. - result.next_page.data[self.merger_id].page += 1 - - return result - - -class MergerAppendDistribute(BaseFeedConfigModel): - """ - Модель мерджера, равномерно распределяющего данные по ключу. - - Attributes: - merger_id уникальный ID мерджера. - type тип объекта - всегда "merger_distribute". - items позиции мерджера. - distribution_key ключ для распределения данных мерджера. - sorting_key ключ сортировки. - sorting_desc флаг сортировки по убыванию. - """ - - merger_id: str - type: Literal["merger_distribute"] - items: List[FeedTypes] - distribution_key: str - sorting_key: Optional[str] = None - sorting_desc: bool = False - - @no_type_check - async def _uniform_distribute(self, data: list) -> list: - # Сортируем записи глобально по `created_at` в порядке убывания - if self.sorting_key: - data = sorted(data, key=lambda x: x[self.sorting_key], reverse=self.sorting_desc) - - # Группируем записи по `distribution_key` - grouped_entries = defaultdict(deque) - for entry in data: - grouped_entries[entry[self.distribution_key]].append(entry) - result = [] - prev_profile_id = None - while any(grouped_entries.values()): - for profile_id in list(grouped_entries.keys()): - if grouped_entries[profile_id]: - # Если текущий `distribution_key` отличается от предыдущего или он последний, берем его - if profile_id != prev_profile_id or len(grouped_entries) == 1: - result.append(grouped_entries[profile_id].popleft()) - prev_profile_id = profile_id - if not grouped_entries[profile_id]: # Если записи закончились, удаляем ключ из группы - del grouped_entries[profile_id] - else: - del grouped_entries[profile_id] - - return result - - async def get_data( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - limit: int, - next_page: FeedResultNextPage, - redis_client: Optional[Union[redis.Redis, AsyncRedis]] = None, - **params: Any, - ) -> FeedResult: - """ - Метод для получения данных методом append. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param redis_client: объект клиента Redis (для конфигурации с view_session мерджером). - :param params: для метода класса. - :return: список данных методом append. - """ - - # Формируем результат append мерджера. - result = FeedResult(data=[], next_page=FeedResultNextPage(data={}), has_next_page=False) - - result_limit = limit - for item in self.items: - # Получаем данные из позиции мерджера. - item_result = await item.get_data( - methods_dict=methods_dict, - user_id=user_id, - limit=result_limit, - next_page=next_page, - redis_client=redis_client, - **params, - ) - - # Добавляем данные позиции к общему результату процентного мерджера. - result.data.extend(item_result.data) - - # Обновляем result_limit - result_limit -= len(item_result.data) - - # Если has_next_page = False, то проверяем has_next_page у позиции и, если необходимо, обновляем. - if not result.has_next_page and item_result.has_next_page: - result.has_next_page = True - - # Обновляем next_page. - result.next_page.data.update(item_result.next_page.data) - - # Если полученных данных хватает, то прерываем итерацию и возвращаем результат. - if result_limit <= 0: - break - - # Распределяем данные равномерно по ключу. - result.data = await self._uniform_distribute(result.data) - return result - - -class SubFeed(BaseFeedConfigModel): - """ - Модель субфида. - - Attributes: - subfeed_id уникальный ID субфида. - type тип объекта - всегда "subfeed". - method_name название клиентского метода для получения данных субфида. - subfeed_params статичные параметры для метода субфида. - shuffle флаг для перемешивания полученных данных мерджера. - """ - - subfeed_id: str - type: Literal["subfeed"] - method_name: str - subfeed_params: Dict[str, Any] = {} - raise_error: Optional[bool] = True - shuffle: bool = False - - async def get_data( - self, - methods_dict: Dict[str, Callable], - user_id: Any, - limit: int, - next_page: FeedResultNextPage, - redis_client: Optional[Union[redis.Redis, AsyncRedis]] = None, - **params: Any, - ) -> FeedResult: - """ - Метод для получения данных из метода субфида. - - :param methods_dict: словарь с используемыми методами. - :param user_id: ID объекта для получения данных (например, ID пользователя). - :param limit: кол-во элементов. - :param next_page: курсор пагинации. - :param redis_client: объект клиента Redis (для конфигурации с view_session мерджером). - :param params: параметры для метода. - :return: список данных. - """ - - # Формируем next_page конкретного субфида. - subfeed_next_page = FeedResultNextPageInside( - page=next_page.data[self.subfeed_id].page if self.subfeed_id in next_page.data else 1, - after=next_page.data[self.subfeed_id].after if self.subfeed_id in next_page.data else None, - ) - - # Формируем params для функции субфида. - method_args = inspect.getfullargspec(methods_dict[self.method_name]).args - method_params: Dict[str, Any] = {} - for arg in method_args: - if arg in params: - method_params[arg] = params[arg] - - # Получаем результат функции клиента в формате SubFeedResult. - try: - method_result = await methods_dict[self.method_name]( - user_id=user_id, - limit=limit, - next_page=subfeed_next_page, - **method_params, - **self.subfeed_params, - ) - except (Exception,) as _: - if self.raise_error: - raise - - method_result = FeedResultClient( - data=[], - next_page=subfeed_next_page, - has_next_page=False, - ) - - if not isinstance(method_result, FeedResultClient): - raise TypeError('SubFeed function must return "FeedResultClient" instance.') - - # Если в конфигурации указано "смешать" данные. - if self.shuffle: - shuffle(method_result.data) - - result = FeedResult( - data=method_result.data, - next_page=FeedResultNextPage(data={self.subfeed_id: method_result.next_page}), - has_next_page=method_result.has_next_page, - ) - return result - - -class FeedConfig(BaseModel): - """ - Модель конфигурации фида. - - Attributes: - version версия конфигурации. - view_session флаг использования механизма расчета всего фида сразу и сохранения в кэш. - session_size размер кэшируемого фида (limit получения данных для сохранения в кэш). - session_live_time срок хранения в кэше для кэшируемого фида (в секундах). - feed мерджер или субфид. - """ - - version: str - feed: FeedTypes - - -# Update Forward Refs -MergerPositional.update_forward_refs() -MergerPercentage.update_forward_refs() -SubFeed.update_forward_refs() -MergerPercentageItem.update_forward_refs() -MergerAppend.update_forward_refs() -MergerAppendDistribute.update_forward_refs() -MergerPercentageGradient.update_forward_refs() -MergerViewSession.update_forward_refs() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..5c79e4e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,27 @@ +from smartfeed.models.base import FeedResult + + +async def make_items(user_id: str, limit: int, next_page: dict, **kwargs) -> FeedResult: + page = next_page.get("page", 1) + start = (page - 1) * limit + data = [{"id": start + i, "title": f"item_{start + i}"} for i in range(limit)] + return FeedResult( + data=data, + next_page={"page": page + 1}, + has_next_page=True, + ) + + +async def make_empty(user_id: str, limit: int, next_page: dict, **kwargs) -> FeedResult: + return FeedResult(data=[], next_page=next_page, has_next_page=False) + + +async def make_error(user_id: str, limit: int, next_page: dict, **kwargs) -> FeedResult: + raise RuntimeError("subfeed error") + + +METHODS = { + "items": make_items, + "empty": make_empty, + "error": make_error, +} diff --git a/tests/fixtures/configs.py b/tests/fixtures/configs.py deleted file mode 100644 index 8c96e4e..0000000 --- a/tests/fixtures/configs.py +++ /dev/null @@ -1,88 +0,0 @@ -from typing import Callable, Dict - -from smartfeed.examples.example_client import ClientMixerClass - -METHODS_DICT: Dict[str, Callable] = { - "ads": ClientMixerClass().example_method, - "followings": ClientMixerClass().example_method, - "empty": ClientMixerClass().empty_method, - "error": ClientMixerClass().error_method, - "doubles": ClientMixerClass().doubles_method, - "posted": ClientMixerClass().keys_method, -} - -PARSING_CONFIG_FIXTURE = { - "version": "1", - "feed": { - "merger_id": "merger_positional_parsing_example", - "type": "merger_positional", - "positions": [1, 3, 15], - "start": 17, - "end": 200, - "step": 2, - "positional": { - "merger_id": "merger_append_parsing_example", - "type": "merger_append", - "items": [ - { - "subfeed_id": "subfeed_merger_append_parsing_example", - "type": "subfeed", - "method_name": "ads", - "subfeed_params": { - "limit_to_return": 10, - }, - }, - { - "merger_id": "merger_percentage_gradient_parsing_example", - "type": "merger_percentage_gradient", - "item_from": { - "percentage": 80, - "data": { - "subfeed_id": "subfeed_from_merger_percentage_gradient_parsing_example", - "type": "subfeed", - "method_name": "followings", - }, - }, - "item_to": { - "percentage": 20, - "data": { - "subfeed_id": "subfeed_to_merger_percentage_gradient_parsing_example", - "type": "subfeed", - "method_name": "followings", - }, - }, - "step": 10, - "size_to_step": 30, - "shuffle": False, - }, - { - "merger_id": "merger_view_session_parsing_example", - "type": "merger_view_session", - "session_size": 800, - "session_live_time": 300, - "data": { - "subfeed_id": "subfeed_merger_view_session_parsing_example", - "type": "subfeed", - "method_name": "followings", - }, - }, - ], - }, - "default": { - "merger_id": "merger_percentage_parsing_example", - "type": "merger_percentage", - "shuffle": False, - "items": [ - { - "percentage": 100, - "data": { - "subfeed_id": "subfeed_merger_percentage_parsing_example", - "type": "subfeed", - "method_name": "followings", - "raise_error": False, - }, - }, - ], - }, - }, -} diff --git a/tests/fixtures/mergers.py b/tests/fixtures/mergers.py deleted file mode 100644 index c6bbf50..0000000 --- a/tests/fixtures/mergers.py +++ /dev/null @@ -1,138 +0,0 @@ -MERGER_APPEND_CONFIG = { - "merger_id": "merger_append_example", - "type": "merger_append", - "items": [ - { - "subfeed_id": "subfeed_merger_append_example", - "type": "subfeed", - "method_name": "ads", - "subfeed_params": { - "limit_to_return": 5, - }, - }, - { - "subfeed_id": "subfeed_2_merger_append_example", - "type": "subfeed", - "method_name": "followings", - }, - ], -} - -MERGER_APPEND_DISTRIBUTE_CONFIG = { - "merger_id": "merger_distribute_example", - "type": "merger_distribute", - "distribution_key": "user_id", - "items": [ - { - "subfeed_id": "subfeed_merger_distribute_example", - "type": "subfeed", - "method_name": "posted", - "subfeed_params": { - "limit_to_return": 25, - }, - }, - { - "subfeed_id": "subfeed_2_merger_distribute_example", - "type": "subfeed", - "method_name": "posted", - "subfeed_params": { - "limit_to_return": 45, - }, - }, - ], -} - -MERGER_PERCENTAGE_CONFIG = { - "merger_id": "merger_percentage_example", - "type": "merger_percentage", - "shuffle": False, - "items": [ - { - "percentage": 40, - "data": { - "subfeed_id": "subfeed_merger_percentage_example", - "type": "subfeed", - "method_name": "followings", - }, - }, - { - "percentage": 60, - "data": { - "subfeed_id": "subfeed_2_merger_percentage_example", - "type": "subfeed", - "method_name": "ads", - }, - }, - ], -} - -MERGER_POSITIONAL_CONFIG = { - "merger_id": "merger_positional_example", - "type": "merger_positional", - "positions": [1, 3, 15], - "start": 17, - "end": 200, - "step": 2, - "positional": { - "subfeed_id": "subfeed_positional_merger_positional_example", - "type": "subfeed", - "method_name": "ads", - "subfeed_params": { - "limit_to_return": 10, - }, - }, - "default": { - "subfeed_id": "subfeed_default_merger_positional_example", - "type": "subfeed", - "method_name": "followings", - }, -} - -MERGER_PERCENTAGE_GRADIENT_CONFIG = { - "merger_id": "merger_percentage_gradient_example", - "type": "merger_percentage_gradient", - "item_from": { - "percentage": 80, - "data": { - "subfeed_id": "subfeed_from_merger_percentage_gradient_example", - "type": "subfeed", - "method_name": "ads", - }, - }, - "item_to": { - "percentage": 20, - "data": { - "subfeed_id": "subfeed_to_merger_percentage_gradient_example", - "type": "subfeed", - "method_name": "followings", - }, - }, - "step": 10, - "size_to_step": 10, - "shuffle": False, -} - -MERGER_VIEW_SESSION_CONFIG = { - "merger_id": "merger_view_session_example", - "type": "merger_view_session", - "session_size": 800, - "session_live_time": 300, - "data": { - "subfeed_id": "subfeed_merger_view_session_example", - "type": "subfeed", - "method_name": "followings", - }, -} - -MERGER_VIEW_SESSION_DUPS_CONFIG = { - "merger_id": "merger_view_session_example", - "type": "merger_view_session", - "deduplicate": True, - "session_size": 10, - "session_live_time": 300, - "data": { - "subfeed_id": "subfeed_merger_view_session_example", - "type": "subfeed", - "method_name": "doubles", - }, -} diff --git a/tests/fixtures/redis.py b/tests/fixtures/redis.py deleted file mode 100644 index b98695e..0000000 --- a/tests/fixtures/redis.py +++ /dev/null @@ -1,10 +0,0 @@ -import pytest -import redis -from redis.asyncio import Redis as AsyncRedis - - -@pytest.fixture(scope="function") -def redis_client(request): - if request.param == "async": - return AsyncRedis(host="localhost", port=6379) - return redis.Redis(host="localhost", port=6379, db=0) diff --git a/tests/fixtures/subfeeds.py b/tests/fixtures/subfeeds.py deleted file mode 100644 index 78ee89c..0000000 --- a/tests/fixtures/subfeeds.py +++ /dev/null @@ -1,27 +0,0 @@ -SUBFEED_CONFIG = { - "subfeed_id": "subfeed_example", - "type": "subfeed", - "method_name": "ads", -} - -SUBFEED_CONFIG_RAISE_ERROR = { - "subfeed_id": "subfeed_example", - "type": "subfeed", - "method_name": "error", -} - -SUBFEED_CONFIG_NO_RAISE_ERROR = { - "subfeed_id": "subfeed_example", - "type": "subfeed", - "method_name": "error", - "raise_error": False, -} - -SUBFEED_WITH_PARAMS_CONFIG = { - "subfeed_id": "subfeed_with_params_example", - "type": "subfeed", - "method_name": "ads", - "subfeed_params": { - "limit_to_return": 10, - }, -} diff --git a/tests/sources.py b/tests/sources.py new file mode 100644 index 0000000..28f954f --- /dev/null +++ b/tests/sources.py @@ -0,0 +1,204 @@ +"""Shared test harness for dedup / pagination correctness. + +The existing test sources are mostly non-stateful (they ignore ``next_page`` and +re-emit the same page forever), which is exactly why the dedup/pagination bugs +were not caught. This module provides: + + * ``ScriptedSource`` -- a *stateful*, offset-paginated source over a FINITE + pool. The pool may contain repeated ids (cross-page duplicates). It records + every id it emits and how many times it was called, so tests can assert + coverage ("no unique item was lost") and fetch counts ("cold build fetched + once"). Finiteness is what lets us prove "no loss": drain to exhaustion and + compare the union of pages against the known unique set. + + * pool factories (unique / duplicate-injected / overlapping). + + * a ``drain`` helper that pages to exhaustion with an infinite-loop guard, plus + invariant assertions (coverage / no-duplicates / page-fullness). + +Cursor convention for ScriptedSource: ``{"pos": }``. +""" + +from __future__ import annotations + +import asyncio +from typing import Any, Dict, List, Literal, Optional + +from smartfeed.models.base import FeedResult +from smartfeed.models.subfeed import SubFeed +from smartfeed.models.wrapper import Wrapper, WrapperCache, WrapperDedup, WrapperRerank +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor + + +# --------------------------------------------------------------------------- +# Source +# --------------------------------------------------------------------------- + + +class ScriptedSource: + """Stateful, offset-paginated source over a finite pool of item dicts.""" + + def __init__(self, pool: List[Dict], latency: float = 0.0, id_key: str = "id"): + self.pool = [dict(it) for it in pool] + self.latency = latency + self.id_key = id_key + self.calls = 0 + self.emitted: List[Any] = [] + + @property + def unique_ids(self) -> set: + return {it[self.id_key] for it in self.pool if self.id_key in it} + + async def __call__(self, user_id, limit, next_page, **kw) -> FeedResult: + self.calls += 1 + pos = (next_page or {}).get("pos", 0) + if self.latency: + await asyncio.sleep(self.latency) + chunk = self.pool[pos : pos + limit] + # Fresh copies so downstream in-place stamping never mutates the pool. + data = [dict(it) for it in chunk] + for it in data: + self.emitted.append(it.get(self.id_key)) + new_pos = pos + len(chunk) + return FeedResult( + data=data, + next_page={"pos": new_pos}, + has_next_page=new_pos < len(self.pool), + ) + + +# --------------------------------------------------------------------------- +# Pool factories (all deterministic -- no RNG, so failures are reproducible) +# --------------------------------------------------------------------------- + + +def unique_pool(n: int, start: int = 0, val_prefix: str = "u") -> List[Dict]: + """Ids start..start+n-1, no duplicates. Isolates *item loss* from dedup.""" + return [{"id": start + k, "val": f"{val_prefix}{start + k}"} for k in range(n)] + + +def dup_pool(n_unique: int, every: int = 4) -> List[Dict]: + """n_unique distinct ids, with an earlier id re-inserted every ``every`` slots. + + The re-inserted copies are *cross-page* duplicates once paginated. The unique + universe is still exactly range(n_unique), so ``assert_full_coverage`` derives + expectations straight from the pool. + """ + pool: List[Dict] = [] + for k in range(n_unique): + pool.append({"id": k, "val": f"v{k}"}) + if k >= 2 and k % every == 0: + dup_id = k - 2 + pool.append({"id": dup_id, "val": f"v{dup_id}"}) # re-emit an earlier id + return pool + + +def adjacent_dup_pool(n_unique: int) -> List[Dict]: + """Each id emitted twice, back-to-back -- both copies land in the same batch, + so within-batch dedup must catch them (no cross-rebuild ambiguity).""" + pool: List[Dict] = [] + for k in range(n_unique): + pool.append({"id": k, "val": f"v{k}"}) + pool.append({"id": k, "val": f"v{k}"}) + return pool + + +# --------------------------------------------------------------------------- +# Context / node builders +# --------------------------------------------------------------------------- + + +def make_ctx(methods: Dict, redis=None, session_id: str = "s1") -> ExecutionContext: + return ExecutionContext(session_id=session_id, methods_dict=methods, redis=redis) + + +def subfeed(subfeed_id: str, method_name: str, **kw) -> SubFeed: + return SubFeed(subfeed_id=subfeed_id, method_name=method_name, **kw) + + +def wrapper( + child, + *, + node_id: str = "w", + session_size: Optional[int] = None, + session_ttl: int = 300, + dedup_key: Optional[str] = None, + missing_key_policy: Literal["error", "keep", "drop"] = "error", + rerank_method: Optional[str] = None, + rerank_raise: bool = True, + cache_key: Optional[str] = None, +) -> Wrapper: + """Build a Wrapper with the requested pipeline stages.""" + cache = None + if session_size is not None: + cache = WrapperCache(session_size=session_size, session_ttl=session_ttl) + dedup = None + if dedup_key is not None: + dedup = WrapperDedup( + dedup_key=dedup_key, + missing_key_policy=missing_key_policy, + ) + rerank = None + if rerank_method is not None: + rerank = WrapperRerank(method_name=rerank_method, raise_error=rerank_raise) + return Wrapper( + node_id=node_id, + cache=cache, + dedup=dedup, + rerank=rerank, + cache_key=cache_key, + data=child, + ) + + +# --------------------------------------------------------------------------- +# Drain + invariant assertions +# --------------------------------------------------------------------------- + + +async def drain(node, ctx, limit: int, max_pages: int = 500, cursor=None) -> List[List]: + """Page until ``has_next_page`` is False. Returns the list of page-data lists. + + Raises AssertionError if pagination does not terminate within ``max_pages`` -- + this is the guard that catches infinite-loop bugs (e.g. shared-cache + continuation re-serving page 1 forever). + """ + cursor = cursor or {} + pages: List[List] = [] + for _ in range(max_pages): + r = await run_executor.run(node, ctx, limit=limit, cursor=cursor) + pages.append(r.data) + if not r.has_next_page: + return pages + cursor = r.next_page + raise AssertionError( + f"pagination did not terminate within {max_pages} pages " f"(infinite loop / non-advancing cursor)" + ) + + +def flat_ids(pages: List[List], key: str = "id") -> List: + return [it[key] for pg in pages for it in pg if isinstance(it, dict) and key in it] + + +def assert_no_duplicates(pages: List[List], key: str = "id") -> None: + ids = flat_ids(pages, key) + seen, dupes = set(), set() + for x in ids: + (dupes if x in seen else seen).add(x) + assert not dupes, f"{len(dupes)} id(s) served more than once: {sorted(dupes)[:20]}" + + +def assert_full_coverage(pages: List[List], expected: set, key: str = "id") -> None: + got = set(flat_ids(pages, key)) + missing = set(expected) - got + extra = got - set(expected) + assert not missing, f"{len(missing)} unique id(s) NEVER served (lost): {sorted(missing)[:20]}" + assert not extra, f"served id(s) outside the source universe: {sorted(extra)[:20]}" + + +def assert_pages_full(pages: List[List], limit: int) -> None: + """Every page except the last must be exactly ``limit`` (dedup must refill, + not short-change, until the source is genuinely exhausted).""" + for i, pg in enumerate(pages[:-1]): + assert len(pg) == limit, f"page {i} returned {len(pg)} items, expected {limit}" diff --git a/tests/test_async_concurrency.py b/tests/test_async_concurrency.py new file mode 100644 index 0000000..46f10c6 --- /dev/null +++ b/tests/test_async_concurrency.py @@ -0,0 +1,278 @@ +"""Test that subfeeds in parallel branches run concurrently via asyncio.gather, +and that the event loop stays responsive under load.""" + +import asyncio +import time +from dataclasses import dataclass, field +from typing import Optional + +import pytest + +from smartfeed.models.base import FeedResult +from smartfeed.models.subfeed import SubFeed +from smartfeed.models.mixers import MergerAppend, MergerPercentage, MergerPercentageItem, MergerPositional +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor + + +# --------------------------------------------------------------------------- +# Helpers: concurrency tracker + loop block monitor +# --------------------------------------------------------------------------- + + +@dataclass +class ConcurrencyTracker: + """Tracks how many leaf calls are in-flight at the same time.""" + + current: int = 0 + peak: int = 0 + _lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + async def enter(self) -> None: + async with self._lock: + self.current += 1 + self.peak = max(self.peak, self.current) + + async def exit(self) -> None: + async with self._lock: + self.current -= 1 + + +class LoopBlockMonitor: + """Detects event-loop blocking by measuring scheduling lag.""" + + def __init__(self, interval: float = 0.01, threshold: float = 0.1): + self.interval = interval + self.threshold = threshold + self.max_lag: float = 0.0 + self._task: Optional[asyncio.Task] = None + self._stop = asyncio.Event() + + async def __aenter__(self): + self._stop.clear() + self._task = asyncio.create_task(self._run()) + return self + + async def __aexit__(self, *exc): + self._stop.set() + if self._task: + await self._task + + async def _run(self): + loop = asyncio.get_running_loop() + expected = loop.time() + self.interval + while not self._stop.is_set(): + await asyncio.sleep(self.interval) + now = loop.time() + lag = max(0.0, now - expected) + self.max_lag = max(self.max_lag, lag) + expected = now + self.interval + + +# --------------------------------------------------------------------------- +# Slow subfeed factory +# --------------------------------------------------------------------------- + + +def make_slow_subfeed_method( + source: str, + latency: float, + tracker: ConcurrencyTracker, + call_counter: dict, +): + """Create an async subfeed method that sleeps (simulating I/O) and tracks concurrency.""" + + async def method(user_id: str, limit: int, next_page: dict, **kw) -> FeedResult: + call_counter[source] = call_counter.get(source, 0) + 1 + page = next_page.get("page", 1) + start = (page - 1) * limit + + await tracker.enter() + try: + await asyncio.sleep(latency) + finally: + await tracker.exit() + + # Each source gets unique id ranges to avoid dedup collisions + offset = hash(source) % 100_000 + data = [{"id": offset + start + i, "val": source} for i in range(limit)] + return FeedResult( + data=data, + next_page={"page": page + 1}, + has_next_page=True, + ) + + return method + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestParallelSubfeeds: + """Verify that sibling subfeeds in a mixer run concurrently.""" + + @pytest.mark.asyncio + async def test_two_subfeeds_run_in_parallel(self): + tracker = ConcurrencyTracker() + calls = {} + latency = 0.05 + + methods = { + "slow_a": make_slow_subfeed_method("a", latency, tracker, calls), + "slow_b": make_slow_subfeed_method("b", latency, tracker, calls), + } + ctx = ExecutionContext(session_id="s1", methods_dict=methods, redis=None) + + node = MergerAppend( + node_id="par", + items=[ + SubFeed(subfeed_id="a", method_name="slow_a"), + SubFeed(subfeed_id="b", method_name="slow_b"), + ], + ) + + t0 = time.monotonic() + result = await run_executor.run(node, ctx, limit=10, cursor={}) + elapsed = time.monotonic() - t0 + + assert len(result.data) == 10 + # Both called + assert calls.get("a", 0) >= 1 + assert calls.get("b", 0) >= 1 + # Peak concurrency > 1 means they ran in parallel + assert tracker.peak > 1, f"Expected parallel execution, peak={tracker.peak}" + # Total time should be ~latency (parallel), not ~2*latency (serial) + assert elapsed < latency * 1.8, f"Took {elapsed:.3f}s, expected < {latency * 1.8:.3f}s" + + @pytest.mark.asyncio + async def test_three_percentage_subfeeds_parallel(self): + tracker = ConcurrencyTracker() + calls = {} + latency = 0.05 + + methods = { + "slow_a": make_slow_subfeed_method("a", latency, tracker, calls), + "slow_b": make_slow_subfeed_method("b", latency, tracker, calls), + "slow_c": make_slow_subfeed_method("c", latency, tracker, calls), + } + ctx = ExecutionContext(session_id="s1", methods_dict=methods, redis=None) + + node = MergerPercentage( + node_id="pct", + items=[ + MergerPercentageItem(percentage=40, data=SubFeed(subfeed_id="a", method_name="slow_a")), + MergerPercentageItem(percentage=30, data=SubFeed(subfeed_id="b", method_name="slow_b")), + MergerPercentageItem(percentage=30, data=SubFeed(subfeed_id="c", method_name="slow_c")), + ], + ) + + t0 = time.monotonic() + result = await run_executor.run(node, ctx, limit=10, cursor={}) + elapsed = time.monotonic() - t0 + + assert len(result.data) == 10 + assert tracker.peak >= 2, f"Expected parallel, peak={tracker.peak}" + assert elapsed < latency * 2.0 + + @pytest.mark.asyncio + async def test_positional_branches_parallel(self): + """Positional's two branches (positional + default) should run in parallel.""" + tracker = ConcurrencyTracker() + calls = {} + latency = 0.05 + + methods = { + "slow_promo": make_slow_subfeed_method("promo", latency, tracker, calls), + "slow_regular": make_slow_subfeed_method("regular", latency, tracker, calls), + } + ctx = ExecutionContext(session_id="s1", methods_dict=methods, redis=None) + + node = MergerPositional( + node_id="pos", + positions=[1, 3, 5], + positional=SubFeed(subfeed_id="promo", method_name="slow_promo"), + default=SubFeed(subfeed_id="regular", method_name="slow_regular"), + ) + + t0 = time.monotonic() + result = await run_executor.run(node, ctx, limit=10, cursor={}) + elapsed = time.monotonic() - t0 + + assert len(result.data) == 10 + assert tracker.peak > 1, f"Expected parallel, peak={tracker.peak}" + assert elapsed < latency * 1.8 + + +class TestEventLoopResponsiveness: + """Verify the event loop stays unblocked during feed execution.""" + + @pytest.mark.asyncio + async def test_loop_not_blocked_under_load(self): + tracker = ConcurrencyTracker() + calls = {} + latency = 0.02 + + methods = {} + subfeeds = [] + for i in range(5): + name = f"source_{i}" + methods[name] = make_slow_subfeed_method(name, latency, tracker, calls) + subfeeds.append(SubFeed(subfeed_id=name, method_name=name)) + + ctx = ExecutionContext(session_id="s1", methods_dict=methods, redis=None) + + node = MergerAppend(node_id="big", items=subfeeds) + + async with LoopBlockMonitor(interval=0.01, threshold=0.05) as monitor: + result = await run_executor.run(node, ctx, limit=20, cursor={}) + + assert len(result.data) == 20 + assert monitor.max_lag < 0.1, f"Event loop blocked: max_lag={monitor.max_lag:.3f}s" + + +class TestNestedParallelism: + """Verify that nested mixer trees still execute leaves in parallel.""" + + @pytest.mark.asyncio + async def test_nested_mixers_parallel(self): + """append(percentage(a, b), positional(c, d)) -- all 4 subfeeds should overlap.""" + tracker = ConcurrencyTracker() + calls = {} + latency = 0.05 + + methods = {f"slow_{x}": make_slow_subfeed_method(x, latency, tracker, calls) for x in ("a", "b", "c", "d")} + ctx = ExecutionContext(session_id="s1", methods_dict=methods, redis=None) + + node = MergerAppend( + node_id="outer", + items=[ + MergerPercentage( + node_id="pct", + items=[ + MergerPercentageItem(percentage=50, data=SubFeed(subfeed_id="a", method_name="slow_a")), + MergerPercentageItem(percentage=50, data=SubFeed(subfeed_id="b", method_name="slow_b")), + ], + ), + MergerPositional( + node_id="pos", + positions=[1, 3], + positional=SubFeed(subfeed_id="c", method_name="slow_c"), + default=SubFeed(subfeed_id="d", method_name="slow_d"), + ), + ], + ) + + t0 = time.monotonic() + result = await run_executor.run(node, ctx, limit=10, cursor={}) + elapsed = time.monotonic() - t0 + + assert len(result.data) == 10 + # All 4 sources should have been called + for x in ("a", "b", "c", "d"): + assert calls.get(x, 0) >= 1 + # At least 2 in-flight at the same time + assert tracker.peak >= 2, f"Expected parallel, peak={tracker.peak}" + # Should take ~1x latency, not ~4x + assert elapsed < latency * 2.5, f"Took {elapsed:.3f}s, suggests serial execution" diff --git a/tests/test_bug_cold_build_stampede.py b/tests/test_bug_cold_build_stampede.py new file mode 100644 index 0000000..8e675d2 --- /dev/null +++ b/tests/test_bug_cold_build_stampede.py @@ -0,0 +1,57 @@ +"""Regression: cold-build thundering herd on the standard (non-shared) cached path. + +Only the cache_key path takes a RedisLock. For a normal cached wrapper, N +concurrent first requests for one session each fetch the full session_size from +the child and each write a different gen; last writer wins, so the losers' gens +no longer match meta and they rebuild again on their next page. Upstream +amplification + cache thrash. + +The source has latency so all N callers enter the cold build before any of them +writes the cache -- a deterministic race. Fails today (N fetches); passes once +the standard path is guarded like the shared path. +""" + +import asyncio + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +N = 5 + + +@pytest.mark.asyncio +async def test_concurrent_cold_build_fetches_child_once(): + src = S.ScriptedSource(S.unique_pool(200), latency=0.05) + node = S.wrapper(S.subfeed("src", "src"), session_size=50) + ctx = S.make_ctx({"src": src}, redis=fakeredis.aioredis.FakeRedis()) + + results = await asyncio.gather(*[run_executor.run(node, ctx, limit=10, cursor={}) for _ in range(N)]) + + assert all(len(r.data) == 10 for r in results) + # One winner should fetch the child; the rest should read the cache it wrote. + assert src.calls == 1, f"expected 1 cold fetch, got {src.calls} (thundering herd)" + + +@pytest.mark.asyncio +async def test_concurrent_cold_build_single_generation(): + src = S.ScriptedSource(S.unique_pool(200), latency=0.05) + node = S.wrapper(S.subfeed("src", "src"), session_size=50) + ctx = S.make_ctx({"src": src}, redis=fakeredis.aioredis.FakeRedis()) + + results = await asyncio.gather(*[run_executor.run(node, ctx, limit=10, cursor={}) for _ in range(N)]) + gens = {r.next_page["w"]["gen"] for r in results} + assert len(gens) == 1, f"concurrent callers got {len(gens)} different generations: {gens}" + + +@pytest.mark.asyncio +async def test_shared_cache_path_already_locks_control(): + """Control: the cache_key path DOES take a lock, so it fetches once.""" + src = S.ScriptedSource(S.unique_pool(200), latency=0.05) + node = S.wrapper(S.subfeed("src", "src"), node_id="w", session_size=50, cache_key="shared") + ctx = S.make_ctx({"src": src}, redis=fakeredis.aioredis.FakeRedis()) + await asyncio.gather(*[run_executor.run(node, ctx, limit=10, cursor={}) for _ in range(N)]) + assert src.calls == 1, f"shared path should fetch once, got {src.calls}" diff --git a/tests/test_bug_lock_loser_gives_up_early.py b/tests/test_bug_lock_loser_gives_up_early.py new file mode 100644 index 0000000..44e7b20 --- /dev/null +++ b/tests/test_bug_lock_loser_gives_up_early.py @@ -0,0 +1,52 @@ +"""Regression: the cold-build lock loser gives up after 5s of +polling while the winner's lock lives 30s. + +`_cold_build_locked` polls 50 x 0.1s and then falls through to an UNLOCKED +`_cold_build`. Any cold build slower than 5s therefore gets a second, unguarded +child fetch racing the real builder: double upstream load and two competing +gens, one of which is orphaned (that client's next page triggers yet another +full rebuild). + +Correct behavior: the loser waits out the winner (poll budget >= lock TTL, or +re-attempt the lock) so the child is fetched exactly once. + +Timing is compressed uniformly (all asyncio.sleep x0.1) so the >5s-nominal +build finishes in <1s of real time; the poll-budget/build-time ratio that +triggers the bug is preserved. +""" + +import asyncio + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +@pytest.mark.asyncio +async def test_slow_cold_build_still_fetches_child_once(monkeypatch): + real_sleep = asyncio.sleep + + async def compressed_sleep(delay: float) -> None: + await real_sleep(delay * 0.1) + + monkeypatch.setattr(asyncio, "sleep", compressed_sleep) + + # Nominal 8s child latency: longer than the old code's 5s poll budget, + # comfortably shorter than the 10s lock TTL the fixed code waits out. + src = S.ScriptedSource(S.unique_pool(100), latency=8.0) + node = S.wrapper(S.subfeed("src", "src"), session_size=50) + ctx = S.make_ctx({"src": src}, redis=fakeredis.aioredis.FakeRedis()) + + a, b = await asyncio.gather( + run_executor.run(node, ctx, limit=10, cursor={}), + run_executor.run(node, ctx, limit=10, cursor={}), + ) + + assert src.calls == 1, ( + f"expected 1 cold fetch (loser waits for the lock holder), got {src.calls}: " + f"the loser gave up polling before the winner finished and rebuilt without the lock" + ) + gens = {a.next_page["w"]["gen"], b.next_page["w"]["gen"]} + assert len(gens) == 1, f"concurrent callers got competing generations: {gens}" diff --git a/tests/test_bug_lock_release_not_atomic.py b/tests/test_bug_lock_release_not_atomic.py new file mode 100644 index 0000000..da105aa --- /dev/null +++ b/tests/test_bug_lock_release_not_atomic.py @@ -0,0 +1,62 @@ +"""Regression: RedisLock release is GET-then-DELETE, not atomic. + +If the lock expires after the release-path GET returned the holder's own token +but before the DELETE executes, and another coroutine acquires the lock in that +window, the DELETE removes the NEW holder's lock: mutual exclusion is silently +lost for a third contender. + +Correct behavior: release must be compare-and-delete in one atomic step (Lua), +so a holder can only ever delete its own token. This test forces the expiry + +re-acquire interleaving deterministically via a hook between GET and DELETE. + +(The related no-TTL-extension concern -- a build outliving the 30s +lock TTL -- is a design decision on the fix and is not encoded here.) +""" + +from typing import Awaitable, Callable, Optional + +import pytest +import fakeredis.aioredis + +from smartfeed.execution.redis_lock import RedisLock + +GetHook = Callable[[str, Optional[bytes]], Awaitable[None]] + + +class HookedRedis: + """Proxy that awaits a hook after each GET returns (to interleave expiry + + re-acquisition between the lock's release GET and DELETE).""" + + def __init__(self, inner): + self._inner = inner + self.on_get: Optional[GetHook] = None + + def __getattr__(self, name): + return getattr(self._inner, name) + + async def get(self, key): + val = await self._inner.get(key) + if self.on_get is not None: + await self.on_get(key, val) + return val + + +@pytest.mark.asyncio +async def test_release_never_deletes_a_successors_lock(): + redis = fakeredis.aioredis.FakeRedis() + hooked = HookedRedis(redis) + + async def expire_and_reacquire(key, val): + hooked.on_get = None # fire once, on A's release GET + await redis.delete("lk") # A's lock TTL expires... + await redis.set("lk", b"token-B") # ...and B acquires in the window + + # HookedRedis duck-types the two Redis calls the lock makes; it is deliberately not a Redis[Any]. + lock_a = RedisLock(hooked, "lk", ttl=30) # pyright: ignore[reportArgumentType] + async with lock_a as acquired: + assert acquired + hooked.on_get = expire_and_reacquire # arm just before release + + assert ( + await redis.get("lk") == b"token-B" + ), "A's release deleted B's lock: GET-then-DELETE must be an atomic compare-and-delete" diff --git a/tests/test_bug_nested_wrapper_priority.py b/tests/test_bug_nested_wrapper_priority.py new file mode 100644 index 0000000..51357ce --- /dev/null +++ b/tests/test_bug_nested_wrapper_priority.py @@ -0,0 +1,65 @@ +"""Regression: _collect_priorities skips a nested Wrapper's own +dedup_priority. + +The walk's generic BaseModel branch does `getattr(child, "data", child)` to unwrap +MergerPercentageItem-style containers -- but a Wrapper is the one BaseNode that +ALSO has `.data`, so a Wrapper sitting directly in a mixer's `items` list (or as +`positional`/`default`) gets unwrapped into its child and its own dedup_priority +override never propagates. + +Documented contract (README "Dedup and dedup_priority"): "a node with a non-zero +dedup_priority overrides its whole subtree" -- for ANY node, wrappers included. + +Both sources emit the same id; the copy from the priority-5 subtree must win the +dedup arbitration even though the priority-0 copy is first-seen. +""" + +import pytest + +from smartfeed.models.base import FeedResult +from smartfeed.models.mixers import MergerAppend, MergerPositional +from smartfeed.models.subfeed import SubFeed +from smartfeed.models.wrapper import Wrapper +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +def _source(who: str): + async def fetch(user_id, limit, next_page, **kw): + return FeedResult(data=[{"id": 1, "who": who}], next_page={}, has_next_page=False) + + return fetch + + +def _ctx(): + return S.make_ctx({"a": _source("a"), "b": _source("b")}) + + +@pytest.mark.asyncio +async def test_wrapper_in_items_list_priority_wins(): + boosted = Wrapper(node_id="boost", dedup_priority=5, data=SubFeed(subfeed_id="b", method_name="b")) + merger = MergerAppend(node_id="mix", items=[SubFeed(subfeed_id="a", method_name="a"), boosted]) + node = S.wrapper(merger, dedup_key="id") + + r = await run_executor.run(node, _ctx(), limit=2, cursor={}) + assert len(r.data) == 1 + assert r.data[0]["who"] == "b", ( + f"the priority-5 wrapper subtree must win dedup, got copy from '{r.data[0]['who']}' " + f"(nested wrapper's dedup_priority was skipped in the priority walk)" + ) + + +@pytest.mark.asyncio +async def test_wrapper_as_positional_child_priority_wins(): + boosted = Wrapper(node_id="boost", dedup_priority=5, data=SubFeed(subfeed_id="b", method_name="b")) + merger = MergerPositional( + node_id="pos", + positions=[2], + positional=boosted, + default=SubFeed(subfeed_id="a", method_name="a"), + ) + node = S.wrapper(merger, dedup_key="id") + + r = await run_executor.run(node, _ctx(), limit=2, cursor={}) + assert len(r.data) == 1 + assert r.data[0]["who"] == "b", f"the priority-5 wrapper subtree must win dedup, got copy from '{r.data[0]['who']}'" diff --git a/tests/test_bug_overfetch_item_loss.py b/tests/test_bug_overfetch_item_loss.py new file mode 100644 index 0000000..2934511 --- /dev/null +++ b/tests/test_bug_overfetch_item_loss.py @@ -0,0 +1,73 @@ +"""Regression: dedup silently drops the overfetched remainder. + +Both dedup paths fetch ``target * overfetch_factor`` items, keep ``target``, and +return the child cursor advanced past *all* fetched items. The surviving-but- +truncated items are consumed from the source and never served on any page. + + * _fetch_and_dedup (cached): wrapper.py:456 (fetch_size) + :479 (data[:target]) + * _passthrough (no cache): wrapper.py:217 (fetch_size) + :234 (data[:limit]) + +These tests drain a finite source and assert full coverage. They fail today +(~75% of items lost at overfetch=4) and must pass once the remainder is +preserved instead of discarded. +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S + + +def _redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.mark.asyncio +async def test_cached_dedup_covers_all_unique_items(): + src = S.ScriptedSource(S.unique_pool(200)) + node = S.wrapper(S.subfeed("src", "src"), session_size=20, dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=200) + S.assert_full_coverage(pages, set(range(200))) + + +@pytest.mark.asyncio +async def test_passthrough_dedup_covers_all_unique_items(): + src = S.ScriptedSource(S.unique_pool(200)) + node = S.wrapper(S.subfeed("src", "src"), dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=200) + S.assert_full_coverage(pages, set(range(200))) + + +@pytest.mark.asyncio +async def test_cached_dedup_covers_all_with_real_duplicates(): + """With a source that genuinely contains duplicates, dedup should remove the + dupes but still surface every *unique* id. Coverage = the unique universe.""" + pool = S.dup_pool(150, every=4) # unique universe is range(150), plus cross-page dupes + src = S.ScriptedSource(pool) + node = S.wrapper(S.subfeed("src", "src"), session_size=20, dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=300) + S.assert_full_coverage(pages, set(range(150))) + + +@pytest.mark.asyncio +async def test_overfetch_1_is_lossless_control(): + """Control: with overfetch_factor=1 there is no discard, so no loss. Proves the + harness is sound and localizes the bug to overfetch>1.""" + src = S.ScriptedSource(S.unique_pool(200)) + node = S.wrapper(S.subfeed("src", "src"), session_size=20, dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=300) + S.assert_full_coverage(pages, set(range(200))) + + +@pytest.mark.asyncio +async def test_no_dedup_is_lossless_control(): + """Control: a plain cached wrapper (no dedup) never overfetches, never loses.""" + src = S.ScriptedSource(S.unique_pool(200)) + node = S.wrapper(S.subfeed("src", "src"), session_size=20) + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=200) + S.assert_full_coverage(pages, set(range(200))) diff --git a/tests/test_bug_passthrough_fresh_scroll.py b/tests/test_bug_passthrough_fresh_scroll.py new file mode 100644 index 0000000..05f5d74 --- /dev/null +++ b/tests/test_bug_passthrough_fresh_scroll.py @@ -0,0 +1,34 @@ +"""Regression: passthrough dedup never resets the seen-set on a +fresh scroll. + +Documented contract (README "Dedup contract", CHANGELOG 1.0.0): "re-sending an +empty cursor (or page 1) starts a FRESH scroll and may repeat items already +shown in a previous scroll". The cached path honors this (`_reset_seen_set` in +`_cold_build`); the passthrough path only ever loads and appends to the +seen-set, so a page refresh (empty cursor) silently filters out everything the +previous scroll showed instead of serving page 1 again. +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +@pytest.mark.asyncio +async def test_empty_cursor_starts_fresh_scroll_on_passthrough(): + src = S.ScriptedSource(S.unique_pool(50)) + node = S.wrapper(S.subfeed("src", "src"), dedup_key="id") # no cache -> passthrough + ctx = S.make_ctx({"src": src}, redis=fakeredis.aioredis.FakeRedis()) + + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + ids_scroll_1 = [it["id"] for it in r1.data] + + # New scroll: empty cursor again (page refresh / F5) + r2 = await run_executor.run(node, ctx, limit=10, cursor={}) + ids_scroll_2 = [it["id"] for it in r2.data] + + assert ids_scroll_2 == ids_scroll_1, ( + f"fresh scroll must restart from page 1, got {ids_scroll_2} " f"(previous scroll's seen-set was never reset)" + ) diff --git a/tests/test_bug_redis_lock_decode.py b/tests/test_bug_redis_lock_decode.py new file mode 100644 index 0000000..541189b --- /dev/null +++ b/tests/test_bug_redis_lock_decode.py @@ -0,0 +1,63 @@ +"""Regression: RedisLock assumes bytes; crashes under decode_responses=True. + +redis_lock.py:35 does ``val.decode()`` in __aexit__. With a Redis client +configured ``decode_responses=True`` (a very common setup), the reply is already +str and release raises AttributeError, breaking the shared-cache cold build. The +rest of the codebase (seen-set load, wrapper.py:269) already handles both +bytes/str, so this is an internal inconsistency. + +Also included: a green characterization test showing the token check itself is +correct (a lock whose value was overwritten by another owner is NOT deleted on +release). The residual issue there is the non-atomic get-then-delete (TOCTOU), +which cannot be reproduced deterministically without concurrency injection and is +documented separately rather than asserted here. +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.execution.redis_lock import RedisLock +from smartfeed.execution import executor as run_executor + + +@pytest.mark.asyncio +async def test_lock_release_with_decode_responses(): + redis = fakeredis.aioredis.FakeRedis(decode_responses=True) + async with RedisLock(redis, "k", ttl=5) as acquired: + assert acquired is True + # Release must have deleted the key without raising. + assert not await redis.exists("k") + + +@pytest.mark.asyncio +async def test_shared_cold_build_with_decode_responses(): + redis = fakeredis.aioredis.FakeRedis(decode_responses=True) + src = S.ScriptedSource(S.unique_pool(100)) + node = S.wrapper(S.subfeed("src", "src"), node_id="w", session_size=20, cache_key="shared") + ctx = S.make_ctx({"src": src}, redis=redis) + r = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(r.data) == 10 + + +@pytest.mark.asyncio +async def test_lock_bytes_mode_is_fine_control(): + redis = fakeredis.aioredis.FakeRedis() # bytes mode (default) + async with RedisLock(redis, "k", ttl=5) as acquired: + assert acquired is True + assert not await redis.exists("k") + + +@pytest.mark.asyncio +async def test_lock_does_not_delete_foreign_token(): + """Characterization: if another owner overwrote the key, release leaves it. + (The token compare-and-delete is correct in the non-racy case.)""" + redis = fakeredis.aioredis.FakeRedis() + lock = RedisLock(redis, "k", ttl=30) + acquired = await lock.__aenter__() + assert acquired is True + # Simulate expiry + re-acquire by a different owner. + await redis.set("k", b"someone-elses-token") + await lock.__aexit__(None, None, None) + # Foreign token must survive -- we must not delete a lock we no longer own. + assert await redis.get("k") == b"someone-elses-token" diff --git a/tests/test_bug_seen_ttl_not_touched.py b/tests/test_bug_seen_ttl_not_touched.py new file mode 100644 index 0000000..ac57d2d --- /dev/null +++ b/tests/test_bug_seen_ttl_not_touched.py @@ -0,0 +1,50 @@ +"""Regression: `_touch_ttl` refreshes cache data+meta but never +the dedup seen-set. + +Every warm page read refreshes the TTL on the data and :meta keys (inactivity +timeout), but the :seen key keeps its original state_ttl countdown. A slow +scroll therefore keeps the cache alive indefinitely while the seen-set silently +expires; the next continuation rebuild then loads an empty seen-set and +re-serves already-shown ids, breaking the documented "each item at most once +per scroll" contract. + +Correct behavior: a warm read that refreshes data/meta must refresh :seen too +(it is the same inactivity signal, and dedup state must not outlive-lag the +cache it guards). +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +@pytest.mark.asyncio +async def test_warm_read_refreshes_seen_set_ttl_alongside_cache(): + redis = fakeredis.aioredis.FakeRedis() + src = S.ScriptedSource(S.unique_pool(30)) + node = S.wrapper(S.subfeed("src", "src"), session_size=10, dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=redis) + + r1 = await run_executor.run(node, ctx, limit=5, cursor={}) # cold build + + # Age every key close to expiry, then serve a warm page. + keys = sorted(k.decode() for k in await redis.keys("*")) + seen_keys = [k for k in keys if k.endswith(":seen")] + assert seen_keys, "dedup wrapper must have written a :seen key" + for k in keys: + await redis.expire(k, 7) + + await run_executor.run(node, ctx, limit=5, cursor=r1.next_page) # warm hit + + # Control: data/meta are refreshed today (documented inactivity-timeout behavior). + base_key = [k for k in keys if not k.endswith((":seen", ":meta"))][0] + assert await redis.ttl(base_key) > 7, "warm read should refresh the cache TTL" + + # The bug: :seen stays on its old countdown while the cache it guards lives on. + seen_ttl = await redis.ttl(seen_keys[0]) + assert seen_ttl > 7, ( + f":seen TTL was not refreshed on the warm read (still {seen_ttl}s) -- " + f"the seen-set can expire mid-scroll while the cache stays alive, causing duplicates" + ) diff --git a/tests/test_bug_shared_cache_continuation.py b/tests/test_bug_shared_cache_continuation.py new file mode 100644 index 0000000..ce05de0 --- /dev/null +++ b/tests/test_bug_shared_cache_continuation.py @@ -0,0 +1,63 @@ +"""Regression: shared cache (cache_key) cannot paginate past session_size. + +When a cache_key wrapper exhausts its per-wrapper cache, the warm path calls +_cold_build with the continuation child_cursor, but _build_shared_base +(wrapper.py:536-539) fast-paths on the already-cached base and returns it, +ignoring the continuation cursor. Every "continuation" re-serves the first +session_size items under a fresh gen -> the feed loops on page 1 forever. + +These tests fail today (non-termination / repeats) and must pass once the shared +base supports continuation. +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +def _redis(): + return fakeredis.aioredis.FakeRedis() + + +def _shared_wrapper(): + return S.wrapper(S.subfeed("src", "src"), node_id="w", session_size=20, cache_key="shared") + + +@pytest.mark.asyncio +async def test_shared_cache_paginates_to_exhaustion(): + src = S.ScriptedSource(S.unique_pool(100)) + node = _shared_wrapper() + ctx = S.make_ctx({"src": src}, redis=_redis()) + # Guard is low: a looping feed re-serves the same 20 items and never ends. + pages = await S.drain(node, ctx, limit=10, max_pages=40) + S.assert_no_duplicates(pages) + S.assert_full_coverage(pages, set(range(100))) + + +@pytest.mark.asyncio +async def test_shared_cache_continuation_advances_past_session(): + """Page 3 (first page after the 20-item session is exhausted) must contain new + ids, not repeat page 1.""" + src = S.ScriptedSource(S.unique_pool(100)) + node = _shared_wrapper() + ctx = S.make_ctx({"src": src}, redis=_redis()) + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + r3 = await run_executor.run(node, ctx, limit=10, cursor=r2.next_page) + ids_seen = {it["id"] for it in r1.data} | {it["id"] for it in r2.data} + ids_p3 = {it["id"] for it in r3.data} + assert not (ids_seen & ids_p3), f"page 3 repeats earlier ids: {sorted(ids_seen & ids_p3)}" + + +@pytest.mark.asyncio +async def test_shared_cache_within_session_is_fine_control(): + """Control: within the first session_size items, shared cache paginates correctly.""" + src = S.ScriptedSource(S.unique_pool(100)) + node = _shared_wrapper() + ctx = S.make_ctx({"src": src}, redis=_redis()) + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + ids = [it["id"] for it in r1.data] + [it["id"] for it in r2.data] + assert ids == list(range(20)) diff --git a/tests/test_bug_shared_cache_dedup.py b/tests/test_bug_shared_cache_dedup.py new file mode 100644 index 0000000..c0cec12 --- /dev/null +++ b/tests/test_bug_shared_cache_dedup.py @@ -0,0 +1,39 @@ +"""Regression: shared cache_key path drops cross-round dedup. + +`_cold_build` (shared branch) calls `_fetch_and_dedup` with seen=None, and +`_dedup` then allocates a FRESH seen set on every refill round (wrapper.py:121). +Round 2's refill re-admits ids already collected in round 1, so duplicates +survive WITHIN one shared segment and are served to the user -- contradicting +README's "the shared base dedups within each segment". + +The identical config without cache_key dedups correctly (control test). +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S + + +POOL = S.dup_pool(20, every=4) # earlier ids re-emitted -> duplicates across refill rounds + + +@pytest.mark.asyncio +async def test_shared_cache_scroll_has_no_duplicates(): + src = S.ScriptedSource(POOL) + node = S.wrapper(S.subfeed("src", "src"), session_size=10, dedup_key="id", cache_key="shared") + ctx = S.make_ctx({"src": src}, redis=fakeredis.aioredis.FakeRedis()) + + pages = await S.drain(node, ctx, limit=5) + S.assert_no_duplicates(pages) + + +@pytest.mark.asyncio +async def test_standard_cache_scroll_has_no_duplicates_control(): + """Control: same config minus cache_key is clean today.""" + src = S.ScriptedSource(POOL) + node = S.wrapper(S.subfeed("src", "src"), session_size=10, dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=fakeredis.aioredis.FakeRedis()) + + pages = await S.drain(node, ctx, limit=5) + S.assert_no_duplicates(pages) diff --git a/tests/test_bug_softfail_cursor_leak.py b/tests/test_bug_softfail_cursor_leak.py new file mode 100644 index 0000000..c80590d --- /dev/null +++ b/tests/test_bug_softfail_cursor_leak.py @@ -0,0 +1,77 @@ +"""Regression: a soft-failed subfeed returns the whole inbound cursor. + +On raise_error=False, SubFeed.execute returns ``next_page=cursor`` -- the entire +feed cursor -- instead of ``{subfeed_id: ...}`` (subfeed.py:39). _merge_cursor +(mixers.py:12-16) does ``merged.update(c)`` per child in child order, so a failed +child that sorts AFTER a healthy sibling dumps all inbound keys and overwrites +the sibling's fresh cursor with the stale inbound value. + +Fails today; passes once the failure path returns a cursor scoped to its own id. +""" + +import pytest + +from tests import sources as S +from smartfeed.models.base import FeedResult +from smartfeed.models.subfeed import SubFeed +from smartfeed.models.mixers import MergerAppend +from smartfeed.execution import executor as run_executor + + +async def good_source(user_id, limit, next_page, **kw): + pos = next_page.get("pos", 0) + data = [{"id": pos + i} for i in range(limit)] + return FeedResult(data=data, next_page={"pos": pos + limit}, has_next_page=True) + + +async def failing_source(user_id, limit, next_page, **kw): + raise RuntimeError("upstream down") + + +METHODS = {"good": good_source, "bad": failing_source} + + +def _node(): + # child order [good, bad] so the failed child's cursor merges LAST. + return MergerAppend( + node_id="top", + items=[ + SubFeed(subfeed_id="good_src", method_name="good"), + SubFeed(subfeed_id="bad_src", method_name="bad", raise_error=False), + ], + ) + + +@pytest.mark.asyncio +async def test_softfail_does_not_clobber_sibling_cursor(): + ctx = S.make_ctx(METHODS, redis=None) + # Inbound cursor carries a STALE position for good_src. + inbound = {"good_src": {"pos": 5}} + r = await run_executor.run(_node(), ctx, limit=10, cursor=inbound) + # good_src is asked for demand 5 (MergerAppend split of limit 10) from pos 5, + # so its fresh cursor is pos 10. The failed sibling must not overwrite it. + assert ( + r.next_page.get("good_src", {}).get("pos") == 10 + ), f"good_src cursor was clobbered by the failed sibling: {r.next_page}" + + +@pytest.mark.asyncio +async def test_softfail_cursor_is_scoped_to_own_id(): + ctx = S.make_ctx(METHODS, redis=None) + inbound = {"good_src": {"pos": 5}, "unrelated": {"x": 1}} + r = await run_executor.run(_node(), ctx, limit=10, cursor=inbound) + # The failed subfeed must not resurrect unrelated inbound cursor keys. + assert "unrelated" not in r.next_page, f"failed subfeed leaked unrelated inbound cursor keys: {r.next_page}" + + +@pytest.mark.asyncio +async def test_softfail_clobber_reserves_items_downstream(): + """The user-facing consequence: because the failed sibling rewinds good_src's + cursor, a subsequent page re-serves items already shown.""" + ctx = S.make_ctx(METHODS, redis=None) + r1 = await run_executor.run(_node(), ctx, limit=10, cursor={}) + r2 = await run_executor.run(_node(), ctx, limit=10, cursor=r1.next_page) + r3 = await run_executor.run(_node(), ctx, limit=10, cursor=r2.next_page) + ids = [it["id"] for pg in (r1.data, r2.data, r3.data) for it in pg] + dupes = sorted({x for x in ids if ids.count(x) > 1}) + assert not dupes, f"items re-served across pages due to cursor clobber: {dupes}" diff --git a/tests/test_bug_stale_waiter.py b/tests/test_bug_stale_waiter.py new file mode 100644 index 0000000..4a102f2 --- /dev/null +++ b/tests/test_bug_stale_waiter.py @@ -0,0 +1,46 @@ +"""Regression: cold-rebuild lock waiter serves the STALE cache batch. + +When a rebuild races (double-fired page request at a cache-exhaustion boundary), +the lock loser polls `_read_meta` and accepts the FIRST meta it sees. The old +data/meta keys are never deleted before a rebuild, so the loser instantly serves +the OLD batch from offset 0 under the OLD gen: the user is re-shown page-1 items +mid-scroll, and the loser's next cursor is stale (the builder writes a new gen), +forcing a full feed restart on the following page. + +Correct behavior: no concurrent response may repeat ids already shown earlier in +this scroll. (Both responses serving the same NEW page is fine -- same cursor, +idempotent retry.) + +The child has latency 0.3s, so the winner is still building when the loser's +0.1s poll finds the stale meta -- the race is deterministic. +""" + +import asyncio + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +@pytest.mark.asyncio +async def test_concurrent_continuation_rebuild_does_not_reserve_shown_items(): + src = S.ScriptedSource(S.unique_pool(100), latency=0.3) + node = S.wrapper(S.subfeed("src", "src"), session_size=20) + ctx = S.make_ctx({"src": src}, redis=fakeredis.aioredis.FakeRedis()) + + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + shown = {it["id"] for it in r1.data} | {it["id"] for it in r2.data} + + # cursor offset == len(batch) -> continuation rebuild; double-fire it + a, b = await asyncio.gather( + run_executor.run(node, ctx, limit=10, cursor=r2.next_page), + run_executor.run(node, ctx, limit=10, cursor=r2.next_page), + ) + ids_a = [it["id"] for it in a.data] + ids_b = [it["id"] for it in b.data] + + assert not (set(ids_a) & shown), f"response A re-served already-shown ids: {sorted(set(ids_a) & shown)}" + assert not (set(ids_b) & shown), f"response B re-served already-shown ids: {sorted(set(ids_b) & shown)}" diff --git a/tests/test_bug_torn_warm_read.py b/tests/test_bug_torn_warm_read.py new file mode 100644 index 0000000..4a2fa3f --- /dev/null +++ b/tests/test_bug_torn_warm_read.py @@ -0,0 +1,86 @@ +"""Regression: torn warm read -- meta and data are two sequential +GETs, so a rebuild landing between them pairs OLD-gen meta with the NEW batch. + +`_execute_with_cache` reads `:meta` (gen matches the cursor -> proceed), then +reads the data key. `_write_cache` replaces both keys atomically in one +pipeline, so a rebuild that lands between the two GETs makes the warm reader +serve the REPLACEMENT batch at the OLD scroll's offset under the OLD gen: +items from a batch the client's gen never belonged to, while page 2 of the +original batch is silently skipped. + +Correct behavior: the served page must be consistent with ONE snapshot -- +either the old batch at the old offset, or a clean rebuild (new gen from +offset 0). This test forces the interleaving deterministically by triggering a +fresh-scroll rebuild from a hook between the two GETs. +""" + +from typing import Awaitable, Callable, Optional + +import pytest +import fakeredis.aioredis + +from smartfeed.models.base import FeedResult +from tests import sources as S +from smartfeed.execution import executor as run_executor + +GetHook = Callable[[str, Optional[bytes]], Awaitable[None]] + + +class HookedRedis: + """Proxy that awaits a hook after each GET returns (to interleave a racing + writer between the wrapper's meta GET and data GET).""" + + def __init__(self, inner): + self._inner = inner + self.on_get: Optional[GetHook] = None + + def __getattr__(self, name): + return getattr(self._inner, name) + + async def get(self, key): + val = await self._inner.get(key) + if self.on_get is not None: + await self.on_get(key, val) + return val + + +class ShiftingSource: + """Returns a disjoint id range on every call, so each rebuild produces a + recognizably different batch.""" + + def __init__(self): + self.calls = 0 + + async def __call__(self, user_id, limit, next_page, **kw) -> FeedResult: + base = self.calls * 100 + self.calls += 1 + data = [{"id": base + i, "val": f"v{base + i}"} for i in range(limit)] + return FeedResult(data=data, next_page={}, has_next_page=False) + + +@pytest.mark.asyncio +async def test_warm_read_is_snapshot_consistent_under_racing_rebuild(): + hooked = HookedRedis(fakeredis.aioredis.FakeRedis()) + node = S.wrapper(S.subfeed("src", "src"), session_size=10) + ctx = S.make_ctx({"src": ShiftingSource()}, redis=hooked) + + r1 = await run_executor.run(node, ctx, limit=5, cursor={}) # batch [0..9], gen g1 + gen_1 = r1.next_page["w"]["gen"] + assert [it["id"] for it in r1.data] == [0, 1, 2, 3, 4] + + async def rebuild_between_reads(key, val): + if not key.endswith(":meta") or val is None: + return + hooked.on_get = None # fire once + # A fresh scroll (empty cursor) rebuilds the batch -> [100..109], new gen. + await run_executor.run(node, ctx, limit=5, cursor={}) + + hooked.on_get = rebuild_between_reads + r2 = await run_executor.run(node, ctx, limit=5, cursor=r1.next_page) + + ids_2 = [it["id"] for it in r2.data] + gen_2 = r2.next_page["w"]["gen"] + assert ids_2 == [5, 6, 7, 8, 9] or gen_2 != gen_1, ( + f"torn read: served {ids_2} from the replacement batch under the old gen -- " + f"meta and data must be read as one snapshot (single MGET/pipeline)" + ) diff --git a/tests/test_bug_variable_page_size.py b/tests/test_bug_variable_page_size.py new file mode 100644 index 0000000..a1652f2 --- /dev/null +++ b/tests/test_bug_variable_page_size.py @@ -0,0 +1,61 @@ +"""Regression: changing `limit` between pages on a cached wrapper skips/repeats. + +The cursor stores a page *number*; the offset is recomputed as (page-1)*limit at +read time (_paginate, wrapper.py:343). If the client changes page size between +requests, the page-number -> offset mapping is inconsistent and items are +skipped (growing limit) or repeated (shrinking limit). + +Fails today; passes once the cursor tracks an absolute offset (or page size is +pinned into the cursor). +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +def _redis(): + return fakeredis.aioredis.FakeRedis() + + +def _cached(): + # session_size large enough that both pages come from one cached batch. + return S.wrapper(S.subfeed("src", "src"), session_size=200) + + +@pytest.mark.asyncio +async def test_growing_limit_does_not_skip(): + src = S.ScriptedSource(S.unique_pool(200)) + node = _cached() + ctx = S.make_ctx({"src": src}, redis=_redis()) + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) # expect ids 0..9 + r2 = await run_executor.run(node, ctx, limit=50, cursor=r1.next_page) # expect 10..59 + served = [it["id"] for it in r1.data] + [it["id"] for it in r2.data] + assert served == list(range(60)), f"expected contiguous 0..59, got gaps: {served[:15]}..." + + +@pytest.mark.asyncio +async def test_shrinking_limit_does_not_repeat(): + src = S.ScriptedSource(S.unique_pool(200)) + node = _cached() + ctx = S.make_ctx({"src": src}, redis=_redis()) + r1 = await run_executor.run(node, ctx, limit=50, cursor={}) # ids 0..49 + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) # expect 50..59 + served = [it["id"] for it in r1.data] + [it["id"] for it in r2.data] + assert len(served) == len( + set(served) + ), f"repeated ids on shrink: {sorted({x for x in served if served.count(x) > 1})}" + + +@pytest.mark.asyncio +async def test_constant_limit_is_fine_control(): + """Control: with a constant limit the page cursor works correctly.""" + src = S.ScriptedSource(S.unique_pool(200)) + node = _cached() + ctx = S.make_ctx({"src": src}, redis=_redis()) + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + served = [it["id"] for it in r1.data] + [it["id"] for it in r2.data] + assert served == list(range(20)) diff --git a/tests/test_config_parsing.py b/tests/test_config_parsing.py new file mode 100644 index 0000000..8cdf5de --- /dev/null +++ b/tests/test_config_parsing.py @@ -0,0 +1,73 @@ +from smartfeed.models.base import BaseNode +from smartfeed.models import FeedConfig, SubFeed, Wrapper + + +class DummyNode(BaseNode): + type: str = "dummy" + node_id: str = "test" + params: dict = {} + + +def test_config_hash_deterministic(): + a = DummyNode(params={"b": 2, "a": 1}) + b = DummyNode(params={"a": 1, "b": 2}) + assert a.config_hash() == b.config_hash() + assert len(a.config_hash()) == 8 + + +def test_config_hash_changes_with_params(): + a = DummyNode(params={"x": 1}) + b = DummyNode(params={"x": 2}) + assert a.config_hash() != b.config_hash() + + +def test_parse_simple_subfeed(): + cfg = FeedConfig.model_validate( + { + "version": "2", + "feed": {"type": "subfeed", "subfeed_id": "x", "method_name": "items"}, + } + ) + assert isinstance(cfg.feed, SubFeed) + assert cfg.feed.subfeed_id == "x" + + +def test_parse_wrapper_with_cache(): + cfg = FeedConfig.model_validate( + { + "version": "2", + "feed": { + "type": "wrapper", + "node_id": "w", + "cache": {"session_size": 100, "session_ttl": 300}, + "data": {"type": "subfeed", "subfeed_id": "x", "method_name": "items"}, + }, + } + ) + assert isinstance(cfg.feed, Wrapper) + assert cfg.feed.cache is not None + assert cfg.feed.cache.session_size == 100 + + +def test_parse_nested_config(): + cfg = FeedConfig.model_validate( + { + "version": "2", + "feed": { + "type": "wrapper", + "node_id": "outer", + "dedup": {"dedup_key": "id"}, + "data": { + "type": "merger_percentage", + "node_id": "mix", + "items": [ + {"percentage": 50, "data": {"type": "subfeed", "subfeed_id": "a", "method_name": "items"}}, + {"percentage": 50, "data": {"type": "subfeed", "subfeed_id": "b", "method_name": "items"}}, + ], + }, + }, + } + ) + assert isinstance(cfg.feed, Wrapper) + assert cfg.feed.dedup is not None + assert cfg.feed.dedup.dedup_key == "id" diff --git a/tests/test_config_validation.py b/tests/test_config_validation.py new file mode 100644 index 0000000..3f7e8dc --- /dev/null +++ b/tests/test_config_validation.py @@ -0,0 +1,138 @@ +"""Config-construction validation: duplicate +subfeed_id/node_id anywhere in the tree, and non-positive gradient step / +size_to_step, are rejected at parse time instead of corrupting state or +crashing at runtime. +""" + +import pytest +from pydantic import ValidationError + +from smartfeed.models import ( + FeedConfig, + MergerAppendDistribute, + MergerPercentage, + MergerPercentageGradient, + MergerPositional, +) + + +def _cfg(feed: dict) -> dict: + return {"version": "2", "feed": feed} + + +def _sub(sid: str) -> dict: + return {"type": "subfeed", "subfeed_id": sid, "method_name": "m"} + + +def test_duplicate_subfeed_ids_rejected(): + feed = {"type": "merger_append", "node_id": "mix", "items": [_sub("a"), _sub("a")]} + with pytest.raises(ValidationError, match="duplicate id 'a'"): + FeedConfig.model_validate(_cfg(feed)) + + +def test_subfeed_id_colliding_with_node_id_rejected(): + """subfeed_id and node_id share one cursor namespace -- a collision across + kinds is just as corrupting as within one kind.""" + feed = {"type": "wrapper", "node_id": "x", "data": _sub("x")} + with pytest.raises(ValidationError, match="duplicate id 'x'"): + FeedConfig.model_validate(_cfg(feed)) + + +def test_duplicate_across_nested_branches_rejected(): + feed = { + "type": "merger_percentage", + "node_id": "mix", + "items": [ + {"percentage": 50, "data": {"type": "wrapper", "node_id": "w1", "data": _sub("a")}}, + {"percentage": 50, "data": {"type": "wrapper", "node_id": "w2", "data": _sub("a")}}, + ], + } + with pytest.raises(ValidationError, match="duplicate id 'a'"): + FeedConfig.model_validate(_cfg(feed)) + + +def test_duplicate_wrapper_node_ids_rejected(): + feed = { + "type": "merger_append", + "node_id": "mix", + "items": [ + {"type": "wrapper", "node_id": "w", "data": _sub("a")}, + {"type": "wrapper", "node_id": "w", "data": _sub("b")}, + ], + } + with pytest.raises(ValidationError, match="duplicate id 'w'"): + FeedConfig.model_validate(_cfg(feed)) + + +def test_unique_ids_parse_ok(): + feed = { + "type": "merger_percentage", + "node_id": "mix", + "items": [ + {"percentage": 50, "data": {"type": "wrapper", "node_id": "w1", "data": _sub("a")}}, + {"percentage": 50, "data": {"type": "wrapper", "node_id": "w2", "data": _sub("b")}}, + ], + } + cfg = FeedConfig.model_validate(_cfg(feed)) + assert isinstance(cfg.feed, MergerPercentage) + assert cfg.feed.node_id == "mix" + + +def _gradient_feed(step: int, size_to_step: int) -> dict: + return { + "type": "merger_percentage_gradient", + "node_id": "g", + "item_from": {"percentage": 80, "data": _sub("a")}, + "item_to": {"percentage": 20, "data": _sub("b")}, + "step": step, + "size_to_step": size_to_step, + } + + +def test_gradient_zero_size_to_step_rejected_at_parse(): + with pytest.raises(ValidationError, match="size_to_step"): + FeedConfig.model_validate(_cfg(_gradient_feed(step=10, size_to_step=0))) + + +def test_gradient_non_positive_step_rejected_at_parse(): + for bad_step in (0, -5): + with pytest.raises(ValidationError, match="step"): + FeedConfig.model_validate(_cfg(_gradient_feed(step=bad_step, size_to_step=10))) + + +def test_gradient_positive_params_parse_ok(): + cfg = FeedConfig.model_validate(_cfg(_gradient_feed(step=10, size_to_step=30))) + assert isinstance(cfg.feed, MergerPercentageGradient) + assert cfg.feed.step == 10 + + +def test_positional_round_trips_via_public_config(): + feed = { + "type": "merger_positional", + "node_id": "pos", + "positions": [1, 3], + "positional": _sub("p"), + "default": _sub("d"), + } + cfg = FeedConfig.model_validate(_cfg(feed)) + assert isinstance(cfg.feed, MergerPositional) + assert cfg.feed.positions == [1, 3] + + +def test_distribute_round_trips_via_public_config(): + feed = { + "type": "merger_distribute", + "node_id": "dist", + "distribution_key": "op", + "sorting_key": "score", + "sorting_desc": True, + "items": [_sub("x"), _sub("y")], + } + cfg = FeedConfig.model_validate(_cfg(feed)) + assert isinstance(cfg.feed, MergerAppendDistribute) + assert cfg.feed.distribution_key == "op" + + +def test_missing_required_field_raises(): + with pytest.raises(ValidationError): + FeedConfig.model_validate(_cfg({"type": "subfeed", "subfeed_id": "a"})) # method_name missing diff --git a/tests/test_continuation.py b/tests/test_continuation.py new file mode 100644 index 0000000..ba626fe --- /dev/null +++ b/tests/test_continuation.py @@ -0,0 +1,180 @@ +"""Tests for Wrapper cache exhaustion and continuation cursor rebuild. + +Scenario: + Wrapper(cache, session_size=20) + limit=10. + - Page 1: cold build -> fetches 20 items, caches them, returns items 0-9 + - Page 2: warm hit -> returns items 10-19 + - Page 3: cache exhausted (start=20 >= 20 cached) -> cold rebuild from + child_cursor stored in meta; new generation spawned. + +The child source tracks its own page via next_page so the rebuild continues +from item 20 onwards with no gap and no overlap. +""" + +import pytest +import fakeredis.aioredis + +from smartfeed.models.base import FeedResult +from smartfeed.models.subfeed import SubFeed +from smartfeed.models.wrapper import Wrapper, WrapperCache +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor + + +# --------------------------------------------------------------------------- +# Infinite stateful source: returns items based on page cursor +# --------------------------------------------------------------------------- + + +async def infinite_source(user_id, limit, next_page, **kw): + """Returns `limit` items starting from (page-1)*limit. IDs are globally unique.""" + page = next_page.get("page", 1) + start = (page - 1) * limit + data = [{"id": start + i, "val": f"item_{start + i}"} for i in range(limit)] + return FeedResult( + data=data, + next_page={"page": page + 1}, + has_next_page=True, + ) + + +METHODS = {"source": infinite_source} + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.fixture +def ctx(redis): + return ExecutionContext(session_id="test_session", methods_dict=METHODS, redis=redis) + + +def _make_wrapper(session_size=20): + return Wrapper( + node_id="w", + cache=WrapperCache(session_size=session_size, session_ttl=300), + data=SubFeed(subfeed_id="src", method_name="source"), + ) + + +# --------------------------------------------------------------------------- +# Helper: collect three pages +# --------------------------------------------------------------------------- + + +async def _three_pages(ctx, session_size=20, limit=10): + node = _make_wrapper(session_size=session_size) + r1 = await run_executor.run(node, ctx, limit=limit, cursor={}) + r2 = await run_executor.run(node, ctx, limit=limit, cursor=r1.next_page) + r3 = await run_executor.run(node, ctx, limit=limit, cursor=r2.next_page) + return r1, r2, r3 + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_pages_1_and_2_served_from_cache(ctx): + """Pages 1 and 2 are served from the same generation (warm hits).""" + node = _make_wrapper(session_size=20) + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + gen1 = r1.next_page["w"]["gen"] + + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + gen2 = r2.next_page["w"]["gen"] + + # Both pages share the same generation id + assert gen1 == gen2 + + +@pytest.mark.asyncio +async def test_page3_triggers_rebuild_with_new_generation(ctx): + """Page 3 exhausts the 20-item cache, triggering a cold rebuild with a new gen.""" + r1, r2, r3 = await _three_pages(ctx) + + gen_before = r2.next_page["w"]["gen"] + gen_after = r3.next_page["w"]["gen"] + + assert gen_before != gen_after, "Page 3 rebuild must produce a new generation id" + + +@pytest.mark.asyncio +async def test_page3_items_continue_from_page2_no_gap(ctx): + """Items on page 3 immediately follow items on page 2 (no gap, no overlap).""" + r1, r2, r3 = await _three_pages(ctx) + + ids_p2 = [item["id"] for item in r2.data] + ids_p3 = [item["id"] for item in r3.data] + + last_p2 = max(ids_p2) + first_p3 = min(ids_p3) + + # No gap: the first item of page 3 immediately follows the last item of page 2 + assert first_p3 == last_p2 + 1, f"Expected page 3 to start at {last_p2 + 1}, got {first_p3}" + + +@pytest.mark.asyncio +async def test_page3_items_no_overlap_with_page1_and_page2(ctx): + """Items on page 3 must not repeat any item from pages 1 or 2.""" + r1, r2, r3 = await _three_pages(ctx) + + seen_ids = {item["id"] for item in r1.data} | {item["id"] for item in r2.data} + p3_ids = {item["id"] for item in r3.data} + + overlap = seen_ids & p3_ids + assert not overlap, f"Page 3 overlaps with earlier pages: {overlap}" + + +@pytest.mark.asyncio +async def test_page3_has_correct_item_count(ctx): + """Page 3 (post-rebuild) still returns exactly `limit` items.""" + _, _, r3 = await _three_pages(ctx) + assert len(r3.data) == 10 + + +@pytest.mark.asyncio +async def test_all_ids_across_three_pages_are_unique(ctx): + """All item IDs across pages 1, 2 and 3 are distinct.""" + r1, r2, r3 = await _three_pages(ctx) + + all_ids = [item["id"] for item in r1.data] + [item["id"] for item in r2.data] + [item["id"] for item in r3.data] + assert len(all_ids) == len(set(all_ids)), "Duplicate IDs found across pages" + + +@pytest.mark.asyncio +async def test_all_ids_are_globally_sequential(ctx): + """Items across pages 1-3 form a gapless sequence starting from 0.""" + r1, r2, r3 = await _three_pages(ctx) + + all_ids = sorted(item["id"] for page in (r1.data, r2.data, r3.data) for item in page) + expected = list(range(30)) + assert all_ids == expected, f"Expected {expected}, got {all_ids}" + + +@pytest.mark.asyncio +async def test_continuation_works_for_smaller_session_size(ctx): + """Continuation rebuild works correctly with session_size=6 and limit=3.""" + node = _make_wrapper(session_size=6) + + # Pages 1 and 2 from cache + r1 = await run_executor.run(node, ctx, limit=3, cursor={}) + r2 = await run_executor.run(node, ctx, limit=3, cursor=r1.next_page) + # Page 3 triggers rebuild + r3 = await run_executor.run(node, ctx, limit=3, cursor=r2.next_page) + + # New generation + assert r3.next_page["w"]["gen"] != r2.next_page["w"]["gen"] + + # No overlap + seen = {item["id"] for item in r1.data} | {item["id"] for item in r2.data} + new = {item["id"] for item in r3.data} + assert not (seen & new) + + # No gap + last_p2 = max(item["id"] for item in r2.data) + first_p3 = min(item["id"] for item in r3.data) + assert first_p3 == last_p2 + 1 diff --git a/tests/test_cursors.py b/tests/test_cursors.py new file mode 100644 index 0000000..031ec76 --- /dev/null +++ b/tests/test_cursors.py @@ -0,0 +1,58 @@ +import pytest +import fakeredis.aioredis + +from smartfeed.models import FeedConfig +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor +from tests.conftest import METHODS + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.fixture +def ctx(redis): + return ExecutionContext(session_id="s1", methods_dict=METHODS, redis=redis) + + +@pytest.mark.asyncio +async def test_cached_wrapper_hides_child_cursors(ctx): + config = FeedConfig.model_validate( + { + "version": "2", + "feed": { + "type": "wrapper", + "node_id": "cached", + "cache": {"session_size": 50, "session_ttl": 300}, + "data": {"type": "subfeed", "subfeed_id": "items", "method_name": "items"}, + }, + } + ) + + result = await run_executor.run(config.feed, ctx, limit=10, cursor={}) + + # Only wrapper cursor, NOT subfeed cursor + assert "cached" in result.next_page + assert "items" not in result.next_page + assert "gen" in result.next_page["cached"] + + +@pytest.mark.asyncio +async def test_uncached_wrapper_exposes_child_cursors(ctx): + config = FeedConfig.model_validate( + { + "version": "2", + "feed": { + "type": "wrapper", + "node_id": "passthrough", + "data": {"type": "subfeed", "subfeed_id": "items", "method_name": "items"}, + }, + } + ) + + result = await run_executor.run(config.feed, ctx, limit=10, cursor={}) + + # Child cursor is visible (no cache to hide it) + assert "items" in result.next_page diff --git a/tests/test_dedup_correctness.py b/tests/test_dedup_correctness.py new file mode 100644 index 0000000..a72a141 --- /dev/null +++ b/tests/test_dedup_correctness.py @@ -0,0 +1,300 @@ +"""Dedup correctness with sources that genuinely contain duplicates. + +Zero-duplicate sources can prove *item loss* but not *dedup correctness*. These +tests inject real duplicates (overlapping sources, cross-page repeats, within- +batch repeats) and assert dedup removes exactly the dupes while surfacing every +unique id. + +These exercise dedup across overlapping sources, cross-page/cross-rebuild repeats, +priority arbitration, and every merger type, at both overfetch=1 and overfetch=4. +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.models.subfeed import SubFeed +from smartfeed.models.mixers import ( + MergerAppend, + MergerAppendDistribute, + MergerPercentage, + MergerPercentageItem, + MergerPositional, +) +from smartfeed.execution import executor as run_executor + + +def _redis(): + return fakeredis.aioredis.FakeRedis() + + +# --------------------------------------------------------------------------- +# Overlapping sources: every unique id exactly once +# --------------------------------------------------------------------------- + + +def _overlap_ctx(): + a = S.ScriptedSource(S.unique_pool(100, start=0)) # ids 0..99 + b = S.ScriptedSource(S.unique_pool(100, start=50)) # ids 50..149 (overlap 50..99) + return a, b, S.make_ctx({"a": a, "b": b}, redis=_redis()) + + +@pytest.mark.asyncio +async def test_overlapping_sources_cached_each_id_once(): + """session_size > universe -> one batch -> within-batch dedup handles overlap.""" + a, b, ctx = _overlap_ctx() + merger = MergerAppend( + node_id="m", + items=[ + SubFeed(subfeed_id="a", method_name="a"), + SubFeed(subfeed_id="b", method_name="b"), + ], + ) + node = S.wrapper(merger, session_size=500, dedup_key="id") + pages = await S.drain(node, ctx, limit=10, max_pages=100) + S.assert_no_duplicates(pages) + S.assert_full_coverage(pages, set(range(150))) + + +@pytest.mark.asyncio +async def test_overlapping_sources_passthrough_each_id_once(): + """Passthrough cross-page dedup via the Redis seen-set (overfetch=1 -> no loss).""" + a, b, ctx = _overlap_ctx() + merger = MergerAppend( + node_id="m", + items=[ + SubFeed(subfeed_id="a", method_name="a"), + SubFeed(subfeed_id="b", method_name="b"), + ], + ) + node = S.wrapper(merger, dedup_key="id") + pages = await S.drain(node, ctx, limit=10, max_pages=200) + S.assert_no_duplicates(pages) + S.assert_full_coverage(pages, set(range(150))) + + +# --------------------------------------------------------------------------- +# Within-batch high-density duplicates collapse +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_adjacent_duplicates_collapse_within_batch(): + src = S.ScriptedSource(S.adjacent_dup_pool(80)) # each id emitted twice, back-to-back + node = S.wrapper(S.subfeed("src", "src"), session_size=500, dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=100) + S.assert_no_duplicates(pages) + S.assert_full_coverage(pages, set(range(80))) + + +# --------------------------------------------------------------------------- +# Cross-page duplicate suppression (passthrough seen-set) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_cross_page_duplicate_suppressed_passthrough(): + # id 3 appears at pos 3 and again at pos 25 (a later page). + pool = S.unique_pool(30) + pool.insert(25, {"id": 3, "val": "v3-again"}) + src = S.ScriptedSource(pool) + node = S.wrapper(S.subfeed("src", "src"), dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=5, max_pages=50) + ids = S.flat_ids(pages) + assert ids.count(3) == 1, f"cross-page duplicate not suppressed: id 3 served {ids.count(3)}x" + + +# --------------------------------------------------------------------------- +# Cross-REBUILD dedup: within one scroll, a shown id must not reappear after a +# rebuild boundary (session-scoped seen-set carried across cold rebuilds). +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_cached_dedup_suppresses_across_rebuild_boundary(): + # id 3 at pos 3 (batch 1) and pos 20 (batch 2, after session_size=20 boundary). + pool = S.unique_pool(20) + pool.append({"id": 3, "val": "v3-again"}) # pos 20 + src = S.ScriptedSource(pool) + node = S.wrapper(S.subfeed("src", "src"), session_size=20, dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=50) + assert S.flat_ids(pages).count(3) == 1 + + +# --------------------------------------------------------------------------- +# Priority arbitration +# --------------------------------------------------------------------------- + + +def _prio_wrapper(): + merger = MergerAppend( + node_id="m", + items=[ + SubFeed(subfeed_id="lo", method_name="x", dedup_priority=1), + SubFeed(subfeed_id="hi", method_name="x", dedup_priority=5), + ], + ) + return S.wrapper(merger, dedup_key="id") + + +def _stamped(id_, source): + return {"id": id_, "_smartfeed_debug_info": {"source": source}} + + +def _winner_source(item): + return (item.get("_smartfeed_debug_info") or {}).get("source") + + +# Regression guard: in-batch priority arbitration must run even when the +# higher-priority copy is NOT the first-seen one. The seen-check skips only true +# cross-page dupes (`key in seen and key not in batch`), so a within-batch duplicate +# still reaches priority arbitration. +@pytest.mark.asyncio +async def test_priority_higher_source_wins_when_listed_second(): + w = _prio_wrapper() + # low-prio at index 0, high-prio dup at index 1: higher priority must win. + out = w._dedup([_stamped(7, "lo"), _stamped(7, "hi")]) + assert len(out) == 1 + assert _winner_source(out[0]) == "hi", f"lower-priority first-seen copy won: {out}" + + +@pytest.mark.asyncio +async def test_priority_higher_source_wins_integration_merger_order(): + """Integration: MergerAppend lists the low-priority source first, so its copy is + first-seen; the high-priority source (listed second) must still win the key.""" + lo = S.ScriptedSource([{"id": 0, "val": "lo"}]) + hi = S.ScriptedSource([{"id": 0, "val": "hi"}]) + merger = MergerAppend( + node_id="m", + items=[ + SubFeed(subfeed_id="lo", method_name="lo", dedup_priority=1), + SubFeed(subfeed_id="hi", method_name="hi", dedup_priority=5), + ], + ) + node = S.wrapper(merger, session_size=100, dedup_key="id") + ctx = S.make_ctx({"lo": lo, "hi": hi}, redis=_redis()) + r = await run_executor.run(node, ctx, limit=10, cursor={}) + kept = [it for it in r.data if it["id"] == 0] + assert len(kept) == 1 + assert kept[0]["val"] == "hi", f"higher-priority source lost to first-seen: {kept}" + + +@pytest.mark.asyncio +async def test_priority_preserved_when_high_arrives_in_refill(): + """Simulates the refill recombination: _dedup(data + refill) where the high- + priority copy only shows up in the refill batch. It must still win, and the + stale copy must be removed (assert the list directly so a fix that keeps the + dupe cannot pass).""" + w = _prio_wrapper() + data = [_stamped(1, "lo"), _stamped(7, "lo")] # first fetch + refill = [_stamped(7, "hi"), _stamped(2, "hi")] # refill batch + out = w._dedup(data + refill) + assert [it["id"] for it in out] == [1, 7, 2], f"expected one copy each in order: {out}" + assert _winner_source(out[1]) == "hi", f"high-priority refill copy lost: {out}" + + +@pytest.mark.asyncio +async def test_priority_defaults_zero_without_source_stamp(): + w = _prio_wrapper() + out = w._dedup([{"id": 8}, _stamped(8, "hi")]) # unstamped (prio 0) then hi (prio 5) + assert len(out) == 1 + assert _winner_source(out[0]) == "hi", f"unstamped prio-0 copy beat higher priority: {out}" + + +@pytest.mark.asyncio +async def test_priority_cross_page_shown_item_wins_documented(): + """Characterization: cross-page, an already-shown (lower-priority) item cannot be + retracted when a higher-priority duplicate appears on a later page.""" + # page 1 shows id 5 from the low-priority source; page 2 has id 5 from high-prio. + lo = S.ScriptedSource([{"id": 5, "val": "lo"}, {"id": 6, "val": "lo"}]) + hi = S.ScriptedSource([{"id": 5, "val": "hi"}, {"id": 7, "val": "hi"}]) + merger = MergerAppend( + node_id="m", + items=[ + SubFeed(subfeed_id="lo", method_name="lo", dedup_priority=1), + SubFeed(subfeed_id="hi", method_name="hi", dedup_priority=5), + ], + ) + node = S.wrapper(merger, dedup_key="id") + ctx = S.make_ctx({"lo": lo, "hi": hi}, redis=_redis()) + r1 = await run_executor.run(node, ctx, limit=1, cursor={}) + # Whatever is shown first for id 5 is final; it is never served twice. + r2 = await run_executor.run(node, ctx, limit=3, cursor=r1.next_page) + ids = [it["id"] for it in r1.data] + [it["id"] for it in r2.data] + assert ids.count(5) == 1 + + +# --------------------------------------------------------------------------- +# Dedup coverage holds through every merger type (regression guard). +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_dedup_over_distribute_covers_all(): + pool = [{"id": k, "grp": k % 5} for k in range(120)] + src = S.ScriptedSource(pool) + merger = MergerAppendDistribute( + node_id="m", items=[SubFeed(subfeed_id="a", method_name="a")], distribution_key="grp" + ) + node = S.wrapper(merger, session_size=20, dedup_key="id") + ctx = S.make_ctx({"a": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=300) + S.assert_full_coverage(pages, set(range(120))) + S.assert_no_duplicates(pages) + + +@pytest.mark.asyncio +async def test_dedup_over_percentage_covers_all(): + src = S.ScriptedSource(S.unique_pool(120)) + merger = MergerPercentage( + node_id="m", + items=[ + MergerPercentageItem(percentage=100, data=SubFeed(subfeed_id="a", method_name="a")), + ], + ) + node = S.wrapper(merger, session_size=20, dedup_key="id") + ctx = S.make_ctx({"a": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=300) + S.assert_full_coverage(pages, set(range(120))) + S.assert_no_duplicates(pages) + + +@pytest.mark.asyncio +async def test_dedup_over_positional_covers_all(): + a = S.ScriptedSource(S.unique_pool(120, start=0)) # ids 0..119 + b = S.ScriptedSource(S.unique_pool(60, start=1000)) # ids 1000..1059 + merger = MergerPositional( + node_id="m", + positions=[1, 3, 5], + positional=SubFeed(subfeed_id="a", method_name="a"), + default=SubFeed(subfeed_id="b", method_name="b"), + ) + node = S.wrapper(merger, session_size=20, dedup_key="id") + ctx = S.make_ctx({"a": a, "b": b}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=300) + S.assert_full_coverage(pages, set(range(120)) | set(range(1000, 1060))) + S.assert_no_duplicates(pages) + + +# --------------------------------------------------------------------------- +# Regression guard: dense duplicates must not produce short non-final +# pages -- the refill loop scans through duplicate runs until the page is full. +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_dense_duplicates_still_fill_pages(): + # Each unique id 1..79 is followed by three copies of a hot id (0): only ~1 in 4 + # fetched items is new, so a page needs several child fetches to fill. + pool = [] + for k in range(1, 80): + pool.append({"id": k}) + pool += [{"id": 0}] * 3 + src = S.ScriptedSource(pool) + node = S.wrapper(S.subfeed("src", "src"), dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=10, max_pages=300) + S.assert_pages_full(pages, limit=10) diff --git a/tests/test_dedup_edge_cases.py b/tests/test_dedup_edge_cases.py new file mode 100644 index 0000000..e8ab4f3 --- /dev/null +++ b/tests/test_dedup_edge_cases.py @@ -0,0 +1,106 @@ +"""Dedup edge cases: missing-key policies, non-dict items, key stringification. + +Contract/characterization tests. Where behavior is arguably wrong (int-vs-str key +collision) the test documents CURRENT behavior, so a +future change is a conscious decision rather than a silent regression. +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.models.base import FeedResult + + +def _redis(): + return fakeredis.aioredis.FakeRedis() + + +# --------------------------------------------------------------------------- +# missing_key_policy +# --------------------------------------------------------------------------- + + +async def _missing_key_source(user_id, limit, next_page, **kw): + # Second item lacks the dedup key "id". + return FeedResult( + data=[{"id": 1, "val": "a"}, {"val": "no-id"}, {"id": 2, "val": "b"}], + next_page={"pos": 0}, + has_next_page=False, + ) + + +@pytest.mark.asyncio +async def test_missing_key_policy_error_raises(): + src = _missing_key_source + node = S.wrapper(S.subfeed("src", "src"), dedup_key="id", missing_key_policy="error") + ctx = S.make_ctx({"src": src}, redis=_redis()) + from smartfeed.execution import executor as run_executor + + with pytest.raises(KeyError, match="Dedup key"): + await run_executor.run(node, ctx, limit=10, cursor={}) + + +@pytest.mark.asyncio +async def test_missing_key_policy_keep_passes_through(): + node = S.wrapper(S.subfeed("src", "src"), dedup_key="id", missing_key_policy="keep") + ctx = S.make_ctx({"src": _missing_key_source}, redis=_redis()) + from smartfeed.execution import executor as run_executor + + r = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(r.data) == 3 # the no-id item is kept un-deduped + + +@pytest.mark.asyncio +async def test_missing_key_policy_drop_removes(): + node = S.wrapper(S.subfeed("src", "src"), dedup_key="id", missing_key_policy="drop") + ctx = S.make_ctx({"src": _missing_key_source}, redis=_redis()) + from smartfeed.execution import executor as run_executor + + r = await run_executor.run(node, ctx, limit=10, cursor={}) + ids = [it.get("id") for it in r.data] + assert ids == [1, 2] # the no-id item is dropped + + +# --------------------------------------------------------------------------- +# non-dict items +# --------------------------------------------------------------------------- + + +async def _non_dict_source(user_id, limit, next_page, **kw): + return FeedResult(data=[{"id": 1}, "a-string", {"id": 2}, 42], next_page={"pos": 0}, has_next_page=False) + + +@pytest.mark.asyncio +async def test_non_dict_items_pass_through_dedup(): + node = S.wrapper(S.subfeed("src", "src"), dedup_key="id") + ctx = S.make_ctx({"src": _non_dict_source}, redis=_redis()) + from smartfeed.execution import executor as run_executor + + r = await run_executor.run(node, ctx, limit=10, cursor={}) + assert "a-string" in r.data and 42 in r.data # non-dicts survive + + +# --------------------------------------------------------------------------- +# key stringification: int 1 vs str "1" +# --------------------------------------------------------------------------- + + +async def _mixed_type_key_source(user_id, limit, next_page, **kw): + return FeedResult( + data=[{"id": 1, "val": "int"}, {"id": "1", "val": "str"}], next_page={"pos": 0}, has_next_page=False + ) + + +@pytest.mark.asyncio +async def test_int_and_str_key_currently_collide_characterization(): + """CURRENT BEHAVIOR: dedup does str(item[key]), so id=1 (int) and id="1" (str) + are treated as the SAME key and one is dropped. See review -- decide whether + mixed-type ids should be distinct. This test pins today's behavior; flip it if + the product decision is that they must be distinct.""" + node = S.wrapper(S.subfeed("src", "src"), dedup_key="id") + ctx = S.make_ctx({"src": _mixed_type_key_source}, redis=_redis()) + from smartfeed.execution import executor as run_executor + + r = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(r.data) == 1, "int 1 and str '1' currently collide via str() coercion" diff --git a/tests/test_dedup_priority.py b/tests/test_dedup_priority.py new file mode 100644 index 0000000..3486496 --- /dev/null +++ b/tests/test_dedup_priority.py @@ -0,0 +1,64 @@ +import pytest +import fakeredis.aioredis + +from smartfeed.models import FeedConfig +from smartfeed.models.base import FeedResult +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor + + +async def promo_source(user_id, limit, next_page, **kw): + data = [{"id": i, "val": "promo"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + +async def regular_source(user_id, limit, next_page, **kw): + data = [{"id": i, "val": "regular"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + +PRIO_METHODS = {"promo": promo_source, "regular": regular_source} + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.fixture +def ctx(redis): + return ExecutionContext(session_id="s1", methods_dict=PRIO_METHODS, redis=redis) + + +@pytest.mark.asyncio +async def test_wrapper_dedup_priority_override(ctx): + """Inner wrapper with dedup_priority=3 overrides child subfeed priorities.""" + config = FeedConfig.model_validate( + { + "version": "2", + "feed": { + "type": "wrapper", + "node_id": "outer", + "dedup": {"dedup_key": "id"}, + "data": { + "type": "merger_append", + "node_id": "top", + "items": [ + {"type": "subfeed", "subfeed_id": "promo", "method_name": "promo", "dedup_priority": 10}, + { + "type": "wrapper", + "node_id": "inner", + "dedup_priority": 3, + "data": {"type": "subfeed", "subfeed_id": "regular", "method_name": "regular"}, + }, + ], + }, + }, + } + ) + + result = await run_executor.run(config.feed, ctx, limit=10, cursor={}) + + # promo(10) > inner wrapper override(3) -> all promo + for item in result.data: + assert item["val"] == "promo" diff --git a/tests/test_distribute.py b/tests/test_distribute.py new file mode 100644 index 0000000..7e40c15 --- /dev/null +++ b/tests/test_distribute.py @@ -0,0 +1,208 @@ +import pytest + +from smartfeed.models.base import FeedResult +from smartfeed.models.mixers import MergerAppendDistribute +from smartfeed.models.subfeed import SubFeed +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor + + +async def make_operators(user_id, limit, next_page, **kw): + """Source producing items with operator_id cycling through op_0, op_1, op_2.""" + data = [{"id": i, "operator_id": f"op_{i % 3}", "val": f"item_{i}"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + +async def make_scored(user_id, limit, next_page, **kw): + """Source with a numeric score field for testing sorting_key. + + Items are in ascending score order so that descending sort visibly reorders them. + op_0 gets scores 1, 3, 5, ... and op_1 gets scores 2, 4, 6, ... + """ + data = [{"id": i, "operator_id": f"op_{i % 2}", "score": i + 1, "val": f"item_{i}"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + +async def make_no_key(user_id, limit, next_page, **kw): + """Source producing items WITHOUT operator_id (to test missing key behaviour).""" + data = [{"id": i, "val": f"item_{i}"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + +METHODS = { + "operators": make_operators, + "scored": make_scored, + "no_key": make_no_key, +} + + +@pytest.fixture +def ctx(): + return ExecutionContext(session_id="s1", methods_dict=METHODS, redis=None) + + +# --------------------------------------------------------------------------- +# Basic round-robin +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_basic_round_robin_no_consecutive_same_operator(ctx): + """Consecutive items should not share the same operator_id (round-robin effect).""" + node = MergerAppendDistribute( + node_id="dist", + items=[SubFeed(subfeed_id="ops", method_name="operators")], + distribution_key="operator_id", + ) + result = await run_executor.run(node, ctx, limit=9, cursor={}) + + assert len(result.data) == 9 + for i in range(len(result.data) - 1): + assert ( + result.data[i]["operator_id"] != result.data[i + 1]["operator_id"] + ), f"Consecutive duplicate operator at positions {i} and {i+1}" + + +@pytest.mark.asyncio +async def test_round_robin_all_operators_represented(ctx): + """All three operators must appear in a full round-robin of 9 items.""" + node = MergerAppendDistribute( + node_id="dist", + items=[SubFeed(subfeed_id="ops", method_name="operators")], + distribution_key="operator_id", + ) + result = await run_executor.run(node, ctx, limit=9, cursor={}) + + operator_ids = {item["operator_id"] for item in result.data} + assert operator_ids == {"op_0", "op_1", "op_2"} + + +@pytest.mark.asyncio +async def test_round_robin_result_length_matches_limit(ctx): + """Result must contain exactly `limit` items.""" + node = MergerAppendDistribute( + node_id="dist", + items=[SubFeed(subfeed_id="ops", method_name="operators")], + distribution_key="operator_id", + ) + for lim in (3, 6, 9): + result = await run_executor.run(node, ctx, limit=lim, cursor={}) + assert len(result.data) == lim + + +# --------------------------------------------------------------------------- +# sorting_key sorts before distributing +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_sorting_key_desc_highest_score_first(ctx): + """With sorting_key='score' and sorting_desc=True, higher scores appear earlier.""" + node = MergerAppendDistribute( + node_id="dist_sorted", + items=[SubFeed(subfeed_id="scored", method_name="scored")], + distribution_key="operator_id", + sorting_key="score", + sorting_desc=True, + ) + result = await run_executor.run(node, ctx, limit=4, cursor={}) + + assert len(result.data) == 4 + # The highest score items per operator group should appear first. + # Verify that score of first item per operator_id group is higher than second. + from collections import defaultdict + + by_op = defaultdict(list) + for item in result.data: + by_op[item["operator_id"]].append(item["score"]) + + for op, scores in by_op.items(): + if len(scores) >= 2: + assert scores[0] >= scores[1], f"Operator {op}: expected descending scores, got {scores}" + + +@pytest.mark.asyncio +async def test_sorting_key_asc_lowest_score_first(ctx): + """With sorting_key='score' and sorting_desc=False, lower scores appear earlier.""" + node = MergerAppendDistribute( + node_id="dist_asc", + items=[SubFeed(subfeed_id="scored", method_name="scored")], + distribution_key="operator_id", + sorting_key="score", + sorting_desc=False, + ) + result = await run_executor.run(node, ctx, limit=4, cursor={}) + + assert len(result.data) == 4 + # All scores in the output must be <= max available score in source + # (weak check: just confirm the sort changed the order vs no sorting) + scores = [item["score"] for item in result.data] + # First item of each operator should have a smaller score than last + assert scores[0] <= scores[-1] or len(set(scores)) == 1 + + +@pytest.mark.asyncio +async def test_no_sorting_key_preserves_source_order(ctx): + """Without sorting_key, source order is preserved before distribution.""" + node_no_sort = MergerAppendDistribute( + node_id="dist_nosort", + items=[SubFeed(subfeed_id="scored", method_name="scored")], + distribution_key="operator_id", + ) + node_sorted = MergerAppendDistribute( + node_id="dist_sorted", + items=[SubFeed(subfeed_id="scored", method_name="scored")], + distribution_key="operator_id", + sorting_key="score", + sorting_desc=True, + ) + + r_nosort = await run_executor.run(node_no_sort, ctx, limit=4, cursor={}) + r_sorted = await run_executor.run(node_sorted, ctx, limit=4, cursor={}) + + ids_nosort = [item["id"] for item in r_nosort.data] + ids_sorted = [item["id"] for item in r_sorted.data] + + # Sorted and unsorted should produce different orderings (source has varying scores) + assert ids_nosort != ids_sorted + + +# --------------------------------------------------------------------------- +# Missing distribution_key raises KeyError +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_missing_distribution_key_raises(ctx): + """Items without the distribution_key should raise KeyError.""" + node = MergerAppendDistribute( + node_id="dist_bad", + items=[SubFeed(subfeed_id="nokey", method_name="no_key")], + distribution_key="operator_id", + ) + with pytest.raises(KeyError): + await run_executor.run(node, ctx, limit=5, cursor={}) + + +# --------------------------------------------------------------------------- +# Multiple subfeeds +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_multiple_subfeeds_merged_before_distribute(ctx): + """Items from multiple subfeeds are pooled before round-robin distribution.""" + node = MergerAppendDistribute( + node_id="dist_multi", + items=[ + SubFeed(subfeed_id="ops_a", method_name="operators"), + SubFeed(subfeed_id="ops_b", method_name="operators"), + ], + distribution_key="operator_id", + ) + result = await run_executor.run(node, ctx, limit=6, cursor={}) + + assert len(result.data) == 6 + # Still no consecutive duplicates + for i in range(len(result.data) - 1): + assert result.data[i]["operator_id"] != result.data[i + 1]["operator_id"] diff --git a/tests/test_gradient.py b/tests/test_gradient.py new file mode 100644 index 0000000..6957759 --- /dev/null +++ b/tests/test_gradient.py @@ -0,0 +1,158 @@ +import pytest + +from smartfeed.models.base import FeedResult +from smartfeed.models.mixers import MergerPercentageGradient, MergerPercentageItem +from smartfeed.models.subfeed import SubFeed +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor + + +async def make_source(prefix, user_id, limit, next_page, **kw): + page = next_page.get("page", 1) + start = (page - 1) * limit + data = [{"id": f"{prefix}_{start + i}", "val": prefix} for i in range(limit)] + return FeedResult(data=data, next_page={"page": page + 1}, has_next_page=True) + + +METHODS = { + "source_a": lambda *a, **kw: make_source("a", *a, **kw), + "source_b": lambda *a, **kw: make_source("b", *a, **kw), +} + + +@pytest.fixture +def ctx(): + return ExecutionContext(session_id="s1", methods_dict=METHODS, redis=None) + + +def _make_gradient(pct_from=80, pct_to=20, step=10, size_to_step=10): + return MergerPercentageGradient( + node_id="grad", + item_from=MergerPercentageItem( + percentage=pct_from, + data=SubFeed(subfeed_id="source_a", method_name="source_a"), + ), + item_to=MergerPercentageItem( + percentage=pct_to, + data=SubFeed(subfeed_id="source_b", method_name="source_b"), + ), + step=step, + size_to_step=size_to_step, + ) + + +@pytest.mark.asyncio +async def test_basic_80_20_page1(ctx): + """On page 1 with 80/20 split and limit=10: 8 from source_a, 2 from source_b.""" + node = _make_gradient(pct_from=80, pct_to=20, step=10, size_to_step=10) + result = await run_executor.run(node, ctx, limit=10, cursor={}) + + assert len(result.data) == 10 + sources = [item["_smartfeed_debug_info"]["source"] for item in result.data] + assert sources.count("source_a") == 8 + assert sources.count("source_b") == 2 + + +@pytest.mark.asyncio +async def test_both_sources_present(ctx): + """Items from both sources appear in the result.""" + node = _make_gradient(pct_from=80, pct_to=20, step=10, size_to_step=10) + result = await run_executor.run(node, ctx, limit=10, cursor={}) + + sources = {item["_smartfeed_debug_info"]["source"] for item in result.data} + assert "source_a" in sources + assert "source_b" in sources + + +@pytest.mark.asyncio +async def test_ratio_shifts_on_later_page(ctx): + """Later pages have a higher proportion of source_b than page 1 (gradient effect).""" + node = _make_gradient(pct_from=80, pct_to=20, step=10, size_to_step=10) + + # Page 1 + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + sources_p1 = [item["_smartfeed_debug_info"]["source"] for item in r1.data] + a_count_p1 = sources_p1.count("source_a") + b_count_p1 = sources_p1.count("source_b") + + # Page 2 (cursor carries page state for the gradient node) + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + sources_p2 = [item["_smartfeed_debug_info"]["source"] for item in r2.data] + a_count_p2 = sources_p2.count("source_a") + b_count_p2 = sources_p2.count("source_b") + + # The gradient should shift: fewer source_a and more source_b on page 2 + assert a_count_p2 < a_count_p1 + assert b_count_p2 > b_count_p1 + + +@pytest.mark.asyncio +async def test_step_parameter_controls_shift(ctx): + """A larger step causes a bigger ratio change between pages.""" + node_small_step = _make_gradient(pct_from=80, pct_to=20, step=5, size_to_step=10) + node_large_step = _make_gradient(pct_from=80, pct_to=20, step=20, size_to_step=10) + + # Compute ratio difference between page 1 and page 2 for each node + def _b_count(node, cursor): + import asyncio + + r = asyncio.get_event_loop().run_until_complete(run_executor.run(node, ctx, limit=10, cursor=cursor)) + sources = [item["_smartfeed_debug_info"]["source"] for item in r.data] + return sources.count("source_b"), r.next_page + + # Use coroutine approach instead + r1_small = await run_executor.run(node_small_step, ctx, limit=10, cursor={}) + r2_small = await run_executor.run(node_small_step, ctx, limit=10, cursor=r1_small.next_page) + + r1_large = await run_executor.run(node_large_step, ctx, limit=10, cursor={}) + r2_large = await run_executor.run(node_large_step, ctx, limit=10, cursor=r1_large.next_page) + + def b_count(result): + sources = [item["_smartfeed_debug_info"]["source"] for item in result.data] + return sources.count("source_b") + + shift_small = b_count(r2_small) - b_count(r1_small) + shift_large = b_count(r2_large) - b_count(r1_large) + + # Larger step => bigger shift in ratio + assert shift_large >= shift_small + + +@pytest.mark.asyncio +async def test_result_length_matches_limit(ctx): + """Result always contains exactly `limit` items.""" + node = _make_gradient() + for lim in (5, 10, 15): + result = await run_executor.run(node, ctx, limit=lim, cursor={}) + assert len(result.data) == lim + + +@pytest.mark.asyncio +async def test_cursor_advances_page(ctx): + """next_page cursor increments the internal page counter for the gradient node.""" + node = _make_gradient() + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + # The gradient stores its own page in cursor under its node_id + assert r1.next_page.get("grad", {}).get("page") == 2 + + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + assert r2.next_page.get("grad", {}).get("page") == 3 + + +@pytest.mark.asyncio +async def test_gradient_eventually_reaches_100_pct_to(ctx): + """After enough pages the ratio fully shifts to source_b (100%).""" + # step=50 will flip to 100% after 2 pages + node = _make_gradient(pct_from=80, pct_to=20, step=50, size_to_step=10) + + cursor = {} + r = None + for _ in range(5): + r = await run_executor.run(node, ctx, limit=10, cursor=cursor) + cursor = r.next_page + assert r is not None + + sources = [item["_smartfeed_debug_info"]["source"] for item in r.data] + # At full shift, all items come from source_b + assert sources.count("source_b") == 10 + assert sources.count("source_a") == 0 diff --git a/tests/test_gradient_demands.py b/tests/test_gradient_demands.py new file mode 100644 index 0000000..d275920 --- /dev/null +++ b/tests/test_gradient_demands.py @@ -0,0 +1,93 @@ +"""The gradient demand computation was rewritten from an O(limit * page) replay +of the whole shift history into an O(limit / size_to_step) walk over the current +page window with a closed-form percentage (a tampered or +legitimately deep `page` must not grow per-request work). + +This file pins (a) exact output equivalence with the original algorithm, kept +verbatim below as the reference, and (b) the bounded-work property. +""" + +from typing import Dict, List + +from smartfeed.models.mixers import MergerPercentageGradient, MergerPercentageItem +from smartfeed.models.subfeed import SubFeed + + +def _reference_demands(pct_from: int, pct_to: int, step: int, size_to_step: int, page: int, limit: int) -> tuple: + """The pre-rewrite algorithm, verbatim -- the behavioral reference.""" + percentage_from = pct_from + percentage_to = pct_to + start_position = limit * (page - 1) + first_iter = True + limit_from = 0 + limit_to = 0 + segments: List[Dict] = [] + + for i in range(size_to_step, limit * page + size_to_step, size_to_step): + if not first_iter and percentage_to < 100: + percentage_from -= step + percentage_to += step + if percentage_to > 100 or percentage_from < 0: + percentage_from = 0 + percentage_to = 100 + + if i > start_position: + iter_limit = (limit * page - start_position) if i > limit * page else (i - start_position) + start_position = i + from_take = iter_limit * percentage_from // 100 + to_take = iter_limit - from_take + limit_from += from_take + limit_to += to_take + segments.append({"limit": iter_limit, "from_take": from_take, "to_take": to_take}) + + if first_iter: + first_iter = False + + return limit_from, limit_to, segments + + +def _node(pct_from: int, pct_to: int, step: int, size_to_step: int) -> MergerPercentageGradient: + return MergerPercentageGradient( + node_id="g", + item_from=MergerPercentageItem(percentage=pct_from, data=SubFeed(subfeed_id="a", method_name="a")), + item_to=MergerPercentageItem(percentage=pct_to, data=SubFeed(subfeed_id="b", method_name="b")), + step=step, + size_to_step=size_to_step, + ) + + +def test_window_rewrite_matches_reference_exactly(): + cases = 0 + for pct_from, pct_to in [(80, 20), (100, 0), (0, 100), (50, 70), (60, 20), (90, 30), (50, 50), (70, 0)]: + for step in (1, 3, 10, 25, 40, 100): + for size_to_step in (1, 2, 3, 7, 10, 30): + node = _node(pct_from, pct_to, step, size_to_step) + for limit in (1, 3, 5, 10, 13): + for page in (1, 2, 3, 5, 8): + got = node._calculate_demands(page, limit) + want = _reference_demands(pct_from, pct_to, step, size_to_step, page, limit) + assert got == want, ( + f"mismatch at from={pct_from} to={pct_to} step={step} " + f"size_to_step={size_to_step} page={page} limit={limit}: {got} != {want}" + ) + cases += 1 + assert cases == 8 * 6 * 6 * 5 * 5 + + +def test_deep_page_work_is_bounded_by_page_size(): + node = _node(80, 20, 10, 10) + limit = 20 + limit_from, limit_to, segments = node._calculate_demands(page=10**6, limit=limit) + # Work is bounded by the page window, not the scroll depth. + assert len(segments) <= limit // 10 + 2 + # By page 10**6 the gradient long since clamped to (0, 100). + assert (limit_from, limit_to) == (0, limit) + + +def test_page_window_is_fully_allocated(): + """from_take + to_take always sum to the page window, whatever the params.""" + node = _node(80, 20, 7, 9) + for page in (1, 2, 4, 100): + limit_from, limit_to, segments = node._calculate_demands(page, 10) + assert limit_from + limit_to == 10 + assert sum(s["limit"] for s in segments) == 10 diff --git a/tests/test_high_priority.py b/tests/test_high_priority.py new file mode 100644 index 0000000..046dc2a --- /dev/null +++ b/tests/test_high_priority.py @@ -0,0 +1,550 @@ +"""High-priority missing tests for SmartFeed v2. + +Covers: +1. Dedup across 3+ pages (with and without cache) +2. Session rebuild continues pagination +3. has_next_page at session boundary +4. Missing dedup key policies (error / drop / keep) +5. Positional with exhausted high-priority subfeed +""" + +import pytest +import fakeredis.aioredis + +from smartfeed.models import ( + FeedResult, + MergerAppend, + MergerPositional, + SubFeed, + Wrapper, + WrapperCache, + WrapperDedup, +) +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor + + +# --------------------------------------------------------------------------- +# Subfeed helpers +# --------------------------------------------------------------------------- + + +async def make_overlapping_a(user_id, limit, next_page, **kw): + """Produces ids 0, 2, 4, 6, ... (page-aware, even numbers).""" + page = next_page.get("page", 1) + start = (page - 1) * limit * 2 # ids 0,2,4,6... + data = [{"id": start + i * 2, "val": "a"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": page + 1}, has_next_page=True) + + +async def make_overlapping_b(user_id, limit, next_page, **kw): + """Produces ids 1, 3, 5, 7, ... with some overlap with source a.""" + page = next_page.get("page", 1) + start = (page - 1) * limit * 2 + 1 # ids 1,3,5... + data = [{"id": start + i * 2, "val": "b"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": page + 1}, has_next_page=True) + + +async def make_overlapping_ab(user_id, limit, next_page, **kw): + """Produces ids that heavily overlap with source a: 0, 2, 4, ...""" + page = next_page.get("page", 1) + start = (page - 1) * limit * 2 + data = [{"id": start + i * 2, "val": "ab_overlap"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": page + 1}, has_next_page=True) + + +async def make_sequential(user_id, limit, next_page, **kw): + """Produces sequentially numbered items across pages (0,1,2,...) -- no gaps.""" + page = next_page.get("page", 1) + start = (page - 1) * limit + data = [{"id": start + i, "val": f"seq_{start + i}"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": page + 1}, has_next_page=True) + + +async def make_promo_limited(user_id, limit, next_page, **kw): + """Returns only 2 items total, then empty.""" + page = next_page.get("page", 1) + if page > 1: + return FeedResult(data=[], next_page={"page": page + 1}, has_next_page=False) + data = [{"id": 1000 + i, "val": "promo"} for i in range(2)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=False) + + +async def make_default_items(user_id, limit, next_page, **kw): + """Returns plenty of default items, page-aware.""" + page = next_page.get("page", 1) + start = (page - 1) * limit + data = [{"id": start + i, "val": "default"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": page + 1}, has_next_page=True) + + +OVERLAP_METHODS = { + "overlap_a": make_overlapping_a, + "overlap_b": make_overlapping_b, + "overlap_ab": make_overlapping_ab, +} + +SEQUENTIAL_METHODS = { + "sequential": make_sequential, +} + +POSITIONAL_METHODS = { + "promo_limited": make_promo_limited, + "default_items": make_default_items, +} + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +# --------------------------------------------------------------------------- +# 1. Dedup across 3+ pages -- no duplicates +# --------------------------------------------------------------------------- + + +class TestDedupAcrossPages: + """Test dedup behaviour across multiple pages. + + Without cache, dedup is applied per-request (per-page) only because there + is no persisted seen-state between paginate calls. + + With cache, the entire session_size chunk is deduped at build time, so + dedup holds across all pages that come from that cached chunk. + """ + + @pytest.mark.asyncio + async def test_dedup_nocache_within_page_only(self): + """Without cache, dedup removes duplicates WITHIN a single page + but cannot prevent duplicates ACROSS pages because seen-state is not + persisted. + + We use two sources that both return the same fixed set of ids on every + call (ignoring pagination). Within a single page, dedup collapses + duplicates. Across pages, the same ids reappear because there is no + persisted seen-state. + """ + + async def make_fixed_a(user_id, limit, next_page, **kw): + """Always returns ids 0..limit-1 regardless of page.""" + data = [{"id": i, "val": "a"} for i in range(limit)] + return FeedResult( + data=data, + next_page={"page": next_page.get("page", 1) + 1}, + has_next_page=True, + ) + + async def make_fixed_b(user_id, limit, next_page, **kw): + """Always returns ids 0..limit-1 (same as a) -- full overlap.""" + data = [{"id": i, "val": "b"} for i in range(limit)] + return FeedResult( + data=data, + next_page={"page": next_page.get("page", 1) + 1}, + has_next_page=True, + ) + + methods = {"fixed_a": make_fixed_a, "fixed_b": make_fixed_b} + ctx = ExecutionContext( + session_id="dedup_nocache", + methods_dict=methods, + redis=None, + ) + node = Wrapper( + node_id="w", + dedup=WrapperDedup(dedup_key="id"), + data=MergerAppend( + node_id="mix", + items=[ + SubFeed(subfeed_id="a", method_name="fixed_a"), + SubFeed(subfeed_id="b", method_name="fixed_b"), + ], + ), + ) + + all_ids = [] + cursor = {} + for _ in range(3): + result = await run_executor.run(node, ctx, limit=10, cursor=cursor) + page_ids = [item["id"] for item in result.data] + # Within each page, no duplicates + assert len(page_ids) == len(set(page_ids)), "Duplicates found within a single page" + all_ids.extend(page_ids) + cursor = result.next_page + + # Across pages, duplicates ARE expected (seen-state not persisted). + # This is the known limitation without cache. + # We assert that cross-page duplicates exist to document the behaviour. + assert len(all_ids) > len(set(all_ids)), ( + "Expected cross-page duplicates without cache, but none found. " + "If the implementation now persists dedup state without cache, " + "this test can be updated." + ) + + @pytest.mark.asyncio + async def test_dedup_with_cache_across_pages(self, redis): + """With cache, dedup is applied to the entire session_size chunk at + cold-build time. Pages served from that chunk have zero duplicates.""" + ctx = ExecutionContext( + session_id="dedup_cached", + methods_dict=OVERLAP_METHODS, + redis=redis, + ) + node = Wrapper( + node_id="w", + cache=WrapperCache(session_size=50, session_ttl=300), + dedup=WrapperDedup(dedup_key="id"), + data=MergerAppend( + node_id="mix", + items=[ + SubFeed(subfeed_id="a", method_name="overlap_a"), + SubFeed(subfeed_id="ab", method_name="overlap_ab"), + ], + ), + ) + + all_ids = [] + cursor = {} + for page_num in range(1, 4): + result = await run_executor.run(node, ctx, limit=10, cursor=cursor) + page_ids = [item["id"] for item in result.data] + # Within each page, no duplicates + assert len(page_ids) == len(set(page_ids)), f"Duplicates in page {page_num}" + all_ids.extend(page_ids) + cursor = result.next_page + + # With cache, dedup holds across ALL pages served from the same session + assert len(all_ids) == len( + set(all_ids) + ), "Duplicates found across cached pages -- dedup should cover the entire session" + + +# --------------------------------------------------------------------------- +# 2. Session rebuild continues pagination +# --------------------------------------------------------------------------- + + +class TestSessionRebuildContinuation: + """Wrapper(cache, session_size=20), limit=10. + + Pages 1-2 come from the cache (20 items). Page 3 triggers a cache rebuild + via the continuation cursor stored in :meta. Items on page 3 should + continue where page 2 left off with no gap and no overlap. + """ + + @pytest.mark.asyncio + async def test_page3_continues_after_rebuild(self, redis): + ctx = ExecutionContext( + session_id="rebuild", + methods_dict=SEQUENTIAL_METHODS, + redis=redis, + ) + node = Wrapper( + node_id="w", + cache=WrapperCache(session_size=20, session_ttl=300), + data=SubFeed(subfeed_id="seq", method_name="sequential"), + ) + + # Page 1 + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(r1.data) == 10 + ids_p1 = [item["id"] for item in r1.data] + + # Page 2 -- still from cache + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + assert len(r2.data) == 10 + ids_p2 = [item["id"] for item in r2.data] + + # Page 3 -- should trigger rebuild (cache exhausted: offset 20 >= 20) + r3 = await run_executor.run(node, ctx, limit=10, cursor=r2.next_page) + assert len(r3.data) == 10, "Page 3 should return items after rebuild" + ids_p3 = [item["id"] for item in r3.data] + + # No overlap between pages + all_ids = ids_p1 + ids_p2 + ids_p3 + assert len(all_ids) == len(set(all_ids)), "Overlap detected between pages after session rebuild" + + # No gap: the maximum id from page 2 + 1 should be the minimum id from page 3 + # (since make_sequential produces strictly sequential ids) + assert ( + min(ids_p3) == max(ids_p2) + 1 + ), f"Gap detected: page 2 ended at {max(ids_p2)}, page 3 starts at {min(ids_p3)}" + + +# --------------------------------------------------------------------------- +# 3. has_next_page at session boundary +# --------------------------------------------------------------------------- + + +class TestHasNextPageAtSessionBoundary: + """Wrapper(cache, session_size=20), limit=10. + + After page 2, all 20 cached items are served. has_next_page should still + be True because the child has more data. Page 3 triggers rebuild and + serves fresh items. + """ + + @pytest.mark.asyncio + async def test_has_next_true_at_boundary(self, redis): + ctx = ExecutionContext( + session_id="boundary", + methods_dict=SEQUENTIAL_METHODS, + redis=redis, + ) + node = Wrapper( + node_id="w", + cache=WrapperCache(session_size=20, session_ttl=300), + data=SubFeed(subfeed_id="seq", method_name="sequential"), + ) + + # Page 1 + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + assert r1.has_next_page is True, "Page 1 should have next page" + + # Page 2 -- last page of cache + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + # The _paginate method sets has_next = end < len(data). + # end = 20, len(data) = 20 => has_next = False from paginate. + # However, the child has more data so the user should be able to + # continue. Verify the system still serves page 3: + # + # Note: _paginate returns has_next=False at the exact boundary. + # The cursor still carries the gen, so on page 3 the wrapper detects + # cache exhaustion and rebuilds. The practical contract is: + # even if has_next_page is False after page 2, requesting page 3 + # with the cursor still works (rebuild path). + + # Page 3 -- rebuild triggered + r3 = await run_executor.run(node, ctx, limit=10, cursor=r2.next_page) + assert len(r3.data) == 10, "Page 3 should return data after rebuild" + assert r3.data[0]["id"] != r1.data[0]["id"], "Page 3 should contain new items, not a repeat of page 1" + + @pytest.mark.asyncio + async def test_page2_has_next_page_reflects_boundary(self, redis): + """Document the exact has_next_page value at the session boundary. + + _paginate sets has_next = end < len(data). + With session_size=20, limit=10: page 2 => end=20, len=20 => False. + This is a known edge: the client sees has_next_page=False but can + still request page 3 (the rebuild path handles it). + """ + ctx = ExecutionContext( + session_id="boundary2", + methods_dict=SEQUENTIAL_METHODS, + redis=redis, + ) + node = Wrapper( + node_id="w", + cache=WrapperCache(session_size=20, session_ttl=300), + data=SubFeed(subfeed_id="seq", method_name="sequential"), + ) + + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + + # Wrapper tracks child_has_next: at boundary, has_next_page=True + # because child has more data (rebuild possible on next request). + assert r2.has_next_page is True + + +# --------------------------------------------------------------------------- +# 4. Missing dedup key policies +# --------------------------------------------------------------------------- + + +class TestMissingDedupKeyPolicies: + """Wrapper(dedup, dedup_key='id') with items that lack the 'id' field. + + Three policies: + - 'error': raises KeyError + - 'drop': item is silently dropped + - 'keep': item is kept in the output + """ + + @staticmethod + async def _make_items_with_missing_key(user_id, limit, next_page, **kw): + """Half of the items have 'id', half do not. Stateful (advances by page) so a + dedup refill fetches genuinely new items instead of regenerating seen ids.""" + page = next_page.get("page", 1) + start = (page - 1) * limit + data = [] + for i in range(limit): + gid = start + i + if gid % 2 == 0: + data.append({"id": gid, "val": f"has_id_{gid}"}) + else: + data.append({"val": f"no_id_{gid}"}) # no 'id' key + return FeedResult(data=data, next_page={"page": page + 1}, has_next_page=True) + + def _make_methods(self): + return {"missing_key": self._make_items_with_missing_key} + + def _make_wrapper(self, policy): + return Wrapper( + node_id="w", + dedup=WrapperDedup(dedup_key="id", missing_key_policy=policy), + data=SubFeed(subfeed_id="src", method_name="missing_key"), + ) + + @pytest.mark.asyncio + async def test_policy_error_raises(self): + ctx = ExecutionContext( + session_id="policy_error", + methods_dict=self._make_methods(), + redis=None, + ) + node = self._make_wrapper("error") + + with pytest.raises(KeyError, match="Dedup key 'id' missing"): + await run_executor.run(node, ctx, limit=10, cursor={}) + + @pytest.mark.asyncio + async def test_policy_drop_removes_items(self): + ctx = ExecutionContext( + session_id="policy_drop", + methods_dict=self._make_methods(), + redis=None, + ) + node = self._make_wrapper("drop") + result = await run_executor.run(node, ctx, limit=10, cursor={}) + + # Only items WITH the 'id' key should survive (refill fills page to limit) + for item in result.data: + assert "id" in item, f"Item without 'id' was not dropped: {item}" + + # Refill loop fetches more items to fill the page after drop + assert len(result.data) == 10 + + @pytest.mark.asyncio + async def test_policy_keep_preserves_items(self): + ctx = ExecutionContext( + session_id="policy_keep", + methods_dict=self._make_methods(), + redis=None, + ) + node = self._make_wrapper("keep") + result = await run_executor.run(node, ctx, limit=10, cursor={}) + + # All items should be preserved + assert len(result.data) == 10 + + items_with_id = [item for item in result.data if "id" in item] + items_without_id = [item for item in result.data if "id" not in item] + assert len(items_with_id) == 5 + assert len(items_without_id) == 5 + + +# --------------------------------------------------------------------------- +# 5. Positional with exhausted high-priority subfeed +# --------------------------------------------------------------------------- + + +class TestPositionalExhaustedPromo: + """Production bug fix scenario. + + Setup: MergerPositional(positions=[1,3,5,7]) with a promo subfeed that + returns only 2 items (has_next_page=False). Default subfeed returns 20. + + The positional merger computes demand as: + pos_count = 4 (slots at positions 1,3,5,7) + default_demand = limit - pos_count = 16 + + When promo returns only 2 of the 4 demanded items, the assemble loop + fills positions 1 and 3 with promo, then falls back to default for + positions 5 and 7. However, default only has 16 items while 18 are + needed (16 non-positional + 2 unfilled positional), so the result is + 18 items total (2 promo + 16 default). + + This documents the current behaviour: the merger does NOT dynamically + increase the default demand to compensate for a partially-exhausted + positional subfeed. + """ + + @pytest.mark.asyncio + async def test_exhausted_promo_fills_with_default(self): + ctx = ExecutionContext( + session_id="exhausted_promo", + methods_dict=POSITIONAL_METHODS, + redis=None, + ) + node = MergerPositional( + node_id="pos", + positions=[1, 3, 5, 7], + positional=SubFeed(subfeed_id="promo", method_name="promo_limited"), + default=SubFeed(subfeed_id="default", method_name="default_items"), + ) + + result = await run_executor.run(node, ctx, limit=20, cursor={}) + + # Build a source map + sources = [] + for item in result.data: + info = item.get("_smartfeed_debug_info", {}) + sources.append(info.get("source")) + + # Promo should appear at index 0 (position 1) and index 2 (position 3) + assert sources[0] == "promo", f"Position 1 should be promo, got {sources[0]}" + assert sources[2] == "promo", f"Position 3 should be promo, got {sources[2]}" + + # Only 2 promo items total + promo_positions = [i for i, src in enumerate(sources) if src == "promo"] + assert ( + len(promo_positions) == 2 + ), f"Expected exactly 2 promo items, got {len(promo_positions)} at indices {promo_positions}" + + # Positions 5 and 7 fall back to default (promo exhausted) + # These are at index 4 and 6 in the result (0-indexed) since only 2 + # promo items appear before them. + for i, src in enumerate(sources): + if i not in (0, 2): + assert src == "default", f"Index {i} should be default, got {src}" + + # Current behaviour: result has fewer than `limit` items because + # the merger under-allocated default demand. + # default_demand = 20 - 4 = 16; actual need = 18 (16 + 2 unfilled promo slots) + # Result: 2 promo + 16 default = 18 + assert len(result.data) == 18, ( + f"Expected 18 items (demand shortfall), got {len(result.data)}. " + "If the implementation now backfills default demand, update this test to expect 20." + ) + + @pytest.mark.asyncio + async def test_exhausted_promo_no_refetch(self): + """Verify the promo subfeed is called only once (not re-fetched for + unfilled positions).""" + call_count = 0 + + async def counting_promo(user_id, limit, next_page, **kw): + nonlocal call_count + call_count += 1 + page = next_page.get("page", 1) + if page > 1: + return FeedResult(data=[], next_page={"page": page + 1}, has_next_page=False) + data = [{"id": 1000 + i, "val": "promo"} for i in range(2)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=False) + + methods = { + "promo_limited": counting_promo, + "default_items": make_default_items, + } + ctx = ExecutionContext( + session_id="no_refetch", + methods_dict=methods, + redis=None, + ) + node = MergerPositional( + node_id="pos", + positions=[1, 3, 5, 7], + positional=SubFeed(subfeed_id="promo", method_name="promo_limited"), + default=SubFeed(subfeed_id="default", method_name="default_items"), + ) + + await run_executor.run(node, ctx, limit=20, cursor={}) + + # The promo subfeed should be called exactly once + assert call_count == 1, f"Promo subfeed was called {call_count} times; expected exactly 1" diff --git a/tests/test_input_validation.py b/tests/test_input_validation.py new file mode 100644 index 0000000..ac56b93 --- /dev/null +++ b/tests/test_input_validation.py @@ -0,0 +1,156 @@ +"""Input validation at the public boundary and on untrusted cursor contents. + +Policy: integrator-level type errors (limit, session_id, cursor type) AND +tampered per-node cursor contents raise loudly (ValueError/TypeError) instead +of crashing mid-slice, serving wrong pages, or silently degrading. +""" + +import decimal + +import pytest +import fakeredis.aioredis + +from smartfeed.manager import FeedManager +from smartfeed.models.base import FeedResult +from smartfeed.models.mixers import MergerPercentageGradient, MergerPercentageItem +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +CONFIG = {"version": "2", "feed": {"type": "subfeed", "subfeed_id": "src", "method_name": "src"}} + + +async def _one_item(user_id, limit, next_page, **kw): + return FeedResult(data=[{"id": 1}], next_page={}, has_next_page=False) + + +def _manager(): + return FeedManager(config=CONFIG, methods_dict={"src": _one_item}, redis_client=None) + + +# -- get_feed boundary (#14) ------------------------------------------------- + + +@pytest.mark.asyncio +async def test_limit_zero_and_negative_raise(): + mgr = _manager() + for bad in (0, -3): + with pytest.raises(ValueError, match="limit"): + await mgr.get_feed(session_id="u", limit=bad) + + +@pytest.mark.asyncio +async def test_limit_non_int_raises(): + with pytest.raises(ValueError, match="limit"): + # Deliberately ill-typed: proves get_feed rejects a non-int limit at runtime. + await _manager().get_feed(session_id="u", limit="10") # pyright: ignore[reportArgumentType] + + +@pytest.mark.asyncio +async def test_cursor_wrong_type_raises(): + with pytest.raises(TypeError, match="cursor"): + # Deliberately ill-typed: proves get_feed rejects a non-dict cursor at runtime. + await _manager().get_feed(session_id="u", limit=10, cursor="abc") # pyright: ignore[reportArgumentType] + + +@pytest.mark.asyncio +async def test_empty_session_id_raises(): + with pytest.raises(ValueError, match="session_id"): + await _manager().get_feed(session_id="", limit=10) + + +# -- cached-wrapper cursor tampering (#13) ------------------------------------ + + +def _cached_wrapper_ctx(): + src = S.ScriptedSource(S.unique_pool(30)) + node = S.wrapper(S.subfeed("src", "src"), session_size=10) + ctx = S.make_ctx({"src": src}, redis=fakeredis.aioredis.FakeRedis()) + return node, ctx + + +@pytest.mark.asyncio +async def test_tampered_offset_rejected(): + node, ctx = _cached_wrapper_ctx() + r1 = await run_executor.run(node, ctx, limit=5, cursor={}) + gen = r1.next_page["w"]["gen"] + for bad_offset in (-4, "5", 2.5, None): + with pytest.raises(ValueError, match="Wrapper 'w'.*offset"): + await run_executor.run(node, ctx, limit=5, cursor={"w": {"offset": bad_offset, "gen": gen}}) + + +@pytest.mark.asyncio +async def test_tampered_gen_type_rejected(): + node, ctx = _cached_wrapper_ctx() + await run_executor.run(node, ctx, limit=5, cursor={}) + with pytest.raises(ValueError, match="Wrapper 'w'.*gen"): + await run_executor.run(node, ctx, limit=5, cursor={"w": {"offset": 5, "gen": 123}}) + + +@pytest.mark.asyncio +async def test_node_cursor_entry_not_dict_rejected(): + node, ctx = _cached_wrapper_ctx() + with pytest.raises(ValueError, match="Wrapper 'w'.*must be a dict"): + await run_executor.run(node, ctx, limit=5, cursor={"w": "junk"}) + + +@pytest.mark.asyncio +async def test_huge_offset_is_valid_and_triggers_continuation(): + """A huge-but-valid int offset is not tampering: it means 'past the batch', + which is the documented cache-exhausted continuation path.""" + node, ctx = _cached_wrapper_ctx() + r1 = await run_executor.run(node, ctx, limit=5, cursor={}) + gen = r1.next_page["w"]["gen"] + r = await run_executor.run(node, ctx, limit=5, cursor={"w": {"offset": 10**9, "gen": gen}}) + assert len(r.data) == 5 # continuation rebuild served a full fresh page + + +@pytest.mark.asyncio +async def test_subfeed_cursor_slice_not_dict_rejected(): + node = S.subfeed("src", "src") + ctx = S.make_ctx({"src": S.ScriptedSource(S.unique_pool(5))}) + with pytest.raises(ValueError, match="SubFeed 'src'"): + await run_executor.run(node, ctx, limit=5, cursor={"src": "junk"}) + + +# -- gradient cursor tampering (#13/#16) --------------------------------------- + + +@pytest.mark.asyncio +async def test_gradient_tampered_page_rejected(): + node = MergerPercentageGradient( + node_id="grad", + item_from=MergerPercentageItem(percentage=80, data=S.subfeed("a", "a")), + item_to=MergerPercentageItem(percentage=20, data=S.subfeed("b", "b")), + step=10, + size_to_step=10, + ) + ctx = S.make_ctx({"a": S.ScriptedSource(S.unique_pool(50)), "b": S.ScriptedSource(S.unique_pool(50, start=100))}) + for bad_cursor in ({"grad": {"page": 0}}, {"grad": {"page": -1}}, {"grad": {"page": "2"}}, {"grad": "junk"}): + with pytest.raises(ValueError, match="grad"): + await run_executor.run(node, ctx, limit=10, cursor=bad_cursor) + + +# -- cache serialization asymmetry (#17) --------------------------------------- + + +async def _decimal_source(user_id, limit, next_page, **kw): + return FeedResult(data=[{"id": 1, "price": decimal.Decimal("9.99")}], next_page={}, has_next_page=False) + + +@pytest.mark.asyncio +async def test_non_serializable_item_with_cache_raises_named_error(): + node = S.wrapper(S.subfeed("src", "src"), session_size=10) + ctx = S.make_ctx({"src": _decimal_source}, redis=fakeredis.aioredis.FakeRedis()) + with pytest.raises(TypeError, match="Wrapper 'w'.*orjson-serializable"): + await run_executor.run(node, ctx, limit=5, cursor={}) + + +@pytest.mark.asyncio +async def test_non_serializable_item_passthrough_works_control(): + """Control: the same item is fine without cache -- the asymmetry is documented, + and the cached path's error above must name the node so it is diagnosable.""" + node = S.wrapper(S.subfeed("src", "src")) + ctx = S.make_ctx({"src": _decimal_source}, redis=None) + r = await run_executor.run(node, ctx, limit=5, cursor={}) + assert len(r.data) == 1 diff --git a/tests/test_invariants.py b/tests/test_invariants.py new file mode 100644 index 0000000..0fbcc73 --- /dev/null +++ b/tests/test_invariants.py @@ -0,0 +1,114 @@ +"""Property/invariant backbone for dedup + pagination. + +Each test drains a FINITE source to exhaustion and asserts one universal +property (coverage / no-duplicates / page-fullness / termination) across the +config matrix, so a single property covers a whole class of configurations. +Uses unique (duplicate-free) pools so coverage/no-duplicate assertions are +unambiguous; duplicate-heavy dedup semantics live in test_dedup_correctness.py. +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S + + +POOL_N = 200 + + +def _redis(): + return fakeredis.aioredis.FakeRedis() + + +def _build(cfg): + """cfg keys: cache(bool), dedup(bool), rerank(bool).""" + child = S.subfeed("src", "src") + node = S.wrapper( + child, + session_size=50 if cfg["cache"] else None, + dedup_key="id" if cfg["dedup"] else None, + rerank_method="identity" if cfg.get("rerank") else None, + ) + return node + + +async def identity_rerank(items, session_id): + return items + + +def _ctx(src): + return S.make_ctx({"src": src, "identity": identity_rerank}, redis=_redis()) + + +def _p(id_, marks=(), **cfg): + return pytest.param(cfg, id=id_, marks=marks) + + +# Coverage must hold on both paths: no item is ever lost. +COVERAGE_CONFIGS = [ + _p("cache-nodedup", cache=True, dedup=False), + _p("passthrough-nodedup", cache=False, dedup=False), + _p("cache-dedup", cache=True, dedup=True), + _p("passthrough-dedup", cache=False, dedup=True), + _p("cache-dedup-rerank", cache=True, dedup=True, rerank=True), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cfg", COVERAGE_CONFIGS) +async def test_coverage_no_item_lost(cfg): + """Draining a finite unique source serves every id exactly once -- nothing lost.""" + src = S.ScriptedSource(S.unique_pool(POOL_N)) + node = _build(cfg) + pages = await S.drain(node, _ctx(src), limit=10, max_pages=200) + S.assert_full_coverage(pages, set(range(POOL_N))) + + +# Same matrix WITHOUT the LOSS marks: item loss shows up as coverage gaps, NOT as +# duplicates or non-termination, so these properties hold for every config. +NODUP_CONFIGS = [ + _p("cache-nodedup", cache=True, dedup=False), + _p("passthrough-nodedup", cache=False, dedup=False), + _p("cache-dedup", cache=True, dedup=True), + _p("passthrough-dedup", cache=False, dedup=True), + _p("cache-dedup-rerank", cache=True, dedup=True, rerank=True), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cfg", NODUP_CONFIGS) +async def test_no_duplicates_across_pages(cfg): + """No id is ever served twice (unique source; loss must not manifest as dupes).""" + src = S.ScriptedSource(S.unique_pool(POOL_N)) + node = _build(cfg) + pages = await S.drain(node, _ctx(src), limit=10, max_pages=200) + S.assert_no_duplicates(pages) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "cfg", + [ + _p("cache-nodedup", cache=True, dedup=False), + _p("passthrough-nodedup", cache=False, dedup=False), + _p("cache-dedup", cache=True, dedup=True), + _p("passthrough-dedup", cache=False, dedup=True), + ], +) +async def test_pages_are_full_until_exhaustion(cfg): + """Every non-final page has exactly `limit` items (loss does not shorten pages, + so this passes even where coverage fails -- keeping the two concerns separate).""" + src = S.ScriptedSource(S.unique_pool(POOL_N)) + node = _build(cfg) + pages = await S.drain(node, _ctx(src), limit=10, max_pages=200) + S.assert_pages_full(pages, limit=10) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cfg", NODUP_CONFIGS) +async def test_pagination_terminates(cfg): + """A finite source must produce a terminating pagination (has_next -> False).""" + src = S.ScriptedSource(S.unique_pool(POOL_N)) + node = _build(cfg) + # drain raises if it does not terminate; reaching this line means it did. + await S.drain(node, _ctx(src), limit=10, max_pages=200) diff --git a/tests/test_manager.py b/tests/test_manager.py new file mode 100644 index 0000000..1e1ada3 --- /dev/null +++ b/tests/test_manager.py @@ -0,0 +1,68 @@ +"""FeedManager-level tests: the sole public entrypoint was +previously exercised only indirectly through executor.run(). +""" + +import pytest +from pydantic import ValidationError + +from smartfeed.manager import FeedManager +from smartfeed.models.base import FeedResult + + +async def make_items(user_id, limit, next_page, **kw): + page = next_page.get("page", 1) + start = (page - 1) * limit + return FeedResult( + data=[{"id": start + i} for i in range(limit)], + next_page={"page": page + 1}, + has_next_page=True, + ) + + +CONFIG = { + "feed": { + "type": "merger_percentage", + "node_id": "mix", + "items": [ + {"percentage": 50, "data": {"type": "subfeed", "subfeed_id": "a", "method_name": "items"}}, + {"percentage": 50, "data": {"type": "subfeed", "subfeed_id": "b", "method_name": "items"}}, + ], + } +} + + +def test_malformed_config_raises_at_construction(): + with pytest.raises(ValidationError): + FeedManager(config={"feed": {"type": "wrapper"}}, methods_dict={}) # node_id/data missing + + +def test_unknown_node_type_raises_at_construction(): + with pytest.raises(ValidationError): + FeedManager(config={"feed": {"type": "no_such_node"}}, methods_dict={}) + + +@pytest.mark.asyncio +async def test_positions_stamped_0_based_contiguous(): + mgr = FeedManager(config=CONFIG, methods_dict={"items": make_items}) + r = await mgr.get_feed(session_id="u", limit=10) + positions = [it["_smartfeed_debug_info"]["smartfeed_position"] for it in r.data] + assert positions == list(range(len(r.data))) + assert len(r.data) == 10 + + +@pytest.mark.asyncio +async def test_cursor_round_trip_through_manager(): + mgr = FeedManager(config=CONFIG, methods_dict={"items": make_items}) + r1 = await mgr.get_feed(session_id="u", limit=10) + r2 = await mgr.get_feed(session_id="u", limit=10, cursor=r1.next_page) + ids1 = [it["id"] for it in r1.data] + ids2 = [it["id"] for it in r2.data] + assert ids2 and ids2 != ids1 # page 2 advanced both sources + + +@pytest.mark.asyncio +async def test_legacy_version_key_is_ignored(): + """Configs carrying the removed top-level `version` tag still parse.""" + mgr = FeedManager(config={"version": "2", **CONFIG}, methods_dict={"items": make_items}) + r = await mgr.get_feed(session_id="u", limit=4) + assert len(r.data) == 4 diff --git a/tests/test_medium_priority.py b/tests/test_medium_priority.py new file mode 100644 index 0000000..edaeec3 --- /dev/null +++ b/tests/test_medium_priority.py @@ -0,0 +1,406 @@ +"""Medium-priority missing tests for SmartFeed v2. + +Covers: +1. SubFeed with subfeed_params forwarding +2. MergerAppend cursor pagination across two pages +3. MergerAppend with one empty child +4. MergerPercentage with one empty source +5. MergerPercentage odd limit (11) -- no off-by-one +6. MergerPositional step-based positions (start=2, end=20, step=3) +7. MergerPositional with empty default subfeed +""" + +import pytest + +from smartfeed.models import ( + FeedResult, + MergerAppend, + MergerPercentage, + MergerPercentageItem, + MergerPositional, + SubFeed, +) +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor +from tests.conftest import METHODS + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _ctx(methods): + return ExecutionContext(session_id="s1", methods_dict=methods, redis=None) + + +# --------------------------------------------------------------------------- +# 1. SubFeed with subfeed_params forwarding +# --------------------------------------------------------------------------- + + +class TestSubFeedParamsForwarding: + """SubFeed.subfeed_params must be forwarded to the underlying method.""" + + @pytest.mark.asyncio + async def test_custom_param_received_by_method(self): + async def make_with_param(user_id, limit, next_page, custom_param=None, **kw): + assert custom_param == 42 + data = [{"id": i, "param": custom_param} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + methods = {"with_param": make_with_param} + ctx = _ctx(methods) + node = SubFeed( + subfeed_id="src", + method_name="with_param", + subfeed_params={"custom_param": 42}, + ) + + result = await run_executor.run(node, ctx, limit=5, cursor={}) + + assert len(result.data) == 5 + for item in result.data: + assert item["param"] == 42 + + @pytest.mark.asyncio + async def test_custom_param_value_is_correct(self): + """Verify the exact value 42 is forwarded, not a default or None.""" + received = {} + + async def capture_param(user_id, limit, next_page, custom_param=None, **kw): + received["custom_param"] = custom_param + return FeedResult( + data=[{"id": 0}], + next_page={"page": 2}, + has_next_page=False, + ) + + methods = {"capture": capture_param} + ctx = _ctx(methods) + node = SubFeed( + subfeed_id="src", + method_name="capture", + subfeed_params={"custom_param": 42}, + ) + + await run_executor.run(node, ctx, limit=1, cursor={}) + + assert received.get("custom_param") == 42 + + +# --------------------------------------------------------------------------- +# 2. MergerAppend cursor pagination +# --------------------------------------------------------------------------- + + +class TestMergerAppendCursorPagination: + """MergerAppend must propagate cursors so page 2 returns different items.""" + + @pytest.mark.asyncio + async def test_page2_returns_different_items(self): + ctx = _ctx(METHODS) + node = MergerAppend( + node_id="append", + items=[ + SubFeed(subfeed_id="a", method_name="items"), + SubFeed(subfeed_id="b", method_name="items"), + ], + ) + + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(r1.data) == 10 + + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + assert len(r2.data) == 10 + + ids_p1 = {item["id"] for item in r1.data} + ids_p2 = {item["id"] for item in r2.data} + + # Page 2 items must differ from page 1 items + assert ids_p1 != ids_p2, "Page 2 returned the same items as page 1" + + @pytest.mark.asyncio + async def test_cursor_carries_both_child_cursors(self): + """The merged cursor must contain entries for both child subfeeds.""" + ctx = _ctx(METHODS) + node = MergerAppend( + node_id="append", + items=[ + SubFeed(subfeed_id="a", method_name="items"), + SubFeed(subfeed_id="b", method_name="items"), + ], + ) + + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + + # Both subfeed cursors must be present so page 2 can continue correctly + assert "a" in r1.next_page, "Cursor is missing child 'a'" + assert "b" in r1.next_page, "Cursor is missing child 'b'" + + +# --------------------------------------------------------------------------- +# 3. MergerAppend with one empty child +# --------------------------------------------------------------------------- + + +class TestMergerAppendOneEmptyChild: + """When one child returns empty, output must contain items from the other.""" + + @pytest.mark.asyncio + async def test_result_contains_non_empty_child_items(self): + ctx = _ctx(METHODS) + node = MergerAppend( + node_id="append", + items=[ + SubFeed(subfeed_id="a", method_name="items"), + SubFeed(subfeed_id="empty", method_name="empty"), + ], + ) + + result = await run_executor.run(node, ctx, limit=10, cursor={}) + + assert len(result.data) > 0, "Expected items from the non-empty child" + sources = [item["_smartfeed_debug_info"]["source"] for item in result.data] + assert all(src == "a" for src in sources), f"Expected all items from 'a', got sources: {sources}" + + @pytest.mark.asyncio + async def test_has_next_page_reflects_non_empty_child(self): + """has_next_page should be True because the non-empty child has more data.""" + ctx = _ctx(METHODS) + node = MergerAppend( + node_id="append", + items=[ + SubFeed(subfeed_id="a", method_name="items"), + SubFeed(subfeed_id="empty", method_name="empty"), + ], + ) + + result = await run_executor.run(node, ctx, limit=10, cursor={}) + + assert result.has_next_page is True + + +# --------------------------------------------------------------------------- +# 4. MergerPercentage with one empty source +# --------------------------------------------------------------------------- + + +class TestMergerPercentageOneEmptySource: + """50/50 split: when one source is empty, all output comes from the other.""" + + @pytest.mark.asyncio + async def test_result_contains_only_non_empty_source(self): + ctx = _ctx(METHODS) + node = MergerPercentage( + node_id="pct", + items=[ + MergerPercentageItem( + percentage=50, + data=SubFeed(subfeed_id="items", method_name="items"), + ), + MergerPercentageItem( + percentage=50, + data=SubFeed(subfeed_id="empty", method_name="empty"), + ), + ], + ) + + result = await run_executor.run(node, ctx, limit=10, cursor={}) + + assert len(result.data) > 0, "Expected items from the non-empty source" + sources = [item["_smartfeed_debug_info"]["source"] for item in result.data] + assert all(src == "items" for src in sources), f"Expected all items from 'items', got sources: {sources}" + + @pytest.mark.asyncio + async def test_no_items_from_empty_source(self): + ctx = _ctx(METHODS) + node = MergerPercentage( + node_id="pct", + items=[ + MergerPercentageItem( + percentage=50, + data=SubFeed(subfeed_id="empty", method_name="empty"), + ), + MergerPercentageItem( + percentage=50, + data=SubFeed(subfeed_id="items", method_name="items"), + ), + ], + ) + + result = await run_executor.run(node, ctx, limit=10, cursor={}) + + sources = [item["_smartfeed_debug_info"]["source"] for item in result.data] + assert "empty" not in sources, "Got items from the empty source" + + +# --------------------------------------------------------------------------- +# 5. MergerPercentage odd limit (11) -- no off-by-one +# --------------------------------------------------------------------------- + + +class TestMergerPercentageOddLimit: + """40/60 split with limit=11 must return exactly 11 items.""" + + @pytest.mark.asyncio + async def test_odd_limit_returns_exactly_11(self): + ctx = _ctx(METHODS) + node = MergerPercentage( + node_id="pct", + items=[ + MergerPercentageItem( + percentage=40, + data=SubFeed(subfeed_id="a", method_name="items"), + ), + MergerPercentageItem( + percentage=60, + data=SubFeed(subfeed_id="b", method_name="items"), + ), + ], + ) + + result = await run_executor.run(node, ctx, limit=11, cursor={}) + + assert ( + len(result.data) == 11 + ), f"Expected exactly 11 items for odd limit with 40/60 split, got {len(result.data)}" + + @pytest.mark.asyncio + async def test_odd_limit_split_sums_to_limit(self): + """The demands for 40% and 60% of 11 must sum to 11 (4 + 7 = 11).""" + ctx = _ctx(METHODS) + node = MergerPercentage( + node_id="pct", + items=[ + MergerPercentageItem( + percentage=40, + data=SubFeed(subfeed_id="a", method_name="items"), + ), + MergerPercentageItem( + percentage=60, + data=SubFeed(subfeed_id="b", method_name="items"), + ), + ], + ) + + result = await run_executor.run(node, ctx, limit=11, cursor={}) + + sources = [item["_smartfeed_debug_info"]["source"] for item in result.data] + count_a = sources.count("a") + count_b = sources.count("b") + + assert count_a + count_b == 11 + # 40% of 11 = 4.4 -> floor 4, remainder 0.4; 60% of 11 = 6.6 -> floor 6, remainder 0.6 + # Remainder distribution gives the 1 leftover to 'b' (highest remainder 60) + assert count_a == 4, f"Expected 4 items from 'a' (40% of 11), got {count_a}" + assert count_b == 7, f"Expected 7 items from 'b' (60% of 11), got {count_b}" + + +# --------------------------------------------------------------------------- +# 6. MergerPositional step-based positions +# --------------------------------------------------------------------------- + + +class TestMergerPositionalStepBased: + """Positions generated via range(start=2, stop=21, step=3) = [2,5,8,11,14,17,20]. + + Items at those 1-indexed positions must come from the positional subfeed. + """ + + _STEP_POSITIONS = list(range(2, 21, 3)) # [2, 5, 8, 11, 14, 17, 20] + + @pytest.mark.asyncio + async def test_positional_items_at_step_positions(self): + ctx = _ctx(METHODS) + node = MergerPositional( + node_id="pos", + positions=self._STEP_POSITIONS, + positional=SubFeed(subfeed_id="positional", method_name="items"), + default=SubFeed(subfeed_id="default", method_name="items"), + ) + + result = await run_executor.run(node, ctx, limit=20, cursor={}) + assert len(result.data) == 20 + + pos_set = set(self._STEP_POSITIONS) + for i, item in enumerate(result.data): + position = i + 1 # 1-indexed + source = item["_smartfeed_debug_info"]["source"] + if position in pos_set: + assert source == "positional", f"Position {position} (index {i}) should be 'positional', got '{source}'" + else: + assert source == "default", f"Position {position} (index {i}) should be 'default', got '{source}'" + + @pytest.mark.asyncio + async def test_step_positions_count(self): + """Verify exactly 7 positional items appear in the output.""" + ctx = _ctx(METHODS) + node = MergerPositional( + node_id="pos", + positions=self._STEP_POSITIONS, + positional=SubFeed(subfeed_id="positional", method_name="items"), + default=SubFeed(subfeed_id="default", method_name="items"), + ) + + result = await run_executor.run(node, ctx, limit=20, cursor={}) + + sources = [item["_smartfeed_debug_info"]["source"] for item in result.data] + assert sources.count("positional") == len(self._STEP_POSITIONS), ( + f"Expected {len(self._STEP_POSITIONS)} positional items, " f"got {sources.count('positional')}" + ) + assert sources.count("default") == 20 - len(self._STEP_POSITIONS) + + +# --------------------------------------------------------------------------- +# 7. MergerPositional with empty default +# --------------------------------------------------------------------------- + + +class TestMergerPositionalEmptyDefault: + """When the default subfeed is empty, only positional items appear.""" + + @pytest.mark.asyncio + async def test_only_positional_items_when_default_empty(self): + ctx = _ctx(METHODS) + positions = [1, 3, 5] + node = MergerPositional( + node_id="pos", + positions=positions, + positional=SubFeed(subfeed_id="positional", method_name="items"), + default=SubFeed(subfeed_id="empty_default", method_name="empty"), + ) + + result = await run_executor.run(node, ctx, limit=10, cursor={}) + + assert len(result.data) > 0, "Expected positional items in the result" + sources = [item["_smartfeed_debug_info"]["source"] for item in result.data] + assert all( + src == "positional" for src in sources + ), f"Expected all items from 'positional', got sources: {sources}" + + @pytest.mark.asyncio + async def test_positional_items_fill_configured_positions(self): + """With empty default, positional items still land at the right positions.""" + ctx = _ctx(METHODS) + positions = [2, 4] + node = MergerPositional( + node_id="pos", + positions=positions, + positional=SubFeed(subfeed_id="positional", method_name="items"), + default=SubFeed(subfeed_id="empty_default", method_name="empty"), + ) + + result = await run_executor.run(node, ctx, limit=10, cursor={}) + + # With no default items, the assemble loop skips non-positional slots + # and only places items at the configured positional slots. + # Positional items appear but non-positional slots are vacant. + for i, item in enumerate(result.data): + source = item["_smartfeed_debug_info"]["source"] + assert source == "positional", f"Index {i} should be 'positional' (only source), got '{source}'" diff --git a/tests/test_merger_append.py b/tests/test_merger_append.py deleted file mode 100644 index e9db5c7..0000000 --- a/tests/test_merger_append.py +++ /dev/null @@ -1,43 +0,0 @@ -import pytest - -from smartfeed.schemas import FeedResultNextPage, FeedResultNextPageInside, MergerAppend -from tests.fixtures.configs import METHODS_DICT -from tests.fixtures.mergers import MERGER_APPEND_CONFIG - - -@pytest.mark.asyncio -async def test_merger_append() -> None: - """ - Тест для проверки получения данных из append мерджера. - """ - - merger_append = MergerAppend.parse_obj(MERGER_APPEND_CONFIG) - merger_append_res = await merger_append.get_data( - methods_dict=METHODS_DICT, - limit=11, - next_page=FeedResultNextPage(data={}), - user_id="x", - ) - - assert merger_append_res.data == ["x_1", "x_2", "x_3", "x_4", "x_5", "x_1", "x_2", "x_3", "x_4", "x_5", "x_6"] - - -@pytest.mark.asyncio -async def test_merger_append_with_item_1_page_2() -> None: - """ - Тест для проверки получения данных из append мерджера с курсором пагинации первого субфида. - """ - - merger_append = MergerAppend.parse_obj(MERGER_APPEND_CONFIG) - merger_append_res = await merger_append.get_data( - methods_dict=METHODS_DICT, - limit=11, - next_page=FeedResultNextPage( - data={"subfeed_merger_append_example": FeedResultNextPageInside(page=2, after="x_5")} - ), - user_id="x", - ) - - assert merger_append_res.data == ["x_6", "x_7", "x_8", "x_9", "x_10", "x_1", "x_2", "x_3", "x_4", "x_5", "x_6"] - assert merger_append_res.next_page.data["subfeed_merger_append_example"].page == 3 - assert merger_append_res.next_page.data["subfeed_merger_append_example"].after == "x_10" diff --git a/tests/test_merger_append_distribute.py b/tests/test_merger_append_distribute.py deleted file mode 100644 index 6bb1782..0000000 --- a/tests/test_merger_append_distribute.py +++ /dev/null @@ -1,56 +0,0 @@ -import pytest - -from smartfeed.schemas import FeedResultNextPage, FeedResultNextPageInside, MergerAppendDistribute -from tests.fixtures.configs import METHODS_DICT -from tests.fixtures.mergers import MERGER_APPEND_DISTRIBUTE_CONFIG - - -@pytest.mark.asyncio -async def test_merger_disturbed_append() -> None: - """ - Тест для проверки получения данных из append мерджера. - """ - - merger_distributed = MergerAppendDistribute.parse_obj(MERGER_APPEND_DISTRIBUTE_CONFIG) - merger_distributed_res = await merger_distributed.get_data( - methods_dict=METHODS_DICT, - limit=20, - next_page=FeedResultNextPage(data={}), - user_id="x", - ) - assert len(merger_distributed_res.data) == 20 - for i in range(len(merger_distributed_res.data) - 1): - assert ( - merger_distributed_res.data[i][merger_distributed.distribution_key] - != merger_distributed_res.data[i + 1][merger_distributed.distribution_key] - ) - - -@pytest.mark.asyncio -async def test_merger_append_with_item_1_page_2() -> None: - """ - Тест для проверки получения данных из append мерджера с курсором пагинации первого субфида. - """ - merger_distributed = MergerAppendDistribute.parse_obj(MERGER_APPEND_DISTRIBUTE_CONFIG) - merger_distributed_res = await merger_distributed.get_data( - methods_dict=METHODS_DICT, - limit=11, - next_page=FeedResultNextPage( - data={ - "subfeed_merger_distribute_example": FeedResultNextPageInside( - page=2, after={"user_id": "x_1", "value": 11} - ) - } - ), - user_id="x", - ) - for i in range(len(merger_distributed_res.data) - 1): - assert ( - merger_distributed_res.data[i][merger_distributed.distribution_key] - != merger_distributed_res.data[i + 1][merger_distributed.distribution_key] - ) - assert merger_distributed_res.next_page.data["subfeed_merger_distribute_example"].page == 3 - assert merger_distributed_res.next_page.data["subfeed_merger_distribute_example"].after == { - "user_id": "x_2", - "value": 22, - } diff --git a/tests/test_merger_percentage.py b/tests/test_merger_percentage.py deleted file mode 100644 index e5ab76e..0000000 --- a/tests/test_merger_percentage.py +++ /dev/null @@ -1,27 +0,0 @@ -import pytest - -from smartfeed.schemas import FeedResultNextPage, FeedResultNextPageInside, MergerPercentage -from tests.fixtures.configs import METHODS_DICT -from tests.fixtures.mergers import MERGER_PERCENTAGE_CONFIG - - -@pytest.mark.asyncio -async def test_merger_percentage() -> None: - """ - Тест для проверки получения данных из процентного мерджера. - """ - - merger_percentage = MergerPercentage.parse_obj(MERGER_PERCENTAGE_CONFIG) - merger_percentage_res = await merger_percentage.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage( - data={ - "subfeed_merger_percentage_example": FeedResultNextPageInside(page=2, after="x_3"), - "subfeed_2_merger_percentage_example": FeedResultNextPageInside(page=3, after="x_20"), - } - ), - user_id="x", - ) - - assert merger_percentage_res.data == ["x_4", "x_21", "x_22", "x_5", "x_23", "x_24", "x_6", "x_25", "x_26", "x_7"] diff --git a/tests/test_merger_percentage_gradient.py b/tests/test_merger_percentage_gradient.py deleted file mode 100644 index eaaba9c..0000000 --- a/tests/test_merger_percentage_gradient.py +++ /dev/null @@ -1,72 +0,0 @@ -import pytest - -from smartfeed.schemas import FeedResultNextPage, FeedResultNextPageInside, MergerPercentageGradient -from tests.fixtures.configs import METHODS_DICT -from tests.fixtures.mergers import MERGER_PERCENTAGE_GRADIENT_CONFIG - - -@pytest.mark.asyncio -async def test_merger_percentage_gradient() -> None: - """ - Тест для проверки получения данных из процентного мерджера с градиентом. - """ - - merger_percentage_gradient = MergerPercentageGradient.parse_obj(MERGER_PERCENTAGE_GRADIENT_CONFIG) - merger_percentage_gradient_res = await merger_percentage_gradient.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage( - data={ - "subfeed_from_merger_percentage_gradient_example": FeedResultNextPageInside(page=2, after="x_3"), - "subfeed_to_merger_percentage_gradient_example": FeedResultNextPageInside(page=3, after="x_20"), - } - ), - user_id="x", - ) - - assert merger_percentage_gradient_res.data == [ - "x_4", - "x_5", - "x_6", - "x_7", - "x_8", - "x_9", - "x_10", - "x_11", - "x_21", - "x_22", - ] - - -@pytest.mark.asyncio -async def test_merger_percentage_gradient_next_page() -> None: - """ - Тест для проверки получения данных из процентного мерджера с градиентом после изменения процента на другой странице. - """ - - merger_percentage_gradient = MergerPercentageGradient.parse_obj(MERGER_PERCENTAGE_GRADIENT_CONFIG) - merger_percentage_gradient_res = await merger_percentage_gradient.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage( - data={ - "merger_percentage_gradient_example": FeedResultNextPageInside(page=2, after=None), - "subfeed_from_merger_percentage_gradient_example": FeedResultNextPageInside(page=2, after="x_3"), - "subfeed_to_merger_percentage_gradient_example": FeedResultNextPageInside(page=3, after="x_20"), - } - ), - user_id="x", - ) - - assert merger_percentage_gradient_res.data == [ - "x_4", - "x_5", - "x_6", - "x_7", - "x_8", - "x_9", - "x_10", - "x_21", - "x_22", - "x_23", - ] diff --git a/tests/test_merger_positional.py b/tests/test_merger_positional.py deleted file mode 100644 index c0f3815..0000000 --- a/tests/test_merger_positional.py +++ /dev/null @@ -1,74 +0,0 @@ -import pytest - -from smartfeed.schemas import FeedResultNextPage, FeedResultNextPageInside, MergerPositional -from tests.fixtures.configs import METHODS_DICT -from tests.fixtures.mergers import MERGER_POSITIONAL_CONFIG - - -@pytest.mark.asyncio -async def test_merger_positional_with_positions() -> None: - """ - Тест для проверки получения данных из позиционного мерджера на основе позиций в конфигурации. - """ - - merger_positional = MergerPositional.parse_obj(MERGER_POSITIONAL_CONFIG) - merger_positional_res = await merger_positional.get_data( - methods_dict=METHODS_DICT, - limit=9, - next_page=FeedResultNextPage( - data={ - "subfeed_positional_merger_positional_example": FeedResultNextPageInside(page=2, after="x_10"), - "subfeed_default_merger_positional_example": FeedResultNextPageInside(page=3, after="x_20"), - } - ), - user_id="x", - ) - - assert merger_positional_res.data == ["x_11", "x_21", "x_12", "x_22", "x_23", "x_24", "x_25", "x_26", "x_27"] - - -@pytest.mark.asyncio -async def test_merger_positional_with_step() -> None: - """ - Тест для проверки получения данных из позиционного мерджера на основе шагов в конфигурации. - """ - - merger_positional = MergerPositional.parse_obj(MERGER_POSITIONAL_CONFIG) - merger_positional_res = await merger_positional.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage( - data={ - "merger_positional_example": FeedResultNextPageInside(page=3, after=None), - "subfeed_positional_merger_positional_example": FeedResultNextPageInside(page=2, after="x_3"), - "subfeed_default_merger_positional_example": FeedResultNextPageInside(page=3, after="x_20"), - } - ), - user_id="x", - ) - - assert merger_positional_res.data == ["x_4", "x_21", "x_5", "x_22", "x_6", "x_23", "x_7", "x_24", "x_8", "x_25"] - - -@pytest.mark.asyncio -async def test_merger_positional_with_empty_default() -> None: - """ - Тест для проверки получения данных из позиционного мерджера на основе шагов в конфигурации. - """ - - merger_positional = MergerPositional.parse_obj(MERGER_POSITIONAL_CONFIG) - merger_positional.default.method_name = "empty" - merger_positional_res = await merger_positional.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage( - data={ - "merger_positional_example": FeedResultNextPageInside(page=3, after=None), - "subfeed_positional_merger_positional_example": FeedResultNextPageInside(page=2, after="x_3"), - "subfeed_default_merger_positional_example": FeedResultNextPageInside(page=3, after="x_20"), - } - ), - user_id="x", - ) - - assert merger_positional_res.data == ["x_4", "x_5", "x_6", "x_7", "x_8"] diff --git a/tests/test_merger_view_session.py b/tests/test_merger_view_session.py deleted file mode 100644 index 78a0566..0000000 --- a/tests/test_merger_view_session.py +++ /dev/null @@ -1,133 +0,0 @@ -import inspect -import json - -import pytest - -from smartfeed.schemas import FeedResultNextPage, FeedResultNextPageInside, MergerViewSession -from tests.fixtures.configs import METHODS_DICT -from tests.fixtures.mergers import MERGER_VIEW_SESSION_CONFIG, MERGER_VIEW_SESSION_DUPS_CONFIG -from tests.fixtures.redis import redis_client - - -@pytest.mark.asyncio -async def test_merger_view_session_no_redis() -> None: - """ - Тест для проверки получения данных из мерджера с кэшированием без клиента Redis. - """ - - merger_vs = MergerViewSession.parse_obj(MERGER_VIEW_SESSION_CONFIG) - with pytest.raises(ValueError): - await merger_vs.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage(data={}), - user_id="x", - ) - - -@pytest.mark.parametrize("redis_client", ["sync", "async"], indirect=True) -@pytest.mark.asyncio -async def test_merger_view_session(redis_client) -> None: - """ - Тест для проверки получения данных из мерджера с кэшированием. - """ - - merger_vs = MergerViewSession.parse_obj(MERGER_VIEW_SESSION_CONFIG) - merger_vs_res = await merger_vs.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage(data={}), - user_id="x", - redis_client=redis_client, - ) - merger_vs_cache = redis_client.get(name="merger_view_session_example_x") - # Для использования синхронной и асинхронной фикстуры в одном тесте проверяем метод get - if inspect.iscoroutine(merger_vs_cache): - merger_vs_cache = json.loads(await merger_vs_cache) - else: - merger_vs_cache = json.loads(merger_vs_cache) - - assert merger_vs_res.data == ["x_1", "x_2", "x_3", "x_4", "x_5", "x_6", "x_7", "x_8", "x_9", "x_10"] - assert len(merger_vs_cache) == merger_vs.session_size - assert merger_vs_cache[:10] == merger_vs_res.data - - -@pytest.mark.parametrize("redis_client", ["sync", "async"], indirect=True) -@pytest.mark.asyncio -async def test_merger_view_session_custom_key(redis_client) -> None: - """ - Тест для проверки получения данных из мерджера с кэшированием по ключу с кастомным постфиксом. - """ - - merger_vs = MergerViewSession.parse_obj(MERGER_VIEW_SESSION_CONFIG) - # Даем дополнительный параметр, который мерджер добавит в ключ кэша. - merger_vs_res = await merger_vs.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage(data={}), - user_id="x", - redis_client=redis_client, - custom_view_session_key="foo", - ) - merger_vs_cache = redis_client.get(name="merger_view_session_example_x_foo") - # Для использования синхронной и асинхронной фикстуры в одном тесте проверяем метод get - if inspect.iscoroutine(merger_vs_cache): - merger_vs_cache = json.loads(await merger_vs_cache) - else: - merger_vs_cache = json.loads(merger_vs_cache) - - assert merger_vs_res.data == ["x_1", "x_2", "x_3", "x_4", "x_5", "x_6", "x_7", "x_8", "x_9", "x_10"] - assert len(merger_vs_cache) == merger_vs.session_size - assert merger_vs_cache[:10] == merger_vs_res.data - - -@pytest.mark.parametrize("redis_client", ["sync", "async"], indirect=True) -@pytest.mark.asyncio -async def test_merger_view_session_next_page(redis_client) -> None: - """ - Тест для проверки получения данных следующей страницы из мерджера с кэшированием. - """ - - merger_vs = MergerViewSession.parse_obj(MERGER_VIEW_SESSION_CONFIG) - merger_vs_res = await merger_vs.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage( - data={"merger_view_session_example": FeedResultNextPageInside(page=2, after=None)} - ), - user_id="x", - redis_client=redis_client, - ) - merger_vs_cache = redis_client.get(name="merger_view_session_example_x") - # Для использования синхронной и асинхронной фикстуры в одном тесте проверяем метод get - if inspect.iscoroutine(merger_vs_cache): - merger_vs_cache = json.loads(await merger_vs_cache) - else: - merger_vs_cache = json.loads(merger_vs_cache) - - assert merger_vs_res.data == ["x_11", "x_12", "x_13", "x_14", "x_15", "x_16", "x_17", "x_18", "x_19", "x_20"] - assert len(merger_vs_cache) == merger_vs.session_size - assert merger_vs_cache[10:20] == merger_vs_res.data - - -@pytest.mark.parametrize("redis_client", ["sync", "async"], indirect=True) -@pytest.mark.asyncio -async def test_merger_view_session_deduplication(redis_client) -> None: - merger_vs = MergerViewSession.parse_obj(MERGER_VIEW_SESSION_DUPS_CONFIG) - merger_vs_res = await merger_vs.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage(data={}), - user_id="x", - redis_client=redis_client, - ) - merger_vs_cache = redis_client.get(name="merger_view_session_example_x") - # Для использования синхронной и асинхронной фикстуры в одном тесте проверяем метод get - if inspect.iscoroutine(merger_vs_cache): - merger_vs_cache = json.loads(await merger_vs_cache) - else: - merger_vs_cache = json.loads(merger_vs_cache) - - assert merger_vs_res.data == [i for i in range(1, 11)] - assert len(merger_vs_cache) == merger_vs.session_size - assert merger_vs_cache[:10] == merger_vs_res.data diff --git a/tests/test_mixers.py b/tests/test_mixers.py new file mode 100644 index 0000000..488752a --- /dev/null +++ b/tests/test_mixers.py @@ -0,0 +1,69 @@ +import pytest +from smartfeed.models.subfeed import SubFeed +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor +from smartfeed.models.mixers import MergerPercentage, MergerPercentageItem, MergerAppend, MergerPositional +from tests.conftest import METHODS + + +@pytest.fixture +def ctx(): + return ExecutionContext(session_id="s1", methods_dict=METHODS, redis=None) + + +@pytest.mark.asyncio +async def test_executor_runs_subfeed(ctx): + node = SubFeed(subfeed_id="items", method_name="items") + result = await run_executor.run(node, ctx, limit=5, cursor={}) + assert len(result.data) == 5 + assert result.data[0]["_smartfeed_debug_info"]["source"] == "items" + + +@pytest.mark.asyncio +async def test_percentage_40_60(ctx): + node = MergerPercentage( + node_id="mix", + items=[ + MergerPercentageItem( + percentage=40, + data=SubFeed(subfeed_id="a", method_name="items"), + ), + MergerPercentageItem( + percentage=60, + data=SubFeed(subfeed_id="b", method_name="items"), + ), + ], + ) + result = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(result.data) == 10 + sources = [item["_smartfeed_debug_info"]["source"] for item in result.data] + assert sources.count("a") == 4 + assert sources.count("b") == 6 + + +@pytest.mark.asyncio +async def test_append_two_subfeeds(ctx): + node = MergerAppend( + node_id="append", + items=[ + SubFeed(subfeed_id="a", method_name="items"), + SubFeed(subfeed_id="b", method_name="items"), + ], + ) + result = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(result.data) == 10 + + +@pytest.mark.asyncio +async def test_positional_inserts_at_positions(ctx): + node = MergerPositional( + node_id="pos", + positions=[1, 3], + positional=SubFeed(subfeed_id="promo", method_name="items"), + default=SubFeed(subfeed_id="regular", method_name="items"), + ) + result = await run_executor.run(node, ctx, limit=5, cursor={}) + sources = [item["_smartfeed_debug_info"]["source"] for item in result.data] + assert sources[0] == "promo" # position 1 + assert sources[1] == "regular" + assert sources[2] == "promo" # position 3 diff --git a/tests/test_pagination_edge_cases.py b/tests/test_pagination_edge_cases.py new file mode 100644 index 0000000..3a8b21a --- /dev/null +++ b/tests/test_pagination_edge_cases.py @@ -0,0 +1,95 @@ +"""Pagination edge cases: has_next_page exactness at exhaustion, empty sources, +limit larger than the pool, and shuffle interacting with cross-page dedup. + +These are green controls -- they pin correct behavior that a regression could +break (a phantom trailing empty page, a lost final item, shuffle defeating the +seen-set). +""" + +import math + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.models.subfeed import SubFeed +from smartfeed.execution import executor as run_executor + + +def _redis(): + return fakeredis.aioredis.FakeRedis() + + +# --------------------------------------------------------------------------- +# has_next_page exactness: no phantom trailing empty page, no lost final item +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.parametrize("n,limit", [(25, 10), (30, 10), (20, 10), (40, 10), (5, 10), (1, 10)]) +async def test_has_next_exact_passthrough(n, limit): + src = S.ScriptedSource(S.unique_pool(n)) + node = S.wrapper(S.subfeed("src", "src")) # passthrough, no cache + ctx = S.make_ctx({"src": src}, redis=None) + pages = await S.drain(node, ctx, limit=limit, max_pages=50) + assert sum(len(p) for p in pages) == n, "not every item was served" + assert all(len(p) > 0 for p in pages), "phantom trailing empty page" + assert len(pages) == max(1, math.ceil(n / limit)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("n,limit", [(25, 10), (30, 10), (20, 10), (45, 10)]) +async def test_has_next_exact_cached(n, limit): + src = S.ScriptedSource(S.unique_pool(n)) + node = S.wrapper(S.subfeed("src", "src"), session_size=20) + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=limit, max_pages=50) + assert sum(len(p) for p in pages) == n + assert all(len(p) > 0 for p in pages), "phantom trailing empty page" + + +# --------------------------------------------------------------------------- +# Empty source +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_empty_source_returns_empty_no_next(): + src = S.ScriptedSource([]) + node = S.wrapper(S.subfeed("src", "src"), session_size=20, dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=_redis()) + r = await run_executor.run(node, ctx, limit=10, cursor={}) + assert r.data == [] + assert r.has_next_page is False + + +# --------------------------------------------------------------------------- +# limit larger than the whole pool +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_limit_larger_than_pool(): + src = S.ScriptedSource(S.unique_pool(5)) + node = S.wrapper(S.subfeed("src", "src")) + ctx = S.make_ctx({"src": src}, redis=None) + pages = await S.drain(node, ctx, limit=10, max_pages=10) + assert [it["id"] for pg in pages for it in pg] == list(range(5)) + + +# --------------------------------------------------------------------------- +# shuffle=True must not defeat cross-page dedup (seen-set) +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_shuffle_does_not_defeat_cross_page_dedup(): + pool = S.unique_pool(30) + pool.insert(25, {"id": 3, "val": "again"}) # id 3 re-emitted on a later page + src = S.ScriptedSource(pool) + node = S.wrapper(SubFeed(subfeed_id="src", method_name="src", shuffle=True), dedup_key="id") + ctx = S.make_ctx({"src": src}, redis=_redis()) + pages = await S.drain(node, ctx, limit=5, max_pages=50) + ids = S.flat_ids(pages) + assert ids.count(3) == 1, "shuffle let a cross-page duplicate through the seen-set" + S.assert_full_coverage(pages, set(range(30))) diff --git a/tests/test_parsing_config.py b/tests/test_parsing_config.py deleted file mode 100644 index 4d789cb..0000000 --- a/tests/test_parsing_config.py +++ /dev/null @@ -1,47 +0,0 @@ -import pytest - -from smartfeed.manager import FeedManager -from smartfeed.schemas import ( - FeedConfig, - MergerAppend, - MergerPercentage, - MergerPercentageGradient, - MergerPercentageItem, - MergerPositional, - MergerViewSession, - SubFeed, -) -from tests.fixtures.configs import METHODS_DICT, PARSING_CONFIG_FIXTURE - - -@pytest.mark.asyncio -async def test_parsing_config() -> None: - """ - Тест для проверки парсинга JSON-файла конфигурации. - """ - - feed_manager = FeedManager(config=PARSING_CONFIG_FIXTURE, methods_dict=METHODS_DICT) - - # Feed Config. - assert isinstance(feed_manager.feed_config, FeedConfig) - # Merger Positional. - assert isinstance(feed_manager.feed_config.feed, MergerPositional) - # Merger Append. - assert isinstance(feed_manager.feed_config.feed.positional, MergerAppend) - # SubFeed with SubFeed Params. - assert isinstance(feed_manager.feed_config.feed.positional.items[0], SubFeed) - # Merger Percentage Gradient. - assert isinstance(feed_manager.feed_config.feed.positional.items[1], MergerPercentageGradient) - # Merger View Session. - assert isinstance(feed_manager.feed_config.feed.positional.items[2], MergerViewSession) - # Merger Percentage. - assert isinstance(feed_manager.feed_config.feed.default, MergerPercentage) - # Merger Percentage Item. - assert isinstance(feed_manager.feed_config.feed.default.items[0], MergerPercentageItem) - # SubFeed without SubFeed Params. - assert isinstance(feed_manager.feed_config.feed.positional.items[1].item_from.data, SubFeed) - assert isinstance(feed_manager.feed_config.feed.positional.items[1].item_to.data, SubFeed) - assert isinstance(feed_manager.feed_config.feed.positional.items[2].data, SubFeed) - # SubFeed with Raise Exception False. - assert isinstance(feed_manager.feed_config.feed.default.items[0].data, SubFeed) - assert feed_manager.feed_config.feed.default.items[0].data.raise_error is False diff --git a/tests/test_redis_live.py b/tests/test_redis_live.py deleted file mode 100644 index 1a23839..0000000 --- a/tests/test_redis_live.py +++ /dev/null @@ -1,167 +0,0 @@ -import asyncio -import json -import pytest -import redis -from redis.asyncio import Redis as AsyncRedis -import time - -from smartfeed.schemas import FeedResultNextPage, MergerViewSession -from tests.fixtures.configs import METHODS_DICT -from tests.fixtures.mergers import MERGER_VIEW_SESSION_CONFIG - - -class RedisReplicationSimulator: - """ - Симулятор задержки репликации Redis для тестирования проблемы кластера. - """ - def __init__(self, real_client): - self.real_client = real_client - self.write_delay = 0.1 # Задержка для имитации репликации - self.pending_writes = {} # Ключи которые только что записали - - def exists(self, cache_key): - return self.real_client.exists(cache_key) - - def set(self, name, value, ex=None): - # Записываем в реальный Redis - result = self.real_client.set(name, value, ex=ex) - # Помечаем что этот ключ только что записан (имитация репликации) - self.pending_writes[name] = time.time() - return result - - def get(self, name): - # Если ключ только что записан (в течение write_delay секунд), возвращаем None - if name in self.pending_writes: - write_time = self.pending_writes[name] - if time.time() - write_time < self.write_delay: - return None # Имитация задержки репликации - else: - # Задержка прошла, можно удалить из pending - del self.pending_writes[name] - - # Обычное чтение из Redis - return self.real_client.get(name) - - -@pytest.mark.asyncio -async def test_redis_replication_delay_problem(): - """ - Тест для воспроизведения проблемы репликации Redis с использованием - RedisReplicationSimulator для имитации задержки. - """ - - # Подключаемся к Redis (должен быть запущен локально) - try: - real_client = redis.Redis(host='localhost', port=6379, db=0) - real_client.ping() # Проверяем соединение - except (redis.ConnectionError, redis.ResponseError): - pytest.skip("Redis not available for live testing") - - # Очищаем тестовый ключ - test_key = "test_merger_view_session_test_user" - real_client.delete(test_key) - - # Используем симулятор задержки репликации - redis_client = RedisReplicationSimulator(real_client) - merger_vs = MergerViewSession.parse_obj(MERGER_VIEW_SESSION_CONFIG) - - print("\n=== Демонстрация проблемы с задержкой репликации ===") - - try: - # Этот вызов должен воспроизвести проблему с оригинальным кодом - result = await merger_vs.get_data( - methods_dict=METHODS_DICT, - limit=10, - next_page=FeedResultNextPage(data={}), - user_id="test_user", - redis_client=redis_client, - ) - - print("✅ Исправление работает! Получили результат без ошибки:") - print(f" Данные: {result.data[:5]}... (показаны первые 5)") - print(f" Размер: {len(result.data)}") - print(f" Есть следующая страница: {result.has_next_page}") - - # Проверяем что получили валидные данные - assert len(result.data) == 10 - assert result.data[0] == "test_user_1" - assert result.has_next_page is True - - except TypeError as e: - if "the JSON object must be str, bytes or bytearray, not NoneType" in str(e): - print("❌ Проблема НЕ исправлена! Все еще получаем TypeError") - raise - else: - print(f"❓ Неожиданная ошибка: {e}") - raise - - finally: - # Очистка - real_client.delete(test_key) - real_client.close() - - -@pytest.mark.asyncio -async def test_redis_multiple_requests(): - """ - Тест множественных запросов для проверки стабильности исправления. - """ - - try: - real_client = redis.Redis(host='localhost', port=6379, db=0) - real_client.ping() - except (redis.ConnectionError, redis.ResponseError): - pytest.skip("Redis not available for live testing") - - test_key = "test_merger_multiple_test_user" - real_client.delete(test_key) - - redis_client = RedisReplicationSimulator(real_client) - merger_vs = MergerViewSession.parse_obj(MERGER_VIEW_SESSION_CONFIG) - - print("\n=== Тест множественных запросов ===") - - try: - # Первый запрос - создает кэш - result1 = await merger_vs.get_data( - methods_dict=METHODS_DICT, - limit=5, - next_page=FeedResultNextPage(data={}), - user_id="test_user", - redis_client=redis_client, - ) - - print(f"Первый запрос: получили {len(result1.data)} элементов") - - # Ждем чтобы задержка репликации прошла - await asyncio.sleep(0.2) - - # Второй запрос - должен использовать кэш - from smartfeed.schemas import FeedResultNextPageInside - result2 = await merger_vs.get_data( - methods_dict=METHODS_DICT, - limit=5, - next_page=FeedResultNextPage( - data={"merger_view_session_example": FeedResultNextPageInside(page=2, after=None)} - ), - user_id="test_user", - redis_client=redis_client, - ) - - print(f"Второй запрос: получили {len(result2.data)} элементов") - print(f"Данные второй страницы: {result2.data}") - - # Проверяем что получили разные данные (пагинация работает) - assert result1.data != result2.data - assert len(result2.data) == 5 - - print("✅ Множественные запросы работают корректно!") - - finally: - real_client.delete(test_key) - real_client.close() - - -if __name__ == "__main__": - # Для запуска напрямую без pytest - asyncio.run(test_redis_replication_delay_problem()) \ No newline at end of file diff --git a/tests/test_redis_lock.py b/tests/test_redis_lock.py new file mode 100644 index 0000000..fdc1e15 --- /dev/null +++ b/tests/test_redis_lock.py @@ -0,0 +1,33 @@ +import pytest +import fakeredis.aioredis + +from smartfeed.execution.redis_lock import RedisLock + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.mark.asyncio +async def test_lock_acquired(redis): + async with RedisLock(redis, "test:lock") as acquired: + assert acquired is True + assert await redis.exists("test:lock") + assert not await redis.exists("test:lock") + + +@pytest.mark.asyncio +async def test_lock_contention(redis): + async with RedisLock(redis, "test:lock") as first: + assert first is True + async with RedisLock(redis, "test:lock") as second: + assert second is False + + +@pytest.mark.asyncio +async def test_lock_released_after_first(redis): + async with RedisLock(redis, "test:lock"): + pass + async with RedisLock(redis, "test:lock") as acquired: + assert acquired is True diff --git a/tests/test_resilience.py b/tests/test_resilience.py new file mode 100644 index 0000000..ee7bc4c --- /dev/null +++ b/tests/test_resilience.py @@ -0,0 +1,245 @@ +"""Tests for resilience: source failures, positional accuracy, rerank error handling.""" + +import pytest +import fakeredis.aioredis + +from smartfeed.models.base import FeedResult +from smartfeed.models.subfeed import SubFeed +from smartfeed.models.wrapper import Wrapper, WrapperCache, WrapperRerank, WrapperDedup +from smartfeed.models.mixers import MergerPercentage, MergerPercentageItem, MergerPositional +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +async def make_source(prefix, user_id, limit, next_page, **kwargs): + page = next_page.get("page", 1) + start = (page - 1) * limit + data = [{"id": f"{prefix}_{start + i}", "val": prefix} for i in range(limit)] + return FeedResult(data=data, next_page={"page": page + 1}, has_next_page=True) + + +async def make_failing_source(user_id, limit, next_page, **kwargs): + raise RuntimeError("source unavailable") + + +async def make_promo(user_id, limit, next_page, **kwargs): + data = [{"id": f"promo_{i}", "val": "promo"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + +async def rerank_reverse(items, session_id): + return list(reversed(items)) + + +async def rerank_crash(items, session_id): + raise RuntimeError("ML service down") + + +METHODS = { + "source_a": lambda *a, **kw: make_source("a", *a, **kw), + "source_b": lambda *a, **kw: make_source("b", *a, **kw), + "failing": make_failing_source, + "promo": make_promo, + "rerank_ok": rerank_reverse, + "rerank_crash": rerank_crash, +} + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.fixture +def ctx(redis): + return ExecutionContext(session_id="test", methods_dict=METHODS, redis=redis) + + +# --------------------------------------------------------------------------- +# 1. Source failure: fill page from remaining source +# --------------------------------------------------------------------------- + + +class TestSourceFailureFallback: + @pytest.mark.asyncio + async def test_one_source_fails_page_filled_from_other(self, ctx): + """If one source in percentage mix fails, the other fills the page.""" + node = Wrapper( + node_id="w", + cache=WrapperCache(session_size=50, session_ttl=300), + data=MergerPercentage( + node_id="mix", + items=[ + MergerPercentageItem( + percentage=50, + data=SubFeed(subfeed_id="good", method_name="source_a"), + ), + MergerPercentageItem( + percentage=50, + data=SubFeed(subfeed_id="bad", method_name="failing", raise_error=False), + ), + ], + ), + ) + result = await run_executor.run(node, ctx, limit=20, cursor={}) + # Should have items from source_a only (source_b failed gracefully) + assert len(result.data) > 0 + for item in result.data: + assert item["val"] == "a" + + @pytest.mark.asyncio + async def test_all_sources_fail_returns_empty(self, ctx): + """If all sources fail, return empty gracefully.""" + node = Wrapper( + node_id="w", + cache=WrapperCache(session_size=50, session_ttl=300), + data=MergerPercentage( + node_id="mix", + items=[ + MergerPercentageItem( + percentage=50, + data=SubFeed(subfeed_id="bad1", method_name="failing", raise_error=False), + ), + MergerPercentageItem( + percentage=50, + data=SubFeed(subfeed_id="bad2", method_name="failing", raise_error=False), + ), + ], + ), + ) + result = await run_executor.run(node, ctx, limit=20, cursor={}) + assert result.data == [] + + +# --------------------------------------------------------------------------- +# 2. Positional: strict position enforcement +# --------------------------------------------------------------------------- + + +class TestPositionalStrictPositions: + @pytest.mark.asyncio + async def test_promo_at_exact_positions(self, ctx): + """Promo items must be at positions 1, 3, 5, 7 (1-indexed).""" + node = MergerPositional( + node_id="pos", + positions=[1, 3, 5, 7], + positional=SubFeed(subfeed_id="promo", method_name="promo"), + default=SubFeed(subfeed_id="regular", method_name="source_a"), + ) + result = await run_executor.run(node, ctx, limit=20, cursor={}) + assert len(result.data) == 20 + + for pos in [1, 3, 5, 7]: + item = result.data[pos - 1] # 1-indexed -> 0-indexed + source = item.get("_smartfeed_debug_info", {}).get("source", "") + assert source == "promo", f"Position {pos}: expected promo, got {source} (id={item.get('id')})" + + @pytest.mark.asyncio + async def test_non_promo_positions_filled_by_default(self, ctx): + """Non-promo positions must be filled by default source.""" + node = MergerPositional( + node_id="pos", + positions=[1, 3, 5, 7], + positional=SubFeed(subfeed_id="promo", method_name="promo"), + default=SubFeed(subfeed_id="regular", method_name="source_a"), + ) + result = await run_executor.run(node, ctx, limit=20, cursor={}) + + non_promo_positions = [i for i in range(20) if (i + 1) not in [1, 3, 5, 7]] + for pos in non_promo_positions: + item = result.data[pos] + source = item.get("_smartfeed_debug_info", {}).get("source", "") + assert source == "regular", f"Position {pos + 1}: expected regular, got {source}" + + @pytest.mark.asyncio + async def test_positional_with_dedup_preserves_positions(self, ctx): + """Even with outer dedup, promo positions must be preserved.""" + node = Wrapper( + node_id="dedup_outer", + dedup=WrapperDedup(dedup_key="id"), + data=MergerPositional( + node_id="pos", + positions=[1, 3, 5, 7], + positional=SubFeed(subfeed_id="promo", method_name="promo", dedup_priority=10), + default=SubFeed(subfeed_id="regular", method_name="source_a", dedup_priority=1), + ), + ) + result = await run_executor.run(node, ctx, limit=20, cursor={}) + assert len(result.data) == 20 + + for pos in [1, 3, 5, 7]: + item = result.data[pos - 1] + source = item.get("_smartfeed_debug_info", {}).get("source", "") + assert source == "promo", f"Position {pos}: expected promo after dedup, got {source}" + + +# --------------------------------------------------------------------------- +# 3. Rerank failure: raise_error=True vs False +# --------------------------------------------------------------------------- + + +class TestRerankFailureHandling: + @pytest.mark.asyncio + async def test_rerank_crash_raise_error_true(self, ctx): + """With raise_error=True (default), rerank failure crashes the request.""" + node = Wrapper( + node_id="w", + cache=WrapperCache(session_size=50, session_ttl=300), + rerank=WrapperRerank(method_name="rerank_crash", raise_error=True), + data=SubFeed(subfeed_id="src", method_name="source_a"), + ) + with pytest.raises(RuntimeError, match="ML service down"): + await run_executor.run(node, ctx, limit=10, cursor={}) + + @pytest.mark.asyncio + async def test_rerank_crash_raise_error_false(self, ctx): + """With raise_error=False, rerank failure keeps original order.""" + node = Wrapper( + node_id="w", + cache=WrapperCache(session_size=50, session_ttl=300), + rerank=WrapperRerank(method_name="rerank_crash", raise_error=False), + data=SubFeed(subfeed_id="src", method_name="source_a"), + ) + result = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(result.data) == 10 + # Items should be in original order (not reversed, since rerank_crash failed) + ids = [item["id"] for item in result.data] + assert ids[0] == "a_0" # original order preserved + + @pytest.mark.asyncio + async def test_rerank_ok_actually_reranks(self, ctx): + """Verify working rerank changes order (control test).""" + node = Wrapper( + node_id="w", + cache=WrapperCache(session_size=50, session_ttl=300), + rerank=WrapperRerank(method_name="rerank_ok"), + data=SubFeed(subfeed_id="src", method_name="source_a"), + ) + result = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(result.data) == 10 + # rerank_ok reverses, so last item of original is first + ids = [item["id"] for item in result.data] + assert ids[0] != "a_0" # order changed + + @pytest.mark.asyncio + async def test_rerank_fail_soft_page2_still_works(self, ctx): + """After fail-soft rerank, page 2 from cache still works.""" + node = Wrapper( + node_id="w", + cache=WrapperCache(session_size=50, session_ttl=300), + rerank=WrapperRerank(method_name="rerank_crash", raise_error=False), + data=SubFeed(subfeed_id="src", method_name="source_a"), + ) + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + assert len(r1.data) == 10 + assert len(r2.data) == 10 + # Different pages + ids1 = {item["id"] for item in r1.data} + ids2 = {item["id"] for item in r2.data} + assert not ids1 & ids2 # no overlap diff --git a/tests/test_shared_lock_contention.py b/tests/test_shared_lock_contention.py new file mode 100644 index 0000000..e1f7b00 --- /dev/null +++ b/tests/test_shared_lock_contention.py @@ -0,0 +1,43 @@ +"""Two wrappers sharing a cache_key race one cold build: the shared-segment lock +must serialize them so the child is fetched exactly once (the _build_shared_base +waiter path had no concurrent coverage). + +The child has latency, so both wrappers are guaranteed to be inside the cold +build window together; their own coldlocks differ (different wrapper config +hashes), so contention lands on the shared-segment lock itself. +""" + +import asyncio + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +async def identity(items, session_id): + return items + + +async def reverse(items, session_id): + return list(reversed(items)) + + +@pytest.mark.asyncio +async def test_concurrent_shared_cold_build_fetches_child_once(): + src = S.ScriptedSource(S.unique_pool(100), latency=0.3) + ctx = S.make_ctx({"src": src, "identity": identity, "reverse": reverse}, redis=fakeredis.aioredis.FakeRedis()) + w_a = S.wrapper(S.subfeed("src", "src"), node_id="a", session_size=20, cache_key="pool", rerank_method="identity") + w_b = S.wrapper(S.subfeed("src", "src"), node_id="b", session_size=20, cache_key="pool", rerank_method="reverse") + + a, b = await asyncio.gather( + run_executor.run(w_a, ctx, limit=10, cursor={}), + run_executor.run(w_b, ctx, limit=10, cursor={}), + ) + + assert src.calls == 1, f"shared base must be fetched once, got {src.calls}" + assert len(a.data) == 10 and len(b.data) == 10 + # Same shared base, per-wrapper rerank: b serves a's window reversed-ish (both + # windows come from the same 20-item segment). + assert {it["id"] for it in a.data} | {it["id"] for it in b.data} <= set(range(20)) diff --git a/tests/test_sharp_edges.py b/tests/test_sharp_edges.py new file mode 100644 index 0000000..93388e6 --- /dev/null +++ b/tests/test_sharp_edges.py @@ -0,0 +1,82 @@ +"""README's documented "sharp edges" pinned as tests. + +Edge 2 (rerank length mismatch always raises) and edges 1/3/4 previously had no +coverage in the exact configurations the README warns about. Edge 5 (cursor +validation) lives in test_input_validation.py. +""" + +import pytest + +from smartfeed.models.base import FeedResult +from smartfeed.models.subfeed import SubFeed +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +# -- edge 1: missing method_name ---------------------------------------------- + + +@pytest.mark.asyncio +async def test_missing_method_raises_keyerror_even_fail_soft(): + """method_name is resolved BEFORE the try block: KeyError regardless of raise_error.""" + node = SubFeed(subfeed_id="s", method_name="no_such_method", raise_error=False) + ctx = S.make_ctx({}) + with pytest.raises(KeyError, match="no_such_method"): + await run_executor.run(node, ctx, limit=5, cursor={}) + + +# -- edge 3: fetch function without **kwargs ----------------------------------- + + +async def no_kwargs_fetch(user_id, limit, next_page): + return FeedResult(data=[{"id": 1}], next_page={}, has_next_page=False) + + +@pytest.mark.asyncio +async def test_no_kwargs_source_raises_typeerror(): + node = S.subfeed("s", "src") + ctx = S.make_ctx({"src": no_kwargs_fetch}) + with pytest.raises(TypeError): + await run_executor.run(node, ctx, limit=5, cursor={}) + + +@pytest.mark.asyncio +async def test_no_kwargs_source_swallowed_to_empty_page_when_fail_soft(): + """The documented footgun: under raise_error=False the signature bug reads + exactly like 'this source had no items today'.""" + node = SubFeed(subfeed_id="s", method_name="src", raise_error=False) + ctx = S.make_ctx({"src": no_kwargs_fetch}) + r = await run_executor.run(node, ctx, limit=5, cursor={}) + assert r.data == [] + assert r.has_next_page is False + + +# -- edge 4: subfeed_params colliding with injected kwargs ---------------------- + + +@pytest.mark.asyncio +async def test_subfeed_params_collision_raises_typeerror(): + async def fine_fetch(user_id, limit, next_page, **kw): + return FeedResult(data=[], next_page={}, has_next_page=False) + + node = SubFeed(subfeed_id="s", method_name="src", subfeed_params={"limit": 3}) + ctx = S.make_ctx({"src": fine_fetch}) + with pytest.raises(TypeError): + await run_executor.run(node, ctx, limit=5, cursor={}) + + +# -- edge 2: rerank length mismatch always raises ------------------------------- + + +@pytest.mark.asyncio +async def test_rerank_length_mismatch_raises_even_fail_soft(): + """rerank.raise_error=False covers only exceptions FROM the callable; the + length contract is enforced unconditionally.""" + + async def drops_one(items, session_id): + return items[:-1] + + node = S.wrapper(S.subfeed("s", "src"), rerank_method="drops_one", rerank_raise=False) + ctx = S.make_ctx({"src": S.ScriptedSource(S.unique_pool(10)), "drops_one": drops_one}) + with pytest.raises(ValueError, match="must return exactly"): + await run_executor.run(node, ctx, limit=5, cursor={}) diff --git a/tests/test_silent_degradation.py b/tests/test_silent_degradation.py new file mode 100644 index 0000000..c19a0d2 --- /dev/null +++ b/tests/test_silent_degradation.py @@ -0,0 +1,30 @@ +"""Documented silent downgrades: these configurations must +degrade gracefully, not crash -- and must not write state they cannot honor. +""" + +import pytest +import fakeredis.aioredis + +from tests import sources as S +from smartfeed.execution import executor as run_executor + + +@pytest.mark.asyncio +async def test_cache_without_redis_degrades_to_passthrough(): + node = S.wrapper(S.subfeed("s", "src"), session_size=20) + ctx = S.make_ctx({"src": S.ScriptedSource(S.unique_pool(50))}, redis=None) + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + assert [it["id"] for it in r1.data] == list(range(10)) + # Passthrough exposes the raw child cursor and keeps paginating without Redis. + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + assert [it["id"] for it in r2.data] == list(range(10, 20)) + + +@pytest.mark.asyncio +async def test_cache_key_without_cache_is_pure_passthrough(): + redis = fakeredis.aioredis.FakeRedis() + node = S.wrapper(S.subfeed("s", "src"), cache_key="shared") # no cache block + ctx = S.make_ctx({"src": S.ScriptedSource(S.unique_pool(50))}, redis=redis) + r = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(r.data) == 10 + assert await redis.keys("*") == [] # no cache/shared/lock keys written diff --git a/tests/test_sub_feed.py b/tests/test_sub_feed.py deleted file mode 100644 index 7da1924..0000000 --- a/tests/test_sub_feed.py +++ /dev/null @@ -1,78 +0,0 @@ -import pytest - -from smartfeed.schemas import FeedResultNextPage, SubFeed -from tests.fixtures.configs import METHODS_DICT -from tests.fixtures.subfeeds import ( - SUBFEED_CONFIG, - SUBFEED_CONFIG_NO_RAISE_ERROR, - SUBFEED_CONFIG_RAISE_ERROR, - SUBFEED_WITH_PARAMS_CONFIG, -) - - -@pytest.mark.asyncio -async def test_sub_feed() -> None: - """ - Тест для проверки получения данных из субфида (без параметров). - """ - - sub_feed = SubFeed.parse_obj(SUBFEED_CONFIG) - sub_feed_data = await sub_feed.get_data( - methods_dict=METHODS_DICT, - limit=15, - next_page=FeedResultNextPage(data={}), - user_id="x", - ) - - assert sub_feed_data.data == [f"x_{i}" for i in range(1, 16)] - - -@pytest.mark.asyncio -async def test_sub_feed_with_params() -> None: - """ - Тест для проверки получения данных из субфида (с параметрами). - """ - - sub_feed = SubFeed.parse_obj(SUBFEED_WITH_PARAMS_CONFIG) - sub_feed_data = await sub_feed.get_data( - methods_dict=METHODS_DICT, - limit=15, - next_page=FeedResultNextPage(data={}), - user_id="x", - ) - - assert sub_feed_data.data == [f"x_{i}" for i in range(1, 11)] - - -@pytest.mark.asyncio -async def test_sub_feed_raise_error() -> None: - """ - Тест для проверки получения данных из субфида (без параметров). - """ - - sub_feed = SubFeed.parse_obj(SUBFEED_CONFIG_RAISE_ERROR) - - with pytest.raises(Exception): - await sub_feed.get_data( - methods_dict=METHODS_DICT, - limit=15, - next_page=FeedResultNextPage(data={}), - user_id="x", - ) - - -@pytest.mark.asyncio -async def test_sub_feed_no_raise_error() -> None: - """ - Тест для проверки получения данных из субфида (без параметров). - """ - - sub_feed = SubFeed.parse_obj(SUBFEED_CONFIG_NO_RAISE_ERROR) - sub_feed_data = await sub_feed.get_data( - methods_dict=METHODS_DICT, - limit=15, - next_page=FeedResultNextPage(data={}), - user_id="x", - ) - - assert sub_feed_data.data == [] diff --git a/tests/test_subfeed.py b/tests/test_subfeed.py new file mode 100644 index 0000000..2e3e3eb --- /dev/null +++ b/tests/test_subfeed.py @@ -0,0 +1,43 @@ +import pytest +from smartfeed.models.subfeed import SubFeed +from tests.conftest import METHODS + + +@pytest.mark.asyncio +async def test_subfeed_returns_data(): + sf = SubFeed(subfeed_id="items", method_name="items") + result = await sf.execute(methods_dict=METHODS, session_id="s1", limit=5, cursor={}) + assert len(result.data) == 5 + assert result.has_next_page is True + + +@pytest.mark.asyncio +async def test_subfeed_stamps_debug_info(): + sf = SubFeed(subfeed_id="my_source", method_name="items") + result = await sf.execute(methods_dict=METHODS, session_id="s1", limit=3, cursor={}) + for item in result.data: + info = item["_smartfeed_debug_info"] + assert info["source"] == "my_source" + + +@pytest.mark.asyncio +async def test_subfeed_raise_error(): + sf = SubFeed(subfeed_id="err", method_name="error") + with pytest.raises(RuntimeError, match="subfeed error"): + await sf.execute(methods_dict=METHODS, session_id="s1", limit=5, cursor={}) + + +@pytest.mark.asyncio +async def test_subfeed_swallow_error(): + sf = SubFeed(subfeed_id="err", method_name="error", raise_error=False) + result = await sf.execute(methods_dict=METHODS, session_id="s1", limit=5, cursor={}) + assert result.data == [] + assert result.has_next_page is False + + +@pytest.mark.asyncio +async def test_subfeed_cursor_propagation(): + sf = SubFeed(subfeed_id="items", method_name="items") + r1 = await sf.execute(methods_dict=METHODS, session_id="s1", limit=5, cursor={}) + r2 = await sf.execute(methods_dict=METHODS, session_id="s1", limit=5, cursor=r1.next_page) + assert r2.data[0]["id"] == 5 # page 2 starts at id=5 diff --git a/tests/test_wrapper_cache.py b/tests/test_wrapper_cache.py new file mode 100644 index 0000000..104883a --- /dev/null +++ b/tests/test_wrapper_cache.py @@ -0,0 +1,77 @@ +import pytest +import fakeredis.aioredis + +from smartfeed.models.wrapper import Wrapper, WrapperCache +from smartfeed.models.subfeed import SubFeed +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor +from tests.conftest import METHODS + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.fixture +def ctx(redis): + return ExecutionContext(session_id="s1", methods_dict=METHODS, redis=redis) + + +def _make_wrapper(node_id="cached", session_size=50, session_ttl=300): + return Wrapper( + node_id=node_id, + cache=WrapperCache(session_size=session_size, session_ttl=session_ttl), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + + +@pytest.mark.asyncio +async def test_cache_cold_build(ctx, redis): + node = _make_wrapper() + result = await run_executor.run(node, ctx, limit=10, cursor={}) + assert len(result.data) == 10 + # Some Redis key should exist + all_keys = list(await redis.keys("sf:*")) + assert len(all_keys) >= 1 + + +@pytest.mark.asyncio +async def test_cache_warm_hit(ctx): + node = _make_wrapper() + r1 = await run_executor.run(node, ctx, limit=10, cursor={}) + r2 = await run_executor.run(node, ctx, limit=10, cursor=r1.next_page) + # Different page + assert r2.data[0]["id"] != r1.data[0]["id"] + assert len(r2.data) == 10 + + +@pytest.mark.asyncio +async def test_generation_id_in_cursor(ctx): + node = _make_wrapper() + result = await run_executor.run(node, ctx, limit=10, cursor={}) + assert "gen" in result.next_page.get("cached", {}) + + +@pytest.mark.asyncio +async def test_stale_gen_resets_to_page1(ctx, redis): + node = _make_wrapper() + await run_executor.run(node, ctx, limit=10, cursor={}) + # Flush to simulate TTL expiry + await redis.flushall() + stale_cursor = {"cached": {"offset": 40, "gen": "stale_nonce"}} + r2 = await run_executor.run(node, ctx, limit=10, cursor=stale_cursor) + # Should reset: served page 1 (offset 0), next cursor points at offset 10 + assert r2.next_page["cached"]["offset"] == 10 + assert r2.next_page["cached"]["gen"] != "stale_nonce" + + +@pytest.mark.asyncio +async def test_wrapper_without_cache_passthrough(ctx): + """Wrapper without cache passes through to child directly.""" + node = Wrapper( + node_id="passthrough", + data=SubFeed(subfeed_id="items", method_name="items"), + ) + result = await run_executor.run(node, ctx, limit=5, cursor={}) + assert len(result.data) == 5 diff --git a/tests/test_wrapper_dedup.py b/tests/test_wrapper_dedup.py new file mode 100644 index 0000000..cedff78 --- /dev/null +++ b/tests/test_wrapper_dedup.py @@ -0,0 +1,86 @@ +import pytest +import fakeredis.aioredis + +from smartfeed.models.wrapper import Wrapper, WrapperDedup +from smartfeed.models.subfeed import SubFeed +from smartfeed.models.mixers import MergerAppend +from smartfeed.models.base import FeedResult +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor + + +async def make_dupes_a(user_id, limit, next_page, **kw): + data = [{"id": i, "val": f"a_{i}"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + +async def make_dupes_b(user_id, limit, next_page, **kw): + data = [{"id": i, "val": f"b_{i}"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + +DEDUP_METHODS = {"dupes_a": make_dupes_a, "dupes_b": make_dupes_b} + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.fixture +def ctx(redis): + return ExecutionContext(session_id="s1", methods_dict=DEDUP_METHODS, redis=redis) + + +@pytest.mark.asyncio +async def test_dedup_removes_duplicates(ctx): + node = Wrapper( + node_id="deduped", + dedup=WrapperDedup(dedup_key="id"), + data=MergerAppend( + node_id="mix", + items=[ + SubFeed(subfeed_id="a", method_name="dupes_a"), + SubFeed(subfeed_id="b", method_name="dupes_b"), + ], + ), + ) + result = await run_executor.run(node, ctx, limit=10, cursor={}) + ids = [item["id"] for item in result.data] + assert len(ids) == len(set(ids)) + + +@pytest.mark.asyncio +async def test_dedup_priority_higher_wins(ctx): + node = Wrapper( + node_id="deduped", + dedup=WrapperDedup(dedup_key="id"), + data=MergerAppend( + node_id="mix", + items=[ + SubFeed(subfeed_id="a", method_name="dupes_a", dedup_priority=5), + SubFeed(subfeed_id="b", method_name="dupes_b", dedup_priority=1), + ], + ), + ) + result = await run_executor.run(node, ctx, limit=10, cursor={}) + for item in result.data: + assert item["val"].startswith("a_") + + +@pytest.mark.asyncio +async def test_dedup_equal_priority_first_seen_wins(ctx): + node = Wrapper( + node_id="deduped", + dedup=WrapperDedup(dedup_key="id"), + data=MergerAppend( + node_id="mix", + items=[ + SubFeed(subfeed_id="a", method_name="dupes_a", dedup_priority=1), + SubFeed(subfeed_id="b", method_name="dupes_b", dedup_priority=1), + ], + ), + ) + result = await run_executor.run(node, ctx, limit=5, cursor={}) + for item in result.data: + assert item["val"].startswith("a_") diff --git a/tests/test_wrapper_full_pipeline.py b/tests/test_wrapper_full_pipeline.py new file mode 100644 index 0000000..9e98ebf --- /dev/null +++ b/tests/test_wrapper_full_pipeline.py @@ -0,0 +1,94 @@ +import pytest +import fakeredis.aioredis + +from smartfeed.models import FeedConfig +from smartfeed.models.base import FeedResult +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor + + +async def source_a(user_id, limit, next_page, **kw): + data = [{"id": i, "val": f"a_{i}"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + +async def source_b(user_id, limit, next_page, **kw): + data = [{"id": i + limit // 2, "val": f"b_{i}"} for i in range(limit)] + return FeedResult(data=data, next_page={"page": 2}, has_next_page=True) + + +async def reverse_rerank(items, session_id): + return list(reversed(items)) + + +PIPELINE_METHODS = {"source_a": source_a, "source_b": source_b, "reverse": reverse_rerank} + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.fixture +def ctx(redis): + return ExecutionContext(session_id="test_full", methods_dict=PIPELINE_METHODS, redis=redis) + + +@pytest.mark.asyncio +async def test_full_pipeline(ctx): + config = FeedConfig.model_validate( + { + "version": "2", + "feed": { + "type": "wrapper", + "node_id": "full", + "cache": {"session_size": 30, "session_ttl": 300}, + "rerank": {"method_name": "reverse"}, + "dedup": { + "dedup_key": "id", + }, + "data": { + "type": "merger_percentage", + "node_id": "mix", + "items": [ + { + "percentage": 50, + "data": { + "type": "subfeed", + "subfeed_id": "a", + "method_name": "source_a", + "dedup_priority": 5, + }, + }, + { + "percentage": 50, + "data": { + "type": "subfeed", + "subfeed_id": "b", + "method_name": "source_b", + "dedup_priority": 1, + }, + }, + ], + }, + }, + } + ) + + r1 = await run_executor.run(config.feed, ctx, limit=10, cursor={}) + + # No duplicate ids + ids = [item["id"] for item in r1.data] + assert len(ids) == len(set(ids)) + + # Has generation_id + assert "gen" in r1.next_page.get("full", {}) + + # Has debug info + for item in r1.data: + info = item["_smartfeed_debug_info"] + assert "source" in info + + # Page 2 from cache + r2 = await run_executor.run(config.feed, ctx, limit=10, cursor=r1.next_page) + assert len(r2.data) > 0 diff --git a/tests/test_wrapper_rerank.py b/tests/test_wrapper_rerank.py new file mode 100644 index 0000000..727a7f2 --- /dev/null +++ b/tests/test_wrapper_rerank.py @@ -0,0 +1,86 @@ +import pytest +import fakeredis.aioredis + +from smartfeed.models.wrapper import Wrapper, WrapperCache, WrapperRerank +from smartfeed.models.subfeed import SubFeed +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor +from tests.conftest import METHODS + + +async def reverse_rerank(items, session_id): + return list(reversed(items)) + + +async def bad_rerank(items, session_id): + return items[:1] # contract violation + + +RERANK_METHODS = {**METHODS, "reverse": reverse_rerank, "bad": bad_rerank} + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.fixture +def ctx(redis): + return ExecutionContext(session_id="s1", methods_dict=RERANK_METHODS, redis=redis) + + +@pytest.mark.asyncio +async def test_rerank_with_cache(ctx): + node = Wrapper( + node_id="reranked", + cache=WrapperCache(session_size=20, session_ttl=300), + rerank=WrapperRerank(method_name="reverse"), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + result = await run_executor.run(node, ctx, limit=5, cursor={}) + assert len(result.data) == 5 + # Rerank reversed the order: first item should have highest original id + id_first = result.data[0]["id"] + id_last = result.data[-1]["id"] + assert id_first > id_last # reversed = descending ids + + +@pytest.mark.asyncio +async def test_rerank_without_cache(ctx): + node = Wrapper( + node_id="reranked", + rerank=WrapperRerank(method_name="reverse"), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + result = await run_executor.run(node, ctx, limit=5, cursor={}) + assert len(result.data) == 5 + # The rerank must actually run: "reverse" flips the order, so a silently + # skipped rerank (ascending ids) fails here. + assert result.data[0]["id"] > result.data[-1]["id"] + + +@pytest.mark.asyncio +async def test_rerank_contract_violation(ctx): + node = Wrapper( + node_id="bad", + cache=WrapperCache(session_size=10, session_ttl=300), + rerank=WrapperRerank(method_name="bad"), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + with pytest.raises(ValueError, match="must return exactly"): + await run_executor.run(node, ctx, limit=5, cursor={}) + + +@pytest.mark.asyncio +async def test_rerank_stamps_rerank_position(ctx): + node = Wrapper( + node_id="reranked", + cache=WrapperCache(session_size=20, session_ttl=300), + rerank=WrapperRerank(method_name="reverse"), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + result = await run_executor.run(node, ctx, limit=5, cursor={}) + for item in result.data: + info = item["_smartfeed_debug_info"]["reranked"] + assert "rerank_position" in info + assert "smartfeed_position" in info diff --git a/tests/test_wrapper_shared_cache.py b/tests/test_wrapper_shared_cache.py new file mode 100644 index 0000000..3181fac --- /dev/null +++ b/tests/test_wrapper_shared_cache.py @@ -0,0 +1,111 @@ +import pytest +import fakeredis.aioredis +from smartfeed.models.wrapper import Wrapper, WrapperCache, WrapperRerank +from smartfeed.models.subfeed import SubFeed +from smartfeed.execution.context import ExecutionContext +from smartfeed.execution import executor as run_executor +from tests.conftest import METHODS + + +async def rerank_reverse(items, session_id): + return list(reversed(items)) + + +async def rerank_identity(items, session_id): + return items + + +SHARED_METHODS = {**METHODS, "rerank_a": rerank_reverse, "rerank_b": rerank_identity} + + +@pytest.fixture +def redis(): + return fakeredis.aioredis.FakeRedis() + + +@pytest.fixture +def ctx(redis): + return ExecutionContext(session_id="s1", methods_dict=SHARED_METHODS, redis=redis) + + +@pytest.mark.asyncio +async def test_shared_cache_two_rerankers(ctx): + w_a = Wrapper( + node_id="a", + cache_key="shared", + cache=WrapperCache(session_size=20, session_ttl=300), + rerank=WrapperRerank(method_name="rerank_a"), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + w_b = Wrapper( + node_id="b", + cache_key="shared", + cache=WrapperCache(session_size=20, session_ttl=300), + rerank=WrapperRerank(method_name="rerank_b"), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + r_a = await run_executor.run(w_a, ctx, limit=5, cursor={}) + r_b = await run_executor.run(w_b, ctx, limit=5, cursor={}) + assert len(r_a.data) == 5 + assert len(r_b.data) == 5 + # Different rerank -> different order + assert r_a.data != r_b.data + + +@pytest.mark.asyncio +async def test_shared_cache_base_written_once(ctx, redis): + """Both wrappers use same cache_key -- base cache key should exist once.""" + w_a = Wrapper( + node_id="a", + cache_key="shared_base", + cache=WrapperCache(session_size=20, session_ttl=300), + rerank=WrapperRerank(method_name="rerank_a"), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + w_b = Wrapper( + node_id="b", + cache_key="shared_base", + cache=WrapperCache(session_size=20, session_ttl=300), + rerank=WrapperRerank(method_name="rerank_b"), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + await run_executor.run(w_a, ctx, limit=5, cursor={}) + await run_executor.run(w_b, ctx, limit=5, cursor={}) + + # Each wrapper writes its own reranked cache under node_id + # But both share the base key (cache_key="shared_base") + all_keys = [k.decode() if isinstance(k, bytes) else k for k in await redis.keys("sf:*")] + # The shared base key should exist + # Shared segments are sf:{sid}:{cache_key}:{child_hash}:{segment} (4 colons); + # the per-wrapper caches sf:{sid}:{cache_key}:{wrapper_hash} have only 3. + segment_keys = [ + k for k in all_keys if ":shared_base:" in k and k.count(":") == 4 and not k.endswith((":meta", ":lock")) + ] + assert len(segment_keys) == 1, f"exactly one shared base segment expected, got {segment_keys}" + + +@pytest.mark.asyncio +async def test_shared_cache_warm_read(ctx, redis): + """After cold build, reading warm cache returns same base data, different rerank.""" + w_a = Wrapper( + node_id="a", + cache_key="shared_warm", + cache=WrapperCache(session_size=20, session_ttl=300), + rerank=WrapperRerank(method_name="rerank_a"), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + w_b = Wrapper( + node_id="b", + cache_key="shared_warm", + cache=WrapperCache(session_size=20, session_ttl=300), + rerank=WrapperRerank(method_name="rerank_b"), + data=SubFeed(subfeed_id="items", method_name="items"), + ) + r_a1 = await run_executor.run(w_a, ctx, limit=5, cursor={}) + # Second call with cursor (warm path for w_a) + r_a2 = await run_executor.run(w_a, ctx, limit=5, cursor=r_a1.next_page) + assert len(r_a2.data) == 5 + + # w_b also reads the shared base -- should work fine + r_b1 = await run_executor.run(w_b, ctx, limit=5, cursor={}) + assert len(r_b1.data) == 5