diff --git a/.env.example b/.env.example index 83262bf..a98fa2c 100644 --- a/.env.example +++ b/.env.example @@ -50,6 +50,7 @@ EMBEDDING_RELATIONSHIPS_DIMENSION=3072 # The following are just examples of the adapter implementation, you can have completely different envs REDIS_HOST="localhost" REDIS_PORT=6379 +REDIS_PASSWORD="" # Worker envs # Can be whatever you prefer @@ -102,9 +103,18 @@ BRAINPAT_TOKEN="your_token" # MultiBrain # Choose to allow or block automatic creation of new brains on requests with non existing new brain_ids BRAIN_CREATION_ALLOWED="true" + +# Production ingress and plugin startup policy. +CORS_ALLOWED_ORIGINS="" +PLUGIN_FAILURE_POLICY="fail" # Choose whether to fallback to default brain if not provided DEFAULT_BRAIN_FALLBACK="true" +# Optional anonymous, read-only documentation search demo (disabled by default) +PUBLIC_DEMO_ENABLED="false" +PUBLIC_DEMO_BRAIN_ID="agentdemo" +PUBLIC_DEMO_MAX_K=10 + # GCP # Used on the oss project for the small LLM but can be changed to any other model GCP_EXTRA_SMALL_LLM_MODEL="gemini-3-flash-preview" @@ -167,6 +177,26 @@ INGEST_ARCHITECT_DENSE_ENTITY_THRESHOLD="12" INGEST_ARCHITECT_DENSE_MAX_CHARS="1200" INGEST_ARCHITECT_PRIOR_CONTEXT="auto" INGEST_ARCHITECT_SCRATCHPAD_TOKEN_CAP="500" +# Search +# SEARCH_ENABLED => true registers GET|POST /retrieve/search, writes tsvector + halfvec HNSW +# (when dim > 2000). Default false keeps memory /retrieve/context identical (dense ∪ ILIKE). +# Requires DATA_DB=postgresql. Search p50 SLO is < 200 ms excluding embed.query RTT. +# SEARCH_USE_DENSE / SEARCH_USE_BM25 => at least one must be true when search is on. +# Both true (default) fuses with SEARCH_FUSION=rrf (or cc with SEARCH_FUSION_ALPHA). +# SEARCH_COMMUNITY_LABELS => hub labels for the communities search channel. +# SEARCH_NEIGHBOR_FANOUT => max 1-hop members per seed when expand=neighbors. +# CONTEXT_PASSAGE_MODE => hybrid|bm25|dense|ilike; only applies when SEARCH_ENABLED=true. +# hybrid uses BM25 ∪ dense RRF on /retrieve/context; ilike freezes the old lexical leg. +SEARCH_ENABLED="false" +SEARCH_USE_DENSE="true" +SEARCH_USE_BM25="true" +SEARCH_FUSION="rrf" +SEARCH_FUSION_ALPHA="0.5" +SEARCH_BM25_K1="1.2" +SEARCH_BM25_B="0.75" +SEARCH_COMMUNITY_LABELS="TYPE,CLASS,TOPIC" +SEARCH_NEIGHBOR_FANOUT="50" +CONTEXT_PASSAGE_MODE="hybrid" GRAPH_DB="networkx" DATA_DB="postgresql" VECTOR_DB="postgresql" @@ -203,4 +233,4 @@ TRACE_HEALTH_INTERVAL_SECONDS=30 TRACE_RESOURCE_INTERVAL_SECONDS=30 TRACE_HEALTH_TIMEOUT_SECONDS=1 TRACE_AGENT_LOOP_ITERATIONS=20 -TRACE_AGENT_TOOL_LOOP_ITERATIONS=20 \ No newline at end of file +TRACE_AGENT_TOOL_LOOP_ITERATIONS=20 diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml deleted file mode 100644 index 4f30e2d..0000000 --- a/.github/workflows/deploy.yaml +++ /dev/null @@ -1,62 +0,0 @@ -name: Deploy per-user on tag - -on: - push: - tags: - - "brainapi@*@*@*@v*" - -jobs: - deploy: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - env: - TF_IN_AUTOMATION: "true" - steps: - - uses: actions/checkout@v4 - - name: Extract tag parts - id: parts - run: | - TAG="${GITHUB_REF_NAME}" - # brainapi@alice@europe-west8@pro@v0.4.2 - IFS='@' read -r APP USER REGION PLAN VERSION <<< "$TAG" - echo "user_slug=$USER" >> $GITHUB_OUTPUT - echo "region=$REGION" >> $GITHUB_OUTPUT - echo "plan=$PLAN" >> $GITHUB_OUTPUT - echo "version=$VERSION" >> $GITHUB_OUTPUT - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: 1.8.5 - cli_config_credentials_token: ${{ secrets.TF_API_TOKEN }} # if using Terraform Cloud - - name: GCP Auth (OIDC) - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER }} - service_account: ${{ secrets.GCP_SA_EMAIL }} - - name: Configure gcloud (for provider ADC) - uses: google-github-actions/setup-gcloud@v2 - - name: Terraform init - working-directory: infra - run: terraform init - - name: Select/Create workspace - working-directory: infra - run: | - WS="${{ steps.parts.outputs.user_slug }}-${{ steps.parts.outputs.region }}" - terraform workspace select "$WS" || terraform workspace new "$WS" - - name: Terraform apply - working-directory: infra - env: - TF_VAR_gcp_project: ${{ secrets.GCP_PROJECT }} - TF_VAR_cloudflare_api_token: ${{ secrets.CF_API_TOKEN }} - TF_VAR_ghcr_username: ${{ secrets.GHCR_USER }} - TF_VAR_ghcr_token: ${{ secrets.GHCR_TOKEN }} - TF_VAR_zone_id: ${{ secrets.CF_ZONE_ID }} - run: | - terraform apply -auto-approve \ - -var "user_slug=${{ steps.parts.outputs.user_slug }}" \ - -var "region=${{ steps.parts.outputs.region }}" \ - -var "plan=${{ steps.parts.outputs.plan }}" \ - -var "version=${{ steps.parts.outputs.version }}" \ - -var-file="plans/${{ steps.parts.outputs.plan }}.tfvars" diff --git a/.github/workflows/heavy-validation.yaml b/.github/workflows/heavy-validation.yaml new file mode 100644 index 0000000..39db041 --- /dev/null +++ b/.github/workflows/heavy-validation.yaml @@ -0,0 +1,135 @@ +name: Heavy production validation + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + actions: read + +jobs: + heavy-smoke-and-restore: + runs-on: [self-hosted, linux, x64, brainapi-heavy] + timeout-minutes: 90 + env: + BRAINAPI_IMAGE: brainapi:ci + BRAINAPI_ENV_FILE: env.heavy.example + REDIS_PASSWORD: ci-redis-password + NEO4J_PASSWORD: ci-neo4j-password + MONGO_PASSWORD: ci-mongo-password + MINIO_ACCESS_KEY: ci-minio-access + MINIO_SECRET_KEY: ci-minio-secret-password + MILVUS_ROOT_PASSWORD: ci-milvus-password + MILVUS_TOKEN: root:ci-milvus-password + BRAINPAT_TOKEN: ci-system-token + BACKUP_DIR: ${{ github.workspace }}/.backup-artifacts + RELEASE_ARTIFACT_DIR: ${{ github.workspace }}/release-artifacts + steps: + - uses: actions/checkout@v4 + - name: Download the exact audited candidate image + timeout-minutes: 45 + uses: actions/github-script@v7 + with: + script: | + const fs = require("fs"); + const shas = [...new Set([ + context.payload.pull_request?.head?.sha, + context.sha, + ].filter(Boolean))]; + const deadline = Date.now() + 40 * 60 * 1000; + while (Date.now() < deadline) { + let run; + for (const sha of shas) { + const response = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: "quality.yaml", + head_sha: sha, + status: "completed", + per_page: 20, + }); + run = response.data.workflow_runs[0]; + if (run) break; + } + if (run) { + if (run.conclusion !== "success") { + core.setFailed(`Quality workflow ${run.id} concluded ${run.conclusion}`); + return; + } + const artifacts = await github.paginate( + github.rest.actions.listWorkflowRunArtifacts, + { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id, + per_page: 100, + } + ); + const artifact = artifacts.find( + (item) => item.name === "light-production-validation" && !item.expired + ); + if (!artifact) { + core.setFailed(`Quality workflow ${run.id} has no candidate image artifact`); + return; + } + const archive = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: "zip", + }); + fs.writeFileSync("quality-artifacts.zip", Buffer.from(archive.data)); + return; + } + await new Promise((resolve) => setTimeout(resolve, 30000)); + } + core.setFailed(`Timed out waiting for successful Quality artifacts for ${shas.join(" or ")}`); + - name: Load and verify the shared candidate image + run: | + unzip -q quality-artifacts.zip -d quality-artifacts + image_archive=$(find quality-artifacts -name brainapi-image.tar.gz -print -quit) + security_file=$(find quality-artifacts -name security.json -print -quit) + test -n "$image_archive" + test -n "$security_file" + gzip -dc "$image_archive" | docker load + expected=$(python -c 'import json,sys; print(json.load(open(sys.argv[1]))["image_digest"])' "$security_file") + actual=$(docker image inspect "$BRAINAPI_IMAGE" --format '{{.Id}}') + test "$actual" = "$expected" + - name: Require runner capacity + run: | + awk '/MemTotal/ { exit !($2 >= 16000000) }' /proc/meminfo + test "$(uname -m)" = x86_64 + - run: docker compose -f deploy/docker-compose.heavy.yaml config -q + - name: Clean heavy startup + run: docker compose --project-name brainapi-heavy-ci -f deploy/docker-compose.heavy.yaml -f deploy/docker-compose.ci.yaml up -d --wait + - name: Heavy product smoke + run: | + curl --fail --retry 20 --retry-delay 5 http://localhost/health + python scripts/production_smoke.py exercise --profile heavy --system-token "$BRAINPAT_TOKEN" --artifact-dir "$RELEASE_ARTIFACT_DIR" + - name: Backup and verify + id: backup + run: | + archive=$(deploy/brainapi-backup backup --profile heavy --project-name brainapi-heavy-ci --env-file deploy/env.heavy.example --compose-file deploy/docker-compose.ci.yaml --backup-dir "$BACKUP_DIR" | tail -1) + deploy/brainapi-backup verify --profile heavy --env-file deploy/env.heavy.example --archive "$archive" + cp "$archive/manifest.json" "$RELEASE_ARTIFACT_DIR/backup-heavy-manifest.json" + echo "archive=$archive" >> "$GITHUB_OUTPUT" + - name: Restore into clean volumes + run: | + docker compose --project-name brainapi-heavy-ci -f deploy/docker-compose.heavy.yaml -f deploy/docker-compose.ci.yaml down --volumes + deploy/brainapi-backup restore --profile heavy --project-name brainapi-heavy-ci --env-file deploy/env.heavy.example --compose-file deploy/docker-compose.ci.yaml --archive "${{ steps.backup.outputs.archive }}" + curl --fail --retry 20 --retry-delay 5 http://localhost/health + python scripts/production_smoke.py verify-restore --profile heavy --system-token "$BRAINPAT_TOKEN" --artifact-dir "$RELEASE_ARTIFACT_DIR" + - name: Logs on failure + if: failure() + run: docker compose --project-name brainapi-heavy-ci -f deploy/docker-compose.heavy.yaml -f deploy/docker-compose.ci.yaml logs --no-color + - name: Clean heavy volumes + if: always() + run: docker compose --project-name brainapi-heavy-ci -f deploy/docker-compose.heavy.yaml -f deploy/docker-compose.ci.yaml down --volumes --remove-orphans + - uses: actions/upload-artifact@v4 + if: always() + with: + name: heavy-production-validation + path: release-artifacts/ diff --git a/.github/workflows/quality.yaml b/.github/workflows/quality.yaml new file mode 100644 index 0000000..345bb02 --- /dev/null +++ b/.github/workflows/quality.yaml @@ -0,0 +1,149 @@ +name: Quality + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@v6 + with: + version: "0.8.22" + - run: pip install poetry==2.1.3 + - name: Verify locks without regeneration + run: | + poetry check --lock + uv lock --check + - run: poetry install + - run: poetry run python -m compileall -q src tests deploy/backup_restore.py + - run: poetry run pytest -q + - name: Audit actual Python environment + run: uvx pip-audit --path .venv/lib/python3.12/site-packages + + benchmark-helpers: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m venv benchmarks/.venv + - run: benchmarks/.venv/bin/pip install -r benchmarks/requirements.txt pytest + - run: benchmarks/.venv/bin/pytest -q benchmarks/locomo/test_sota_helpers.py benchmarks/beam/test_beam_helpers.py + + frontend: + runs-on: ubuntu-latest + strategy: + matrix: + project: [console, tui] + defaults: + run: + working-directory: ${{ matrix.project }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22.22.0" + cache: npm + cache-dependency-path: ${{ matrix.project }}/package-lock.json + - run: npm ci + - if: matrix.project == 'console' + run: npm audit --omit=dev --audit-level=high + - if: matrix.project == 'tui' + run: npm run typecheck + - run: npm run build + + rust: + runs-on: ubuntu-latest + defaults: + run: + working-directory: mcp-stdio-http-bridge + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.85.0 + - run: cargo test --locked + + image-light-smoke: + runs-on: ubuntu-latest + needs: [python, benchmark-helpers, frontend, rust] + env: + BRAINAPI_IMAGE: brainapi:ci + BRAINAPI_ENV_FILE: env.light.example + REDIS_PASSWORD: ci-redis-password + POSTGRES_PASSWORD: ci-postgres-password + POSTGRES_USERNAME: brainapi + BRAINPAT_TOKEN: ci-system-token + BACKUP_DIR: ${{ github.workspace }}/.backup-artifacts + RELEASE_ARTIFACT_DIR: ${{ github.workspace }}/release-artifacts + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - name: Validate Compose and secret rejection + run: | + docker compose -f deploy/docker-compose.light.yaml config -q + if env -u REDIS_PASSWORD docker compose -f deploy/docker-compose.light.yaml config -q; then exit 1; fi + - name: Build exact test image + run: docker build --platform linux/amd64 --load -t "$BRAINAPI_IMAGE" . + - name: Export exact candidate image + run: | + mkdir -p "$RELEASE_ARTIFACT_DIR" + docker save "$BRAINAPI_IMAGE" | gzip -1 > "$RELEASE_ARTIFACT_DIR/brainapi-image.tar.gz" + - name: Generate image SBOM + uses: anchore/sbom-action@v0 + with: + image: brainapi:ci + format: spdx-json + output-file: brainapi.spdx.json + - name: Audit actual image + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: brainapi:ci + severity: HIGH,CRITICAL + ignore-unfixed: true + exit-code: "1" + format: json + output: trivy.json + - name: Light profile smoke + run: | + docker compose --project-name brainapi-ci -f deploy/docker-compose.light.yaml -f deploy/docker-compose.ci.yaml up -d --wait + curl --fail --retry 12 --retry-delay 5 http://localhost/health + python scripts/production_smoke.py exercise --profile light --system-token "$BRAINPAT_TOKEN" --artifact-dir "$RELEASE_ARTIFACT_DIR" + - name: Light backup and clean restore + run: | + archive=$(deploy/brainapi-backup backup --profile light --project-name brainapi-ci --env-file deploy/env.light.example --compose-file deploy/docker-compose.ci.yaml --backup-dir "$BACKUP_DIR" | tail -1) + deploy/brainapi-backup verify --profile light --env-file deploy/env.light.example --archive "$archive" + cp "$archive/manifest.json" "$RELEASE_ARTIFACT_DIR/backup-light-manifest.json" + docker compose --project-name brainapi-ci -f deploy/docker-compose.light.yaml -f deploy/docker-compose.ci.yaml down --volumes + deploy/brainapi-backup restore --profile light --project-name brainapi-ci --env-file deploy/env.light.example --compose-file deploy/docker-compose.ci.yaml --archive "$archive" + curl --fail --retry 12 --retry-delay 5 http://localhost/health + python scripts/production_smoke.py verify-restore --profile light --system-token "$BRAINPAT_TOKEN" --artifact-dir "$RELEASE_ARTIFACT_DIR" + - name: Record exact-image security evidence + run: | + cp brainapi.spdx.json "$RELEASE_ARTIFACT_DIR/brainapi.spdx.json" + python scripts/write_security_artifact.py --trivy trivy.json --sbom "$RELEASE_ARTIFACT_DIR/brainapi.spdx.json" --image "$BRAINAPI_IMAGE" --output "$RELEASE_ARTIFACT_DIR/security.json" + - name: Logs on failure + if: failure() + run: docker compose --project-name brainapi-ci -f deploy/docker-compose.light.yaml -f deploy/docker-compose.ci.yaml logs --no-color + - name: Clean CI volumes + if: always() + run: docker compose --project-name brainapi-ci -f deploy/docker-compose.light.yaml -f deploy/docker-compose.ci.yaml down --volumes --remove-orphans + - uses: actions/upload-artifact@v4 + if: always() + with: + name: light-production-validation + path: | + release-artifacts/ + trivy.json diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..13601e3 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,163 @@ +name: Publish tagged container + +on: + push: + tags: ["v*"] + +permissions: + contents: read + checks: read + actions: read + packages: write + id-token: write + +jobs: + validate-tag: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.tag.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: tag + shell: bash + run: | + if [[ "$GITHUB_REF_NAME" =~ ^v([0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?)$ ]]; then + echo "version=${BASH_REMATCH[1]}" >> "$GITHUB_OUTPUT" + else + echo "Unsupported release tag: $GITHUB_REF_NAME" >&2 + exit 1 + fi + - name: Require tag version to match project metadata + run: | + project_version=$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml","rb"))["project"]["version"])') + tag_version="${{ steps.tag.outputs.version }}" + normalized_version="${tag_version/-rc./rc}" + test "$project_version" = "$normalized_version" + - name: Require the tagged commit to be on main + run: git merge-base --is-ancestor "$GITHUB_SHA" origin/main + + verify-required-checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Require successful quality and heavy validation for this commit + uses: actions/github-script@v7 + with: + script: | + const required = [ + "python", + "benchmark-helpers", + "frontend (console)", + "frontend (tui)", + "rust", + "image-light-smoke", + "heavy-smoke-and-restore", + ]; + const runs = await github.paginate(github.rest.checks.listForRef, { + owner: context.repo.owner, + repo: context.repo.repo, + ref: context.sha, + per_page: 100, + }); + const failed = required.filter((name) => + !runs.some((run) => run.name === name && run.conclusion === "success") + ); + if (failed.length) { + core.setFailed(`Missing successful checks for ${context.sha}: ${failed.join(", ")}`); + } + - name: Download exact-commit validation artifacts + uses: actions/github-script@v7 + with: + script: | + const fs = require("fs"); + const required = [ + ["quality.yaml", "light-production-validation", "light.zip"], + ["heavy-validation.yaml", "heavy-production-validation", "heavy.zip"], + ]; + for (const [workflow, artifactName, output] of required) { + const response = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: workflow, + head_sha: context.sha, + status: "completed", + per_page: 100, + }); + const run = response.data.workflow_runs.find( + (item) => item.conclusion === "success" + ); + if (!run) { + core.setFailed(`No successful ${workflow} run found for ${context.sha}`); + return; + } + const artifacts = await github.paginate( + github.rest.actions.listWorkflowRunArtifacts, + { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id, + per_page: 100, + } + ); + const artifact = artifacts.find( + (item) => item.name === artifactName && !item.expired + ); + if (!artifact) { + core.setFailed(`Missing ${artifactName} on workflow run ${run.id}`); + return; + } + const archive = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: "zip", + }); + fs.writeFileSync(output, Buffer.from(archive.data)); + } + - name: Enforce recorded product, restore, security, and benchmark gates + run: | + mkdir -p gate-input/light gate-input/heavy gate-artifacts + unzip -q light.zip -d gate-input/light + unzip -q heavy.zip -d gate-input/heavy + python scripts/assemble_release_artifacts.py --light gate-input/light --heavy gate-input/heavy --output gate-artifacts + python scripts/check_release_readiness.py gate-artifacts + - uses: actions/upload-artifact@v4 + with: + name: release-evidence-${{ github.ref_name }} + path: gate-artifacts/ + + publish: + runs-on: ubuntu-latest + needs: [validate-tag, verify-required-checks] + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: release-evidence-${{ github.ref_name }} + path: gate-artifacts + - name: Load and verify the gated image + run: | + gzip -dc gate-artifacts/brainapi-image.tar.gz | docker load + expected=$(python -c 'import json; print(json.load(open("gate-artifacts/security.json"))["image_digest"])') + actual=$(docker image inspect brainapi:ci --format '{{.Id}}') + test "$actual" = "$expected" + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: publish + name: Publish the unchanged validated image + run: | + target="ghcr.io/lumen-labs/brainapi:$GITHUB_REF_NAME" + docker tag brainapi:ci "$target" + docker push "$target" + digest=$(docker image inspect "$target" --format '{{index .RepoDigests 0}}' | cut -d@ -f2) + test -n "$digest" + echo "$digest" > image-digest.txt + - uses: actions/upload-artifact@v4 + with: + name: release-image-digest-${{ needs.validate-tag.outputs.version }} + path: image-digest.txt diff --git a/.github/workflows/staging.yaml b/.github/workflows/staging.yaml deleted file mode 100644 index a5c52af..0000000 --- a/.github/workflows/staging.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: Staging - -on: - push: - branches: - - staging - -jobs: - build-and-release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: { fetch-depth: 0 } - - uses: actions/setup-node@v4 - - run: "[ NOT IMPLEMENTED YET ]" diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml deleted file mode 100644 index ff3a0b2..0000000 --- a/.github/workflows/tags.yaml +++ /dev/null @@ -1,76 +0,0 @@ -name: Build and Push Container - -on: - push: - branches: - - main - -jobs: - build-and-push: - runs-on: ubuntu-latest - permissions: - contents: write - packages: write - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install Poetry - run: pip install poetry==2.1.3 - - - name: Refresh poetry.lock - run: poetry lock - - - name: Get version - id: version - run: echo "VERSION=$(grep '^version = ' pyproject.toml | sed 's/version = "\(.*\)"/\1/')" >> $GITHUB_OUTPUT - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GHCR_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/lumen-labs/brainapi - tags: | - type=raw,value=latest,enable={{is_default_branch}} - - - name: Build and push Docker image - uses: docker/build-push-action@v5 - with: - context: . - platforms: linux/amd64 - push: true - tags: | - ghcr.io/lumen-labs/brainapi:v${{ steps.version.outputs.VERSION }} - ghcr.io/lumen-labs/brainapi:latest - build-args: | - BUILD_DATE=${{ github.event.head_commit.timestamp }} - BUILD_SHA=${{ github.sha }} - CACHE_BUST=${{ github.run_number }} - secrets: | - HF_TOKEN=${{ secrets.HF_TOKEN }} - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Create Git tag (only on main branch) - if: github.ref == 'refs/heads/main' - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git tag -a v${{ steps.version.outputs.VERSION }} -m "Release v${{ steps.version.outputs.VERSION }}" - git push origin v${{ steps.version.outputs.VERSION }} diff --git a/Dockerfile b/Dockerfile index e27db4a..d3c941c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,18 @@ # syntax=docker/dockerfile:1.4 -# ── Stage 1: builder ──────────────────────────────────────── -FROM python:3.11-slim AS builder +# ── Stage 1: Console builder ──────────────────────────────── +FROM node:22.22.0-bookworm-slim AS console-builder + +WORKDIR /console +COPY console/package.json console/package-lock.json ./ +RUN npm ci +COPY console/ ./ +RUN npm run build + +# ── Stage 2: Python builder ───────────────────────────────── +FROM python:3.11.14-slim-bookworm AS builder + +ARG INSTALL_LOCAL_ML=false ENV PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ @@ -12,7 +23,8 @@ ENV PYTHONUNBUFFERED=1 \ POETRY_NO_INTERACTION=1 \ POETRY_VIRTUALENVS_IN_PROJECT=true \ HF_HOME=/app/.cache \ - SENTENCE_TRANSFORMERS_HOME=/app/.cache + SENTENCE_TRANSFORMERS_HOME=/app/.cache \ + TIKTOKEN_CACHE_DIR=/app/.cache/tiktoken RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -25,24 +37,25 @@ WORKDIR /app COPY pyproject.toml poetry.lock ./ -RUN poetry sync --no-root \ - && TORCH_VER=$(/app/.venv/bin/python -c "import torch; print(torch.__version__.split('+')[0])") \ - && /app/.venv/bin/pip install --no-cache-dir --no-deps --force-reinstall \ - "torch==${TORCH_VER}" \ - --index-url https://download.pytorch.org/whl/cpu \ +RUN if [ "$INSTALL_LOCAL_ML" = "true" ]; then \ + poetry sync --no-root --only main --with local-ml; \ + else \ + poetry sync --no-root --only main; \ + fi \ + && mkdir -p "$TIKTOKEN_CACHE_DIR" \ + && /app/.venv/bin/python -c "import tiktoken; tiktoken.get_encoding('cl100k_base')" \ && rm -rf /root/.cache /tmp/* COPY src/ ./src/ COPY scripts/preload_docker_models.py ./scripts/ -RUN --mount=type=secret,id=HF_TOKEN,required=false \ - export HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \ - && export HUGGING_FACE_HUB_TOKEN="$HF_TOKEN" \ - && /app/.venv/bin/python scripts/preload_docker_models.py \ +RUN if [ "$INSTALL_LOCAL_ML" = "true" ]; then \ + /app/.venv/bin/python scripts/preload_docker_models.py; \ + fi \ && rm -rf /root/.cache /tmp/* -# ── Stage 2: runtime ──────────────────────────────────────── -FROM python:3.11-slim +# ── Stage 3: runtime ──────────────────────────────────────── +FROM python:3.11.14-slim-bookworm ARG BUILD_DATE ARG BUILD_SHA @@ -55,19 +68,18 @@ ENV PYTHONUNBUFFERED=1 \ PATH="/app/.venv/bin:$PATH" \ HF_HOME=/app/.cache \ SENTENCE_TRANSFORMERS_HOME=/app/.cache \ + TIKTOKEN_CACHE_DIR=/app/.cache/tiktoken \ PIP_NO_CACHE_DIR=1 \ PIP_RETRIES=8 \ PIP_TIMEOUT=900 \ PIP_DEFAULT_TIMEOUT=900 -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ curl \ + util-linux \ && rm -rf /var/lib/apt/lists/* \ - && GOSU_VERSION=1.19 \ - && dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \ - && curl -fsSL -o /usr/local/bin/gosu \ - "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${dpkgArch}" \ - && chmod +x /usr/local/bin/gosu \ + && python -m pip uninstall -y setuptools wheel \ && groupadd -r appuser && useradd -r -g appuser -m appuser WORKDIR /app @@ -76,6 +88,7 @@ COPY --from=builder /app/.venv /app/.venv COPY --from=builder /app/.cache /app/.cache COPY --from=builder /app/src /app/src COPY --from=builder /app/pyproject.toml /app/ +COPY --from=console-builder /console/dist /app/console/dist COPY entrypoint.sh ./ RUN chmod +x /app/entrypoint.sh @@ -86,7 +99,7 @@ VOLUME ["/app/plugins"] EXPOSE 8000 HEALTHCHECK --interval=30s --timeout=30s --start-period=120s --retries=5 \ - CMD curl -f http://localhost:8000/docs || exit 1 + CMD curl -f http://localhost:8000/health || exit 1 USER root ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/README.md b/README.md index 23b5f73..c5b7777 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,8 @@ BrainAPI follows an **append-only philosophy**: nothing is deleted from the grap - **`accurate`** (default) — the full swarm runs: Scout extracts every entity, the Observations agent writes notes, and the Janitor validates and deduplicates the Architect's relationships. - **`lightweight`** — a faster, cheaper path that extracts only the most important entities and skips the heavier validation steps. Switch with `--pipeline lightweight` or `PIPELINE_MODE` in `.env`. +Search (`GET|POST /retrieve/search`) is **off by default**. Set `SEARCH_ENABLED=true` to register the hit-list API. BM25 requires `DATA_DB=postgresql`; dense-only search (`SEARCH_USE_BM25=false`) is supported by either production profile. PostgreSQL writes BM25 `tsvector` indexes and uses halfvec HNSW when embeddings are >2000-d. Default light-profile retrieval is both channels fused (`SEARCH_FUSION=rrf`). The search p50 SLO is **< 200 ms excluding `embed.query`** — ask for `profile_stages` to split embed RTT from retrieve. `/retrieve/context` stays dense ∪ ILIKE unless search is on and `CONTEXT_PASSAGE_MODE` is `hybrid` / `bm25` / `dense`. + --- ## 🔎 Retrieving Knowledge @@ -218,10 +220,11 @@ Once your data is in, BrainAPI exposes purpose-built retrieval endpoints (REST, | Endpoint | Method | What you get | | ---------------------------- | ------ | ---------------------------------------------------------------------------------------------------------------- | -| `/retrieve/context` | `POST` | **Relevant information for a piece of text** — graph triples, passages, historical context, and optional provenance | +| `/retrieve/context` | `POST` | **Relevant information for a piece of text** — one-shot graph triples, passages, historical context, and optional provenance; p50 target < 1000 ms | +| `/retrieve/search` | `GET`/`POST` | **Ranked search hits** (opt-in: `SEARCH_ENABLED=true`) — passages (BM25 ∪ dense) by default; optional `entities` / `events` / `communities` channels and `expand=neighbors`; p50 < 200 ms excluding embed | | `/retrieve/entity/status` | `GET` | **Existence check** for a specific entity — returns whether it exists, its node, relationships, and observations | -| `/retrieve/entity/synergies` | `GET` | **Sibling entities** of the same type — shared event hubs + embedding similarity (`top_k`, `labels`, `polarity`) | -| `/retrieve/recommend` | `GET`/`POST` | **Train-free recommendations** — synergies plus asymmetric walks, multi-interest, optional attribute prefs | +| `/retrieve/entity/synergies` | `GET` | **Sibling entities (Preview)** — shared event hubs + embedding similarity (`top_k`, `labels`, `polarity`); quality thresholds are not yet release-stable | +| `/retrieve/recommend` | `GET`/`POST` | **Train-free recommendations (Preview)** — synergies plus asymmetric walks, multi-interest, and optional attribute preferences; endpoint compatibility is retained while the quality gate is defined | **Get context for a question:** @@ -242,7 +245,7 @@ curl "http://localhost:8000/retrieve/entity/status?target=Emily" \ # → { "exists": true, "node": {...}, "relationships": [...], "observations": [...] } ``` -**Find synergies (same-type siblings):** +**Find synergies (same-type siblings, Preview):** ```sh curl "http://localhost:8000/retrieve/entity/synergies?target=Neural%20Networks%20101&top_k=20" \ @@ -250,7 +253,6 @@ curl "http://localhost:8000/retrieve/entity/synergies?target=Neural%20Networks%2 # → similar entities ranked by association_score ``` - **Rank recommendations (product / next-item style):** ```sh @@ -260,7 +262,7 @@ curl "http://localhost:8000/retrieve/recommend?target=u01&top_k=20&labels=PRODUC ``` -These three cover the most common needs, but there are more (`/retrieve/hops`, `/retrieve/entities/neighbors`, `/retrieve/text-chunks`, …). Start with [Context retrieval](https://brainapi.lumen-labs.ai/docs/v2/retrieval/context) and the [docs hub](https://brainapi.lumen-labs.ai/docs/v2). +These endpoints cover the most common needs, but there are more (`/retrieve/hops`, `/retrieve/entities/neighbors`, `/retrieve/text-chunks`, …). Start with [Context retrieval](https://brainapi.lumen-labs.ai/docs/v2/retrieval/context) and the [docs hub](https://brainapi.lumen-labs.ai/docs/v2). --- diff --git a/benchmarks/.env.example b/benchmarks/.env.example index 5cbb772..313d245 100644 --- a/benchmarks/.env.example +++ b/benchmarks/.env.example @@ -1,6 +1,9 @@ BRAINAPI_URL=http://localhost:8000 BRAINPAT_TOKEN= +# Search eval (default searchbenchsmoke). Must start with searchbench. +# SEARCH_BRAIN_ID=searchbenchsmoke + # Dual profile: product (greedy) | sota (deepseek-v4-flash + SC + gap-fill) # BENCH_PROFILE=product # BENCH_PROFILE=sota diff --git a/benchmarks/AGENTS.md b/benchmarks/AGENTS.md index a742471..a07584a 100644 --- a/benchmarks/AGENTS.md +++ b/benchmarks/AGENTS.md @@ -8,8 +8,9 @@ | **LongMemEval** | `benchmarks/longmemeval/` | `./longmemeval.sh` | `benchmarks.longmemeval` | | **BEAM** | `benchmarks/beam/` | `./beam.sh` | `benchmarks.beam` | | **RecSys** | `benchmarks/recsys/` | `./recsys.sh` | `benchmarks.recsys` | +| **Search** | `benchmarks/search/` | `./search.sh` | `benchmarks.search` | -All harnesses talk to a running BrainAPI over HTTP only (no `src/` imports). RecSys uses `POST /ingest/structured` (KB write) + **train-free** `GET /retrieve/recommend` on brain **`demorecsys` only** — never LoCoMo/BEAM brains. Optional: `plugins/features-rec` for attribute prefs; `plugins/recsys-gnn` LightGCN via `--backend lightgcn`. +All harnesses talk to a running BrainAPI over HTTP only (no `src/` imports, except Search `mapping.py` re-exporting the product catalog mapper). RecSys uses `POST /ingest/structured` (KB write) + **train-free** `GET /retrieve/recommend` on brain **`demorecsys` only** — never LoCoMo/BEAM brains. Optional: `plugins/features-rec` for attribute prefs; `plugins/recsys-gnn` LightGCN via `--backend lightgcn`. Search uses `POST /ingest/` + `POST /retrieve/search` on **`searchbench*` only** — never LoCoMo/BEAM/recsys brains, never `/retrieve/context` as the metric path. ### LoCoMo @@ -35,6 +36,28 @@ Held-out next-item HitRate/Recall@K via **train-free graph recommend** (default) Protocol: [`docs/research/16-recsys-eval-protocol.md`](../docs/research/16-recsys-eval-protocol.md). Upserts only `benchmarks.recsys` — never wipe or write to `beam1m1clean` / `locomoconv26*`. +### Search + +Hybrid BM25 + dense ranking via `POST /retrieve/search`, plus optional graph channels (`entities` / `events` / `communities`, `expand=neighbors`). Flow: ingest docs (chunk+embed; LLM enrichment skipped by default) → optional structured catalog triples (`--ingest-graph`) → map `DOCID` markers to chunk ids → search with `profile_stages`. Gold matches chunk ids or `hit.id == doc_id`. Brain: `searchbenchsmoke` (any id must start with `searchbench`). Server must have `SEARCH_ENABLED=true`, `DATA_DB=postgresql`, `BRAIN_CREATION_ALLOWED=true`. + +```bash +./search.sh smoke +./search.sh evaluate --fusion rrf +./search.sh download --name esci +./search.sh download --name wands +./search.sh evaluate --dataset data/search_esci.jsonl --brain searchbenchesci --run search-esci +./search.sh --brain searchbenchwands evaluate --dataset data/search_wands.jsonl --run search-wands-passages-k50 --channels passages --k 50 --ks 5,10,20,50 +./search.sh --brain searchbenchwands evaluate --dataset data/search_wands.jsonl --run search-wands-passages-k50 --channels passages --k 50 --ks 5,10,20,50 --skip-ingest +./search.sh --brain searchbenchwandsgraph evaluate --dataset data/search_wands.jsonl --run search-wandsgraph-communities-k50 --channels communities --k 50 --ks 5,10,20,50 --ingest-graph +./search.sh evaluate --dataset data/search_esci.jsonl --brain searchbenchesci --ingest-graph --channels passages,entities,communities +./search.sh evaluate --rerank plugin:cross-encoder +./search.sh report --run +``` + +Catalog downloads are a **subset** of Amazon ESCI (KDD Cup 2022, US Task 1 / small_version test) and WANDS (Wayfair). Default caps: 80 queries, 2000 products, 40 candidates/query. Frozen WANDS quality slice: `data/search_wands.jsonl` (66 queries / 2000 docs); download refuses overwrite. Gold is graded (ESCI E/S/C/I; WANDS Exact/Partial/Irrelevant). WANDS control is passages k=50 `rerank=none` on `searchbenchwands`, then `--skip-ingest`. Opt-in catalog graph is brain `searchbenchwandsgraph` only (`--ingest-graph`, `channels=communities`); ledger rows are an **architecture demo**, not a claim vs 0.823 / ESCI 0.500. `--ingest-graph` is refused on frozen `searchbenchwands` / `searchbenchesci74` / `searchbenchescies` / `searchbenchesciltr2`. Pass `--enrich` only if you want Scout/Architect on ingest. Use dedicated `searchbench*` brains — never `demorecsys`. Graph mapping is product `src/core/search/catalog_graph.py` (generic HAS events + CLASS/TYPE/ATTR hubs). Do not score recs HitRate as search. + +Metrics: Recall@{5,10,20}, nDCG@10, MRR; p50/p95 retrieve **excluding** embed RTT. No LLM judge. Protocol: [`docs/research/18-search-eval-protocol.md`](../docs/research/18-search-eval-protocol.md). Upserts only `benchmarks.search` — never wipe or write to `locomoconv*`, `beam*`, `demorecsys`. + ### BEAM Typical flow: `download` → `ingest` → `evaluate` → `report` via `./beam.sh` or `.venv/bin/python -m beam`. @@ -55,6 +78,7 @@ Typical flow: `download` → `ingest` → `evaluate` → `report` via `./beam.sh - LoCoMo harness upserts `benchmarks.locomo` via `write_report` → `update_reports_json` when report `status` is `ok`. - BEAM harness upserts `benchmarks.beam` the same way (field: `headline_score`, continuous `[0,1]`). - RecSys harness upserts `benchmarks.recsys` only (`hit_rate@K` / `recall@K`); it must not touch other suite keys. +- Search harness upserts `benchmarks.search` only (`ndcg@10` / `recall@10` / `p50_retrieve_ms`); it must not touch other suite keys. - After any completed evaluate/report that produces a successful scored run, ensure the suite entry in `REPORTS.json` reflects it. - If you write or patch `runs//report.json` manually, update `REPORTS.json` yourself (same schema; upsert by `run_id` under the suite). - When adding a new suite, add a `benchmarks.` key (`name` + `leaderboard`) and upsert the same way. diff --git a/benchmarks/README.md b/benchmarks/README.md index 536302b..26bf46a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -8,6 +8,7 @@ Standalone HTTP-only harnesses (no `src/` imports): | [LongMemEval](https://github.com/xiaowu0162/LongMemEval) | `longmemeval/` | `./longmemeval.sh` | | [BEAM](https://github.com/mohammadtavakoli78/BEAM) | `beam/` | `./beam.sh` | | RecSys (train-free `/retrieve/recommend`) | `recsys/` | `./recsys.sh` | +| Search | `search/` | `./search.sh` | Shared setup: `requirements.txt`, `.env` (`BRAINPAT_TOKEN`, LLM keys). Results ledger: [`REPORTS.json`](REPORTS.json). Agent notes: [`AGENTS.md`](AGENTS.md). @@ -362,3 +363,56 @@ Protocol: [`docs/research/16-recsys-eval-protocol.md`](../docs/research/16-recsy - **Headline**: `hit_rate@K` / `recall@K` (K typically 10/20). - Also: intra-list diversity, type coverage; report `model=graph-recommend` or `lightgcn`. - Successful scored runs upsert `benchmarks.recsys` only — do not touch other suite keys. + +--- + +# Search benchmark for BrainAPI + +Standalone HTTP-only harness for `POST /retrieve/search` (hybrid BM25 + dense). It does **not** import `src/` and does **not** call `/retrieve/context`. + +Protocol: [`docs/research/18-search-eval-protocol.md`](../docs/research/18-search-eval-protocol.md). + +## Prerequisites + +- BrainAPI running with `SEARCH_ENABLED=true`, `DATA_DB=postgresql`, `BRAIN_CREATION_ALLOWED=true` +- `BRAINPAT_TOKEN` in `benchmarks/.env` +- Brain id starting with `searchbench` (default `searchbenchsmoke`) + +A 404 on `/retrieve/search` means search is disabled — fix the server env; it is not a ranking miss. + +## Quickstart + +```bash +cd benchmarks +./search.sh dataset-stats +./search.sh smoke +./search.sh evaluate --fusion rrf +./search.sh evaluate --rerank plugin:cross-encoder +./search.sh report --run +``` + +Toy corpus: [`data/search_toy.jsonl`](data/search_toy.jsonl) (keyword + paraphrase queries; gold via `DOCID` markers). Search ingest defaults to **chunk + embed only** (`skip_enrichment`); pass `--enrich` for Scout/Architect. + +Product-search corpora (LLM-free ingest; dedicated `searchbench*` brains, not `demorecsys`): + +```bash +./search.sh download --name esci +./search.sh download --name wands +./search.sh dataset-stats --dataset data/search_esci.jsonl +./search.sh evaluate --dataset data/search_esci.jsonl --brain searchbenchesci --run search-esci +./search.sh --brain searchbenchwands evaluate --dataset data/search_wands.jsonl --run search-wands-passages-k50 --channels passages --k 50 --ks 5,10,20,50 +./search.sh --brain searchbenchwands evaluate --dataset data/search_wands.jsonl --run search-wands-passages-k50 --channels passages --k 50 --ks 5,10,20,50 --skip-ingest +./search.sh --brain searchbenchwandsgraph evaluate --dataset data/search_wands.jsonl --run search-wandsgraph-communities-k50 --channels communities --k 50 --ks 5,10,20,50 --ingest-graph +``` + +`download` writes a **subset** under `data/` (gitignored): Amazon ESCI / Shopping Queries (KDD Cup 2022, US Task 1 test) and WANDS (Wayfair). Defaults: 80 queries, 2000 products, 40 candidates per query. Frozen WANDS quality slice is `data/search_wands.jsonl` (**66** queries / **2000** docs); `--force` cannot clobber it. Raw caches live in `data/esci/` and `data/wands/`. nDCG@10 is graded (ESCI E=1 / S=0.1 / C=0.01 / I=0; WANDS Exact=1 / Partial=0.5 / Irrelevant=0). WANDS first-stage control: passages, k=50, `rerank=none`, brain `searchbenchwands`, then `--skip-ingest`. Opt-in catalog graph: new brain `searchbenchwandsgraph` only; architecture demo, do not mix with ESCI nDCG or frozen WANDS 0.823. + +Optional plugins (core hybrid works if they are absent; unknown `plugin:` is 400): + +- `plugins/search-rerank` — `--rerank plugin:cross-encoder` (K≤10) +- `plugins/search-splade` — `--channels plugin:splade` after `POST /search-splade/index` +- `plugins/search-colbert` — `--channels plugin:colbert` after `POST /search-colbert/index` + +Metrics: Recall@{5,10,20}, nDCG@10, MRR; p50/p95 of retrieve **excluding** embed RTT. Successful `evaluate` upserts `benchmarks.search` in [`REPORTS.json`](REPORTS.json). Smoke does not publish scores. + +Never ingest into or wipe `locomoconv*`, `beam*`, or `demorecsys`. diff --git a/benchmarks/REPORTS.json b/benchmarks/REPORTS.json index 94a721a..0d56170 100644 --- a/benchmarks/REPORTS.json +++ b/benchmarks/REPORTS.json @@ -1,6 +1,6 @@ { "schema_version": 2, - "updated_at": "2026-08-05T20:07:44.710028+00:00", + "updated_at": "2026-08-21T07:53:39.332214+00:00", "description": "BrainAPI benchmark results. Top published scores across suites. Updated when a suite evaluate/report completes successfully.", "benchmarks": { "locomo": { @@ -803,6 +803,1792 @@ "recorded_at": "2026-08-05T20:07:44.707984+00:00" } ] + }, + "search": { + "name": "Search (hybrid BM25 + dense)", + "representative_run_id": "search-wands-passages-k50", + "representative_scope": "WANDS frozen slice: 66 queries / 2000 documents", + "leaderboard": [ + { + "run_id": "search-20260818T111250Z", + "brain": "searchbenchsmoke", + "dataset": "search_toy.jsonl", + "fusion": "rrf", + "rerank": "none", + "n_queries": 10, + "ndcg@10": 1.0, + "recall@10": 1.0, + "mrr": 1.0, + "p50_retrieve_ms": 25.1575, + "p95_retrieve_ms": 27.67875, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-20260818T111250Z/report.json", + "recorded_at": "2026-08-18T11:13:57.784280+00:00" + }, + { + "run_id": "search-toy-cc", + "brain": "searchbenchsmoke", + "dataset": "search_toy.jsonl", + "fusion": "cc", + "rerank": "none", + "n_queries": 10, + "ndcg@10": 1.0, + "recall@10": 1.0, + "mrr": 1.0, + "p50_retrieve_ms": 24.2875, + "p95_retrieve_ms": 41.931949999999986, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-toy-cc/report.json", + "recorded_at": "2026-08-18T11:15:19.039999+00:00" + }, + { + "run_id": "search-esci-slice-allcols", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages", + "entities", + "communities" + ], + "n_queries": 11, + "ndcg@10": 0.6815199154995802, + "recall@10": 0.4656565656565656, + "mrr": 0.8484848484848485, + "p50_retrieve_ms": 108.159, + "p95_retrieve_ms": 129.32150000000001, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-allcols/report.json", + "recorded_at": "2026-08-18T13:02:36.140118+00:00" + }, + { + "run_id": "search-esci-slice-passages", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "n_queries": 11, + "ndcg@10": 0.7579469533596556, + "recall@10": 0.5106060606060606, + "mrr": 0.8484848484848485, + "p50_retrieve_ms": 29.279, + "p95_retrieve_ms": 153.97, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-passages/report.json", + "recorded_at": "2026-08-18T13:02:47.533040+00:00" + }, + { + "run_id": "search-esci-slice-entities", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "entities" + ], + "n_queries": 11, + "ndcg@10": 0.5323459350114446, + "recall@10": 0.4151515151515151, + "mrr": 0.6673553719008264, + "p50_retrieve_ms": 52.75, + "p95_retrieve_ms": 75.479, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-entities/report.json", + "recorded_at": "2026-08-18T13:31:39.862006+00:00" + }, + { + "run_id": "search-esci-slice-communities", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "communities" + ], + "n_queries": 11, + "ndcg@10": 0.0, + "recall@10": 0.0, + "mrr": 0.0, + "p50_retrieve_ms": 51.536, + "p95_retrieve_ms": 58.432500000000005, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-communities/report.json", + "recorded_at": "2026-08-18T13:31:42.972981+00:00" + }, + { + "run_id": "search-esci-slice-pe", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages", + "entities" + ], + "n_queries": 11, + "ndcg@10": 0.6815199154995802, + "recall@10": 0.4656565656565656, + "mrr": 0.8484848484848485, + "p50_retrieve_ms": 74.312, + "p95_retrieve_ms": 91.006, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-pe/report.json", + "recorded_at": "2026-08-18T13:31:46.700938+00:00" + }, + { + "run_id": "search-esci-slice-entities-after", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "entities" + ], + "n_queries": 11, + "ndcg@10": 0.6549492067418874, + "recall@10": 0.4818181818181818, + "mrr": 0.7568181818181817, + "p50_retrieve_ms": 51.071, + "p95_retrieve_ms": 74.023, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-entities-after/report.json", + "recorded_at": "2026-08-18T13:36:54.694299+00:00" + }, + { + "run_id": "search-esci-slice-communities-after", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "communities" + ], + "n_queries": 11, + "ndcg@10": 0.4776363794553365, + "recall@10": 0.35000000000000003, + "mrr": 0.8484848484848485, + "p50_retrieve_ms": 129.202, + "p95_retrieve_ms": 152.889, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-communities-after/report.json", + "recorded_at": "2026-08-18T13:39:09.162227+00:00" + }, + { + "run_id": "search-esci-slice-passages-smoke", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "n_queries": 11, + "ndcg@10": 0.7579469533596556, + "recall@10": 0.5106060606060606, + "mrr": 0.8484848484848485, + "p50_retrieve_ms": 27.735, + "p95_retrieve_ms": 32.483000000000004, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-passages-smoke/report.json", + "recorded_at": "2026-08-18T13:37:03.038349+00:00" + }, + { + "run_id": "search-esci-slice-pec-after", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages", + "entities", + "communities" + ], + "n_queries": 11, + "ndcg@10": 0.6526549564168582, + "recall@10": 0.496969696969697, + "mrr": 0.816017316017316, + "p50_retrieve_ms": 183.946, + "p95_retrieve_ms": 216.89249999999998, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-pec-after/report.json", + "recorded_at": "2026-08-18T13:39:52.408131+00:00" + }, + { + "run_id": "search-esci-slice-passages-ce", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "plugin:cross-encoder", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 11, + "ndcg@10": 0.7375861975343849, + "ndcg@20": null, + "ndcg": null, + "recall@10": 0.5106060606060606, + "recall@20": 0.8469696969696969, + "mrr": 0.8030303030303031, + "p50_retrieve_ms": 30.029, + "p95_retrieve_ms": 76.32499999999999, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-passages-ce/report.json", + "recorded_at": "2026-08-18T14:20:28.184907+00:00" + }, + { + "run_id": "search-esci-slice-passages-pool", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": true, + "n_queries": 11, + "ndcg@10": 0.7884517684883318, + "ndcg@20": 0.8114424261982733, + "ndcg": 0.8137804024940007, + "recall@10": 0.5348484848484848, + "recall@20": 0.8469696969696969, + "mrr": 0.9090909090909091, + "p50_retrieve_ms": 22.721, + "p95_retrieve_ms": 75.3825, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-passages-pool/report.json", + "recorded_at": "2026-08-18T14:18:44.171034+00:00" + }, + { + "run_id": "search-esci-slice-ce-pool", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "none", + "rerank": "plugin:cross-encoder", + "channels": [ + "rank-pool-ce" + ], + "rank_pool": true, + "rank_pool_ce": true, + "n_queries": 11, + "ndcg@10": 0.5976277406374534, + "ndcg@20": 0.668518640484444, + "ndcg": 0.7862048091602294, + "recall@10": 0.398637349324982, + "recall@20": 0.6598423817505625, + "mrr": 0.8333333333333333, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-ce-pool/report.json", + "recorded_at": "2026-08-18T14:20:28.034427+00:00" + }, + { + "run_id": "search-esci-74-ce-pool", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "plugin:cross-encoder", + "channels": [ + "rank-pool-ce" + ], + "rank_pool": true, + "rank_pool_ce": true, + "n_queries": 74, + "ndcg@10": 0.5492314171926193, + "ndcg@20": 0.6311374744057928, + "ndcg": 0.7174501167408996, + "recall@10": 0.4907400145298616, + "recall@20": 0.7577102510384188, + "mrr": 0.8306570306570307, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ce-pool/report.json", + "recorded_at": "2026-08-18T14:29:17.114211+00:00" + }, + { + "run_id": "search-esci-74-ce-pool-ft", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "plugin:cross-encoder", + "channels": [ + "rank-pool-ce" + ], + "rank_pool": true, + "rank_pool_ce": true, + "n_queries": 74, + "ndcg@10": 0.5832632846066277, + "ndcg@20": 0.6690889170924552, + "ndcg": 0.7451408010463653, + "recall@10": 0.4893954738298456, + "recall@20": 0.7641337775411228, + "mrr": 0.8932519057519057, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ce-pool-ft/report.json", + "recorded_at": "2026-08-18T14:32:48.672213+00:00" + }, + { + "run_id": "search-esci-74-passages", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "ndcg@10": 0.49528253069041633, + "ndcg@20": 0.5490094202707876, + "ndcg": 0.5490094202707876, + "recall@10": 0.3769307112873572, + "recall@20": 0.5920965629425753, + "mrr": 0.7580598455598455, + "p50_retrieve_ms": 70.9225, + "p95_retrieve_ms": 94.17495, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages/report.json", + "recorded_at": "2026-08-18T15:38:50.867049+00:00" + }, + { + "run_id": "search-esci-74-passages-pool", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": true, + "rank_pool_ce": false, + "n_queries": 74, + "ndcg@10": 0.5458756376131827, + "ndcg@20": 0.5785048208487761, + "ndcg": 0.6013933046523523, + "recall@10": 0.42064799804369324, + "recall@20": 0.5920965629425753, + "mrr": 0.8254504504504505, + "p50_retrieve_ms": 65.447, + "p95_retrieve_ms": 91.44999999999999, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-pool/report.json", + "recorded_at": "2026-08-18T15:41:37.970033+00:00" + }, + { + "run_id": "search-esci-slice-passages-gate", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 11, + "ndcg@10": 0.7579469533596556, + "ndcg@20": 0.7962544743266965, + "ndcg": 0.7962544743266965, + "recall@10": 0.5106060606060606, + "recall@20": 0.8469696969696969, + "mrr": 0.8484848484848485, + "p50_retrieve_ms": 25.549, + "p95_retrieve_ms": 41.532, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-passages-gate/report.json", + "recorded_at": "2026-08-18T16:47:11.911155+00:00" + }, + { + "run_id": "search-esci-slice-pec-lists", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages", + "entities", + "communities" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 11, + "ndcg@10": 0.6526549564168582, + "ndcg@20": 0.6261389422887412, + "ndcg": 0.683063799169122, + "recall@10": 0.496969696969697, + "recall@20": 0.6530303030303031, + "mrr": 0.816017316017316, + "p50_retrieve_ms": 160.231, + "p95_retrieve_ms": 198.8395, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-pec-lists/report.json", + "recorded_at": "2026-08-18T16:47:26.669145+00:00" + }, + { + "run_id": "search-esci-slice-entities-neighbors", + "brain": "searchbenchesci20", + "dataset": "search_esci_slice.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "entities" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 11, + "ndcg@10": 0.6549492067418874, + "ndcg@20": 0.7173885785786568, + "ndcg": 0.7173885785786568, + "recall@10": 0.4818181818181818, + "recall@20": 0.8217171717171716, + "mrr": 0.7568181818181817, + "p50_retrieve_ms": 223.289, + "p95_retrieve_ms": 271.031, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-slice-entities-neighbors/report.json", + "recorded_at": "2026-08-18T16:48:04.631525+00:00" + }, + { + "run_id": "search-esci-74-passages-control", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "ndcg@10": 0.49528253069041633, + "ndcg@20": 0.5490094202707876, + "ndcg": 0.5490094202707876, + "recall@10": 0.3769307112873572, + "recall@20": 0.5920965629425753, + "mrr": 0.7580598455598455, + "p50_retrieve_ms": 61.7115, + "p95_retrieve_ms": 91.52104999999995, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-control/report.json", + "recorded_at": "2026-08-18T17:13:00.463193+00:00" + }, + { + "run_id": "search-esci-74-ce-pool-l12-4class", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "4class-weighted-ce", + "channels": [ + "rank-pool-4class" + ], + "rank_pool": true, + "rank_pool_ce": true, + "n_queries": 74, + "ndcg@10": 0.5968432958924662, + "ndcg@20": 0.6782580445181766, + "ndcg": 0.7467109659912253, + "recall@10": 0.5088965287461584, + "recall@20": 0.7673275730894211, + "mrr": 0.8351464398761695, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ce-pool-l12-4class/report.json", + "recorded_at": "2026-08-18T17:44:17.593025+00:00" + }, + { + "run_id": "search-esci-74-ce-pool-4class", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "4class-weighted-ce", + "channels": [ + "rank-pool-4class" + ], + "rank_pool": true, + "rank_pool_ce": true, + "n_queries": 74, + "ndcg@10": 0.5826375707951627, + "ndcg@20": 0.6696127031036031, + "ndcg": 0.7419804620261266, + "recall@10": 0.49798997460620914, + "recall@20": 0.7668403956654921, + "mrr": 0.8317567567567568, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ce-pool-4class/report.json", + "recorded_at": "2026-08-18T17:54:14.073711+00:00" + }, + { + "run_id": "search-esci-74-ce-pool-4class-nowt", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "4class-weighted-ce", + "channels": [ + "rank-pool-4class" + ], + "rank_pool": true, + "rank_pool_ce": true, + "n_queries": 74, + "ndcg@10": 0.6096378704103984, + "ndcg@20": 0.6881968860231006, + "ndcg": 0.7594213054854562, + "recall@10": 0.5249760000665886, + "recall@20": 0.7631438693842529, + "mrr": 0.8530036338859869, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ce-pool-4class-nowt/report.json", + "recorded_at": "2026-08-18T18:20:20.750483+00:00" + }, + { + "run_id": "search-esci-74-ce-pool-4class-e2", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "4class-weighted-ce", + "channels": [ + "rank-pool-4class" + ], + "rank_pool": true, + "rank_pool_ce": true, + "n_queries": 74, + "ndcg@10": 0.6169746541917066, + "ndcg@20": 0.6945278030601721, + "ndcg": 0.7652624576380807, + "recall@10": 0.5214445444747297, + "recall@20": 0.7719306369596903, + "mrr": 0.8525418275418275, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ce-pool-4class-e2/report.json", + "recorded_at": "2026-08-18T18:48:00.939749+00:00" + }, + { + "run_id": "search-esci-74-passages-k50", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4998984443226263, + "ndcg@20": 0.5522842574567189, + "ndcg@50": 0.6378451547901237, + "ndcg": 0.6378451547901237, + "recall@10": 0.378732513089159, + "recall@20": 0.589865760711773, + "recall@50": 0.833954370548067, + "mrr": 0.7645849420849421, + "pool_coverage": null, + "p50_retrieve_ms": 60.091, + "p95_retrieve_ms": 88.02255, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-k50/report.json", + "recorded_at": "2026-08-18T20:31:55.998500+00:00" + }, + { + "run_id": "search-esci-74-passages-k100", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 100, + "ndcg@10": 0.4998055248060819, + "ndcg@20": 0.5523315287868084, + "ndcg@50": 0.6375525242924566, + "ndcg": 0.6564185982755066, + "recall@10": 0.378732513089159, + "recall@20": 0.5913672622132746, + "recall@50": 0.830951367545064, + "mrr": 0.76497727997728, + "pool_coverage": null, + "p50_retrieve_ms": 57.553, + "p95_retrieve_ms": 81.81569999999999, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-k100/report.json", + "recorded_at": "2026-08-18T20:32:27.553496+00:00" + }, + { + "run_id": "search-esci-74-passages-k50-pool", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": true, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.5668024300832123, + "ndcg@20": 0.624102711461205, + "ndcg@50": 0.6826188564511254, + "ndcg": 0.6947261855882534, + "recall@10": 0.4507852781809733, + "recall@20": 0.6686543312372134, + "recall@50": 0.8324528690465655, + "mrr": 0.8271396396396397, + "pool_coverage": 0.778222315718655, + "p50_retrieve_ms": 60.1315, + "p95_retrieve_ms": 86.49954999999997, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-k50-pool/report.json", + "recorded_at": "2026-08-18T20:33:02.203247+00:00" + }, + { + "run_id": "search-esci-74-passages-k100-pool", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": true, + "rank_pool_ce": false, + "n_queries": 74, + "k": 100, + "ndcg@10": 0.5737527941225129, + "ndcg@20": 0.6352462971440838, + "ndcg@50": 0.7092403328584962, + "ndcg": 0.7183777959320791, + "recall@10": 0.46580029319598837, + "recall@20": 0.6964864396614271, + "recall@50": 0.9062083174986402, + "mrr": 0.836599099099099, + "pool_coverage": 0.8462618325454071, + "p50_retrieve_ms": 58.5245, + "p95_retrieve_ms": 82.5653, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-k100-pool/report.json", + "recorded_at": "2026-08-18T20:33:39.620042+00:00" + }, + { + "run_id": "search-esci-74-passages-k50-ce", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "harness:cross-encoder", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4478964444706324, + "ndcg@20": 0.5146313751051845, + "ndcg@50": 0.6060742524325241, + "ndcg": 0.6060742524325241, + "recall@10": 0.331273452580599, + "recall@20": 0.5538599992903688, + "recall@50": 0.833954370548067, + "mrr": 0.7197929141477528, + "pool_coverage": null, + "p50_retrieve_ms": 60.091, + "p95_retrieve_ms": 88.02255, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-k50-ce/report.json", + "recorded_at": "2026-08-18T20:34:39.854467+00:00" + }, + { + "run_id": "search-esci-74-passages-k50-ce-l12", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "harness:cross-encoder", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4674585131694284, + "ndcg@20": 0.5084668214514952, + "ndcg@50": 0.6140738833603318, + "ndcg": 0.6140738833603318, + "recall@10": 0.3412172585548375, + "recall@20": 0.5191872363321448, + "recall@50": 0.833954370548067, + "mrr": 0.7063349063349063, + "pool_coverage": null, + "p50_retrieve_ms": 60.091, + "p95_retrieve_ms": 88.02255, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-k50-ce-l12/report.json", + "recorded_at": "2026-08-18T20:35:44.462471+00:00" + }, + { + "run_id": "search-esci-74-passages-k50-qrewrite", + "brain": "searchbenchesci74", + "dataset": "search_esci_74_qrewrite.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4998984443226263, + "ndcg@20": 0.5522842574567189, + "ndcg@50": 0.6378451547901237, + "ndcg": 0.6378451547901237, + "recall@10": 0.378732513089159, + "recall@20": 0.589865760711773, + "recall@50": 0.833954370548067, + "mrr": 0.7645849420849421, + "pool_coverage": null, + "p50_retrieve_ms": 59.432500000000005, + "p95_retrieve_ms": 82.09889999999999, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-k50-qrewrite/report.json", + "recorded_at": "2026-08-18T21:18:51.308816+00:00" + }, + { + "run_id": "search-esci-74-dense-ance-k50", + "brain": "harness-local", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "none", + "channels": [ + "harness-dense" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.3960504272587966, + "ndcg@20": 0.445560860839057, + "ndcg@50": 0.5370854731938006, + "ndcg": 0.5370854731938006, + "recall@10": 0.3015013697338196, + "recall@20": 0.4872087291019431, + "recall@50": 0.7503156876126188, + "mrr": 0.6471417846417846, + "pool_coverage": null, + "p50_retrieve_ms": 74.9370945944778, + "p95_retrieve_ms": 74.9370945944778, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-dense-ance-k50/report.json", + "recorded_at": "2026-08-18T21:22:24.054560+00:00" + }, + { + "run_id": "search-esci-74-passages-k50-ce-retrieved", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "harness:cross-encoder", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4648895994453083, + "ndcg@20": 0.5245865140860362, + "ndcg@50": 0.6126753191137789, + "ndcg": 0.6126753191137789, + "recall@10": 0.35140463046557896, + "recall@20": 0.5410788442403298, + "recall@50": 0.833954370548067, + "mrr": 0.7120610588001892, + "pool_coverage": null, + "p50_retrieve_ms": 60.091, + "p95_retrieve_ms": 88.02255, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-k50-ce-retrieved/report.json", + "recorded_at": "2026-08-18T22:38:31.141079+00:00" + }, + { + "run_id": "search-esci-74-colbert-k50", + "brain": "harness-local-colbert", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "none", + "channels": [ + "harness-colbert" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.43412701768229023, + "ndcg@20": 0.4815601734708754, + "ndcg@50": 0.5523204205368764, + "ndcg": 0.5523204205368764, + "recall@10": 0.3110168068124024, + "recall@20": 0.5023603484326227, + "recall@50": 0.7138820979266567, + "mrr": 0.6689285195333583, + "pool_coverage": null, + "p50_retrieve_ms": 10063.364791662614, + "p95_retrieve_ms": 10063.364791662614, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-colbert-k50/report.json", + "recorded_at": "2026-08-18T22:51:16.979167+00:00" + }, + { + "run_id": "search-esci-74-bge-base-k50", + "brain": "harness-local", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "none", + "channels": [ + "harness-dense" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.44136708015333975, + "ndcg@20": 0.5041842920107446, + "ndcg@50": 0.5892299329783445, + "ndcg": 0.5892299329783445, + "recall@10": 0.32179328339461255, + "recall@20": 0.5251414009467592, + "recall@50": 0.7989247943561789, + "mrr": 0.7471303646913403, + "pool_coverage": null, + "p50_retrieve_ms": 408.0482843509762, + "p95_retrieve_ms": 408.04828435097625, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-bge-base-k50/report.json", + "recorded_at": "2026-08-18T22:52:53.742680+00:00" + }, + { + "run_id": "search-esci-74-union-bge-k50", + "brain": "harness-local-union", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "harness-union" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.47918374395281466, + "ndcg@20": 0.5381296158524957, + "ndcg@50": 0.6141255447421892, + "ndcg": 0.6141255447421892, + "recall@10": 0.36314016383757297, + "recall@20": 0.5774058852814317, + "recall@50": 0.8271070974286027, + "mrr": 0.7653163449499656, + "pool_coverage": null, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-union-bge-k50/report.json", + "recorded_at": "2026-08-19T08:10:06.928384+00:00" + }, + { + "run_id": "search-esci-74-exhaustive-ce", + "brain": "harness-local-exhaustive", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "4class-weighted-ce", + "channels": [ + "exhaustive-4class" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4162185585932262, + "ndcg@20": 0.4392534244105597, + "ndcg@50": 0.5042093175793891, + "ndcg": 0.5042093175793891, + "recall@10": 0.26585896457775815, + "recall@20": 0.4060812399456508, + "recall@50": 0.6029270278490679, + "mrr": 0.6578079129936715, + "pool_coverage": null, + "p50_retrieve_ms": 94253.11211881155, + "p95_retrieve_ms": 94253.11211881155, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-exhaustive-ce/report.json", + "recorded_at": "2026-08-19T10:05:59.463503+00:00" + }, + { + "run_id": "search-esci-74-catalog-ce-k50", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "plugin:cross-encoder", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4674585131694284, + "ndcg@20": 0.5084668214514952, + "ndcg@50": 0.6140738833603318, + "ndcg": 0.6140738833603318, + "recall@10": 0.3412172585548375, + "recall@20": 0.5191872363321448, + "recall@50": 0.833954370548067, + "mrr": 0.7063349063349063, + "pool_coverage": null, + "p50_retrieve_ms": 59.308499999999995, + "p95_retrieve_ms": 86.88884999999999, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-catalog-ce-k50/report.json", + "recorded_at": "2026-08-19T10:17:45.568452+00:00" + }, + { + "run_id": "search-esci-74-cascade-tail-k50", + "brain": "harness-local-cascade", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "none", + "channels": [ + "harness-cascade" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4998984443226263, + "ndcg@20": 0.6194622913116794, + "ndcg@50": 0.6686100079902446, + "ndcg": 0.6686100079902446, + "recall@10": 0.378732513089159, + "recall@20": 0.7504072036031975, + "recall@50": 0.8885964311815054, + "mrr": 0.7679931554931555, + "pool_coverage": null, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-cascade-tail-k50/report.json", + "recorded_at": "2026-08-19T10:32:17.738702+00:00" + }, + { + "run_id": "search-esci-74-ce-pool-deberta-base", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "4class-weighted-ce", + "channels": [ + "rank-pool-4class" + ], + "rank_pool": true, + "rank_pool_ce": true, + "n_queries": 74, + "k": 40, + "ndcg@10": 0.6492728519674476, + "ndcg@20": 0.7101117922903459, + "ndcg@50": null, + "ndcg": 0.7772540710194906, + "recall@10": 0.5183145850778, + "recall@20": 0.7604145345598889, + "recall@50": null, + "mrr": 0.8646712599298806, + "pool_coverage": null, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ce-pool-deberta-base/report.json", + "recorded_at": "2026-08-19T14:00:25.594211+00:00" + }, + { + "run_id": "search-esci-74-passages-k50-ce-deberta", + "brain": "searchbenchsmoke", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "harness:cross-encoder", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.5101053946025641, + "ndcg@20": 0.5568741362838021, + "ndcg@50": 0.6365973490191127, + "ndcg": 0.6365973490191127, + "recall@10": 0.36281504203381093, + "recall@20": 0.5467544580912761, + "recall@50": 0.833954370548067, + "mrr": 0.7457139784725992, + "pool_coverage": null, + "p50_retrieve_ms": 60.091, + "p95_retrieve_ms": 88.02255, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-k50-ce-deberta/report.json", + "recorded_at": "2026-08-19T14:09:19.540811+00:00" + }, + { + "run_id": "search-esci-es-passages-k50", + "brain": "searchbenchescies", + "dataset": "search_esci_es.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 62, + "k": 50, + "ndcg@10": 0.5774393479403104, + "ndcg@20": 0.6303553323648456, + "ndcg@50": 0.7327853549091549, + "ndcg": 0.7327853549091549, + "recall@10": 0.35328224222955834, + "recall@20": 0.5758587932370467, + "recall@50": 0.9141280526823616, + "mrr": 0.8704493087557603, + "pool_coverage": null, + "p50_retrieve_ms": 84.13, + "p95_retrieve_ms": 96.16295000000001, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-es-passages-k50/report.json", + "recorded_at": "2026-08-19T16:34:42.965337+00:00" + }, + { + "run_id": "search-italian-smoke", + "brain": "searchbenchitsmoke", + "dataset": "search_italian_smoke.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 3, + "k": 20, + "ndcg@10": 1.0, + "ndcg@20": 1.0, + "ndcg@50": null, + "ndcg": 1.0, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@50": null, + "mrr": 1.0, + "pool_coverage": null, + "p50_retrieve_ms": 25.33, + "p95_retrieve_ms": 29.3206, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-italian-smoke/report.json", + "recorded_at": "2026-08-19T16:37:37.730780+00:00" + }, + { + "run_id": "search-italian-smoke-inflect", + "brain": "searchbenchitsmoke", + "dataset": "search_italian_smoke_inflect.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 3, + "k": 20, + "ndcg@10": 1.0, + "ndcg@20": 1.0, + "ndcg@50": null, + "ndcg": 1.0, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@50": null, + "mrr": 1.0, + "pool_coverage": null, + "p50_retrieve_ms": 21.494, + "p95_retrieve_ms": 23.211199999999998, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-italian-smoke-inflect/report.json", + "recorded_at": "2026-08-19T16:37:46.108861+00:00" + }, + { + "run_id": "search-italian-smoke-inflect-fts", + "brain": "searchbenchitsmoke", + "dataset": "search_italian_smoke_inflect.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 3, + "k": 20, + "ndcg@10": 1.0, + "ndcg@20": 1.0, + "ndcg@50": null, + "ndcg": 1.0, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@50": null, + "mrr": 1.0, + "pool_coverage": null, + "p50_retrieve_ms": 33.56, + "p95_retrieve_ms": 36.8936, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-italian-smoke-inflect-fts/report.json", + "recorded_at": "2026-08-19T16:44:05.699330+00:00" + }, + { + "run_id": "search-italian-minilm-pipeline", + "brain": "searchbenchitmmini", + "dataset": "search_italian_smoke.jsonl", + "fusion": "none", + "rerank": "none", + "channels": [ + "harness-dense" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 3, + "k": 10, + "ndcg@10": 1.0, + "ndcg@20": 1.0, + "ndcg@50": null, + "ndcg": 1.0, + "recall@10": 1.0, + "recall@20": null, + "recall@50": null, + "mrr": 1.0, + "pool_coverage": null, + "p50_retrieve_ms": 188.19916666689096, + "p95_retrieve_ms": 188.19916666689096, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-italian-minilm-pipeline/report.json", + "recorded_at": "2026-08-19T19:27:33.792776+00:00" + }, + { + "run_id": "search-esci-es-spell-k50", + "brain": "searchbenchescies", + "dataset": "search_esci_es_spell.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 62, + "k": 50, + "ndcg@10": 0.5950327313726855, + "ndcg@20": 0.6464666665333806, + "ndcg@50": 0.7424579237775919, + "ndcg": 0.7424579237775919, + "recall@10": 0.36220476611566177, + "recall@20": 0.5832920760052671, + "recall@50": 0.9143064382925891, + "mrr": 0.8865783410138248, + "pool_coverage": null, + "p50_retrieve_ms": 92.655, + "p95_retrieve_ms": 107.2923, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-es-spell-k50/report.json", + "recorded_at": "2026-08-19T19:28:14.068481+00:00" + }, + { + "run_id": "search-italian-smoke-extras-locale", + "brain": "searchbenchitsmoke", + "dataset": "search_italian_smoke.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 3, + "k": 10, + "ndcg@10": 1.0, + "ndcg@20": 1.0, + "ndcg@50": null, + "ndcg": 1.0, + "recall@10": 1.0, + "recall@20": null, + "recall@50": null, + "mrr": 1.0, + "pool_coverage": null, + "p50_retrieve_ms": 17.126, + "p95_retrieve_ms": 23.894000000000002, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-italian-smoke-extras-locale/report.json", + "recorded_at": "2026-08-19T19:30:47.530480+00:00" + }, + { + "run_id": "search-italian-smoke-extras-nope", + "brain": "searchbenchitsmoke", + "dataset": "search_italian_smoke.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 3, + "k": 10, + "ndcg@10": 0.0, + "ndcg@20": 0.0, + "ndcg@50": null, + "ndcg": 0.0, + "recall@10": 0.0, + "recall@20": null, + "recall@50": null, + "mrr": 0.0, + "pool_coverage": null, + "p50_retrieve_ms": 22.721, + "p95_retrieve_ms": 24.2753, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-italian-smoke-extras-nope/report.json", + "recorded_at": "2026-08-19T19:29:31.676418+00:00" + }, + { + "run_id": "search-esci-74-passages-k50-extras-sanity", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4998984443226263, + "ndcg@20": 0.5522842574567189, + "ndcg@50": 0.6378451547901237, + "ndcg": 0.6378451547901237, + "recall@10": 0.378732513089159, + "recall@20": 0.589865760711773, + "recall@50": 0.833954370548067, + "mrr": 0.7645849420849421, + "pool_coverage": null, + "p50_retrieve_ms": 63.1325, + "p95_retrieve_ms": 88.86579999999998, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-passages-k50-extras-sanity/report.json", + "recorded_at": "2026-08-19T19:35:44.545077+00:00" + }, + { + "run_id": "search-italian-smoke-inflect-fts-or", + "brain": "searchbenchitsmoke", + "dataset": "search_italian_smoke_inflect.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 3, + "k": 20, + "ndcg@10": 1.0, + "ndcg@20": 1.0, + "ndcg@50": null, + "ndcg": 1.0, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@50": null, + "mrr": 1.0, + "pool_coverage": null, + "p50_retrieve_ms": 26.364, + "p95_retrieve_ms": 42.744899999999994, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-italian-smoke-inflect-fts-or/report.json", + "recorded_at": "2026-08-19T20:16:24.241357+00:00" + }, + { + "run_id": "search-esci-74-cc-a03", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "cc", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.48296913346675313, + "ndcg@20": 0.5353434819101804, + "ndcg@50": 0.6235227747628354, + "ndcg": 0.6235227747628354, + "recall@10": 0.37594464118364007, + "recall@20": 0.5859902261910772, + "recall@50": 0.8324528690465655, + "mrr": 0.7492780202562811, + "pool_coverage": null, + "p50_retrieve_ms": 62.8885, + "p95_retrieve_ms": 89.98204999999999, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-cc-a03/report.json", + "recorded_at": "2026-08-19T20:18:18.361436+00:00" + }, + { + "run_id": "search-esci-74-cc-a05", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "cc", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4861505982058926, + "ndcg@20": 0.5412686017716648, + "ndcg@50": 0.62608241624179, + "ndcg": 0.62608241624179, + "recall@10": 0.37673766109430706, + "recall@20": 0.5919104321112831, + "recall@50": 0.8324528690465655, + "mrr": 0.7604021879021879, + "pool_coverage": null, + "p50_retrieve_ms": 61.432, + "p95_retrieve_ms": 92.56864999999999, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-cc-a05/report.json", + "recorded_at": "2026-08-19T20:18:41.196658+00:00" + }, + { + "run_id": "search-esci-74-cc-a07", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "cc", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4925393217389386, + "ndcg@20": 0.5491258060245199, + "ndcg@50": 0.6321844176410597, + "ndcg": 0.6321844176410597, + "recall@10": 0.3780890124456584, + "recall@20": 0.5964401922862047, + "recall@50": 0.833954370548067, + "mrr": 0.7630083655083656, + "pool_coverage": null, + "p50_retrieve_ms": 60.5155, + "p95_retrieve_ms": 89.04719999999998, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-cc-a07/report.json", + "recorded_at": "2026-08-19T20:19:05.271923+00:00" + }, + { + "run_id": "search-esci-74-literal-fill-k50", + "brain": "searchbenchesci74", + "dataset": "search_esci_74.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.4998984443226263, + "ndcg@20": 0.45027838324634667, + "ndcg@50": 0.5436008239474691, + "ndcg": 0.5436008239474691, + "recall@10": 0.378732513089159, + "recall@20": 0.3929861145019264, + "recall@50": 0.6549969089908488, + "mrr": 0.7639237451737452, + "pool_coverage": null, + "p50_retrieve_ms": 49.019999999999996, + "p95_retrieve_ms": 74.27314999999999, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-literal-fill-k50/report.json", + "recorded_at": "2026-08-19T20:20:15.131585+00:00" + }, + { + "run_id": "search-esci-74-ltr-head-k50", + "brain": "harness-local-ltr", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "ltr-pairwise", + "channels": [ + "harness-ltr-head" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.5146015046757452, + "ndcg@20": 0.5567887112159354, + "ndcg@50": 0.6406685515140392, + "ndcg": 0.6406685515140392, + "recall@10": 0.384165323643092, + "recall@20": 0.5830174861607277, + "recall@50": 0.833954370548067, + "mrr": 0.7525579150579151, + "pool_coverage": null, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ltr-head-k50/report.json", + "recorded_at": "2026-08-19T21:09:47.411344+00:00" + }, + { + "run_id": "search-esci-74-ltr-cefeat-k50", + "brain": "harness-local-ltr", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "ltr-pairwise", + "channels": [ + "harness-ltr-head" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.5242780297400812, + "ndcg@20": 0.5655195290162985, + "ndcg@50": 0.6456032400547804, + "ndcg": 0.6456032400547804, + "recall@10": 0.38343544674019964, + "recall@20": 0.5687827599719494, + "recall@50": 0.833954370548067, + "mrr": 0.7792567567567568, + "pool_coverage": null, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ltr-cefeat-k50/report.json", + "recorded_at": "2026-08-20T07:14:39.761450+00:00" + }, + { + "run_id": "search-esci-74-ltr-deberta-k50", + "brain": "harness-local-ltr", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "ltr-pairwise", + "channels": [ + "harness-ltr-head" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.5420964444237363, + "ndcg@20": 0.580013195834899, + "ndcg@50": 0.6509103805030941, + "ndcg": 0.6509103805030941, + "recall@10": 0.3872531182148088, + "recall@20": 0.5863055009035371, + "recall@50": 0.833954370548067, + "mrr": 0.7744787644787645, + "pool_coverage": null, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ltr-deberta-k50/report.json", + "recorded_at": "2026-08-20T07:43:07.017767+00:00" + }, + { + "run_id": "search-esci-74-ltr-deberta-lgbm-k50", + "brain": "harness-local-ltr", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "ltr-lightgbm", + "channels": [ + "harness-ltr-head" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.5159432058815566, + "ndcg@20": 0.5658120484311862, + "ndcg@50": 0.6415336394479464, + "ndcg": 0.6415336394479464, + "recall@10": 0.37878019077485103, + "recall@20": 0.5918308018978374, + "recall@50": 0.833954370548067, + "mrr": 0.7098312215959275, + "pool_coverage": null, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ltr-deberta-lgbm-k50/report.json", + "recorded_at": "2026-08-20T07:48:40.794647+00:00" + }, + { + "run_id": "search-esci-ltr200-passages-k50", + "brain": "searchbenchesciltr2", + "dataset": "search_esci_ltr200.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 170, + "k": 50, + "ndcg@10": 0.38688497520690734, + "ndcg@20": 0.4428785087667844, + "ndcg@50": 0.5191838961643084, + "ndcg": 0.5191838961643084, + "recall@10": 0.3668275379939918, + "recall@20": 0.5506847180507305, + "recall@50": 0.7435914662537475, + "mrr": 0.6599425680048517, + "pool_coverage": null, + "p50_retrieve_ms": 76.07849999999999, + "p95_retrieve_ms": 152.37659999999983, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-ltr200-passages-k50/report.json", + "recorded_at": "2026-08-20T10:45:30.571169+00:00" + }, + { + "run_id": "search-esci-74-ltr-deberta-train200", + "brain": "harness-local-ltr", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "ltr-pairwise", + "channels": [ + "harness-ltr-head" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.5436096369070966, + "ndcg@20": 0.5825504636416924, + "ndcg@50": 0.6525856561608427, + "ndcg": 0.6525856561608427, + "recall@10": 0.3911684808472767, + "recall@20": 0.5850220289937974, + "recall@50": 0.833954370548067, + "mrr": 0.7621138996138996, + "pool_coverage": null, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ltr-deberta-train200/report.json", + "recorded_at": "2026-08-20T10:57:34.082897+00:00" + }, + { + "run_id": "search-esci-74-ltr-deberta-lgbm-train200", + "brain": "harness-local-ltr", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "ltr-lightgbm", + "channels": [ + "harness-ltr-head" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.5333470812074246, + "ndcg@20": 0.5739148370361776, + "ndcg@50": 0.6521013344978932, + "ndcg": 0.6521013344978932, + "recall@10": 0.38938710235182716, + "recall@20": 0.5909398273461974, + "recall@50": 0.833954370548067, + "mrr": 0.7517797970384177, + "pool_coverage": null, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ltr-deberta-lgbm-train200/report.json", + "recorded_at": "2026-08-20T11:33:18.933082+00:00" + }, + { + "run_id": "search-esci-74-ltr-deberta-hybrid170", + "brain": "harness-local-ltr", + "dataset": "search_esci_74.jsonl", + "fusion": "none", + "rerank": "ltr-pairwise", + "channels": [ + "harness-ltr-head" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 74, + "k": 50, + "ndcg@10": 0.5304449022081842, + "ndcg@20": 0.5753013243082012, + "ndcg@50": 0.6452218148321592, + "ndcg": 0.6452218148321592, + "recall@10": 0.38518108116293304, + "recall@20": 0.5882930387092911, + "recall@50": 0.833954370548067, + "mrr": 0.7646718146718147, + "pool_coverage": null, + "p50_retrieve_ms": 0.0, + "p95_retrieve_ms": 0.0, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-esci-74-ltr-deberta-hybrid170/report.json", + "recorded_at": "2026-08-20T11:57:34.067863+00:00" + }, + { + "run_id": "search-wands-passages-k50", + "brain": "searchbenchwands", + "dataset": "search_wands.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 66, + "k": 50, + "ndcg@10": 0.823176135584089, + "ndcg@20": 0.8120944688355884, + "ndcg@50": 0.8223686140560469, + "ndcg": 0.8223686140560469, + "recall@10": 0.26880378393536286, + "recall@20": 0.46518737110842373, + "recall@50": 0.8365552576078893, + "mrr": 0.9245931537598204, + "pool_coverage": null, + "p50_retrieve_ms": 86.595, + "p95_retrieve_ms": 96.93674999999999, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-wands-passages-k50/report.json", + "recorded_at": "2026-08-20T13:17:40.353988+00:00" + }, + { + "run_id": "search-wandsgraph-communities-k50", + "brain": "searchbenchwandsgraph", + "dataset": "search_wands.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "communities" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 66, + "k": 50, + "ndcg@10": 0.4094084735740818, + "ndcg@20": 0.36455789387786824, + "ndcg@50": 0.325224291157982, + "ndcg": 0.4570937299953597, + "recall@10": 0.14824773180036338, + "recall@20": 0.20624528979792137, + "recall@50": 0.28701452056715215, + "mrr": 0.6737914862914863, + "pool_coverage": null, + "p50_retrieve_ms": 5609.237499999999, + "p95_retrieve_ms": 11856.0285, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-wandsgraph-communities-k50/report.json", + "recorded_at": "2026-08-21T03:12:00.249304+00:00", + "protocol": "architecture demo: isolated catalog graph on searchbenchwandsgraph; not a quality-default claim vs frozen WANDS passages 0.823 or ESCI 0.500", + "claim": "architecture-demo" + }, + { + "run_id": "search-wandsgraph-passages-k50", + "brain": "searchbenchwandsgraph", + "dataset": "search_wands.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 66, + "k": 50, + "ndcg@10": 0.8230998576010691, + "ndcg@20": 0.812056754200934, + "ndcg@50": 0.8223486981324697, + "ndcg": 0.8223486981324697, + "recall@10": 0.26880378393536286, + "recall@20": 0.46518737110842373, + "recall@50": 0.8365552576078893, + "mrr": 0.9245931537598204, + "pool_coverage": null, + "p50_retrieve_ms": 98.604, + "p95_retrieve_ms": 115.12925, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-wandsgraph-passages-k50/report.json", + "recorded_at": "2026-08-21T03:12:36.455586+00:00", + "protocol": "architecture demo: isolated catalog graph on searchbenchwandsgraph; not a quality-default claim vs frozen WANDS passages 0.823 or ESCI 0.500", + "claim": "architecture-demo" + }, + { + "run_id": "search-wandsgraph-passages-nodeid-k50", + "brain": "searchbenchwandsgraph", + "dataset": "search_wands.jsonl", + "fusion": "rrf", + "rerank": "none", + "channels": [ + "passages" + ], + "rank_pool": false, + "rank_pool_ce": false, + "n_queries": 66, + "k": 50, + "ndcg@10": 0.8230998576010691, + "ndcg@20": 0.812056754200934, + "ndcg@50": 0.8223486981324697, + "ndcg": 0.8223486981324697, + "recall@10": 0.26880378393536286, + "recall@20": 0.46518737110842373, + "recall@50": 0.8365552576078893, + "mrr": 0.9245931537598204, + "pool_coverage": null, + "p50_retrieve_ms": 84.0175, + "p95_retrieve_ms": 105.24199999999999, + "git_sha": "f42c4a8eaadb3c0cc79f78c34b0e1a70534bee10", + "report_path": "runs/search-wandsgraph-passages-nodeid-k50/report.json", + "recorded_at": "2026-08-21T07:53:39.311033+00:00", + "protocol": "architecture demo: isolated catalog graph on searchbenchwandsgraph; not a quality-default claim vs frozen WANDS passages 0.823 or ESCI 0.500", + "claim": "architecture-demo" + } + ] } } } diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt index 40855cb..54e2c03 100644 --- a/benchmarks/requirements.txt +++ b/benchmarks/requirements.txt @@ -3,5 +3,8 @@ openai>=1.40.0,<2.0.0 python-dotenv>=1.0.0,<2.0.0 rich>=13.0.0,<15.0.0 datasets>=2.19.0,<4.0.0 +huggingface_hub>=0.23.0,<1.0.0 +pyarrow>=15.0.0,<20.0.0 scipy>=1.11.0,<2.0.0 json-repair>=0.30.0,<1.0.0 +lightgbm>=4.3.0,<5.0.0 diff --git a/benchmarks/search.sh b/benchmarks/search.sh new file mode 100755 index 0000000..1d53022 --- /dev/null +++ b/benchmarks/search.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")" && pwd)" +PYTHON="${ROOT}/.venv/bin/python" +PARENT="${ROOT}/../.venv/bin/python" +if [[ ! -x "$PYTHON" ]]; then + echo "Missing ${PYTHON}. Create it with:" + echo " cd benchmarks && python3 -m venv .venv && .venv/bin/pip install -r requirements.txt" + exit 1 +fi +for arg in "$@"; do + if [[ "$arg" == "rank-pool-ce" || "$arg" == "finetune-ce" || "$arg" == "pool-first-stage" || "$arg" == "rerank-retrieved" || "$arg" == "finetune-dense" || "$arg" == "local-dense" || "$arg" == "mine-retrieved-lists" || "$arg" == "finetune-4class" || "$arg" == "colbert-local" || "$arg" == "rank-corpus" || "$arg" == "ltr-head" || "$arg" == "backfill-entity-text" ]]; then + if [[ -x "$PARENT" ]]; then + PYTHON="$PARENT" + fi + if [[ "$arg" == "backfill-entity-text" && -f "${HOME}/.brainapi/source/.env" ]]; then + set -a + # shellcheck disable=SC1091 + source "${HOME}/.brainapi/source/.env" + set +a + export SEARCH_ENABLED="${SEARCH_ENABLED:-true}" + export DATA_DB="${DATA_DB:-postgresql}" + export VECTOR_DB="${VECTOR_DB:-postgresql}" + export GRAPH_DB="${GRAPH_DB:-networkx}" + export ENV="${ENV:-development}" + fi + break + fi +done +export PYTHONPATH="${ROOT}${PYTHONPATH:+:$PYTHONPATH}" +exec "$PYTHON" -m search "$@" diff --git a/benchmarks/search/__init__.py b/benchmarks/search/__init__.py new file mode 100644 index 0000000..8d8cd25 --- /dev/null +++ b/benchmarks/search/__init__.py @@ -0,0 +1 @@ +"""Search eval harness against searchbench* brains.""" diff --git a/benchmarks/search/__main__.py b/benchmarks/search/__main__.py new file mode 100644 index 0000000..5f6e430 --- /dev/null +++ b/benchmarks/search/__main__.py @@ -0,0 +1,4 @@ +from search.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/search/backfill_entities.py b/benchmarks/search/backfill_entities.py new file mode 100644 index 0000000..3c159d5 --- /dev/null +++ b/benchmarks/search/backfill_entities.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from typing import Any, Callable + +from search.config import FROZEN_STRUCTURED_BRAINS, validate_brain_id +from search.mapping import catalog_entity_backfill_rows, entity_uuid + + +def refuse_entity_backfill(brain_id: str) -> str: + bid = validate_brain_id(brain_id) + if bid in FROZEN_STRUCTURED_BRAINS: + raise SystemExit( + f"Refusing ENTITY text backfill on frozen brain {bid}. " + "Use searchbenchwandsgraph only (never wipe frozen structured brains)." + ) + return bid + + +def backfill_rows_from_docs(docs: list[dict[str, Any]]) -> list[dict[str, str]]: + return catalog_entity_backfill_rows(docs) + + +def apply_entity_text_backfill( + *, + brain_id: str, + rows: list[dict[str, str]], + graph: Any, + embeddings: Any, + vector_store: Any, + is_item: Callable[[str, list[str] | None], bool] | None = None, +) -> dict[str, Any]: + bid = refuse_entity_backfill(brain_id) + if is_item is None: + from src.core.search.graph_channels import is_item_entity as is_item + from src.constants.embeddings import Vector + from src.core.agents.scout_agent import ScoutEntity + from src.core.search.catalog_graph import node_embed_text + + updated = 0 + missing = 0 + skipped = 0 + for row in rows: + uuid = str(row.get("uuid") or entity_uuid(str(row.get("doc_id") or ""))).strip() + blob = str(row.get("search_text") or "").strip() + if not uuid or not blob: + skipped += 1 + continue + node = graph.get_by_uuid(uuid, bid) + if node is None: + missing += 1 + continue + labels = list(getattr(node, "labels", None) or []) + if not is_item(uuid, labels): + skipped += 1 + continue + props = dict(getattr(node, "properties", None) or {}) + props["search_text"] = blob + graph.update_node( + uuid, + bid, + new_properties={"search_text": blob}, + ) + scout = ScoutEntity( + uuid=uuid, + name=str(getattr(node, "name", "") or row.get("name") or uuid), + type="ENTITY", + properties=props, + ) + text, _ = node_embed_text(scout) + vector = embeddings.embed_text(text) + if not isinstance(vector, Vector): + skipped += 1 + continue + existing = props.get("v_id") + if existing: + vector.id = str(existing) + vector.metadata = { + "labels": labels or ["ENTITY"], + "name": scout.name, + "uuid": uuid, + } + v_ids = vector_store.add_vectors([vector], store="nodes", brain_id=bid) + if v_ids: + graph.update_node( + uuid, + bid, + new_properties={"search_text": blob, "v_id": v_ids[0]}, + ) + updated += 1 + return { + "brain_id": bid, + "n_rows": len(rows), + "updated": updated, + "missing": missing, + "skipped": skipped, + } diff --git a/benchmarks/search/catalog.py b/benchmarks/search/catalog.py new file mode 100644 index 0000000..43340ba --- /dev/null +++ b/benchmarks/search/catalog.py @@ -0,0 +1,743 @@ +from __future__ import annotations + +import csv +from collections import defaultdict +from pathlib import Path +from typing import Any, Iterable, Iterator +from urllib.request import urlretrieve + +from search.config import DATA_DIR +from search.dataset import DOC_MARKER_PREFIX, write_records + +ESCI_NAME = "esci" +WANDS_NAME = "wands" +JDSEARCH_NAME = "jdsearch" +SUPPORTED_CATALOGS = (ESCI_NAME, WANDS_NAME, JDSEARCH_NAME) +DOWNLOAD_ALL_CATALOGS = (ESCI_NAME, WANDS_NAME) +ESCI_LOCALES = ("us", "es", "jp") + +ESCI_JSONL = DATA_DIR / "search_esci.jsonl" +WANDS_JSONL = DATA_DIR / "search_wands.jsonl" +JDSEARCH_JSONL = DATA_DIR / "search_jdsearch.jsonl" +ESCI_CACHE_DIR = DATA_DIR / "esci" +WANDS_CACHE_DIR = DATA_DIR / "wands" + +ESCI_GITHUB_BASE = ( + "https://github.com/amazon-science/esci-data/raw/main/" + "shopping_queries_dataset" +) +ESCI_HF_REPO = "khanmu2003/amazon-shopping-queries-dataset" +ESCI_EXAMPLES_FILE = "shopping_queries_dataset_examples.parquet" +ESCI_PRODUCTS_FILE = "shopping_queries_dataset_products.parquet" + +WANDS_GITHUB_BASE = "https://raw.githubusercontent.com/wayfair/WANDS/main/dataset" + +ESCI_GAINS = {"E": 1.0, "S": 0.1, "C": 0.01, "I": 0.0} +WANDS_GAINS = {"Exact": 1.0, "Partial": 0.5, "Irrelevant": 0.0} + +DEFAULT_MAX_QUERIES = 80 +DEFAULT_MAX_DOCS = 2000 +DEFAULT_CANDIDATES_PER_QUERY = 40 +PRODUCT_TEXT_MAX_CHARS = 4000 +FROZEN_JSONL_ALWAYS = ("search_esci_74.jsonl",) +FROZEN_JSONL_IF_EXISTS = ("search_wands.jsonl", "search_jdsearch.jsonl") + + +def normalize_esci_locale(locale: str) -> str: + loc = (locale or "us").strip().lower() + if loc not in ESCI_LOCALES: + raise ValueError( + f"Unknown ESCI locale {locale!r}. Supported: {ESCI_LOCALES}. " + "ESCI has no Italian (it) split." + ) + return loc + + +def catalog_overwrite_blocked(path: Path) -> bool: + resolved = path.resolve() + for name in FROZEN_JSONL_ALWAYS: + if resolved == (DATA_DIR / name).resolve(): + return True + for name in FROZEN_JSONL_IF_EXISTS: + frozen = (DATA_DIR / name).resolve() + if resolved == frozen and frozen.exists() and frozen.stat().st_size > 0: + return True + return False + + +def catalog_jsonl_path(name: str, *, locale: str = "us") -> Path: + key = name.strip().lower() + if key == ESCI_NAME: + loc = normalize_esci_locale(locale) + if loc == "us": + return ESCI_JSONL + return DATA_DIR / f"search_esci_{loc}.jsonl" + if key == WANDS_NAME: + return WANDS_JSONL + if key == JDSEARCH_NAME: + return JDSEARCH_JSONL + raise ValueError(f"Unknown catalog {name!r}. Supported: {SUPPORTED_CATALOGS}") + + +def prepare_catalog( + name: str, + *, + out_path: Path | None = None, + force: bool = False, + max_queries: int = DEFAULT_MAX_QUERIES, + max_docs: int = DEFAULT_MAX_DOCS, + candidates_per_query: int = DEFAULT_CANDIDATES_PER_QUERY, + locale: str = "us", + split: str = "test", + holdout_qids: set[str] | None = None, +) -> Path: + key = name.strip().lower() + dest = out_path or catalog_jsonl_path(key, locale=locale) + if dest.exists() and dest.stat().st_size > 0 and not force: + return dest + if key == ESCI_NAME: + rows = prepare_esci_rows( + max_queries=max_queries, + max_docs=max_docs, + candidates_per_query=candidates_per_query, + locale=locale, + split=split, + force=force, + holdout_qids=holdout_qids, + ) + elif key == WANDS_NAME: + rows = prepare_wands_rows( + max_queries=max_queries, + max_docs=max_docs, + candidates_per_query=candidates_per_query, + force=force, + ) + elif key == JDSEARCH_NAME: + from search.jdsearch import prepare_jdsearch_bundle + + return prepare_jdsearch_bundle( + dest, + max_queries=max_queries, + max_docs=max_docs, + candidates_per_query=candidates_per_query, + force=force, + ) + else: + raise ValueError(f"Unknown catalog {name!r}. Supported: {SUPPORTED_CATALOGS}") + return write_records(rows, dest) + + +def prepare_esci_rows( + *, + max_queries: int = DEFAULT_MAX_QUERIES, + max_docs: int = DEFAULT_MAX_DOCS, + candidates_per_query: int = DEFAULT_CANDIDATES_PER_QUERY, + locale: str = "us", + split: str = "test", + force: bool = False, + holdout_qids: set[str] | None = None, +) -> list[dict[str, Any]]: + locale = normalize_esci_locale(locale) + examples_path, products_path = download_esci(force=force) + selected, needed_ids = _select_catalog( + _esci_judgments(examples_path, locale=locale, split=split), + max_queries=max_queries, + max_docs=max_docs, + candidates_per_query=candidates_per_query, + slice_name=f"esci-{locale}", + dataset=ESCI_NAME, + gains=ESCI_GAINS, + label_key="esci_label", + query_text_key="query", + holdout_qids=holdout_qids, + ) + products = _esci_products(products_path, needed_ids, locale=locale) + return _docs_and_queries( + selected, + products=products, + doc_ids=needed_ids, + dataset=ESCI_NAME, + title_key="product_title", + description_key="product_description", + extra_fields=extra_fields_from_catalog( + products, + title_key="product_title", + description_key="product_description", + ), + ) + + +def prepare_wands_rows( + *, + max_queries: int = DEFAULT_MAX_QUERIES, + max_docs: int = DEFAULT_MAX_DOCS, + candidates_per_query: int = DEFAULT_CANDIDATES_PER_QUERY, + force: bool = False, +) -> list[dict[str, Any]]: + product_path, query_path, label_path = download_wands(force=force) + queries = _wands_queries(query_path) + selected, needed_ids = _select_catalog( + _wands_judgments(label_path, queries), + max_queries=max_queries, + max_docs=max_docs, + candidates_per_query=candidates_per_query, + slice_name="wands", + dataset=WANDS_NAME, + gains=WANDS_GAINS, + label_key="label", + query_text_key="query", + query_slice_key="query_class", + ) + products = _wands_products(product_path, needed_ids) + return _docs_and_queries( + selected, + products=products, + doc_ids=needed_ids, + dataset=WANDS_NAME, + title_key="product_name", + description_key="product_description", + extra_fields=( + ("Class", "product_class"), + ("Category", "category_hierarchy"), + ("Hierarchy", "category_hierarchy"), + ("Features", "product_features"), + ), + ) + + +def download_esci(*, force: bool = False) -> tuple[Path, Path]: + ESCI_CACHE_DIR.mkdir(parents=True, exist_ok=True) + examples = ESCI_CACHE_DIR / ESCI_EXAMPLES_FILE + products = ESCI_CACHE_DIR / ESCI_PRODUCTS_FILE + _ensure_file( + examples, + url=f"{ESCI_GITHUB_BASE}/{ESCI_EXAMPLES_FILE}", + hf_filename=ESCI_EXAMPLES_FILE, + force=force, + ) + _ensure_file( + products, + url=f"{ESCI_GITHUB_BASE}/{ESCI_PRODUCTS_FILE}", + hf_filename=ESCI_PRODUCTS_FILE, + force=force, + ) + return examples, products + + +def download_wands(*, force: bool = False) -> tuple[Path, Path, Path]: + WANDS_CACHE_DIR.mkdir(parents=True, exist_ok=True) + product = WANDS_CACHE_DIR / "product.csv" + query = WANDS_CACHE_DIR / "query.csv" + label = WANDS_CACHE_DIR / "label.csv" + for path, name in ( + (product, "product.csv"), + (query, "query.csv"), + (label, "label.csv"), + ): + _download_url( + f"{WANDS_GITHUB_BASE}/{name}", + path, + force=force, + ) + return product, query, label + + +def _stringify_catalog_value(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value.strip() + if isinstance(value, (list, tuple)): + parts = [_stringify_catalog_value(item) for item in value] + return " ".join(part for part in parts if part) + if isinstance(value, dict): + parts = [] + for key, item in value.items(): + text = _stringify_catalog_value(item) + if text: + parts.append(f"{key}: {text}") + return "; ".join(parts) + return str(value).strip() + + +def extra_fields_from_catalog( + products: dict[str, dict[str, Any]], + *, + title_key: str, + description_key: str, + skip: tuple[str, ...] = ("product_id",), +) -> tuple[tuple[str, str], ...]: + keys: set[str] = set() + for row in products.values(): + keys.update(str(key) for key in row.keys()) + extras: list[tuple[str, str]] = [] + skipped = {title_key, description_key, *skip} + for key in sorted(keys): + if key in skipped: + continue + label = key.replace("product_", "").replace("_", " ").title() + extras.append((label, key)) + return tuple(extras) + + +def format_product_text( + doc_id: str, + *, + title: str = "", + description: str = "", + extras: Iterable[tuple[str, str]] = (), + max_chars: int = PRODUCT_TEXT_MAX_CHARS, +) -> str: + lines = [f"{DOC_MARKER_PREFIX}{doc_id}."] + title = (title or "").strip() + if title: + lines.append(f"Title: {title}") + for label, value in extras: + text = (value or "").strip() + if text: + lines.append(f"{label}: {text}") + description = (description or "").strip() + if description: + lines.append(f"Description: {description}") + body = "\n".join(lines).strip() + if len(body) > max_chars: + return body[: max_chars - 1].rstrip() + "…" + return body + + +def _holdout_ids(holdout_qids: Iterable[str] | None) -> set[str]: + out: set[str] = set() + for raw in holdout_qids or []: + qid = str(raw or "").strip() + if not qid: + continue + out.add(qid) + lower = qid.lower() + if lower.startswith("esci-"): + out.add(qid[5:]) + else: + out.add(f"esci-{qid}") + return out + + +def _select_catalog( + judgments: list[dict[str, Any]], + *, + max_queries: int, + max_docs: int, + candidates_per_query: int, + slice_name: str, + dataset: str, + gains: dict[str, float], + label_key: str, + query_text_key: str, + query_slice_key: str | None = None, + holdout_qids: Iterable[str] | None = None, +) -> tuple[list[dict[str, Any]], set[str]]: + by_query: dict[str, list[dict[str, Any]]] = defaultdict(list) + query_meta: dict[str, dict[str, Any]] = {} + for row in judgments: + qid = str(row.get("query_id") or "").strip() + pid = str(row.get("product_id") or "").strip() + if not qid or not pid: + continue + by_query[qid].append(row) + if qid not in query_meta: + query_meta[qid] = row + + blocked = _holdout_ids(holdout_qids) + eligible = [ + qid + for qid in sorted(by_query, key=_sort_key) + if qid not in blocked and f"esci-{qid}" not in blocked + ] + selected_qids = eligible[: max(1, max_queries)] + needed_ids: set[str] = set() + queries: list[dict[str, Any]] = [] + + for qid in selected_qids: + ranked = _rank_candidates(by_query[qid], gains, label_key) + if candidates_per_query > 0: + ranked = ranked[:candidates_per_query] + gold_grades: dict[str, float] = {} + gold_doc_ids: list[str] = [] + candidate_doc_ids: list[str] = [] + candidate_grades: dict[str, float] = {} + for item in ranked: + pid = str(item["product_id"]) + label = str(item.get(label_key) or "") + gain = float(gains.get(label, 0.0)) + candidate_doc_ids.append(pid) + candidate_grades[pid] = gain + if pid not in needed_ids and len(needed_ids) >= max_docs: + continue + needed_ids.add(pid) + if gain > 0: + gold_grades[pid] = gain + gold_doc_ids.append(pid) + if not gold_doc_ids: + continue + meta = query_meta[qid] + slice_value = slice_name + if query_slice_key: + slice_value = str(meta.get(query_slice_key) or slice_name) + queries.append( + { + "type": "query", + "qid": f"{dataset}-{qid}", + "query": str(meta.get(query_text_key) or "").strip(), + "gold_doc_ids": gold_doc_ids, + "gold_grades": gold_grades, + "candidate_doc_ids": candidate_doc_ids, + "candidate_grades": candidate_grades, + "slice": slice_value, + } + ) + return queries, needed_ids + + +def _qid_to_source_id(qid: str, dataset: str) -> str: + text = str(qid or "").strip() + prefix = f"{dataset}-" + if text.lower().startswith(prefix.lower()): + return text[len(prefix) :] + return text + + +def attach_esci_candidate_pools( + rows: list[dict[str, Any]], + *, + locale: str = "us", + split: str = "test", + candidates_per_query: int = DEFAULT_CANDIDATES_PER_QUERY, +) -> list[dict[str, Any]]: + from search.dataset import split_corpus + + locale = normalize_esci_locale(locale) + docs, queries = split_corpus(rows) + if not queries: + return rows + wanted = { + _qid_to_source_id(str(query.get("qid") or ""), ESCI_NAME) + for query in queries + } + wanted.discard("") + examples_path, products_path = download_esci(force=False) + judgments = [ + row + for row in _esci_judgments(examples_path, locale=locale, split=split) + if str(row.get("query_id") or "").strip() in wanted + ] + selected, needed_ids = _select_catalog( + judgments, + max_queries=max(1, len(wanted)), + max_docs=max(len(needed_ids_from_docs(docs)) + 4000, 1), + candidates_per_query=candidates_per_query, + slice_name=f"esci-{locale}", + dataset=ESCI_NAME, + gains=ESCI_GAINS, + label_key="esci_label", + query_text_key="query", + ) + by_source = { + _qid_to_source_id(str(row.get("qid") or ""), ESCI_NAME): row + for row in selected + } + existing_ids = {str(doc.get("doc_id") or "") for doc in docs} + missing = needed_ids - existing_ids + for query in queries: + source = _qid_to_source_id(str(query.get("qid") or ""), ESCI_NAME) + attached = by_source.get(source) + if not attached: + query.setdefault("candidate_doc_ids", list(query.get("gold_doc_ids") or [])) + query.setdefault( + "candidate_grades", + dict(query.get("gold_grades") or {}), + ) + continue + query["candidate_doc_ids"] = list(attached.get("candidate_doc_ids") or []) + query["candidate_grades"] = dict(attached.get("candidate_grades") or {}) + missing.update(set(query["candidate_doc_ids"]) - existing_ids) + if not missing: + return docs + queries + products = _esci_products(products_path, missing, locale=locale) + extra = _docs_and_queries( + [], + products=products, + doc_ids=missing, + dataset=ESCI_NAME, + title_key="product_title", + description_key="product_description", + extra_fields=extra_fields_from_catalog( + products, + title_key="product_title", + description_key="product_description", + ), + ) + extra_docs = [row for row in extra if row.get("type") == "doc"] + return docs + extra_docs + queries + + +def needed_ids_from_docs(docs: list[dict[str, Any]]) -> set[str]: + return {str(doc.get("doc_id") or "") for doc in docs if doc.get("doc_id")} + + +def _docs_and_queries( + queries: list[dict[str, Any]], + *, + products: dict[str, dict[str, Any]], + doc_ids: set[str], + dataset: str, + title_key: str, + description_key: str, + extra_fields: tuple[tuple[str, str], ...] = (), +) -> list[dict[str, Any]]: + docs: list[dict[str, Any]] = [] + field_aliases = { + "brand": "brand", + "color": "color", + "class": "class", + "category": "class", + "hierarchy": "hierarchy", + "category hierarchy": "hierarchy", + "features": "features", + "locale": "locale", + "description": "description", + "price": "price", + "rating": "rating", + } + for pid in sorted(doc_ids, key=_sort_key): + catalog = products.get(pid) or {} + extras = [ + (label, _stringify_catalog_value(catalog.get(field))) + for label, field in extra_fields + ] + title = _stringify_catalog_value(catalog.get(title_key)) or pid + description = _stringify_catalog_value(catalog.get(description_key)) + row: dict[str, Any] = { + "type": "doc", + "doc_id": pid, + "text": format_product_text( + pid, + title=title, + description=description, + extras=extras, + ), + "title": title, + "description": description, + "ingest": "chunks", + "dataset": dataset, + } + for label, field in extra_fields: + mapped = field_aliases.get(label.strip().lower()) + if not mapped: + continue + value = _stringify_catalog_value(catalog.get(field)) + if value and mapped not in row: + row[mapped] = value + docs.append(row) + queries = [q for q in queries if q.get("query") and q.get("gold_doc_ids")] + queries.sort(key=lambda row: str(row["qid"])) + return docs + queries + + +def _rank_candidates( + rows: list[dict[str, Any]], + gains: dict[str, float], + label_key: str, +) -> list[dict[str, Any]]: + def key(row: dict[str, Any]) -> tuple[float, str]: + label = str(row.get(label_key) or "") + return (-float(gains.get(label, 0.0)), str(row.get("product_id") or "")) + + return sorted(rows, key=key) + + +def _esci_judgments( + path: Path, + *, + locale: str, + split: str, +) -> list[dict[str, Any]]: + wanted_locale = locale.strip().lower() + wanted_split = split.strip().lower() + rows: list[dict[str, Any]] = [] + for item in _iter_parquet(path): + if int(item.get("small_version") or 0) != 1: + continue + if str(item.get("product_locale") or "").strip().lower() != wanted_locale: + continue + if str(item.get("split") or "").strip().lower() != wanted_split: + continue + rows.append( + { + "query_id": item.get("query_id"), + "query": item.get("query"), + "product_id": item.get("product_id"), + "esci_label": item.get("esci_label"), + } + ) + return rows + + +def _esci_products( + path: Path, + needed_ids: set[str], + *, + locale: str = "us", +) -> dict[str, dict[str, Any]]: + catalog: dict[str, dict[str, Any]] = {} + if not needed_ids: + return catalog + wanted_locale = locale.strip().lower() + for item in _iter_parquet(path): + pid = str(item.get("product_id") or "").strip() + if pid not in needed_ids or pid in catalog: + continue + item_locale = str(item.get("product_locale") or "").strip().lower() + if item_locale and item_locale != wanted_locale: + continue + catalog[pid] = item + if len(catalog) >= len(needed_ids): + break + return catalog + + +def _wands_queries(path: Path) -> dict[str, dict[str, str]]: + out: dict[str, dict[str, str]] = {} + for row in _iter_tsv(path): + qid = str(row.get("query_id") or "").strip() + if not qid: + continue + out[qid] = { + "query_id": qid, + "query": str(row.get("query") or "").strip(), + "query_class": str(row.get("query_class") or "wands").strip() + or "wands", + } + return out + + +def _wands_products(path: Path, needed_ids: set[str]) -> dict[str, dict[str, Any]]: + catalog: dict[str, dict[str, Any]] = {} + if not needed_ids: + return catalog + for row in _iter_tsv(path): + pid = str(row.get("product_id") or "").strip() + if pid not in needed_ids: + continue + catalog[pid] = row + if len(catalog) >= len(needed_ids): + break + return catalog + + +def _wands_judgments( + path: Path, + queries: dict[str, dict[str, str]], +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for row in _iter_tsv(path): + qid = str(row.get("query_id") or "").strip() + pid = str(row.get("product_id") or "").strip() + if not qid or not pid: + continue + meta = queries.get(qid) or {} + rows.append( + { + "query_id": qid, + "query": meta.get("query") or "", + "query_class": meta.get("query_class") or "wands", + "product_id": pid, + "label": str(row.get("label") or "").strip(), + } + ) + return rows + + +def _iter_tsv(path: Path) -> Iterator[dict[str, str]]: + with path.open("r", encoding="utf-8", newline="") as fh: + reader = csv.DictReader(fh, delimiter="\t") + for row in reader: + if not row: + continue + yield {str(k): ("" if v is None else str(v)) for k, v in row.items()} + + +def _iter_parquet(path: Path) -> Iterator[dict[str, Any]]: + try: + import pyarrow.parquet as pq + except ImportError as exc: + raise SystemExit( + "pyarrow is required to read ESCI parquet files. " + "Install benchmarks/requirements.txt." + ) from exc + parquet = pq.ParquetFile(path) + for batch in parquet.iter_batches(batch_size=8192): + for row in batch.to_pylist(): + yield row + + +def _ensure_file( + dest: Path, + *, + url: str, + hf_filename: str, + force: bool, +) -> Path: + if dest.exists() and dest.stat().st_size > 64 and not force: + if not _is_git_lfs_pointer(dest): + return dest + try: + _download_url(url, dest, force=True) + if dest.exists() and dest.stat().st_size > 64 and not _is_git_lfs_pointer(dest): + return dest + except Exception: + pass + return _download_hf(hf_filename, dest, force=True) + + +def _download_hf(filename: str, dest: Path, *, force: bool) -> Path: + try: + from huggingface_hub import hf_hub_download + except ImportError as exc: + raise SystemExit( + "huggingface_hub is required to download ESCI. " + "Install benchmarks/requirements.txt." + ) from exc + cached = hf_hub_download( + repo_id=ESCI_HF_REPO, + filename=filename, + repo_type="dataset", + force_download=force, + ) + src = Path(cached) + dest.parent.mkdir(parents=True, exist_ok=True) + if src.resolve() != dest.resolve(): + dest.write_bytes(src.read_bytes()) + return dest + + +def _download_url(url: str, dest: Path, *, force: bool) -> Path: + dest.parent.mkdir(parents=True, exist_ok=True) + if dest.exists() and dest.stat().st_size > 0 and not force: + return dest + tmp = dest.with_suffix(dest.suffix + ".tmp") + urlretrieve(url, tmp) + tmp.replace(dest) + return dest + + +def _is_git_lfs_pointer(path: Path) -> bool: + try: + head = path.read_bytes()[:80] + except OSError: + return False + return head.startswith(b"version https://git-lfs.github.com/spec/v1") + + +def _sort_key(value: str) -> tuple[int, str | int]: + text = str(value) + if text.isdigit(): + return (0, int(text)) + return (1, text) diff --git a/benchmarks/search/cli.py b/benchmarks/search/cli.py new file mode 100644 index 0000000..8ae12d1 --- /dev/null +++ b/benchmarks/search/cli.py @@ -0,0 +1,1563 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from rich.console import Console +from rich.table import Table + +from search.catalog import ( + DOWNLOAD_ALL_CATALOGS, + SUPPORTED_CATALOGS, + catalog_jsonl_path, + catalog_overwrite_blocked, + prepare_catalog, +) +from search.client import BrainAPIClient, SearchDisabledError +from search.config import BENCHMARKS_ROOT, DATA_DIR, DEFAULT_BRAIN_ID, Settings, validate_brain_id +from search.dataset import dataset_stats, load_records, split_corpus +from search.evaluate import assert_wandsgraph_node_join, ensure_run_dir, evaluate_search +from search.mapping import load_interaction_rows +from search.report import print_report_table, write_report + +console = Console() + + +def _parse_channels(args: argparse.Namespace) -> list[str] | None: + raw = getattr(args, "channels", None) + if not raw: + return None + items = [item.strip() for item in str(raw).split(",") if item.strip()] + return items or None + + +def _parse_label_list(raw: str | None) -> list[str] | None: + if not raw: + return None + items = [item.strip() for item in str(raw).split(",") if item.strip()] + return items or None + + +def _parse_extras(raw: str | None) -> dict[str, str] | None: + if not raw: + return None + try: + data = json.loads(raw) + except json.JSONDecodeError as exc: + raise SystemExit(f"--extras must be a JSON object: {exc}") from exc + if not isinstance(data, dict): + raise SystemExit("--extras must be a JSON object") + return {str(key): str(value) for key, value in data.items() if value is not None} or None + + +def _resolve_dataset_path(args: argparse.Namespace, settings: Settings) -> Path: + if getattr(args, "dataset", None): + return Path(args.dataset) + return settings.dataset_path + + +def _skip_enrichment(args: argparse.Namespace) -> bool: + return not bool(getattr(args, "enrich", False)) + + +def _eval_kwargs(args: argparse.Namespace) -> dict: + interactions = None + path = getattr(args, "interactions", None) + if path: + interactions = load_interaction_rows(Path(path)) + return { + "fusion": args.fusion, + "fusion_alpha": getattr(args, "fusion_alpha", None), + "rerank": args.rerank, + "mode": getattr(args, "mode", None) or "default", + "channels": _parse_channels(args), + "node_labels": _parse_label_list(getattr(args, "node_labels", None)), + "community_labels": _parse_label_list(getattr(args, "community_labels", None)), + "expand": getattr(args, "expand", None) or "none", + "skip_enrichment": _skip_enrichment(args), + "ingest_graph": bool(getattr(args, "ingest_graph", False)), + "skip_ingest": bool(getattr(args, "skip_ingest", False)), + "rank_pool": bool(getattr(args, "rank_pool", False)), + "personalize": bool(getattr(args, "personalize", False)), + "interactions": interactions, + "extras": _parse_extras(getattr(args, "extras", None)), + } + + +def cmd_download(args: argparse.Namespace, settings: Settings) -> int: + from search.config import DATA_DIR + from search.finetune_esci_4class import held_out_query_ids + + names = [item.strip().lower() for item in str(args.name).split(",") if item.strip()] + if len(names) == 1 and names[0] in {"all", "*"}: + names = list(DOWNLOAD_ALL_CATALOGS) + unknown = [name for name in names if name not in SUPPORTED_CATALOGS] + if unknown: + console.print( + f"[red]Unknown dataset[/red] {unknown!r}. " + f"Supported: {', '.join(SUPPORTED_CATALOGS)}" + ) + return 1 + holdout: set[str] = set() + holdout_raw = str(getattr(args, "holdout_dataset", None) or "").strip() + if holdout_raw: + holdout_path = Path(holdout_raw) + if not holdout_path.is_absolute(): + holdout_path = (BENCHMARKS_ROOT / holdout_path).resolve() + if not holdout_path.exists(): + console.print(f"[red]Missing holdout dataset[/red] {holdout_path}") + return 1 + holdout = held_out_query_ids(holdout_path) + console.print(f"[cyan]Holdout[/cyan] {holdout_path.name} n={len(holdout)}") + protected = { + (DATA_DIR / "search_esci.jsonl").resolve(), + (DATA_DIR / "search_esci_74.jsonl").resolve(), + } + timeout = 0 + for name in names: + try: + default_out = catalog_jsonl_path(name, locale=args.locale) + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + return 1 + out = Path(args.out) if args.out and len(names) == 1 else default_out + if not out.is_absolute(): + out = (BENCHMARKS_ROOT / out).resolve() + split = str(args.split or "test") + if getattr(args, "dry_stats", False): + if name != "jdsearch": + console.print("[red]--dry-stats is only implemented for --name jdsearch[/red]") + return 1 + from search.jdsearch import collect_jdsearch_stats, print_jdsearch_stats + + console.print(f"[cyan]JDsearch stats[/cyan] (no JSONL write)") + stats = collect_jdsearch_stats() + print_jdsearch_stats(stats) + timeout += 1 + continue + if catalog_overwrite_blocked(out): + console.print(f"[red]Refusing to overwrite[/red] {out}") + return 1 + if split == "train" and out.resolve() in protected: + console.print( + f"[red]Refusing to overwrite[/red] {out} with split=train. " + "Pass --out data/search_esci_ltr200.jsonl" + ) + return 1 + console.print(f"[cyan]Downloading[/cyan] {name} split={split} → {out}") + path = prepare_catalog( + name, + out_path=out, + force=args.force, + max_queries=args.max_queries, + max_docs=args.max_docs, + candidates_per_query=args.candidates_per_query, + locale=args.locale, + split=split, + holdout_qids=holdout or None, + ) + rows = load_records(path) + stats = dataset_stats(rows) + console.print(f"[green]Ready[/green] {path}") + console.print({key: value for key, value in stats.items() if key != "doc_ids"}) + timeout += 1 + return 0 if timeout else 1 + + +def cmd_dataset_stats(args: argparse.Namespace, settings: Settings) -> int: + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + rows = load_records(path) + stats = dataset_stats(rows) + table = Table(title=f"Search dataset — {path.name}") + table.add_column("field") + table.add_column("value") + table.add_row("n_docs", str(stats["n_docs"])) + table.add_row("n_queries", str(stats["n_queries"])) + table.add_row("slices", json.dumps(stats["slices"])) + table.add_row("graded", str(stats.get("graded"))) + console.print(table) + return 0 + + +def cmd_smoke(args: argparse.Namespace, settings: Settings) -> int: + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + rows = load_records(path) + docs, queries = split_corpus(rows) + skip_ingest = bool(getattr(args, "skip_ingest", False)) + if skip_ingest and settings.brain_id == "searchbenchwandsgraph": + queries = queries[: max(1, min(args.limit_queries, len(queries)))] + else: + docs = docs[: args.limit] + queries = queries[: max(1, min(args.limit_queries, len(queries)))] + limited = [{"type": "doc", **doc} for doc in docs] + [ + {"type": "query", **query} for query in queries + ] + console.print( + f"[cyan]Smoke[/cyan] brain={settings.brain_id} " + f"docs={len(docs)} queries={len(queries)}" + ) + with BrainAPIClient(settings) as client: + try: + result = evaluate_search( + client, + limited, + ks=(5, 10, 20), + timeout_s=args.timeout, + k=args.k, + dataset_name=path.name, + **_eval_kwargs(args), + ) + except SearchDisabledError as exc: + console.print(f"[red]{exc}[/red]") + return 1 + if result.get("search_error"): + console.print(f"[red]{result['search_error']}[/red]") + return 1 + if result.get("ingest", {}).get("status") not in {"completed", "partial_failed"}: + console.print("[red]Smoke ingest failed[/red]") + console.print(result.get("ingest")) + return 1 + if int(result.get("n_docs_mapped") or 0) < 1: + console.print("[red]Smoke could not map DOCID markers to chunks[/red]") + return 1 + if int(result.get("n_queries") or 0) < 1: + console.print("[red]Smoke search returned no scored queries[/red]") + return 1 + try: + join = assert_wandsgraph_node_join(client, result) + except SystemExit as exc: + console.print(f"[red]{exc}[/red]") + return 1 + if not join.get("skipped"): + console.print( + f"node_id={join.get('node_id')} neighbors={join.get('status_code')} " + f"count={join.get('neighbor_count')}" + ) + first = (result.get("queries") or [{}])[0] + console.print( + f"retrieve/search({first.get('qid')}) -> {first.get('n_hits')} hits " + f"recall@10={((first.get('metrics') or {}).get('recall@10'))}" + ) + console.print("[green]Smoke passed[/green]") + return 0 + + +def cmd_backfill_entity_text(args: argparse.Namespace, settings: Settings) -> int: + from search.backfill_entities import apply_entity_text_backfill, refuse_entity_backfill + from search.mapping import catalog_entity_backfill_rows + + try: + brain_id = refuse_entity_backfill(settings.brain_id) + except SystemExit as exc: + console.print(f"[red]{exc}[/red]") + return 1 + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + docs, _ = split_corpus(load_records(path)) + limit = getattr(args, "limit", None) + if limit: + docs = docs[: int(limit)] + rows = catalog_entity_backfill_rows(docs) + console.print( + f"[cyan]ENTITY backfill[/cyan] brain={brain_id} docs={len(rows)}" + ) + import os + from dotenv import load_dotenv + + for env_path in ( + Path.home() / ".brainapi" / "source" / ".env", + Path(__file__).resolve().parents[2] / ".env", + ): + if env_path.exists(): + load_dotenv(env_path, override=False) + os.environ.setdefault("DATA_DB", "postgresql") + os.environ.setdefault("VECTOR_DB", "postgresql") + os.environ.setdefault("GRAPH_DB", "networkx") + os.environ.setdefault("SEARCH_ENABLED", "true") + os.environ.setdefault("ENV", "development") + from src.core.instances import ( + embeddings_adapter, + graph_adapter, + vector_store_adapter, + ) + + summary = apply_entity_text_backfill( + brain_id=brain_id, + rows=rows, + graph=graph_adapter, + embeddings=embeddings_adapter, + vector_store=vector_store_adapter, + ) + console.print(summary) + if int(summary.get("updated") or 0) < 1: + console.print("[red]No ENTITY nodes updated[/red]") + return 1 + console.print("[green]ENTITY backfill done[/green]") + return 0 + + +def cmd_evaluate(args: argparse.Namespace, settings: Settings) -> int: + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + rows = load_records(path) + run_id, run_dir = ensure_run_dir(settings, args.run) + ks = tuple(int(x) for x in args.ks.split(",") if x.strip()) or (5, 10, 20) + console.print( + f"[cyan]Evaluating[/cyan] brain={settings.brain_id} " + f"dataset={path.name} run={run_id} fusion={args.fusion or 'rrf'}" + ) + with BrainAPIClient(settings) as client: + try: + result = evaluate_search( + client, + rows, + ks=ks, + timeout_s=args.timeout, + k=args.k, + dataset_name=path.name, + limit_docs=getattr(args, "limit_docs", None), + limit_queries=getattr(args, "limit_queries", None), + **_eval_kwargs(args), + ) + except SearchDisabledError as exc: + console.print(f"[red]{exc}[/red]") + return 1 + report = write_report(run_dir, result) + print_report_table(report) + console.print(f"[green]Wrote[/green] {run_dir / 'report.json'}") + if result.get("search_error"): + console.print(f"[red]{result['search_error']}[/red]") + return 0 if report.get("status") == "ok" else 1 + + +def cmd_report(args: argparse.Namespace, settings: Settings) -> int: + run_dir = settings.runs_dir / args.run + eval_path = run_dir / "eval.json" + if not eval_path.exists(): + console.print(f"[red]Missing[/red] {eval_path}") + return 1 + eval_result = json.loads(eval_path.read_text(encoding="utf-8")) + report = write_report(run_dir, eval_result) + print_report_table(report) + return 0 if report.get("status") == "ok" else 1 + + +def cmd_finetune_ce(args: argparse.Namespace, settings: Settings) -> int: + from search.finetune_esci_ce import finetune + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + out = Path(args.out) + console.print( + f"[cyan]Fine-tune CE[/cyan] dataset={path.name} out={out} " + f"label_mode={args.label_mode} fields={args.fields}" + ) + meta = finetune( + jsonl_path=path, + out_dir=out, + max_pairs=int(args.max_pairs), + epochs=int(args.epochs), + batch_size=int(args.batch_size), + label_mode=str(args.label_mode), + fields=str(args.fields), + base=str(args.base), + lr=float(args.lr), + max_length=int(getattr(args, "max_length", 256)), + ) + console.print(json.dumps(meta, indent=2)) + return 0 + + +def cmd_miss_strata(args: argparse.Namespace, settings: Settings) -> int: + from search.miss_strata import main_classify + + path = _resolve_dataset_path(args, settings) + eval_path = settings.runs_dir / args.from_run / "eval.json" + if not eval_path.exists(): + console.print(f"[red]Missing[/red] {eval_path}") + return 1 + out = Path(args.out) if args.out else settings.runs_dir / args.from_run / "miss_strata.json" + taxonomy = main_classify(eval_path=eval_path, dataset_path=path, out_path=out) + table = Table(title=f"Miss strata {args.from_run}") + table.add_column("field") + table.add_column("value") + for key in ( + "n_queries", + "n_gold", + "n_in_top10", + "n_in_11_to_k", + "n_missed", + "stratum_counts", + "n_total_miss", + "n_rewritable_total_miss", + "run_query_side", + "rewritable_qids", + "total_miss_qids", + ): + table.add_row(key, str(taxonomy.get(key))) + console.print(table) + console.print(f"[green]Wrote[/green] {out}") + return 0 + + +def cmd_query_rewrite(args: argparse.Namespace, settings: Settings) -> int: + from search.miss_strata import main_classify, write_rewritten_jsonl + + path = _resolve_dataset_path(args, settings) + eval_path = settings.runs_dir / args.from_run / "eval.json" + if not eval_path.exists(): + console.print(f"[red]Missing[/red] {eval_path}") + return 1 + taxonomy = main_classify(eval_path=eval_path, dataset_path=path) + if not taxonomy.get("run_query_side"): + console.print( + "[yellow]No rewritable total-miss qids (skip Task 2).[/yellow] " + f"total_miss={taxonomy.get('total_miss_qids')}" + ) + return 0 + dest = Path(args.out) if args.out else path.with_name("search_esci_74_qrewrite.jsonl") + rows = load_records(path) + write_rewritten_jsonl(rows, taxonomy, dest) + console.print( + f"[green]Wrote[/green] {dest} qids={taxonomy.get('rewritable_qids')}" + ) + return 0 + + +def cmd_spell_normalize(args: argparse.Namespace, settings: Settings) -> int: + from search.miss_strata import write_spell_jsonl + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + dest = Path(args.out) if args.out else path.with_name("search_esci_es_spell.jsonl") + rows = load_records(path) + summary = write_spell_jsonl(rows, dest) + console.print( + f"[green]Wrote[/green] {dest} changed_queries={summary.get('n_queries_changed')}" + ) + return 0 + + +def cmd_finetune_dense(args: argparse.Namespace, settings: Settings) -> int: + from search.finetune_esci_dense import finetune_dense + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + eval_path = Path(args.from_eval) if args.from_eval else None + out = Path(args.out) + console.print( + f"[cyan]Fine-tune dense ANCE[/cyan] dataset={path.name} out={out} " + f"(not Reddy 0.857)" + ) + meta = finetune_dense( + jsonl_path=path, + out_dir=out, + eval_path=eval_path, + max_pairs=int(args.max_pairs), + epochs=int(args.epochs), + batch_size=int(args.batch_size), + fields=str(args.fields), + base=str(args.base), + lr=float(args.lr), + max_length=int(args.max_length), + ) + console.print(json.dumps(meta, indent=2)) + return 0 + + +def cmd_local_dense(args: argparse.Namespace, settings: Settings) -> int: + from search.local_dense import run_local_dense + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + rows = load_records(path) + run_id, run_dir = ensure_run_dir(settings, args.run) + ks = tuple(int(x) for x in str(args.ks).split(",") if x.strip()) or (5, 10, 20, 50) + model_name = str(args.model) + protocol = None + if "bge" in model_name.lower(): + protocol = ( + "local BGE-base dual-encoder over search_esci_74.jsonl; " + "not MiniLM ANCE; no query: prefix; not Reddy 0.857" + ) + if "paraphrase-multilingual-minilm" in model_name.lower() or "italian_smoke" in path.name: + protocol = ( + "local paraphrase-multilingual-MiniLM-L12-v2 over " + "search_italian_smoke.jsonl; pipeline only (hits returned); " + "not text-embedding-3-large quality; not Reddy; not ES n=62" + ) + console.print( + f"[cyan]Local dense retrieve[/cyan] dataset={path.name} run={run_id} " + f"model={model_name} brain={settings.brain_id}" + ) + result = run_local_dense( + rows, + model_name=str(args.model), + dataset_name=path.name, + k=int(args.k), + ks=ks, + brain_id=settings.brain_id, + protocol=protocol, + ) + report = write_report(run_dir, result) + print_report_table(report) + console.print(f"[green]Wrote[/green] {run_dir / 'report.json'}") + return 0 if report.get("status") == "ok" else 1 + + +def cmd_mine_retrieved_lists(args: argparse.Namespace, settings: Settings) -> int: + from search.mine_retrieved_lists import mine + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + out = Path(args.out) + console.print( + f"[cyan]Mine retrieved lists[/cyan] dataset={path.name} out={out} " + f"(unlabeled=I; hold out test qids)" + ) + meta = mine( + jsonl_path=path, + out_path=out, + max_queries=int(args.max_queries), + k=int(args.k), + seed=int(args.seed), + ) + console.print(json.dumps(meta, indent=2)) + return 0 + + +def cmd_export_hybrid_lists(args: argparse.Namespace, settings: Settings) -> int: + from search.export_hybrid_lists import PROTECTED_OUT_NAMES, export_hybrid_lists + from search.list_overlap import load_eval_run + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + holdout_raw = str(getattr(args, "holdout_dataset", None) or "").strip() + holdout = Path(holdout_raw) if holdout_raw else DATA_DIR / "search_esci_74.jsonl" + if not holdout.is_absolute(): + holdout = (BENCHMARKS_ROOT / holdout).resolve() + if not holdout.exists(): + console.print(f"[red]Missing holdout[/red] {holdout}") + return 1 + out = Path(args.out) + if not out.is_absolute(): + out = (BENCHMARKS_ROOT / out).resolve() + if out.name in PROTECTED_OUT_NAMES: + console.print(f"[red]Refusing to overwrite[/red] {out}") + return 1 + run_dir = settings.runs_dir / str(args.from_run) + if not (run_dir / "eval.json").exists(): + console.print(f"[red]Missing eval[/red] {run_dir / 'eval.json'}") + return 1 + eval_result = load_eval_run(run_dir) + console.print( + f"[cyan]Export hybrid lists[/cyan] from={args.from_run} " + f"dataset={path.name} holdout={holdout.name} out={out} " + f"(unlabeled=I; source=hybrid-k50)" + ) + try: + meta = export_hybrid_lists( + eval_result=eval_result, + dataset_path=path, + holdout_path=holdout, + out_path=out, + k=int(args.k), + ) + except ValueError as exc: + console.print(f"[red]{exc}[/red]") + return 1 + console.print(json.dumps(meta, indent=2)) + return 0 + + +def cmd_finetune_4class(args: argparse.Namespace, settings: Settings) -> int: + from search.finetune_esci_4class import finetune + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + lists_path = Path(args.from_lists) if args.from_lists else None + if lists_path is not None and not lists_path.exists(): + console.print(f"[red]Missing[/red] {lists_path}") + return 1 + out = Path(args.out) + if not out.is_absolute(): + out = (BENCHMARKS_ROOT / out).resolve() + base_model = str(args.base) + base_path = Path(base_model) + if not base_path.is_absolute(): + maybe_base = (BENCHMARKS_ROOT / base_path).resolve() + if maybe_base.exists(): + base_path = maybe_base + base_model = str(base_path) + if out.exists() and base_path.exists() and out.resolve() == base_path.resolve(): + console.print(f"[red]Refusing to overwrite base checkpoint[/red] {out}") + return 1 + lists_source = str(getattr(args, "lists_source", None) or "").strip() + if lists_path is not None: + source = lists_source or "retrieved-bm25" + else: + source = "pool" + console.print( + f"[cyan]Fine-tune 4-class CE[/cyan] dataset={path.name} out={out} " + f"source={source}" + ) + meta = finetune( + jsonl_path=path, + out_dir=out, + max_pairs=int(args.max_pairs), + epochs=int(args.epochs), + batch_size=int(args.batch_size), + max_length=int(args.max_length), + base_model=base_model, + seed=int(args.seed), + ckpt_every=int(args.ckpt_every), + use_class_weights=bool(args.class_weights), + lists_path=lists_path, + lists_source=source if lists_path is not None else None, + ) + console.print(json.dumps(meta, indent=2)) + return 0 + + +def cmd_colbert_local(args: argparse.Namespace, settings: Settings) -> int: + from search.local_colbert import run_local_colbert + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + rows = load_records(path) + run_id, run_dir = ensure_run_dir(settings, args.run) + ks = tuple(int(x) for x in str(args.ks).split(",") if x.strip()) or (5, 10, 20, 50) + console.print( + f"[cyan]Local ColBERT MaxSim[/cyan] dataset={path.name} run={run_id} " + f"(not fused with passages; not Reddy 0.857)" + ) + result = run_local_colbert( + rows, + dataset_name=path.name, + k=int(args.k), + ks=ks, + brain_id="harness-local-colbert", + ) + report = write_report(run_dir, result) + print_report_table(report) + console.print(f"[green]Wrote[/green] {run_dir / 'report.json'}") + return 0 if report.get("status") == "ok" else 1 + + +def _existing_path(path: Path) -> Path: + if path.exists(): + return path + from search.config import BENCHMARKS_ROOT + + alt = BENCHMARKS_ROOT / path + return alt if alt.exists() else path + + +def cmd_rank_corpus(args: argparse.Namespace, settings: Settings) -> int: + from search.rank_corpus import DEFAULT_MODEL, load_predict, run_exhaustive_ce + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + model_dir = _existing_path(Path(args.model) if args.model else DEFAULT_MODEL) + if not model_dir.exists(): + console.print(f"[red]Missing model[/red] {model_dir}") + return 1 + rows = load_records(path) + run_id, run_dir = ensure_run_dir(settings, args.run) + ks = tuple(int(x) for x in str(args.ks).split(",") if x.strip()) or (5, 10, 20, 50) + console.print( + f"[cyan]Exhaustive 4-class CE[/cyan] dataset={path.name} run={run_id} " + f"model={model_dir} (protocol exhaustive-catalog; not Reddy 0.857)" + ) + predict, model_name = load_predict(model_dir, max_length=int(args.max_length)) + result = run_exhaustive_ce( + rows, + predict=predict, + model_name=model_name, + dataset_name=path.name, + k=int(args.k), + ks=ks, + brain_id="harness-local-exhaustive", + ) + report = write_report(run_dir, result) + print_report_table(report) + console.print(f"[green]Wrote[/green] {run_dir / 'report.json'}") + return 0 if report.get("status") == "ok" else 1 + + +def cmd_list_overlap(args: argparse.Namespace, settings: Settings) -> int: + from search.list_overlap import load_eval_run, summarize_overlap + + names = [item.strip() for item in str(args.against_runs).split(",") if item.strip()] + passages = load_eval_run(settings.runs_dir / args.passages_run) + passages["run_id"] = args.passages_run + sidecars: dict = {} + for name in names: + eval_result = load_eval_run(settings.runs_dir / name) + eval_result["run_id"] = name + sidecars[name] = eval_result + queries = None + dataset = getattr(args, "dataset", None) + if dataset: + path = Path(dataset) + if not path.exists(): + path = _existing_path(path) + if path.exists(): + _, queries = split_corpus(load_records(path)) + summary = summarize_overlap( + passages, + sidecars, + k=int(args.k), + queries=queries, + ) + run_id, run_dir = ensure_run_dir(settings, args.run) + out = run_dir / "overlap.json" + out.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + headline = { + name: { + "unique_gold_hits": row["unique_gold_hits"], + "queries_with_unique": row["queries_with_unique"], + "n_queries": row["n_queries"], + } + for name, row in summary["runs"].items() + } + console.print(json.dumps(headline, indent=2)) + console.print(f"[green]Wrote[/green] {out}") + return 0 + + +def cmd_union_lists(args: argparse.Namespace, settings: Settings) -> int: + from search.list_overlap import load_eval_run, run_union + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + names = [item.strip() for item in str(args.from_runs).split(",") if item.strip()] + evals = [] + for name in names: + eval_result = load_eval_run(settings.runs_dir / name) + eval_result["run_id"] = name + evals.append(eval_result) + rows = load_records(path) + run_id, run_dir = ensure_run_dir(settings, args.run) + ks = tuple(int(x) for x in str(args.ks).split(",") if x.strip()) or (5, 10, 20, 50) + console.print( + f"[cyan]Harness RRF union[/cyan] from={names} dataset={path.name} " + f"run={run_id} (not live graph; not Reddy 0.857)" + ) + result = run_union( + evals, + rows, + dataset_name=path.name, + k=int(args.k), + ks=ks, + brain_id="harness-local-union", + run_names=names, + ) + report = write_report(run_dir, result) + print_report_table(report) + console.print(f"[green]Wrote[/green] {run_dir / 'report.json'}") + return 0 if report.get("status") == "ok" else 1 + + +def cmd_cascade_lists(args: argparse.Namespace, settings: Settings) -> int: + from search.list_overlap import load_eval_run, run_cascade + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + passages = load_eval_run(settings.runs_dir / args.passages_run) + passages["run_id"] = args.passages_run + names = [item.strip() for item in str(args.from_runs).split(",") if item.strip()] + sidecars = [] + for name in names: + eval_result = load_eval_run(settings.runs_dir / name) + eval_result["run_id"] = name + sidecars.append(eval_result) + rows = load_records(path) + run_id, run_dir = ensure_run_dir(settings, args.run) + ks = tuple(int(x) for x in str(args.ks).split(",") if x.strip()) or (5, 10, 20, 50) + console.print( + f"[cyan]Frozen-head cascade[/cyan] passages={args.passages_run} " + f"sidecars={names} head_k={args.head_k} dataset={path.name} " + f"run={run_id} (not live graph; not Reddy 0.857)" + ) + result = run_cascade( + passages, + sidecars, + rows, + dataset_name=path.name, + k=int(args.k), + head_k=int(args.head_k), + ks=ks, + brain_id="harness-local-cascade", + run_names=[args.passages_run, *names], + ) + report = write_report(run_dir, result) + print_report_table(report) + console.print(f"[green]Wrote[/green] {run_dir / 'report.json'}") + return 0 if report.get("status") == "ok" else 1 + + +def cmd_ltr_head(args: argparse.Namespace, settings: Settings) -> int: + from search.list_overlap import load_eval_run + from search.ltr_head import HEAD_RANKNET, PAIR_UNLABELED_ZERO, ce_cache_name, run_ltr_head + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + if not path.is_absolute(): + alt = (BENCHMARKS_ROOT / path).resolve() + if alt.exists(): + path = alt + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + passages = load_eval_run(settings.runs_dir / args.from_run) + passages["run_id"] = args.from_run + rows = load_records(path) + run_id, run_dir = ensure_run_dir(settings, args.run) + ks = tuple(int(x) for x in str(args.ks).split(",") if x.strip()) or (5, 10, 20, 50) + policy = str(getattr(args, "pair_policy", None) or PAIR_UNLABELED_ZERO) + head = str(getattr(args, "ltr_model", None) or HEAD_RANKNET) + ce_raw = str(getattr(args, "ce_model", None) or "").strip() + ce_model = Path(ce_raw) if ce_raw else None + if ce_model is not None and not ce_model.is_absolute(): + ce_model = (BENCHMARKS_ROOT / ce_model).resolve() + if ce_model is not None and not ce_model.exists(): + console.print(f"[red]Missing CE model[/red] {ce_model}") + return 1 + ce_cache = None + if ce_model is not None: + ce_cache = settings.runs_dir / args.from_run / ce_cache_name(ce_model) + train_run = str(getattr(args, "train_from_run", None) or "").strip() + train_eval = None + train_rows = None + train_ce_cache = None + train_dataset_name = None + if train_run: + train_ds = str(getattr(args, "train_dataset", None) or "").strip() + if not train_ds: + console.print("[red]--train-dataset is required with --train-from-run[/red]") + return 1 + train_path = Path(train_ds) + if not train_path.is_absolute(): + train_path = (BENCHMARKS_ROOT / train_path).resolve() + if not train_path.exists(): + console.print(f"[red]Missing train dataset[/red] {train_path}") + return 1 + train_eval = load_eval_run(settings.runs_dir / train_run) + train_eval["run_id"] = train_run + train_rows = load_records(train_path) + train_dataset_name = train_path.name + if ce_model is not None: + train_ce_cache = settings.runs_dir / train_run / ce_cache_name(ce_model) + mode = "apply" if train_run else "CV" + console.print( + f"[cyan]LTR head {mode}[/cyan] from={args.from_run} dataset={path.name} " + f"train={train_run or 'folds'} {train_dataset_name or ''} " + f"run={run_id} folds={args.folds} pair={policy} head={head} " + f"ce={ce_model or 'none'} (not live; not Reddy 0.857)" + ) + result = run_ltr_head( + passages, + rows, + dataset_name=path.name, + k=int(args.k), + ks=ks, + n_folds=int(args.folds), + brain_id="harness-local-ltr", + source_run=args.from_run, + pair_policy=policy, + ce_model=ce_model, + ce_cache_path=ce_cache, + ltr_head=head, + train_eval_result=train_eval, + train_rows=train_rows, + train_source_run=train_run or None, + train_ce_cache_path=train_ce_cache, + ) + report = write_report(run_dir, result) + print_report_table(report) + overlap = result.get("overlap_only_metrics") or {} + console.print( + f"overlap-only ndcg@10={overlap.get('ndcg@10')} " + f"recall@10={overlap.get('recall@10')}" + ) + weights = result.get("ltr_mean_weights") or {} + if weights: + kind = "gain importances" if head == "lightgbm" else "weights" + console.print(f"mean CV {kind} {weights}") + if result.get("ltr_train_run"): + console.print( + f"trained on {result.get('ltr_n_train_queries')} queries " + f"from {result.get('ltr_train_run')}" + ) + console.print(f"[green]Wrote[/green] {run_dir / 'report.json'}") + return 0 if report.get("status") == "ok" else 1 + + +def cmd_pool_first_stage(args: argparse.Namespace, settings: Settings) -> int: + from search.pool_first_stage import main as pool_main + + argv = ["--dataset", str(_resolve_dataset_path(args, settings))] + if args.run: + argv.extend(["--run", str(args.run)]) + if args.variant: + argv.extend(["--variant", str(args.variant)]) + if args.expand_out: + argv.extend(["--expand-out", str(args.expand_out)]) + return int(pool_main(argv)) + + +def cmd_rerank_retrieved(args: argparse.Namespace, settings: Settings) -> int: + from search.rerank_retrieved import run_ce_on_retrieved + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + source_dir = settings.runs_dir / args.from_run + eval_path = source_dir / "eval.json" + if not eval_path.exists(): + console.print(f"[red]Missing[/red] {eval_path}") + return 1 + eval_result = json.loads(eval_path.read_text(encoding="utf-8")) + eval_result["run_id"] = args.from_run + rows = load_records(path) + run_id, run_dir = ensure_run_dir(settings, args.run) + ks = None + if getattr(args, "ks", None): + ks = tuple(int(x) for x in str(args.ks).split(",") if x.strip()) + console.print( + f"[cyan]Harness CE on retrieved hits[/cyan] from={args.from_run} " + f"dataset={path.name} run={run_id} (not Reddy 0.857)" + ) + result = run_ce_on_retrieved( + eval_result, + rows, + dataset_name=path.name, + ks=ks, + brain_id=settings.brain_id, + ) + report = write_report(run_dir, result) + print_report_table(report) + console.print(f"[green]Wrote[/green] {run_dir / 'report.json'}") + return 0 if report.get("status") == "ok" else 1 + + +def cmd_rank_pool_ce(args: argparse.Namespace, settings: Settings) -> int: + from search.rank_pool import run_ce_on_pool + + path = _resolve_dataset_path(args, settings) + if not path.exists(): + console.print(f"[red]Missing[/red] {path}") + return 1 + rows = load_records(path) + run_id, run_dir = ensure_run_dir(settings, args.run) + console.print( + f"[cyan]Ranking-in-pool CE[/cyan] dataset={path.name} run={run_id} " + f"(cite Reddy 0.857 only if protocol is CE-on-pool; n is not ~4477)" + ) + result = run_ce_on_pool( + rows, + dataset_name=path.name, + brain_id=settings.brain_id, + ) + report = write_report(run_dir, result) + print_report_table(report) + console.print(f"[green]Wrote[/green] {run_dir / 'report.json'}") + if result.get("ce_model"): + console.print( + f"ce_model={result.get('ce_model')} " + f"missing_text={result.get('ce_missing_text')}" + ) + return 0 if report.get("status") == "ok" else 1 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="search", + description=( + "Labeled search eval on searchbench*: ingest docs → " + "POST /retrieve/search (Recall/nDCG/MRR + retrieve latency)" + ), + ) + parser.add_argument( + "--env-file", + type=Path, + default=None, + help="Optional dotenv path (default benchmarks/.env)", + ) + parser.add_argument( + "--brain", + default=None, + help=f"Brain id (default {DEFAULT_BRAIN_ID}; must start with searchbench)", + ) + sub = parser.add_subparsers(dest="command", required=True) + + p_download = sub.add_parser( + "download", + help="Download Amazon ESCI and/or WANDS into search JSONL", + ) + p_download.add_argument( + "--name", + default="all", + help="esci, wands, jdsearch, or all (comma-separated; all = esci,wands)", + ) + p_download.add_argument("--out", type=str, default=None) + p_download.add_argument("--force", action="store_true") + p_download.add_argument( + "--dry-stats", + action="store_true", + dest="dry_stats", + help="JDsearch only: print label histogram from the tar/cache without writing JSONL", + ) + p_download.add_argument("--max-queries", type=int, default=80) + p_download.add_argument("--max-docs", type=int, default=2000) + p_download.add_argument("--candidates-per-query", type=int, default=40) + p_download.add_argument( + "--locale", + default="us", + help="ESCI locale: us, es, or jp (default us). Not Italian.", + ) + p_download.add_argument( + "--split", + default="test", + help="ESCI split (default test; small_version / Task 1 only)", + ) + p_download.add_argument( + "--holdout-dataset", + dest="holdout_dataset", + default="", + help="JSONL whose qids are skipped (e.g. data/search_esci_74.jsonl)", + ) + p_download.set_defaults(func=cmd_download) + + p_stats = sub.add_parser("dataset-stats", help="Summarize search JSONL") + p_stats.add_argument("--dataset", type=str, default=None) + p_stats.set_defaults(func=cmd_dataset_stats) + + p_smoke = sub.add_parser( + "smoke", help="Ingest a few docs then POST /retrieve/search" + ) + p_smoke.add_argument("--dataset", type=str, default=None) + p_smoke.add_argument("--limit", type=int, default=2) + p_smoke.add_argument("--limit-queries", type=int, default=2) + p_smoke.add_argument("--k", type=int, default=10) + p_smoke.add_argument("--fusion", choices=["rrf", "cc"], default=None) + p_smoke.add_argument("--fusion-alpha", dest="fusion_alpha", type=float, default=None) + p_smoke.add_argument( + "--rerank", + default=None, + help="none or plugin: (e.g. plugin:cross-encoder). Unknown name is 400.", + ) + p_smoke.add_argument( + "--channels", + default=None, + help=( + "Comma-separated channels (default passages): " + "passages,entities,events,communities and/or plugin:." + ), + ) + p_smoke.add_argument("--node-labels", dest="node_labels", default=None) + p_smoke.add_argument("--community-labels", dest="community_labels", default=None) + p_smoke.add_argument( + "--expand", + choices=["none", "neighbors"], + default="none", + ) + p_smoke.add_argument( + "--ingest-graph", + action="store_true", + help="Also write deterministic catalog triples (entity uuid = doc_id).", + ) + p_smoke.add_argument( + "--interactions", + type=str, + default=None, + help="Optional interaction JSONL (EVENT+happened_at) on this searchbench brain.", + ) + p_smoke.add_argument("--timeout", type=float, default=600.0) + p_smoke.add_argument( + "--enrich", + action="store_true", + help="Run Scout/Architect LLM ingest. Default skips enrichment (chunk+embed only).", + ) + p_smoke.add_argument( + "--skip-ingest", + action="store_true", + help="Reuse chunks already on this searchbench brain (no POST /ingest/).", + ) + p_smoke.add_argument( + "--personalize", + action="store_true", + dest="personalize", + help="Send each query's target field on POST /retrieve/search. Default omit.", + ) + p_smoke.add_argument( + "--target-from-query", + action="store_true", + dest="personalize", + help="Alias for --personalize.", + ) + p_smoke.set_defaults(func=cmd_smoke) + + p_bf = sub.add_parser( + "backfill-entity-text", + help="ENTITY-only search_text + nodes vector refresh (not architect, not frozen brains)", + ) + p_bf.add_argument("--dataset", type=str, default="data/search_wands.jsonl") + p_bf.add_argument("--limit", type=int, default=None) + p_bf.set_defaults(func=cmd_backfill_entity_text) + + p_eval = sub.add_parser( + "evaluate", + help="Ingest corpus, search all queries, write report + ledger", + ) + p_eval.add_argument("--dataset", type=str, default=None) + p_eval.add_argument("--run", type=str, default=None) + p_eval.add_argument("--ks", type=str, default="5,10,20") + p_eval.add_argument("--k", type=int, default=20) + p_eval.add_argument("--fusion", choices=["rrf", "cc"], default=None) + p_eval.add_argument("--fusion-alpha", dest="fusion_alpha", type=float, default=None) + p_eval.add_argument( + "--rerank", + default=None, + help="none or plugin: (e.g. plugin:cross-encoder). Unknown name is 400.", + ) + p_eval.add_argument( + "--mode", + choices=["default", "catalog"], + default="default", + help="default keeps RERANK_MAX_K=10. catalog retrieves deeper and reranks up to 50.", + ) + p_eval.add_argument( + "--channels", + default=None, + help=( + "Comma-separated channels (default passages): " + "passages,entities,events,communities and/or plugin:." + ), + ) + p_eval.add_argument("--node-labels", dest="node_labels", default=None) + p_eval.add_argument("--community-labels", dest="community_labels", default=None) + p_eval.add_argument( + "--expand", + choices=["none", "neighbors"], + default="none", + ) + p_eval.add_argument( + "--ingest-graph", + action="store_true", + help="Also write deterministic catalog triples (entity uuid = doc_id).", + ) + p_eval.add_argument( + "--interactions", + type=str, + default=None, + help="Optional interaction JSONL (EVENT+happened_at) on this searchbench brain.", + ) + p_eval.add_argument("--timeout", type=float, default=600.0) + p_eval.add_argument( + "--enrich", + action="store_true", + help="Run Scout/Architect LLM ingest. Default skips enrichment (chunk+embed only).", + ) + p_eval.add_argument("--limit-docs", dest="limit_docs", type=int, default=None) + p_eval.add_argument("--limit-queries", dest="limit_queries", type=int, default=None) + p_eval.add_argument( + "--skip-ingest", + action="store_true", + help="Reuse chunks already on this searchbench brain (no POST /ingest/).", + ) + p_eval.add_argument( + "--rank-pool", + action="store_true", + dest="rank_pool", + help=( + "Restrict hits to each query's candidate_doc_ids (including I) " + "and score pool nDCG@20. Do not average with shared-corpus nDCG@10." + ), + ) + p_eval.add_argument( + "--personalize", + action="store_true", + dest="personalize", + help="Send each query's target field on POST /retrieve/search. Default omit.", + ) + p_eval.add_argument( + "--target-from-query", + action="store_true", + dest="personalize", + help="Alias for --personalize.", + ) + p_eval.add_argument( + "--extras", + default=None, + help='Optional equality filter JSON object, e.g. {"locale":"it"}', + ) + p_eval.set_defaults(func=cmd_evaluate) + + p_report = sub.add_parser("report", help="Rebuild report from eval.json") + p_report.add_argument("--run", type=str, required=True) + p_report.set_defaults(func=cmd_report) + + p_pool = sub.add_parser( + "rank-pool-ce", + help="Score labeled candidate pools with MiniLM CE (no search API)", + ) + p_pool.add_argument("--dataset", type=str, default=None) + p_pool.add_argument("--run", type=str, default=None) + p_pool.set_defaults(func=cmd_rank_pool_ce) + + p_rr = sub.add_parser( + "rerank-retrieved", + help="Harness CE over stored first-stage hits (not production RERANK_MAX_K)", + ) + p_rr.add_argument("--from-run", dest="from_run", type=str, required=True) + p_rr.add_argument("--dataset", type=str, default=None) + p_rr.add_argument("--run", type=str, default=None) + p_rr.add_argument("--ks", type=str, default=None) + p_rr.set_defaults(func=cmd_rerank_retrieved) + + p_ft = sub.add_parser( + "finetune-ce", + help="Fine-tune a pool CE on ESCI US train (test qids held out)", + ) + p_ft.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_ft.add_argument( + "--out", + type=str, + default="data/models/esci-minilm-graded", + ) + p_ft.add_argument("--max-pairs", type=int, default=80000) + p_ft.add_argument("--epochs", type=int, default=1) + p_ft.add_argument("--batch-size", type=int, default=32) + p_ft.add_argument( + "--label-mode", + choices=["binary", "graded", "multiclass"], + default="graded", + ) + p_ft.add_argument("--fields", choices=["title", "catalog"], default="catalog") + p_ft.add_argument( + "--base", + default="cross-encoder/ms-marco-MiniLM-L-6-v2", + ) + p_ft.add_argument("--lr", type=float, default=7e-6) + p_ft.add_argument("--max-length", type=int, default=256) + p_ft.set_defaults(func=cmd_finetune_ce) + + p_ms = sub.add_parser( + "miss-strata", + help="Classify k=50 hits: head-ok / rank-too-low / total-miss", + ) + p_ms.add_argument("--from-run", dest="from_run", type=str, required=True) + p_ms.add_argument("--dataset", type=str, default=None) + p_ms.add_argument("--out", type=str, default=None) + p_ms.set_defaults(func=cmd_miss_strata) + + p_qw = sub.add_parser( + "query-rewrite", + help="Rewrite pathological total-miss queries only (skip esci-72)", + ) + p_qw.add_argument("--from-run", dest="from_run", type=str, required=True) + p_qw.add_argument("--dataset", type=str, default=None) + p_qw.add_argument("--out", type=str, default=None) + p_qw.set_defaults(func=cmd_query_rewrite) + + p_sp = sub.add_parser( + "spell-normalize", + help="Harness-only NFKC/punctuation/accent normalize of query text", + ) + p_sp.add_argument("--dataset", type=str, default="data/search_esci_es.jsonl") + p_sp.add_argument("--out", type=str, default="data/search_esci_es_spell.jsonl") + p_sp.set_defaults(func=cmd_spell_normalize) + + p_ftd = sub.add_parser( + "finetune-dense", + help="Fine-tune a dual encoder with k=50 hard-negative bank (test qids held out)", + ) + p_ftd.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_ftd.add_argument( + "--out", + type=str, + default="data/models/esci-minilm-dense-ance", + ) + p_ftd.add_argument( + "--from-eval", + type=str, + default="runs/search-esci-74-passages-k50/eval.json", + ) + p_ftd.add_argument("--max-pairs", type=int, default=20000) + p_ftd.add_argument("--epochs", type=int, default=1) + p_ftd.add_argument("--batch-size", type=int, default=32) + p_ftd.add_argument("--fields", choices=["title", "catalog"], default="catalog") + p_ftd.add_argument( + "--base", + default="sentence-transformers/all-MiniLM-L6-v2", + ) + p_ftd.add_argument("--lr", type=float, default=2e-5) + p_ftd.add_argument("--max-length", type=int, default=256) + p_ftd.set_defaults(func=cmd_finetune_dense) + + p_ld = sub.add_parser( + "local-dense", + help="Retrieve the JSONL corpus with a local dual encoder (not BrainAPI)", + ) + p_ld.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_ld.add_argument("--run", type=str, default="search-esci-74-dense-ance-k50") + p_ld.add_argument("--model", type=str, required=True) + p_ld.add_argument("--k", type=int, default=50) + p_ld.add_argument("--ks", type=str, default="5,10,20,50") + p_ld.set_defaults(func=cmd_local_dense) + + p_mine = sub.add_parser( + "mine-retrieved-lists", + help="BM25 top-k train lists with ESCI labels; unlabeled=I", + ) + p_mine.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_mine.add_argument("--out", type=str, default="data/esci_retrieved_lists.jsonl") + p_mine.add_argument("--max-queries", type=int, default=6000) + p_mine.add_argument("--k", type=int, default=50) + p_mine.add_argument("--seed", type=int, default=11) + p_mine.set_defaults(func=cmd_mine_retrieved_lists) + + p_exh = sub.add_parser( + "export-hybrid-lists", + help="Export stored hybrid k=50 hits as 4-class rows; unlabeled=I", + ) + p_exh.add_argument( + "--from-run", + dest="from_run", + default="search-esci-ltr200-passages-k50", + ) + p_exh.add_argument("--dataset", type=str, default="data/search_esci_ltr200.jsonl") + p_exh.add_argument( + "--holdout-dataset", + dest="holdout_dataset", + default="data/search_esci_74.jsonl", + ) + p_exh.add_argument("--out", type=str, default="data/esci_hybrid_lists_ltr200.jsonl") + p_exh.add_argument("--k", type=int, default=50) + p_exh.set_defaults(func=cmd_export_hybrid_lists) + + p_ft4 = sub.add_parser( + "finetune-4class", + help="Fine-tune 4-class L-12 CE (pool or retrieved-bm25 lists)", + ) + p_ft4.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_ft4.add_argument( + "--out", + type=str, + default="data/models/esci-minilm-l12-retrieved", + ) + p_ft4.add_argument("--from-lists", type=str, default=None) + p_ft4.add_argument( + "--lists-source", + dest="lists_source", + default="", + help="Label for --from-lists (hybrid-k50 or retrieved-bm25)", + ) + p_ft4.add_argument("--max-pairs", type=int, default=80000) + p_ft4.add_argument("--epochs", type=int, default=1) + p_ft4.add_argument("--batch-size", type=int, default=32) + p_ft4.add_argument("--max-length", type=int, default=192) + p_ft4.add_argument("--ckpt-every", type=int, default=200) + p_ft4.add_argument("--class-weights", action="store_true") + p_ft4.add_argument( + "--base", + default="cross-encoder/ms-marco-MiniLM-L-12-v2", + ) + p_ft4.add_argument("--seed", type=int, default=11) + p_ft4.set_defaults(func=cmd_finetune_4class) + + p_cb = sub.add_parser( + "colbert-local", + help="ColBERT MaxSim sidecar over the JSONL corpus (not BrainAPI)", + ) + p_cb.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_cb.add_argument("--run", type=str, default="search-esci-74-colbert-k50") + p_cb.add_argument("--k", type=int, default=50) + p_cb.add_argument("--ks", type=str, default="5,10,20,50") + p_cb.set_defaults(func=cmd_colbert_local) + + p_rc = sub.add_parser( + "rank-corpus", + help="Score every JSONL doc with 4-class CE (exhaustive-catalog, not production)", + ) + p_rc.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_rc.add_argument("--run", type=str, default="search-esci-74-exhaustive-ce") + p_rc.add_argument( + "--model", + type=str, + default="data/models/esci-minilm-l12-4class-nowt-e2", + ) + p_rc.add_argument("--k", type=int, default=50) + p_rc.add_argument("--ks", type=str, default="5,10,20,50") + p_rc.add_argument("--max-length", dest="max_length", type=int, default=192) + p_rc.set_defaults(func=cmd_rank_corpus) + + p_ov = sub.add_parser( + "list-overlap", + help="Count gold ASINs in sidecar top-k missing from passages", + ) + p_ov.add_argument( + "--passages-run", + dest="passages_run", + default="search-esci-74-passages-k50", + ) + p_ov.add_argument( + "--against-runs", + dest="against_runs", + default="search-esci-74-bge-base-k50,search-esci-74-colbert-k50", + ) + p_ov.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_ov.add_argument("--run", type=str, default="search-esci-74-list-overlap") + p_ov.add_argument("--k", type=int, default=50) + p_ov.set_defaults(func=cmd_list_overlap) + + p_un = sub.add_parser( + "union-lists", + help="RRF-union stored first-stage lists (harness only, not live graph)", + ) + p_un.add_argument( + "--from-runs", + dest="from_runs", + default="search-esci-74-passages-k50,search-esci-74-bge-base-k50", + ) + p_un.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_un.add_argument("--run", type=str, default="search-esci-74-union-bge-k50") + p_un.add_argument("--k", type=int, default=50) + p_un.add_argument("--ks", type=str, default="5,10,20,50") + p_un.set_defaults(func=cmd_union_lists) + + p_cs = sub.add_parser( + "cascade-lists", + help="Keep passages top-10; inject sidecar unique golds into ranks 11-50", + ) + p_cs.add_argument( + "--passages-run", + dest="passages_run", + default="search-esci-74-passages-k50", + ) + p_cs.add_argument( + "--from-runs", + dest="from_runs", + default="search-esci-74-bge-base-k50,search-esci-74-colbert-k50", + ) + p_cs.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_cs.add_argument("--run", type=str, default="search-esci-74-cascade-tail-k50") + p_cs.add_argument("--k", type=int, default=50) + p_cs.add_argument("--head-k", dest="head_k", type=int, default=10) + p_cs.add_argument("--ks", type=str, default="5,10,20,50") + p_cs.set_defaults(func=cmd_cascade_lists) + + p_ltr = sub.add_parser( + "ltr-head", + help="Query-grouped CV LTR on stored hybrid k=50 (BM25/dense rank + title overlap)", + ) + p_ltr.add_argument( + "--from-run", + dest="from_run", + default="search-esci-74-passages-k50", + ) + p_ltr.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_ltr.add_argument("--run", type=str, default="search-esci-74-ltr-head-k50") + p_ltr.add_argument("--k", type=int, default=50) + p_ltr.add_argument("--ks", type=str, default="5,10,20,50") + p_ltr.add_argument("--folds", type=int, default=5) + p_ltr.add_argument( + "--pair-policy", + dest="pair_policy", + default="unlabeled_zero", + choices=["unlabeled_zero", "other_query_neg"], + ) + p_ltr.add_argument( + "--ce-model", + dest="ce_model", + default="", + help="Optional 4-class CE dir; ce_gain is a feature, not the ranker", + ) + p_ltr.add_argument( + "--ltr-model", + dest="ltr_model", + default="ranknet", + choices=["ranknet", "lightgbm"], + help="Linear RankNet (default) or gated LightGBM lambdarank", + ) + p_ltr.add_argument( + "--train-from-run", + dest="train_from_run", + default="", + help="Fit on this run's lists; apply to --from-run (no CV on eval qids)", + ) + p_ltr.add_argument( + "--train-dataset", + dest="train_dataset", + default="", + help="JSONL for --train-from-run", + ) + p_ltr.set_defaults(func=cmd_ltr_head) + + p_fs = sub.add_parser( + "pool-first-stage", + help="Local fielded BM25 over the JSONL corpus (not Reddy 0.857)", + ) + p_fs.add_argument("--dataset", type=str, default="data/search_esci_74.jsonl") + p_fs.add_argument("--run", type=str, default="search-esci-74-fielded-bm25") + p_fs.add_argument( + "--variant", + default="all,title,title-boost,rm3,rm3-title-boost", + ) + p_fs.add_argument( + "--expand-out", + type=str, + default="data/search_esci_74_rm3.jsonl", + ) + p_fs.set_defaults(func=cmd_pool_first_stage) + + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + settings = Settings.load(args.env_file) + if args.brain: + settings.brain_id = validate_brain_id(args.brain) + return int(args.func(args, settings)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/search/client.py b/benchmarks/search/client.py new file mode 100644 index 0000000..5f693b3 --- /dev/null +++ b/benchmarks/search/client.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Any + +import httpx + +from search.config import TERMINAL_TASK_STATUSES, Settings + + +class SearchDisabledError(RuntimeError): + pass + + +@dataclass +class TimedResult: + data: Any + latency_ms: float + status_code: int = 200 + + +@dataclass +class BrainAPIClient: + settings: Settings + timeout: float = 120.0 + max_retries: int = 5 + _client: httpx.Client = field(init=False, repr=False) + + def __post_init__(self) -> None: + self.settings.require_brainapi() + self._client = httpx.Client( + base_url=self.settings.brainapi_url, + headers={ + "Authorization": f"Bearer {self.settings.brainpat_token}", + "Content-Type": "application/json", + "X-Brain-ID": self.settings.brain_id, + }, + timeout=self.timeout, + ) + + def close(self) -> None: + self._client.close() + + def __enter__(self) -> "BrainAPIClient": + return self + + def __exit__(self, *args: object) -> None: + self.close() + + def _request( + self, + method: str, + path: str, + *, + json: dict | None = None, + params: dict | None = None, + ) -> TimedResult: + last_error: Exception | None = None + for attempt in range(self.max_retries): + started = time.perf_counter() + try: + response = self._client.request( + method, path, json=json, params=params + ) + latency_ms = (time.perf_counter() - started) * 1000 + if ( + response.status_code == 404 + and path.rstrip("/").endswith("/retrieve/search") + ): + detail = "" + try: + detail = response.text[:500] + except Exception: + detail = "" + raise SearchDisabledError( + "POST /retrieve/search returned 404. Set SEARCH_ENABLED=true " + "(and DATA_DB=postgresql) on the API under test. " + f"body={detail!r}" + ) from None + if response.status_code in (429, 500, 502, 503, 504): + wait = min(2**attempt * 0.5, 30) + time.sleep(wait) + last_error = httpx.HTTPStatusError( + f"HTTP {response.status_code}", + request=response.request, + response=response, + ) + continue + if response.status_code >= 400: + detail = "" + try: + detail = response.text[:500] + except Exception: + detail = "" + raise RuntimeError( + f"{method} {path} returned {response.status_code}. " + f"body={detail!r}" + ) from None + if response.status_code == 204 or not response.content: + return TimedResult( + data=None, + latency_ms=latency_ms, + status_code=response.status_code, + ) + return TimedResult( + data=response.json(), + latency_ms=latency_ms, + status_code=response.status_code, + ) + except SearchDisabledError: + raise + except (httpx.TransportError, httpx.TimeoutException) as exc: + last_error = exc + wait = min(2**attempt * 0.5, 30) + time.sleep(wait) + raise RuntimeError( + f"Request failed after retries: {method} {path}: {last_error}" + ) + + def ingest_structured( + self, + triples: list[dict[str, Any]], + brain_id: str | None = None, + *, + mode: str = "deterministic", + ) -> TimedResult: + bid = brain_id or self.settings.brain_id + body = { + "mode": mode, + "brain_id": bid, + "data": triples, + } + return self._request("POST", "/ingest/structured", json=body) + + def ingest_text( + self, + text: str, + brain_id: str | None = None, + *, + skip_enrichment: bool = False, + meta_keys: dict[str, Any] | None = None, + ) -> TimedResult: + bid = brain_id or self.settings.brain_id + body: dict[str, Any] = { + "data": {"data_type": "text", "text_data": text}, + "brain_id": bid, + "observate_for": [], + "preferred_extraction_entities": [], + "skip_enrichment": skip_enrichment, + } + if meta_keys: + body["meta_keys"] = meta_keys + return self._request("POST", "/ingest/", json=body) + + def get_task(self, task_id: str, brain_id: str | None = None) -> TimedResult: + bid = brain_id or self.settings.brain_id + return self._request( + "GET", f"/tasks/{task_id}", params={"brain_id": bid} + ) + + def wait_for_task( + self, + task_id: str, + brain_id: str | None = None, + *, + timeout_s: float = 600.0, + poll_interval_s: float = 2.0, + ) -> TimedResult: + bid = brain_id or self.settings.brain_id + started = time.perf_counter() + while True: + result = self.get_task(task_id, bid) + status = (result.data or {}).get("status", "unknown") + if status in TERMINAL_TASK_STATUSES: + return TimedResult( + data=result.data, + latency_ms=(time.perf_counter() - started) * 1000, + status_code=result.status_code, + ) + if time.perf_counter() - started > timeout_s: + raise TimeoutError( + f"Task {task_id} did not finish within {timeout_s}s " + f"(last status={status})" + ) + time.sleep(poll_interval_s) + + def search( + self, + query: str, + brain_id: str | None = None, + *, + k: int = 20, + fusion: str | None = None, + fusion_alpha: float | None = None, + rerank: str | None = None, + mode: str | None = None, + profile_stages: bool = True, + channels: list[str] | None = None, + node_labels: list[str] | None = None, + community_labels: list[str] | None = None, + expand: str | None = None, + extras: dict[str, str] | None = None, + target: str | None = None, + ) -> TimedResult: + bid = brain_id or self.settings.brain_id + body: dict[str, Any] = { + "query": query, + "brain_id": bid, + "k": k, + "channels": channels or ["passages"], + "profile_stages": profile_stages, + } + if fusion: + body["fusion"] = fusion + if fusion_alpha is not None: + body["fusion_alpha"] = fusion_alpha + if rerank: + body["rerank"] = rerank + if mode and mode != "default": + body["mode"] = mode + if node_labels: + body["node_labels"] = node_labels + if community_labels: + body["community_labels"] = community_labels + if expand and expand != "none": + body["expand"] = expand + if extras: + body["extras"] = extras + if target: + body["target"] = target + return self._request("POST", "/retrieve/search", json=body) + + def get_neighbors( + self, + uuid: str, + brain_id: str | None = None, + *, + limit: int = 10, + look_for: str | None = None, + ) -> TimedResult: + bid = brain_id or self.settings.brain_id + params: dict[str, Any] = { + "uuid": uuid, + "brain_id": bid, + "limit": limit, + } + if look_for: + params["look_for"] = look_for + return self._request("GET", "/retrieve/entities/neighbors", params=params) + + def list_text_chunks( + self, + brain_id: str | None = None, + *, + limit: int = 100, + skip: int = 0, + query_text: str | None = None, + ) -> TimedResult: + bid = brain_id or self.settings.brain_id + params: dict[str, Any] = { + "brain_id": bid, + "limit": limit, + "skip": skip, + } + if query_text: + params["query_text"] = query_text + return self._request("GET", "/retrieve/text-chunks", params=params) diff --git a/benchmarks/search/config.py b/benchmarks/search/config.py new file mode 100644 index 0000000..20484e0 --- /dev/null +++ b/benchmarks/search/config.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +from dotenv import load_dotenv + +BENCHMARKS_ROOT = Path(__file__).resolve().parent.parent +DATA_DIR = BENCHMARKS_ROOT / "data" +RUNS_DIR = BENCHMARKS_ROOT / "runs" +DEFAULT_DATASET_PATH = DATA_DIR / "search_toy.jsonl" +DEFAULT_BRAIN_ID = "searchbenchsmoke" +REQUIRED_PREFIX = "searchbench" +FORBIDDEN_BRAINS = frozenset( + { + "beam1m1clean", + "demorecsys", + "locomoconv26", + "locomoconv26clean", + "locomoconv26nostorm", + } +) +FORBIDDEN_PREFIXES = ( + "beam", + "demorecsys", + "lme", + "locomoconv", + "longmemeval", +) +FROZEN_STRUCTURED_BRAINS = frozenset( + { + "searchbenchwands", + "searchbenchesci74", + "searchbenchescies", + "searchbenchesciltr2", + } +) +TERMINAL_TASK_STATUSES = frozenset({"completed", "failed", "partial_failed"}) + + +def validate_brain_id(brain_id: str) -> str: + bid = (brain_id or "").strip() + if not bid: + raise SystemExit("brain_id is required.") + lowered = bid.lower() + forbidden = {item.lower() for item in FORBIDDEN_BRAINS} + if lowered in forbidden or lowered.startswith(FORBIDDEN_PREFIXES): + raise SystemExit( + f"Refusing brain_id={bid!r}. Use a dedicated searchbench* brain " + f"(default {DEFAULT_BRAIN_ID}). Never wipe eval memory or recsys brains." + ) + if not lowered.startswith(REQUIRED_PREFIX): + raise SystemExit( + f"Refusing brain_id={bid!r}. Search eval brains must start with " + f"{REQUIRED_PREFIX!r} (default {DEFAULT_BRAIN_ID})." + ) + return bid + + +@dataclass +class Settings: + brainapi_url: str + brainpat_token: str + dataset_path: Path + runs_dir: Path + brain_id: str + + @classmethod + def load(cls, env_file: Path | None = None) -> "Settings": + if env_file is None: + env_file = BENCHMARKS_ROOT / ".env" + if env_file.exists(): + load_dotenv(env_file, override=False) + else: + load_dotenv(override=False) + + brain_id = validate_brain_id( + os.getenv("SEARCH_BRAIN_ID", DEFAULT_BRAIN_ID) or DEFAULT_BRAIN_ID + ) + return cls( + brainapi_url=os.getenv("BRAINAPI_URL", "http://localhost:8000").rstrip("/"), + brainpat_token=os.getenv("BRAINPAT_TOKEN", ""), + dataset_path=Path( + os.getenv("SEARCH_DATASET_PATH", str(DEFAULT_DATASET_PATH)) + ), + runs_dir=Path(os.getenv("BENCH_RUNS_DIR", str(RUNS_DIR))), + brain_id=brain_id, + ) + + def require_brainapi(self) -> None: + if not self.brainpat_token: + raise SystemExit( + "BRAINPAT_TOKEN is required. Copy benchmarks/.env.example to " + "benchmarks/.env and set it." + ) diff --git a/benchmarks/search/dataset.py b/benchmarks/search/dataset.py new file mode 100644 index 0000000..3a75241 --- /dev/null +++ b/benchmarks/search/dataset.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from typing import Any + +DOC_MARKER_PREFIX = "DOCID " + + +def doc_marker(doc_id: str) -> str: + return f"{DOC_MARKER_PREFIX}{doc_id}" + + +def write_records(rows: list[dict[str, Any]], path: Path) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as fh: + for row in rows: + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + return path + + +def load_records(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + rows.append(json.loads(line)) + return rows + + +def split_corpus( + rows: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + docs: list[dict[str, Any]] = [] + queries: list[dict[str, Any]] = [] + for row in rows: + kind = str(row.get("type") or "").strip().lower() + if kind == "doc": + docs.append(row) + elif kind == "query": + queries.append(row) + else: + raise ValueError(f"Unknown record type {kind!r} in {row!r}") + return docs, queries + + +def dataset_stats(rows: list[dict[str, Any]]) -> dict[str, Any]: + docs, queries = split_corpus(rows) + slices = Counter(str(q.get("slice") or "unspecified") for q in queries) + return { + "n_docs": len(docs), + "n_queries": len(queries), + "slices": dict(slices), + "doc_ids": [str(d.get("doc_id") or "") for d in docs], + "graded": any(bool(q.get("gold_grades")) for q in queries), + } + + +def map_doc_ids_to_chunks( + docs: list[dict[str, Any]], + chunks: list[dict[str, Any]], +) -> dict[str, set[str]]: + mapping: dict[str, set[str]] = {str(doc["doc_id"]): set() for doc in docs} + for chunk in chunks: + text = str(chunk.get("text") or "") + chunk_id = str(chunk.get("id") or "") + if not chunk_id: + continue + for doc in docs: + doc_id = str(doc.get("doc_id") or "") + marker = str(doc.get("marker") or doc_marker(doc_id)) + if marker and marker in text: + mapping.setdefault(doc_id, set()).add(chunk_id) + return mapping diff --git a/benchmarks/search/evaluate.py b/benchmarks/search/evaluate.py new file mode 100644 index 0000000..e194bf2 --- /dev/null +++ b/benchmarks/search/evaluate.py @@ -0,0 +1,747 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from search.client import BrainAPIClient, SearchDisabledError, TimedResult +from search.config import FROZEN_STRUCTURED_BRAINS, Settings +from search.dataset import map_doc_ids_to_chunks, split_corpus +from search.mapping import docs_to_triples, interactions_to_triples +from search.metrics import ( + aggregate_query_metrics, + mrr, + ndcg_at_k, + recall_at_k, + retrieve_latency_ms, +) + + +def ensure_run_dir(settings: Settings, run_id: str | None = None) -> tuple[str, Path]: + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + rid = run_id or f"search-{stamp}" + run_dir = settings.runs_dir / rid + run_dir.mkdir(parents=True, exist_ok=True) + return rid, run_dir + + +def _hit_ids(payload: dict[str, Any] | None) -> list[str]: + hits = (payload or {}).get("hits") or [] + ids: list[str] = [] + for hit in hits: + if not isinstance(hit, dict): + continue + hid = hit.get("id") + if hid: + ids.append(str(hid)) + return ids + + +def _hit_dump( + payload: dict[str, Any] | None, + chunk_to_doc: dict[str, str] | None = None, +) -> list[dict[str, Any]]: + mapping = chunk_to_doc or {} + rows: list[dict[str, Any]] = [] + for hit in (payload or {}).get("hits") or []: + if not isinstance(hit, dict): + continue + hid = hit.get("id") + if not hid: + continue + raw_id = str(hid) + node_id = str(hit.get("node_id") or "").strip() or None + rows.append( + { + "id": raw_id, + "channel": str(hit.get("channel") or ""), + "doc_id": mapping.get(raw_id, raw_id), + "node_id": node_id, + } + ) + return rows + + +_DOC_EXTRAS_KEYS = ("brand", "color", "locale") + + +def doc_meta_keys(doc: dict[str, Any]) -> dict[str, str] | None: + meta: dict[str, str] = {} + extras = doc.get("extras") + if isinstance(extras, dict): + for key, value in extras.items(): + if value is None: + continue + meta[str(key)] = str(value) + for key in _DOC_EXTRAS_KEYS: + value = doc.get(key) + if value is None: + continue + meta[str(key)] = str(value) + return meta or None + + +def ingest_docs( + client: BrainAPIClient, + docs: list[dict[str, Any]], + *, + timeout_s: float, + skip_enrichment: bool = True, +) -> dict[str, Any]: + tasks: list[dict[str, Any]] = [] + for doc in docs: + text = str(doc.get("text") or "") + accepted = client.ingest_text( + text, + skip_enrichment=skip_enrichment, + meta_keys=doc_meta_keys(doc), + ) + task_id = str((accepted.data or {}).get("task_id") or "") + if not task_id: + tasks.append( + { + "doc_id": doc.get("doc_id"), + "status": "failed", + "error": "ingest response missing task_id", + } + ) + continue + waited = client.wait_for_task( + task_id, + timeout_s=timeout_s, + poll_interval_s=0.25 if skip_enrichment else 2.0, + ) + status = str((waited.data or {}).get("status") or "unknown") + tasks.append( + { + "doc_id": doc.get("doc_id"), + "task_id": task_id, + "status": status, + } + ) + statuses = [str(item.get("status")) for item in tasks] + if not tasks: + overall = "failed" + elif all(status == "completed" for status in statuses): + overall = "completed" + elif any(status == "completed" for status in statuses): + overall = "partial_failed" + else: + overall = "failed" + return {"status": overall, "n_docs": len(docs), "tasks": tasks} + + +def ingest_triples( + client: BrainAPIClient, + triples: list[dict[str, Any]], + *, + timeout_s: float, + chunk_size: int = 20000, +) -> dict[str, Any]: + if not triples: + return {"status": "skipped", "n_triples": 0, "tasks": []} + tasks: list[dict[str, Any]] = [] + chunk = max(1, int(chunk_size)) + for start in range(0, len(triples), chunk): + batch = triples[start : start + chunk] + submitted = client.ingest_structured(batch) + task_id = str((submitted.data or {}).get("task_id") or "") + if not task_id: + tasks.append( + { + "status": "failed", + "error": "structured ingest response missing task_id", + } + ) + continue + waited = client.wait_for_task(task_id, timeout_s=timeout_s) + status = str((waited.data or {}).get("status") or "unknown") + tasks.append({"task_id": task_id, "status": status}) + statuses = [str(item.get("status")) for item in tasks] + if not tasks: + overall = "failed" + elif all(status == "completed" for status in statuses): + overall = "completed" + elif any(status == "completed" for status in statuses): + overall = "partial_failed" + else: + overall = "failed" + return {"status": overall, "n_triples": len(triples), "tasks": tasks} + + +def list_all_text_chunks(client: BrainAPIClient) -> list[dict[str, Any]]: + chunks: list[dict[str, Any]] = [] + skip = 0 + page = 100 + total = None + while True: + result = client.list_text_chunks(limit=page, skip=skip) + payload = result.data or {} + batch = payload.get("data") or [] + if not isinstance(batch, list): + break + chunks.extend([item for item in batch if isinstance(item, dict)]) + total = payload.get("total") + skip += len(batch) + if not batch: + break + if total is not None and skip >= int(total): + break + return chunks + + +def invert_doc_chunks(doc_to_chunks: dict[str, set[str]]) -> dict[str, str]: + chunk_to_doc: dict[str, str] = {} + for doc_id, chunks in (doc_to_chunks or {}).items(): + for chunk_id in chunks or set(): + chunk_to_doc[str(chunk_id)] = str(doc_id) + return chunk_to_doc + + +def canonicalize_hit_ids( + ranked: list[str], + chunk_to_doc: dict[str, str], +) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for hid in ranked: + key = chunk_to_doc.get(str(hid), str(hid)) + if not key or key in seen: + continue + seen.add(key) + out.append(key) + return out + + +def unique_doc_counts( + raw_ids: list[str], + chunk_to_doc: dict[str, str] | None = None, + *, + k: int = 20, +) -> tuple[int, int]: + head = [str(item) for item in raw_ids[:k] if item] + mapping = chunk_to_doc or {} + canonical = [mapping.get(item, item) for item in head] + return len(set(head)), len({item for item in canonical if item}) + + +def channel_id_lists( + payload: dict[str, Any] | None, + hits: list[dict[str, str]] | None = None, +) -> dict[str, list[str]]: + raw = (payload or {}).get("channel_lists") or {} + if not isinstance(raw, dict): + raw = {} + out: dict[str, list[str]] = { + "dense_ids": [str(item) for item in (raw.get("dense") or []) if item], + "bm25_ids": [str(item) for item in (raw.get("bm25") or []) if item], + "entity_ids": [str(item) for item in (raw.get("entities") or []) if item], + "community_ids": [str(item) for item in (raw.get("communities") or []) if item], + "event_ids": [str(item) for item in (raw.get("events") or []) if item], + "neighbor_ids": [str(item) for item in (raw.get("neighbors") or []) if item], + "passage_ids": [], + } + by_channel: dict[str, list[str]] = {} + for hit in hits or []: + channel = str(hit.get("channel") or "") + hid = str(hit.get("id") or "") + if not channel or not hid: + continue + by_channel.setdefault(channel, []).append(hid) + out["passage_ids"] = list(by_channel.get("passages") or []) + if not out["entity_ids"]: + out["entity_ids"] = list(by_channel.get("entities") or []) + if not out["community_ids"]: + out["community_ids"] = list(by_channel.get("communities") or []) + if not out["event_ids"]: + out["event_ids"] = list(by_channel.get("events") or []) + if not out["neighbor_ids"]: + out["neighbor_ids"] = list(by_channel.get("neighbors") or []) + return out + + +def gold_chunk_ids( + query: dict[str, Any], + doc_to_chunks: dict[str, set[str]], +) -> set[str]: + gold: set[str] = set() + for doc_id in query.get("gold_doc_ids") or []: + gold.update(doc_to_chunks.get(str(doc_id)) or set()) + for chunk_id in query.get("gold_chunk_ids") or []: + gold.add(str(chunk_id)) + grades = query.get("gold_grades") or {} + if isinstance(grades, dict): + for doc_id, gain in grades.items(): + if float(gain or 0) <= 0: + continue + gold.update(doc_to_chunks.get(str(doc_id)) or set()) + return gold + + +def gold_chunk_grades( + query: dict[str, Any], + doc_to_chunks: dict[str, set[str]], +) -> dict[str, float]: + grades: dict[str, float] = {} + raw = query.get("gold_grades") or {} + if isinstance(raw, dict) and raw: + for doc_id, gain in raw.items(): + value = float(gain or 0) + if value <= 0: + continue + for chunk_id in doc_to_chunks.get(str(doc_id)) or set(): + grades[chunk_id] = max(grades.get(chunk_id, 0.0), value) + return grades + for chunk_id in gold_chunk_ids(query, doc_to_chunks): + grades[chunk_id] = 1.0 + return grades + + +def gold_hit_ids( + query: dict[str, Any], + doc_to_chunks: dict[str, set[str]], +) -> set[str]: + gold: set[str] = set() + for doc_id in query.get("gold_doc_ids") or []: + gold.add(str(doc_id)) + grades = query.get("gold_grades") or {} + if isinstance(grades, dict): + for doc_id, gain in grades.items(): + if float(gain or 0) <= 0: + continue + gold.add(str(doc_id)) + chunk_to_doc = invert_doc_chunks(doc_to_chunks) + for chunk_id in query.get("gold_chunk_ids") or []: + gold.add(chunk_to_doc.get(str(chunk_id), str(chunk_id))) + if gold: + return gold + return { + chunk_to_doc.get(chunk_id, chunk_id) + for chunk_id in gold_chunk_ids(query, doc_to_chunks) + } + + +def gold_hit_grades( + query: dict[str, Any], + doc_to_chunks: dict[str, set[str]], +) -> dict[str, float]: + grades: dict[str, float] = {} + raw = query.get("gold_grades") or {} + if isinstance(raw, dict) and raw: + for doc_id, gain in raw.items(): + value = float(gain or 0) + if value <= 0: + continue + key = str(doc_id) + grades[key] = max(grades.get(key, 0.0), value) + return grades + for doc_id in gold_hit_ids(query, doc_to_chunks): + grades[str(doc_id)] = 1.0 + return grades + + +def candidate_pool_ids(query: dict[str, Any]) -> list[str]: + raw = query.get("candidate_doc_ids") or [] + if raw: + return [str(item) for item in raw] + return [str(item) for item in (query.get("gold_doc_ids") or [])] + + +def candidate_pool_grades(query: dict[str, Any]) -> dict[str, float]: + raw = query.get("candidate_grades") or {} + if isinstance(raw, dict) and raw: + return {str(doc_id): float(gain) for doc_id, gain in raw.items()} + grades = { + str(doc_id): float(gain) + for doc_id, gain in (query.get("gold_grades") or {}).items() + } + for doc_id in query.get("gold_doc_ids") or []: + grades.setdefault(str(doc_id), 1.0) + return grades + + +def ingested_doc_ids(doc_to_chunks: dict[str, set[str]]) -> set[str]: + return { + str(doc_id) + for doc_id, chunks in (doc_to_chunks or {}).items() + if chunks + } + + +def filter_ranked_to_pool( + ranked: list[str], + pool: list[str], +) -> list[str]: + allowed = {str(item) for item in pool} + return [item for item in ranked if str(item) in allowed] + + +def score_search_result( + result: TimedResult, + *, + gold: set[str], + ks: tuple[int, ...], + grades: dict[str, float] | None = None, + chunk_to_doc: dict[str, str] | None = None, + pool_ids: list[str] | None = None, +) -> dict[str, Any]: + payload = result.data if isinstance(result.data, dict) else {} + ranked = _hit_ids(payload) + hits = _hit_dump(payload, chunk_to_doc) + n_unique_raw, n_unique_canonical = unique_doc_counts( + ranked, chunk_to_doc, k=20 + ) + unique_k = max(ks) if ks else 20 + n_retrieve_raw, n_retrieve_canonical = unique_doc_counts( + ranked, chunk_to_doc, k=unique_k + ) + lists = channel_id_lists(payload, hits) + if chunk_to_doc: + ranked = canonicalize_hit_ids(ranked, chunk_to_doc) + if pool_ids: + ranked = filter_ranked_to_pool(ranked, pool_ids) + retrieve_ms, embed_ms = retrieve_latency_ms( + payload.get("stage_timings") if isinstance(payload, dict) else None, + result.latency_ms, + ) + ndcg_ks = tuple(dict.fromkeys((*ks, 10, 20))) + metrics = {f"recall@{cut}": recall_at_k(ranked, gold, cut) for cut in ks} + for cut in ndcg_ks: + metrics[f"ndcg@{cut}"] = ndcg_at_k(ranked, gold, cut, grades=grades) + full_k = max(len(ranked), 1) + metrics["ndcg"] = ndcg_at_k(ranked, gold, full_k, grades=grades) + metrics["mrr"] = mrr(ranked, gold) + return { + "hit_ids": ranked, + "hits": hits, + "metrics": metrics, + "retrieve_ms": retrieve_ms, + "embed_ms": embed_ms, + "client_wall_ms": result.latency_ms, + "n_hits": len(ranked), + "n_unique_docs_raw": n_unique_raw, + "n_unique_docs_canonical": n_unique_canonical, + "n_unique_docs_retrieve_raw": n_retrieve_raw, + "n_unique_docs_retrieve_canonical": n_retrieve_canonical, + "unique_docs_k": unique_k, + "facets": payload.get("facets") if isinstance(payload, dict) else None, + "response_node_ids": ( + list(payload.get("node_ids") or []) if isinstance(payload, dict) else [] + ), + **lists, + } + + +WANDSGRAPH_BRAIN = "searchbenchwandsgraph" + + +def assert_wandsgraph_node_join( + client: BrainAPIClient, + result: dict[str, Any], + *, + neighbor_limit: int = 5, +) -> dict[str, Any]: + if client.settings.brain_id != WANDSGRAPH_BRAIN: + return {"skipped": True, "brain_id": client.settings.brain_id} + queries = result.get("queries") or [] + if not queries: + raise SystemExit("searchbenchwandsgraph node_id join needs at least one query") + first = queries[0] + hits = first.get("hits") or [] + matched = [ + hit + for hit in hits + if isinstance(hit, dict) + and hit.get("node_id") + and str(hit.get("node_id")) == str(hit.get("doc_id") or "") + ] + if not matched: + raise SystemExit( + "searchbenchwandsgraph passage hits missing node_id matching DOCID/doc_id. " + f"sample={hits[:3]!r}" + ) + node_id = str(matched[0]["node_id"]) + timed = client.get_neighbors(node_id, limit=neighbor_limit) + if timed.status_code != 200: + raise SystemExit( + f"GET /retrieve/entities/neighbors?uuid={node_id} returned {timed.status_code}" + ) + payload = timed.data if isinstance(timed.data, dict) else {} + return { + "skipped": False, + "node_id": node_id, + "status_code": timed.status_code, + "neighbor_count": payload.get("count"), + "qid": first.get("qid"), + } + + +def evaluate_search( + client: BrainAPIClient, + rows: list[dict[str, Any]], + *, + ks: tuple[int, ...] = (5, 10, 20), + timeout_s: float = 600.0, + k: int = 20, + fusion: str | None = None, + fusion_alpha: float | None = None, + rerank: str | None = None, + mode: str | None = None, + channels: list[str] | None = None, + node_labels: list[str] | None = None, + community_labels: list[str] | None = None, + expand: str | None = None, + dataset_name: str = "search_toy.jsonl", + limit_docs: int | None = None, + limit_queries: int | None = None, + skip_enrichment: bool = True, + ingest_graph: bool = False, + skip_ingest: bool = False, + rank_pool: bool = False, + personalize: bool = False, + interactions: list[dict[str, Any]] | None = None, + extras: dict[str, str] | None = None, +) -> dict[str, Any]: + brain_id = client.settings.brain_id + if (ingest_graph or interactions) and brain_id in FROZEN_STRUCTURED_BRAINS: + raise SystemExit( + f"Refusing structured graph/interaction ingest on frozen brain {brain_id}. " + "Use searchbenchwandsgraph (or another non-frozen searchbench*)." + ) + docs, queries = split_corpus(rows) + if limit_docs is not None: + docs = docs[:limit_docs] + if limit_queries is not None: + queries = queries[:limit_queries] + + if skip_ingest: + ingest = { + "status": "completed", + "n_docs": len(docs), + "tasks": [], + "reused": True, + } + else: + ingest = ingest_docs( + client, + docs, + timeout_s=timeout_s, + skip_enrichment=skip_enrichment, + ) + graph_ingest: dict[str, Any] = {"status": "skipped", "n_triples": 0, "tasks": []} + if ingest_graph and docs: + graph_ingest = ingest_triples( + client, + docs_to_triples(docs), + timeout_s=timeout_s, + ) + interaction_ingest: dict[str, Any] = { + "status": "skipped", + "n_triples": 0, + "tasks": [], + } + if interactions: + interaction_ingest = ingest_triples( + client, + interactions_to_triples(interactions), + timeout_s=timeout_s, + ) + chunks = list_all_text_chunks(client) if ingest["status"] != "failed" else [] + doc_to_chunks = map_doc_ids_to_chunks(docs, chunks) + mapped = sum(1 for ids in doc_to_chunks.values() if ids) + chunk_to_doc = invert_doc_chunks(doc_to_chunks) + + per_query: list[dict[str, Any]] = [] + search_error: str | None = None + event_probe: dict[str, Any] | None = None + for query in queries: + gold = gold_hit_ids(query, doc_to_chunks) + pool = candidate_pool_ids(query) if rank_pool else [] + grades = ( + candidate_pool_grades(query) + if rank_pool + else gold_hit_grades(query, doc_to_chunks) + ) + target = None + if personalize: + raw_target = str(query.get("target") or "").strip() + if raw_target: + target = raw_target + try: + result = client.search( + str(query.get("query") or ""), + k=k, + fusion=fusion, + fusion_alpha=fusion_alpha, + rerank=rerank, + mode=mode, + channels=channels, + node_labels=node_labels, + community_labels=community_labels, + expand=expand, + extras=extras, + target=target, + profile_stages=True, + ) + except SearchDisabledError as exc: + search_error = str(exc) + break + except RuntimeError as exc: + search_error = str(exc) + break + scored = score_search_result( + result, + gold=gold, + ks=ks, + grades=grades, + chunk_to_doc=chunk_to_doc, + pool_ids=pool or None, + ) + ingested = ingested_doc_ids(doc_to_chunks) + missing_brain = [pid for pid in pool if pid not in ingested] if pool else [] + retrieved_in_pool = list(scored.get("hit_ids") or []) + coverage = ( + (len(retrieved_in_pool) / len(pool)) if pool else None + ) + per_query.append( + { + "qid": query.get("qid"), + "query": query.get("query"), + "slice": query.get("slice") or "unspecified", + "gold_doc_ids": list(query.get("gold_doc_ids") or []), + "gold_grades": { + str(doc_id): float(gain) + for doc_id, gain in grades.items() + if float(gain or 0) > 0 + }, + "gold_chunk_ids": sorted(gold), + "candidate_doc_ids": pool, + "pool_size": len(pool) if rank_pool else None, + "pool_coverage": coverage, + "missing_from_brain": missing_brain, + **scored, + } + ) + + if not queries and interactions and search_error is None: + names: list[str] = [] + seen: set[str] = set() + for row in interactions: + behavior = str(row.get("behavior") or "").strip() + if behavior and behavior not in seen: + seen.add(behavior) + names.append(behavior) + hit_counts: list[dict[str, Any]] = [] + for name in names[:5]: + try: + result = client.search( + name, + k=k, + fusion=fusion, + rerank=rerank, + mode=mode, + channels=channels or ["events"], + expand=expand, + profile_stages=True, + ) + except SearchDisabledError as exc: + search_error = str(exc) + break + except RuntimeError as exc: + search_error = str(exc) + break + payload = result.data if isinstance(result.data, dict) else {} + hits = payload.get("hits") or [] + hit_counts.append({"query": name, "n_hits": len(hits)}) + event_probe = { + "n_event_queries": len(hit_counts), + "queries": hit_counts, + "n_hits": sum(int(item.get("n_hits") or 0) for item in hit_counts), + } + + metrics = aggregate_query_metrics(per_query, ks=ks) if per_query else {} + if per_query and any( + row.get("n_unique_docs_canonical") is not None for row in per_query + ): + n_raw = [ + float(row.get("n_unique_docs_raw") or 0) for row in per_query + ] + n_canon = [ + float(row.get("n_unique_docs_canonical") or 0) for row in per_query + ] + metrics["unique_docs@20_raw"] = sum(n_raw) / len(n_raw) + metrics["unique_docs@20_canonical"] = sum(n_canon) / len(n_canon) + retrieve_k = int(per_query[0].get("unique_docs_k") or 20) + n_ret = [ + float(row.get("n_unique_docs_retrieve_canonical") or 0) + for row in per_query + ] + metrics[f"unique_docs@{retrieve_k}_canonical"] = sum(n_ret) / len(n_ret) + if rank_pool and per_query: + coverages = [ + float(row["pool_coverage"]) + for row in per_query + if row.get("pool_coverage") is not None + ] + if coverages: + metrics["pool_coverage"] = sum(coverages) / len(coverages) + metrics["missing_from_brain"] = sum( + len(row.get("missing_from_brain") or []) for row in per_query + ) / len(per_query) + status = "ok" + if docs and ingest["status"] not in {"completed", "partial_failed"}: + status = "failed" + if docs and mapped < 1: + status = "failed" + if queries and not per_query: + status = "failed" + if not queries and interactions: + if event_probe is None or int(event_probe.get("n_hits") or 0) < 1: + status = "failed" + elif search_error is None: + status = "ok" + if search_error: + status = "failed" + + protocol = None + if brain_id == "searchbenchwandsgraph": + protocol = ( + "architecture demo: isolated catalog graph on searchbenchwandsgraph; " + "not a quality-default claim vs frozen WANDS passages 0.823 or ESCI 0.500" + ) + if brain_id == "searchbenchjdslice": + protocol = ( + "architecture demo: JDsearch rank-pool slice on searchbenchjdslice; " + "not TEM 0.219; not frozen WANDS 0.823 or ESCI 0.500" + ) + + return { + "status": status, + "brain_id": client.settings.brain_id, + "dataset": dataset_name, + "protocol": protocol, + "fusion": fusion or "rrf", + "fusion_alpha": fusion_alpha, + "rerank": rerank or "none", + "mode": mode or "default", + "channels": channels or ["passages"], + "expand": expand or "none", + "k": k, + "ks": list(ks), + "n_docs": len(docs), + "n_queries": len(per_query), + "n_docs_mapped": mapped, + "skip_enrichment": skip_enrichment, + "ingest_graph": ingest_graph, + "skip_ingest": skip_ingest, + "rank_pool": rank_pool, + "personalize": personalize, + "extras": extras, + "ingest": ingest, + "graph_ingest": graph_ingest, + "interaction_ingest": interaction_ingest, + "event_probe": event_probe, + "search_error": search_error, + "metrics": metrics, + "queries": per_query, + } diff --git a/benchmarks/search/export_hybrid_lists.py b/benchmarks/search/export_hybrid_lists.py new file mode 100644 index 0000000..ff50c08 --- /dev/null +++ b/benchmarks/search/export_hybrid_lists.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from search.dataset import load_records, split_corpus +from search.finetune_esci_4class import LABELS, LABEL_TO_ID, held_out_query_ids +from search.mine_retrieved_lists import is_held_out, write_lists + +PROTECTED_OUT_NAMES = { + "esci_retrieved_lists.jsonl", + "search_esci.jsonl", + "search_esci_74.jsonl", +} + + +def label_from_gain(gain: float) -> str: + value = float(gain or 0.0) + if value >= 0.99: + return "E" + if value >= 0.05: + return "S" + if value > 0.0: + return "C" + return "I" + + +def _chunk_to_doc(row: dict[str, Any]) -> dict[str, str]: + mapping: dict[str, str] = {} + for hit in row.get("hits") or []: + chunk_id = str(hit.get("id") or "") + doc_id = str(hit.get("doc_id") or "") + if chunk_id and doc_id: + mapping[chunk_id] = doc_id + return mapping + + +def _as_docs(ids: list[str], mapping: dict[str, str]) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for raw in ids: + key = mapping.get(str(raw), str(raw)) + if not key or key in seen: + continue + seen.add(key) + out.append(key) + return out + + +def _queries_by_qid(queries: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for row in queries: + qid = str(row.get("qid") or "") + if qid: + out[qid] = row + return out + + +def _passage_for(doc: dict[str, Any] | None, pid: str) -> str: + if not doc: + return pid + text = str(doc.get("text") or "").strip() + if text: + return text + title = str(doc.get("title") or "").strip() + if title: + return f"Title: {title}" + return pid + + +def rows_from_eval( + eval_result: dict[str, Any], + *, + docs: list[dict[str, Any]], + queries: list[dict[str, Any]], + holdout: set[str], + k: int = 50, +) -> list[dict[str, Any]]: + docs_by_id = { + str(doc.get("doc_id") or ""): doc + for doc in docs + if str(doc.get("doc_id") or "") + } + query_rows = _queries_by_qid(queries) + cut = max(1, int(k)) + rows: list[dict[str, Any]] = [] + for row in eval_result.get("queries") or []: + qid = str(row.get("qid") or "") + if not qid or is_held_out(qid, holdout): + continue + source = query_rows.get(qid) or row + query = str(source.get("query") or row.get("query") or "") + grades = source.get("gold_grades") or row.get("gold_grades") or {} + if not isinstance(grades, dict): + grades = {} + mapping = _chunk_to_doc(row) + hit_ids = [str(item) for item in (row.get("hit_ids") or []) if item] + pids = _as_docs(hit_ids, mapping)[:cut] + for pid in pids: + gain = float(grades.get(pid) or 0.0) + label = label_from_gain(gain) + if label not in LABEL_TO_ID: + label = "I" + rows.append( + { + "query_id": qid, + "query": query, + "product_id": pid, + "label": label, + "passage": _passage_for(docs_by_id.get(pid), pid), + } + ) + return rows + + +def export_hybrid_lists( + *, + eval_result: dict[str, Any], + dataset_path: Path, + holdout_path: Path, + out_path: Path, + k: int = 50, +) -> dict[str, Any]: + name = out_path.name + if name in PROTECTED_OUT_NAMES: + raise ValueError(f"refusing to overwrite {out_path}") + holdout = held_out_query_ids(holdout_path) + docs, queries = split_corpus(load_records(dataset_path)) + rows = rows_from_eval( + eval_result, + docs=docs, + queries=queries, + holdout=holdout, + k=k, + ) + write_lists(rows, out_path) + counts = {label: 0 for label in LABELS} + by_q: dict[str, list[str]] = {} + for row in rows: + counts[str(row["label"])] = counts.get(str(row["label"]), 0) + 1 + by_q.setdefault(str(row["query_id"]), []).append(str(row["label"])) + n_with_i = sum(1 for labels in by_q.values() if "I" in labels) + meta = { + "n_queries": len(by_q), + "n_pairs": len(rows), + "n_holdout_qids": len(holdout), + "k": int(k), + "label_counts": counts, + "n_lists_with_i": n_with_i, + "source": "hybrid-k50", + "out": str(out_path), + "dataset": str(dataset_path), + "holdout": str(holdout_path), + } + meta_path = out_path.with_name(out_path.stem + ".meta.json") + meta_path.write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8") + return meta diff --git a/benchmarks/search/finetune_esci_4class.py b/benchmarks/search/finetune_esci_4class.py new file mode 100644 index 0000000..4fa11c0 --- /dev/null +++ b/benchmarks/search/finetune_esci_4class.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import argparse +import json +import random +from collections import Counter +from pathlib import Path +from typing import Any, Sequence + +from search.catalog import ( + ESCI_CACHE_DIR, + ESCI_EXAMPLES_FILE, + ESCI_PRODUCTS_FILE, + extra_fields_from_catalog, + format_product_text, + _esci_products, + _iter_parquet, + _stringify_catalog_value, +) +from search.dataset import load_records, split_corpus + +LABELS = ("E", "S", "C", "I") +LABEL_TO_ID = {label: index for index, label in enumerate(LABELS)} +CLASS_GAINS = (1.0, 0.1, 0.01, 0.0) +DEFAULT_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2" +DEFAULT_OUT = Path("data/models/esci-minilm-l12-4class") + + +def pick_device() -> str: + import torch + + if torch.backends.mps.is_available(): + return "mps" + if torch.cuda.is_available(): + return "cuda" + return "cpu" + + +def held_out_query_ids(jsonl_path: Path) -> set[str]: + _, queries = split_corpus(load_records(jsonl_path)) + out: set[str] = set() + for row in queries: + qid = str(row.get("qid") or "") + if qid.lower().startswith("esci-"): + out.add(qid[5:]) + out.add(qid) + return out + + +def product_passage(catalog: dict[str, Any], extra_fields: Sequence[tuple[str, str]]) -> str: + pid = str(catalog.get("product_id") or "unknown") + title = _stringify_catalog_value(catalog.get("product_title")) or pid + extras = [ + (label, _stringify_catalog_value(catalog.get(field))) + for label, field in extra_fields + ] + return format_product_text( + pid, + title=title, + description=_stringify_catalog_value(catalog.get("product_description")), + extras=extras, + ) + + +def weighted_scores( + class_probs: Sequence[Sequence[float]] | Sequence[float], + gains: Sequence[float] = CLASS_GAINS, +) -> list[float]: + rows = list(class_probs) + if not rows: + return [] + first = rows[0] + if isinstance(first, (int, float)): + rows = [rows] + gain = [float(value) for value in gains] + scores: list[float] = [] + for row in rows: + total = 0.0 + for index, value in enumerate(row): + if index >= len(gain): + break + total += float(value) * gain[index] + scores.append(total) + return scores + + +def rank_doc_ids(doc_ids: Sequence[str], scores: Sequence[float]) -> list[str]: + indexed = list(range(len(doc_ids))) + indexed.sort(key=lambda index: (-float(scores[index]), str(doc_ids[index]))) + return [str(doc_ids[index]) for index in indexed] + + +def persist_tokenizer_max_length(model: Any, out_dir: Path) -> None: + max_length = getattr(model, "max_length", None) + tokenizer = getattr(model, "tokenizer", None) + if not max_length or tokenizer is None: + return + tokenizer.model_max_length = int(max_length) + tokenizer.save_pretrained(str(out_dir)) + + +def class_weights(counts: Sequence[int]) -> list[float]: + values = [max(1, int(count)) for count in counts] + total = float(sum(values)) + n_labels = float(len(values)) + return [total / (n_labels * float(count)) for count in values] + + +def load_train_rows( + *, + holdout: set[str], + locale: str = "us", + max_pairs: int = 180000, + seed: int = 11, +) -> list[dict[str, Any]]: + examples_path = ESCI_CACHE_DIR / ESCI_EXAMPLES_FILE + wanted_locale = locale.strip().lower() + rows: list[dict[str, Any]] = [] + for item in _iter_parquet(examples_path): + if int(item.get("small_version") or 0) != 1: + continue + if str(item.get("product_locale") or "").strip().lower() != wanted_locale: + continue + if str(item.get("split") or "").strip().lower() != "train": + continue + qid = str(item.get("query_id") or "").strip() + if not qid or qid in holdout: + continue + pid = str(item.get("product_id") or "").strip() + query = str(item.get("query") or "").strip() + label = str(item.get("esci_label") or "").strip().upper() + if not pid or not query or label not in LABEL_TO_ID: + continue + rows.append( + { + "query": query, + "product_id": pid, + "label_id": LABEL_TO_ID[label], + "label": label, + } + ) + rng = random.Random(seed) + rng.shuffle(rows) + return rows[: max(1, max_pairs)] + + +def load_retrieved_rows(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + with path.open(encoding="utf-8") as fh: + for line in fh: + raw = line.strip() + if not raw: + continue + item = json.loads(raw) + label = str(item.get("label") or "").strip().upper() + query = str(item.get("query") or "").strip() + pid = str(item.get("product_id") or "").strip() + if not query or not pid or label not in LABEL_TO_ID: + continue + row = { + "query": query, + "product_id": pid, + "label": label, + "label_id": LABEL_TO_ID[label], + } + passage = str(item.get("passage") or "").strip() + if passage: + row["passage"] = passage + rows.append(row) + return rows + + +def attach_passages( + rows: list[dict[str, Any]], + *, + locale: str = "us", +) -> list[tuple[str, str, int]]: + products_path = ESCI_CACHE_DIR / ESCI_PRODUCTS_FILE + needed = {str(row["product_id"]) for row in rows} + products = _esci_products(products_path, needed, locale=locale) + extra_fields = extra_fields_from_catalog( + products, + title_key="product_title", + description_key="product_description", + ) + pairs: list[tuple[str, str, int]] = [] + for row in rows: + catalog = products.get(str(row["product_id"])) or {"product_id": row["product_id"]} + passage = product_passage(catalog, extra_fields) + pairs.append((str(row["query"]), passage, int(row["label_id"]))) + return pairs + + +def train_loop( + model, + loader, + *, + epochs: int, + loss_fct, + warmup_steps: int, + lr: float, + out_dir: Path, + ckpt_every: int = 200, +) -> int: + import torch + from torch.optim import AdamW + from transformers import get_linear_schedule_with_warmup + + loader.collate_fn = model.smart_batching_collate + model.model.to(model._target_device) + optimizer = AdamW(model.model.parameters(), lr=lr) + total_steps = max(1, len(loader) * epochs) + scheduler = get_linear_schedule_with_warmup( + optimizer, + num_warmup_steps=warmup_steps, + num_training_steps=total_steps, + ) + out_dir.mkdir(parents=True, exist_ok=True) + global_step = 0 + model.model.train() + for _epoch in range(epochs): + for features, labels in loader: + optimizer.zero_grad(set_to_none=True) + logits = model.model(**features).logits + loss = loss_fct(logits, labels) + loss.backward() + torch.nn.utils.clip_grad_norm_(model.model.parameters(), 1.0) + optimizer.step() + scheduler.step() + global_step += 1 + if global_step % 50 == 0 or global_step == total_steps: + print( + f"step {global_step}/{total_steps} loss={float(loss.detach().cpu()):.4f}", + flush=True, + ) + if global_step % ckpt_every == 0: + model.save(str(out_dir)) + persist_tokenizer_max_length(model, out_dir) + (out_dir / "train_step.json").write_text( + json.dumps( + { + "global_step": global_step, + "loss": float(loss.detach().cpu()), + } + ) + + "\n", + encoding="utf-8", + ) + print(f"checkpoint {out_dir} step={global_step}", flush=True) + model.save(str(out_dir)) + persist_tokenizer_max_length(model, out_dir) + return global_step + + +def finetune( + *, + jsonl_path: Path, + out_dir: Path, + max_pairs: int = 80000, + epochs: int = 1, + batch_size: int = 32, + max_length: int = 192, + base_model: str = DEFAULT_MODEL, + seed: int = 11, + ckpt_every: int = 200, + use_class_weights: bool = False, + lists_path: Path | None = None, + lists_source: str | None = None, +) -> dict[str, Any]: + import torch + from sentence_transformers import CrossEncoder, InputExample + from torch.utils.data import DataLoader + + holdout = held_out_query_ids(jsonl_path) + source = "pool" + if lists_path is not None: + source = str(lists_source or "").strip() or "retrieved-bm25" + rows = load_retrieved_rows(lists_path) + rng = random.Random(seed) + rng.shuffle(rows) + rows = rows[: max(1, max_pairs)] + if rows and all(str(row.get("passage") or "").strip() for row in rows): + pairs = [ + (str(row["query"]), str(row["passage"]), int(row["label_id"])) + for row in rows + ] + else: + pairs = attach_passages(rows) + else: + rows = load_train_rows(holdout=holdout, max_pairs=max_pairs, seed=seed) + pairs = attach_passages(rows) + label_counts = Counter(label_id for _, _, label_id in pairs) + counts = [int(label_counts.get(index, 0)) for index in range(len(LABELS))] + weights = class_weights(counts) if use_class_weights else [1.0] * len(LABELS) + examples = [ + InputExample(texts=[query, passage], label=label_id) + for query, passage, label_id in pairs + ] + device = pick_device() + model = CrossEncoder( + base_model, + num_labels=len(LABELS), + max_length=max_length, + device=device, + automodel_args={"ignore_mismatched_sizes": True}, + default_activation_function=torch.nn.Identity(), + ) + loader = DataLoader(examples, shuffle=True, batch_size=batch_size) + warmup = max(10, len(loader) // 10) + loss_fct = torch.nn.CrossEntropyLoss( + weight=torch.tensor(weights, dtype=torch.float32, device=model._target_device), + label_smoothing=0.1, + ) + steps = train_loop( + model, + loader, + epochs=epochs, + loss_fct=loss_fct, + warmup_steps=warmup, + lr=2e-5, + out_dir=out_dir, + ckpt_every=ckpt_every, + ) + meta = { + "n_pairs": len(pairs), + "n_steps": steps, + "label_counts": {LABELS[index]: counts[index] for index in range(len(LABELS))}, + "class_weights": {LABELS[index]: weights[index] for index in range(len(LABELS))}, + "class_gains": {LABELS[index]: CLASS_GAINS[index] for index in range(len(LABELS))}, + "n_holdout_qids": len(holdout), + "epochs": epochs, + "batch_size": batch_size, + "max_length": max_length, + "model": str(out_dir), + "base": base_model, + "device": device, + "loss": "4-class-ce-label-smoothing-0.1", + "use_class_weights": use_class_weights, + "score": "1.0*P(E)+0.1*P(S)+0.01*P(C)+0.0*P(I)", + "source": source, + "lists_path": str(lists_path) if lists_path is not None else None, + } + payload = json.dumps(meta, indent=2) + "\n" + (out_dir / "train_meta.json").write_text(payload, encoding="utf-8") + (out_dir / "finetune_meta.json").write_text(payload, encoding="utf-8") + return meta + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Fine-tune a 4-class ESCI cross-encoder (not Exact→1 MiniLM-L-6)" + ) + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--max-pairs", type=int, default=80000) + parser.add_argument("--epochs", type=int, default=1) + parser.add_argument("--batch-size", type=int, default=32) + parser.add_argument("--max-length", type=int, default=192) + parser.add_argument("--ckpt-every", type=int, default=200) + parser.add_argument("--class-weights", action="store_true") + parser.add_argument("--base", default=DEFAULT_MODEL) + parser.add_argument("--seed", type=int, default=11) + parser.add_argument("--from-lists", type=Path, default=None) + parser.add_argument("--lists-source", default="") + args = parser.parse_args(argv) + meta = finetune( + jsonl_path=args.dataset, + out_dir=args.out, + max_pairs=args.max_pairs, + epochs=args.epochs, + batch_size=args.batch_size, + max_length=args.max_length, + base_model=args.base, + seed=args.seed, + ckpt_every=args.ckpt_every, + use_class_weights=bool(args.class_weights), + lists_path=args.from_lists, + lists_source=( + str(args.lists_source).strip() or "retrieved-bm25" + if args.from_lists is not None + else None + ), + ) + print(json.dumps(meta, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/search/finetune_esci_ce.py b/benchmarks/search/finetune_esci_ce.py new file mode 100644 index 0000000..8dfebef --- /dev/null +++ b/benchmarks/search/finetune_esci_ce.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import argparse +import json +import random +from pathlib import Path +from typing import Any + +from search.catalog import ( + ESCI_CACHE_DIR, + ESCI_EXAMPLES_FILE, + ESCI_GAINS, + ESCI_PRODUCTS_FILE, + PRODUCT_TEXT_MAX_CHARS, + _esci_products, + _iter_parquet, + _stringify_catalog_value, +) +from search.dataset import load_records, split_corpus + +DEFAULT_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2" +REDDY_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2" +CLASS_ORDER = ("E", "S", "C", "I") +CLASS_TO_INDEX = {label: index for index, label in enumerate(CLASS_ORDER)} + + +def held_out_query_ids(jsonl_path: Path) -> set[str]: + _, queries = split_corpus(load_records(jsonl_path)) + out: set[str] = set() + for row in queries: + qid = str(row.get("qid") or "") + if qid.lower().startswith("esci-"): + out.add(qid[5:]) + out.add(qid) + return out + + +def product_passage(catalog: dict[str, Any], *, fields: str) -> str: + title = _stringify_catalog_value(catalog.get("product_title")) or "" + if fields == "title": + return title + parts: list[str] = [] + if title: + parts.append(title) + brand = _stringify_catalog_value(catalog.get("product_brand")) + if brand: + parts.append(f"Brand: {brand}") + color = _stringify_catalog_value(catalog.get("product_color")) + if color: + parts.append(f"Color: {color}") + bullets = _stringify_catalog_value(catalog.get("product_bullet_point")) + if bullets: + parts.append(f"Bullets: {bullets}") + description = _stringify_catalog_value(catalog.get("product_description")) + if description: + parts.append(f"Description: {description}") + body = "\n".join(parts).strip() + if len(body) > PRODUCT_TEXT_MAX_CHARS: + return body[: PRODUCT_TEXT_MAX_CHARS - 1].rstrip() + "…" + return body + + +def load_train_rows( + *, + holdout: set[str], + locale: str = "us", + max_pairs: int = 80000, + seed: int = 7, +) -> list[dict[str, Any]]: + examples_path = ESCI_CACHE_DIR / ESCI_EXAMPLES_FILE + wanted_locale = locale.strip().lower() + rows: list[dict[str, Any]] = [] + for item in _iter_parquet(examples_path): + if int(item.get("small_version") or 0) != 1: + continue + if str(item.get("product_locale") or "").strip().lower() != wanted_locale: + continue + if str(item.get("split") or "").strip().lower() != "train": + continue + qid = str(item.get("query_id") or "").strip() + if not qid or qid in holdout: + continue + pid = str(item.get("product_id") or "").strip() + query = str(item.get("query") or "").strip() + if not pid or not query: + continue + label = str(item.get("esci_label") or "").strip().upper() + if label not in ESCI_GAINS: + continue + rows.append( + { + "query": query, + "product_id": pid, + "label": label, + "gain": float(ESCI_GAINS[label]), + "binary": 1.0 if label == "E" else 0.0, + "class_index": CLASS_TO_INDEX[label], + } + ) + rng = random.Random(seed) + rng.shuffle(rows) + return rows[: max(1, max_pairs)] + + +def load_train_pairs( + *, + holdout: set[str], + locale: str = "us", + max_pairs: int = 30000, + seed: int = 7, + fields: str = "title", + label_mode: str = "binary", +) -> list[tuple[str, str, float]]: + rows = load_train_rows( + holdout=holdout, locale=locale, max_pairs=max_pairs, seed=seed + ) + products_path = ESCI_CACHE_DIR / ESCI_PRODUCTS_FILE + needed = {str(row["product_id"]) for row in rows} + products = _esci_products(products_path, needed, locale=locale) + pairs: list[tuple[str, str, float]] = [] + for row in rows: + catalog = products.get(str(row["product_id"])) or {} + passage = product_passage(catalog, fields=fields) or str(row["product_id"]) + if label_mode == "binary": + target = float(row["binary"]) + elif label_mode == "graded": + target = float(row["gain"]) + else: + target = float(row["class_index"]) + pairs.append((row["query"], passage, target)) + return pairs + + +def finetune( + *, + jsonl_path: Path, + out_dir: Path, + max_pairs: int = 80000, + epochs: int = 1, + batch_size: int = 32, + label_mode: str = "graded", + fields: str = "catalog", + base: str = DEFAULT_MODEL, + lr: float = 7e-6, + max_length: int = 256, +) -> dict[str, Any]: + from sentence_transformers import CrossEncoder, InputExample + from torch.utils.data import DataLoader + + if label_mode not in {"binary", "graded", "multiclass"}: + raise ValueError(f"Unknown label_mode {label_mode!r}") + if fields not in {"title", "catalog"}: + raise ValueError(f"Unknown fields {fields!r}") + holdout = held_out_query_ids(jsonl_path) + pairs = load_train_pairs( + holdout=holdout, + max_pairs=max_pairs, + fields=fields, + label_mode=label_mode, + ) + if label_mode == "multiclass": + examples = [ + InputExample(texts=[query, passage], label=int(target)) + for query, passage, target in pairs + ] + model = CrossEncoder(base, num_labels=len(CLASS_ORDER), max_length=max_length) + else: + examples = [ + InputExample(texts=[query, passage], label=float(target)) + for query, passage, target in pairs + ] + model = CrossEncoder( + base, + num_labels=1, + max_length=max_length, + default_activation_function=None, + ) + n_pos = sum( + 1 + for _, _, target in pairs + if ( + int(target) == CLASS_TO_INDEX["E"] + if label_mode == "multiclass" + else float(target) >= 1.0 + ) + ) + loader = DataLoader(examples, shuffle=True, batch_size=batch_size) + warmup = max(10, len(loader) // 10) + model.fit( + train_dataloader=loader, + epochs=epochs, + warmup_steps=warmup, + optimizer_params={"lr": lr}, + show_progress_bar=True, + ) + out_dir.mkdir(parents=True, exist_ok=True) + model.save(str(out_dir)) + counts: dict[str, int] = {} + if label_mode == "multiclass": + for _, _, target in pairs: + label = CLASS_ORDER[int(target)] + counts[label] = counts.get(label, 0) + 1 + meta = { + "n_pairs": len(pairs), + "n_exact": n_pos, + "n_holdout_qids": len(holdout), + "epochs": epochs, + "model": str(out_dir), + "base": base, + "label_mode": label_mode, + "fields": fields, + "lr": lr, + "max_length": max_length, + "class_counts": counts, + "class_order": list(CLASS_ORDER) if label_mode == "multiclass" else None, + } + (out_dir / "finetune_meta.json").write_text( + json.dumps(meta, indent=2) + "\n", encoding="utf-8" + ) + return meta + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="finetune_esci_ce") + parser.add_argument( + "--dataset", + type=Path, + default=Path("data/search_esci_74.jsonl"), + ) + parser.add_argument( + "--out", + type=Path, + default=Path("data/models/esci-minilm-graded"), + ) + parser.add_argument("--max-pairs", type=int, default=80000) + parser.add_argument("--epochs", type=int, default=1) + parser.add_argument("--batch-size", type=int, default=32) + parser.add_argument( + "--label-mode", + choices=["binary", "graded", "multiclass"], + default="graded", + ) + parser.add_argument("--fields", choices=["title", "catalog"], default="catalog") + parser.add_argument("--base", default=DEFAULT_MODEL) + parser.add_argument("--lr", type=float, default=7e-6) + parser.add_argument("--max-length", type=int, default=256) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + meta = finetune( + jsonl_path=args.dataset, + out_dir=args.out, + max_pairs=args.max_pairs, + epochs=args.epochs, + batch_size=args.batch_size, + label_mode=args.label_mode, + fields=args.fields, + base=args.base, + lr=args.lr, + max_length=args.max_length, + ) + print(json.dumps(meta, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/search/finetune_esci_dense.py b/benchmarks/search/finetune_esci_dense.py new file mode 100644 index 0000000..b090811 --- /dev/null +++ b/benchmarks/search/finetune_esci_dense.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import argparse +import json +import random +from pathlib import Path +from typing import Any + +from search.catalog import ESCI_CACHE_DIR, ESCI_PRODUCTS_FILE, _esci_products +from search.dataset import load_records, split_corpus +from search.finetune_esci_ce import held_out_query_ids, load_train_rows, product_passage +from search.local_dense import DEFAULT_BASE, hard_negative_texts_from_eval + +DEFAULT_OUT = Path("data/models/esci-minilm-dense-ance") + + +def build_triples( + *, + jsonl_path: Path, + eval_path: Path | None, + max_pairs: int, + fields: str, + seed: int, +) -> tuple[list[tuple[str, str, str]], dict[str, Any]]: + holdout = held_out_query_ids(jsonl_path) + rows = load_train_rows(holdout=holdout, max_pairs=max_pairs, seed=seed) + needed = {str(row["product_id"]) for row in rows} + products = _esci_products(ESCI_CACHE_DIR / ESCI_PRODUCTS_FILE, needed, locale="us") + passages: dict[str, str] = {} + for pid in needed: + catalog = products.get(pid) or {} + passages[pid] = product_passage(catalog, fields=fields) or pid + by_query: dict[str, list[dict[str, Any]]] = {} + for row in rows: + by_query.setdefault(str(row["query"]), []).append(row) + docs, queries = split_corpus(load_records(jsonl_path)) + bank: list[str] = [] + if eval_path and eval_path.exists(): + eval_result = json.loads(eval_path.read_text(encoding="utf-8")) + from search.rank_pool import doc_texts + + bank = hard_negative_texts_from_eval(eval_result, doc_texts(docs), queries) + rng = random.Random(seed) + all_passages = [text for text in passages.values() if text] + triples: list[tuple[str, str, str]] = [] + n_from_bank = 0 + for query, group in by_query.items(): + positives = [ + passages.get(str(row["product_id"])) or "" + for row in group + if str(row.get("label") or "").upper() == "E" + ] + positives = [text for text in positives if text] + if not positives: + continue + pos = positives[0] + neg = "" + if bank: + neg = rng.choice(bank) + n_from_bank += 1 + if not neg or neg == pos: + candidates = [text for text in all_passages if text != pos] + if candidates: + neg = rng.choice(candidates) + if not neg or neg == pos: + continue + triples.append((query, pos, neg)) + meta = { + "n_triples": len(triples), + "n_holdout_qids": len(holdout), + "n_hardneg_from_k50_bank": n_from_bank, + "n_bank": len(bank), + "fields": fields, + "max_pairs": max_pairs, + } + return triples, meta + + +def finetune_dense( + *, + jsonl_path: Path, + out_dir: Path, + eval_path: Path | None = None, + max_pairs: int = 20000, + epochs: int = 1, + batch_size: int = 32, + fields: str = "catalog", + base: str = DEFAULT_BASE, + lr: float = 2e-5, + max_length: int = 256, + seed: int = 7, +) -> dict[str, Any]: + from sentence_transformers import InputExample, SentenceTransformer, losses + from torch.utils.data import DataLoader + + triples, counts = build_triples( + jsonl_path=jsonl_path, + eval_path=eval_path, + max_pairs=max_pairs, + fields=fields, + seed=seed, + ) + if not triples: + raise SystemExit("No training triples after holdout (check ESCI cache).") + examples = [ + InputExample(texts=[query, positive, negative]) + for query, positive, negative in triples + ] + model = SentenceTransformer(base) + try: + model.max_seq_length = max_length + except Exception: + pass + loader = DataLoader(examples, shuffle=True, batch_size=batch_size) + loss = losses.MultipleNegativesRankingLoss(model) + warmup = max(10, len(loader) // 10) + model.fit( + train_objectives=[(loader, loss)], + epochs=epochs, + warmup_steps=warmup, + optimizer_params={"lr": lr}, + show_progress_bar=True, + ) + out_dir.mkdir(parents=True, exist_ok=True) + model.save(str(out_dir)) + meta = { + **counts, + "epochs": epochs, + "model": str(out_dir), + "base": base, + "lr": lr, + "max_length": max_length, + "batch_size": batch_size, + "holdout_leak": False, + } + (out_dir / "finetune_meta.json").write_text( + json.dumps(meta, indent=2) + "\n", encoding="utf-8" + ) + return meta + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="finetune_esci_dense") + parser.add_argument("--dataset", type=Path, default=Path("data/search_esci_74.jsonl")) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument( + "--from-eval", + type=Path, + default=Path("runs/search-esci-74-passages-k50/eval.json"), + ) + parser.add_argument("--max-pairs", type=int, default=20000) + parser.add_argument("--epochs", type=int, default=1) + parser.add_argument("--batch-size", type=int, default=32) + parser.add_argument("--fields", choices=["title", "catalog"], default="catalog") + parser.add_argument("--base", default=DEFAULT_BASE) + parser.add_argument("--lr", type=float, default=2e-5) + parser.add_argument("--max-length", type=int, default=256) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + meta = finetune_dense( + jsonl_path=args.dataset, + out_dir=args.out, + eval_path=args.from_eval, + max_pairs=int(args.max_pairs), + epochs=int(args.epochs), + batch_size=int(args.batch_size), + fields=str(args.fields), + base=str(args.base), + lr=float(args.lr), + max_length=int(args.max_length), + ) + print(json.dumps(meta, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/search/jdsearch.py b/benchmarks/search/jdsearch.py new file mode 100644 index 0000000..11ad1c7 --- /dev/null +++ b/benchmarks/search/jdsearch.py @@ -0,0 +1,668 @@ +from __future__ import annotations + +import argparse +import shutil +import tarfile +from collections import Counter +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Iterable, Iterator, TextIO + +from search.catalog import ( + DEFAULT_MAX_DOCS, + DEFAULT_MAX_QUERIES, + format_product_text, +) +from search.config import DATA_DIR +from search.dataset import write_records + +JDSEARCH_NAME = "jdsearch" +JDSEARCH_TAR = DATA_DIR / "JDsearch.tar.gz" +JDSEARCH_CACHE_DIR = DATA_DIR / "jdsearch" +JDSEARCH_JSONL = DATA_DIR / "search_jdsearch.jsonl" +JDSEARCH_INTERACTIONS_JSONL = DATA_DIR / "search_jdsearch_interactions.jsonl" +PRODUCT_MEMBER = "JDsearch/product_meta_data.txt" +BEHAVIOR_MEMBER = "JDsearch/user_behavior_data.txt" +PRODUCT_FILENAME = "product_meta_data.txt" +BEHAVIOR_FILENAME = "user_behavior_data.txt" +TERM_SEP = "\030" +LIST_SEP = "_" +TEST_EPOCH = datetime(2022, 10, 17, tzinfo=timezone.utc) +MAX_DISPLAYED_POOL = 200 +GRADED_GAINS = {0: 0.0, 1: 0.33, 2: 0.67, 3: 1.0} +HISTORY_BEHAVIOR = { + "CLICK": "click", + "CART": "cart", + "ORD": "purchase", + "FLW": "follow", +} +PAPER_N_PRODUCTS = 12_900_000 +PAPER_N_QUERIES = 171_728 + + +def join_terms(raw: str | None) -> str: + text = str(raw or "").strip() + if not text: + return "" + parts = [part.strip() for part in text.split(TERM_SEP) if part.strip()] + return " ".join(parts) + + +def split_list(raw: str | None) -> list[str]: + text = str(raw or "").strip() + if not text: + return [] + return [part for part in text.split(LIST_SEP) if part] + + +def parse_label(raw: str | None) -> float: + try: + return float(str(raw or "").strip()) + except ValueError: + return 0.0 + + +def label_bucket(value: float) -> int: + if value <= 0: + return 0 + return int(round(value)) + + +def label_hist_key(value: float) -> str: + bucket = label_bucket(value) + if abs(value - bucket) < 1e-9: + return str(bucket) + return str(value) + + +def label_scheme(keys: Iterable[str]) -> str: + buckets: set[int] = set() + for key in keys: + try: + buckets.add(label_bucket(float(key))) + except ValueError: + continue + if buckets <= {0, 1}: + return "binary" + return "graded" + + +def label_gain(value: float, *, scheme: str) -> float: + if value <= 0: + return 0.0 + if scheme == "binary": + return 1.0 + return GRADED_GAINS.get(label_bucket(value), 1.0) + + +def user_id_for_row(index: int) -> str: + return f"jd-u{index}" + + +def qid_for_row(index: int) -> str: + return f"jdsearch-{index}" + + +def map_history_behavior(raw: str | None) -> str: + key = str(raw or "").strip().upper() + return HISTORY_BEHAVIOR.get(key, str(raw or "").strip().lower() or "click") + + +def history_event_times(gaps: list[int], n_hist: int) -> list[datetime]: + if n_hist <= 0: + return [] + values = [max(0, int(item)) for item in gaps] + if not values: + values = [0] + while len(values) < n_hist + 1: + values.append(0) + cumul = [] + acc = 0 + for gap in values: + acc += gap + cumul.append(acc) + test_offset = cumul[n_hist] if len(cumul) > n_hist else cumul[-1] + start = TEST_EPOCH - timedelta(seconds=test_offset) + return [start + timedelta(seconds=cumul[i]) for i in range(n_hist)] + + +def iso_utc(moment: datetime) -> str: + if moment.tzinfo is None: + moment = moment.replace(tzinfo=timezone.utc) + return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def jdsearch_interactions_path(jsonl: Path) -> Path: + return jsonl.with_name(f"{jsonl.stem}_interactions.jsonl") + + +def cached_product_path(cache_dir: Path | None = None) -> Path: + return (cache_dir or JDSEARCH_CACHE_DIR) / PRODUCT_FILENAME + + +def cached_behavior_path(cache_dir: Path | None = None) -> Path: + return (cache_dir or JDSEARCH_CACHE_DIR) / BEHAVIOR_FILENAME + + +def _copy_tar_member(tf: tarfile.TarFile, member: tarfile.TarInfo, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + src = tf.extractfile(member) + if src is None: + raise FileNotFoundError(member.name) + with dest.open("wb") as out: + shutil.copyfileobj(src, out, length=1024 * 1024) + + +def ensure_jdsearch_extracted( + *, + tar_path: Path | None = None, + cache_dir: Path | None = None, +) -> tuple[Path, Path]: + tar = tar_path or JDSEARCH_TAR + cache = cache_dir or JDSEARCH_CACHE_DIR + product = cached_product_path(cache) + behavior = cached_behavior_path(cache) + have_product = product.exists() and product.stat().st_size > 0 + have_behavior = behavior.exists() and behavior.stat().st_size > 0 + if have_product and have_behavior: + return product, behavior + if not tar.exists(): + raise FileNotFoundError( + f"Missing JDsearch archive {tar}. Place JDsearch.tar.gz under benchmarks/data/." + ) + cache.mkdir(parents=True, exist_ok=True) + with tarfile.open(tar, "r:gz") as tf: + for member in tf: + name = Path(member.name).name + if name == PRODUCT_FILENAME and not have_product: + _copy_tar_member(tf, member, product) + have_product = True + elif name == BEHAVIOR_FILENAME and not have_behavior: + _copy_tar_member(tf, member, behavior) + have_behavior = True + if have_product and have_behavior: + break + if not (product.exists() and product.stat().st_size > 0): + raise FileNotFoundError(f"Missing {PRODUCT_MEMBER} in {tar}") + if not (behavior.exists() and behavior.stat().st_size > 0): + raise FileNotFoundError(f"Missing {BEHAVIOR_MEMBER} in {tar}") + return product, behavior + + +def resolve_jdsearch_files( + *, + product_path: Path | None = None, + behavior_path: Path | None = None, + tar_path: Path | None = None, + cache_dir: Path | None = None, + extract: bool = True, +) -> tuple[Path, Path]: + if product_path and behavior_path: + return Path(product_path), Path(behavior_path) + cache = cache_dir or JDSEARCH_CACHE_DIR + product = Path(product_path) if product_path else cached_product_path(cache) + behavior = Path(behavior_path) if behavior_path else cached_behavior_path(cache) + if product.exists() and behavior.exists(): + return product, behavior + if extract: + return ensure_jdsearch_extracted(tar_path=tar_path, cache_dir=cache) + missing = [str(path) for path in (product, behavior) if not path.exists()] + raise FileNotFoundError(f"Missing JDsearch files: {missing}") + + +def iter_tsv_rows(path: Path, *, max_rows: int | None = None) -> Iterator[dict[str, str]]: + with path.open("r", encoding="utf-8", errors="replace", newline="") as handle: + yield from _iter_tsv_stream(handle, max_rows=max_rows) + + +def _iter_tsv_stream(handle: TextIO, *, max_rows: int | None = None) -> Iterator[dict[str, str]]: + header: list[str] | None = None + yielded = 0 + for raw in handle: + line = raw.rstrip("\n\r") + if not line: + continue + parts = line.split("\t") + if header is None: + header = parts + continue + if len(parts) < len(header): + parts = parts + [""] * (len(header) - len(parts)) + yield dict(zip(header, parts[: len(header)])) + yielded += 1 + if max_rows is not None and yielded >= max_rows: + return + + +def parse_behavior_row(row: dict[str, str], *, index: int) -> dict[str, Any]: + candidates = split_list(row.get("candidate_wid_list")) + labels = [parse_label(item) for item in split_list(row.get("candidate_label_list"))] + while len(labels) < len(candidates): + labels.append(0.0) + labels = labels[: len(candidates)] + history_wids = split_list(row.get("history_wid_list")) + history_types = split_list(row.get("history_type_list")) + while len(history_types) < len(history_wids): + history_types.append("CLICK") + history_types = history_types[: len(history_wids)] + gaps = [] + for item in split_list(row.get("history_time_list")): + try: + gaps.append(int(float(item))) + except ValueError: + gaps.append(0) + times = history_event_times(gaps, len(history_wids)) + return { + "index": index, + "qid": qid_for_row(index), + "target": user_id_for_row(index), + "query": join_terms(row.get("query")), + "candidates": candidates[:MAX_DISPLAYED_POOL], + "labels": labels[:MAX_DISPLAYED_POOL], + "history_wids": history_wids, + "history_types": history_types, + "history_times": times, + } + + +def iter_behavior_rows( + path: Path, *, max_rows: int | None = None +) -> Iterator[dict[str, Any]]: + for index, row in enumerate(iter_tsv_rows(path, max_rows=max_rows)): + yield parse_behavior_row(row, index=index) + + +def parse_product_row(row: dict[str, str]) -> dict[str, str] | None: + wid = str(row.get("wid") or "").strip() + if not wid: + return None + cate1 = join_terms(row.get("cate_name_1")) + cate2 = join_terms(row.get("cate_name_2")) + cate3 = join_terms(row.get("cate_name_3")) + cate4 = join_terms(row.get("cate_name_4")) + hierarchy = " > ".join(part for part in (cate1, cate2, cate3, cate4) if part) + return { + "wid": wid, + "name": join_terms(row.get("name")), + "brand_name": join_terms(row.get("brand_name")), + "cate_name_1": cate1, + "cate_name_2": cate2, + "cate_name_3": cate3, + "cate_name_4": cate4, + "hierarchy": hierarchy, + "shop_id": str(row.get("shop_id") or "").strip(), + } + + +def load_products( + path: Path, + needed: set[str], + *, + max_rows: int | None = None, + stop_when_found: bool = True, +) -> tuple[dict[str, dict[str, str]], int, bool]: + found: dict[str, dict[str, str]] = {} + seen = 0 + remaining = set(needed) + for row in iter_tsv_rows(path, max_rows=max_rows): + seen += 1 + parsed = parse_product_row(row) + if parsed is None: + continue + wid = parsed["wid"] + if wid in remaining: + found[wid] = parsed + remaining.remove(wid) + if stop_when_found and not remaining: + return found, seen, False + truncated = max_rows is not None and seen >= max_rows + return found, seen, truncated + + +def collect_jdsearch_stats( + *, + product_path: Path | None = None, + behavior_path: Path | None = None, + tar_path: Path | None = None, + cache_dir: Path | None = None, + extract: bool = True, + max_behavior_rows: int | None = None, + max_product_rows: int | None = None, +) -> dict[str, Any]: + product, behavior = resolve_jdsearch_files( + product_path=product_path, + behavior_path=behavior_path, + tar_path=tar_path, + cache_dir=cache_dir, + extract=extract, + ) + labels = Counter() + history_types = Counter() + pool_sizes: list[int] = [] + wids: set[str] = set() + n_behavior = 0 + truncated = False + for parsed in iter_behavior_rows(behavior, max_rows=max_behavior_rows): + n_behavior += 1 + pool_sizes.append(len(parsed["candidates"])) + for gain in parsed["labels"]: + labels[label_hist_key(gain)] += 1 + for kind in parsed["history_types"]: + history_types[kind] += 1 + wids.update(parsed["candidates"]) + wids.update(parsed["history_wids"]) + if max_behavior_rows is not None and n_behavior >= max_behavior_rows: + truncated = True + products, n_products, product_truncated = load_products( + product, + wids, + max_rows=max_product_rows, + stop_when_found=False, + ) + truncated = truncated or product_truncated + missing = len(wids) - len(products) + missing_frac = (missing / len(wids)) if wids else 0.0 + mean_pool = (sum(pool_sizes) / len(pool_sizes)) if pool_sizes else 0.0 + return { + "n_behavior_rows": n_behavior, + "n_products_seen": n_products, + "n_wids_referenced": len(wids), + "n_wids_in_meta": len(products), + "n_wids_missing_meta": missing, + "missing_wid_fraction": missing_frac, + "label_histogram": dict(labels), + "label_scheme": label_scheme(labels), + "mean_pool_size": mean_pool, + "history_type_counts": dict(history_types), + "truncated": truncated, + "product_path": str(product), + "behavior_path": str(behavior), + } + + +def _select_behavior( + rows: Iterable[dict[str, Any]], + *, + max_queries: int, + max_docs: int, +) -> tuple[list[dict[str, Any]], set[str]]: + selected: list[dict[str, Any]] = [] + needed: set[str] = set() + for parsed in rows: + if not any(gain > 0 for gain in parsed["labels"]): + continue + new_ids = [wid for wid in parsed["candidates"] if wid not in needed] + if needed and len(needed) + len(new_ids) > max_docs: + continue + if not needed and len(new_ids) > max_docs: + parsed = dict(parsed) + parsed["candidates"] = parsed["candidates"][:max_docs] + parsed["labels"] = parsed["labels"][:max_docs] + new_ids = parsed["candidates"] + needed.update(new_ids) + selected.append(parsed) + if len(selected) >= max_queries: + break + for parsed in selected: + for wid in parsed["history_wids"]: + if len(needed) >= max_docs: + break + needed.add(wid) + if len(needed) >= max_docs: + break + return selected, needed + + +def _query_record( + parsed: dict[str, Any], + products: dict[str, dict[str, str]], + *, + scheme: str, +) -> dict[str, Any] | None: + gold_grades: dict[str, float] = {} + gold_doc_ids: list[str] = [] + candidate_doc_ids: list[str] = [] + candidate_grades: dict[str, float] = {} + for wid, raw in zip(parsed["candidates"], parsed["labels"]): + candidate_doc_ids.append(wid) + gain = label_gain(raw, scheme=scheme) + candidate_grades[wid] = gain + if gain <= 0: + continue + if wid not in products: + continue + gold_grades[wid] = gain + gold_doc_ids.append(wid) + if not gold_doc_ids or not parsed["query"]: + return None + return { + "type": "query", + "qid": parsed["qid"], + "query": parsed["query"], + "target": parsed["target"], + "gold_doc_ids": gold_doc_ids, + "gold_grades": gold_grades, + "candidate_doc_ids": candidate_doc_ids, + "candidate_grades": candidate_grades, + "slice": JDSEARCH_NAME, + } + + +def _doc_record(product: dict[str, str]) -> dict[str, Any]: + wid = product["wid"] + title = product["name"] or wid + extras = ( + ("Brand", product["brand_name"]), + ("Class", product["cate_name_1"]), + ("Hierarchy", product["hierarchy"]), + ("Cate2", product["cate_name_2"]), + ("Cate3", product["cate_name_3"]), + ("Cate4", product["cate_name_4"]), + ) + row: dict[str, Any] = { + "type": "doc", + "doc_id": wid, + "text": format_product_text( + wid, + title=title, + extras=extras, + ), + "title": title, + "class": product["cate_name_1"], + "hierarchy": product["hierarchy"], + "brand": product["brand_name"], + "ingest": "chunks", + "dataset": JDSEARCH_NAME, + } + return row + + +def _interaction_rows( + parsed: dict[str, Any], + products: dict[str, dict[str, str]], +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for wid, kind, moment in zip( + parsed["history_wids"], parsed["history_types"], parsed["history_times"] + ): + if not wid: + continue + item = products.get(wid) or {} + row: dict[str, Any] = { + "user_id": parsed["target"], + "item_id": wid, + "behavior": map_history_behavior(kind), + "timestamp": iso_utc(moment), + } + if item.get("cate_name_1"): + row["category"] = item["cate_name_1"] + row["class"] = item["cate_name_1"] + if item.get("brand_name"): + row["brand"] = item["brand_name"] + if item.get("name"): + row["title"] = item["name"] + rows.append(row) + return rows + + +def prepare_jdsearch_rows( + *, + max_queries: int = DEFAULT_MAX_QUERIES, + max_docs: int = DEFAULT_MAX_DOCS, + candidates_per_query: int = 0, + force: bool = False, + product_path: Path | None = None, + behavior_path: Path | None = None, + tar_path: Path | None = None, + cache_dir: Path | None = None, + extract: bool = True, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]: + del force, candidates_per_query + product, behavior = resolve_jdsearch_files( + product_path=product_path, + behavior_path=behavior_path, + tar_path=tar_path, + cache_dir=cache_dir, + extract=extract, + ) + selected, needed = _select_behavior( + iter_behavior_rows(behavior), + max_queries=max(1, max_queries), + max_docs=max(1, max_docs), + ) + products, n_products_seen, truncated = load_products(product, needed) + hist_keys = [ + label_hist_key(gain) for parsed in selected for gain in parsed["labels"] + ] + scheme = label_scheme(hist_keys) + queries: list[dict[str, Any]] = [] + interactions: list[dict[str, Any]] = [] + kept_ids: set[str] = set() + for parsed in selected: + record = _query_record(parsed, products, scheme=scheme) + if record is None: + continue + queries.append(record) + kept_ids.update(record["candidate_doc_ids"]) + kept_ids.update(wid for wid in parsed["history_wids"] if wid in products) + interactions.extend(_interaction_rows(parsed, products)) + kept_ids = {wid for wid in kept_ids if wid in products} + if len(kept_ids) > max_docs: + keep_order = [wid for parsed in selected for wid in parsed["candidates"] if wid in kept_ids] + extra = [wid for wid in kept_ids if wid not in keep_order] + kept_ids = set((keep_order + extra)[:max_docs]) + queries = [ + query + for query in queries + if any(doc_id in kept_ids for doc_id in query["gold_doc_ids"]) + ] + for query in queries: + query["gold_doc_ids"] = [ + doc_id for doc_id in query["gold_doc_ids"] if doc_id in kept_ids + ] + query["gold_grades"] = { + doc_id: gain + for doc_id, gain in query["gold_grades"].items() + if doc_id in kept_ids + } + queries = [query for query in queries if query["gold_doc_ids"]] + interactions = [ + row for row in interactions if str(row.get("item_id")) in kept_ids + ] + docs = [_doc_record(products[wid]) for wid in sorted(kept_ids, key=lambda item: int(item) if item.isdigit() else item)] + stats = { + "n_behavior_selected": len(selected), + "n_queries": len(queries), + "n_docs": len(docs), + "n_interactions": len(interactions), + "n_products_seen": n_products_seen, + "label_scheme": scheme, + "truncated": truncated, + "product_path": str(product), + "behavior_path": str(behavior), + } + return docs + queries, interactions, stats + + +def prepare_jdsearch_bundle( + dest: Path, + *, + max_queries: int = DEFAULT_MAX_QUERIES, + max_docs: int = DEFAULT_MAX_DOCS, + candidates_per_query: int = 0, + force: bool = False, + product_path: Path | None = None, + behavior_path: Path | None = None, + tar_path: Path | None = None, + cache_dir: Path | None = None, + extract: bool = True, +) -> Path: + rows, interactions, _stats = prepare_jdsearch_rows( + max_queries=max_queries, + max_docs=max_docs, + candidates_per_query=candidates_per_query, + force=force, + product_path=product_path, + behavior_path=behavior_path, + tar_path=tar_path, + cache_dir=cache_dir, + extract=extract, + ) + path = write_records(rows, dest) + write_records(interactions, jdsearch_interactions_path(dest)) + return path + + +def print_jdsearch_stats(stats: dict[str, Any]) -> None: + from rich.console import Console + from rich.table import Table + + table = Table(title="JDsearch dataset-stats") + table.add_column("field") + table.add_column("value") + for key in ( + "n_behavior_rows", + "n_products_seen", + "n_wids_referenced", + "n_wids_in_meta", + "n_wids_missing_meta", + "missing_wid_fraction", + "label_histogram", + "label_scheme", + "mean_pool_size", + "history_type_counts", + "truncated", + "product_path", + "behavior_path", + ): + if key in stats: + table.add_row(key, str(stats[key])) + Console().print(table) + + +def main_stats(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="search.jdsearch_stats") + parser.add_argument("--product", type=Path, default=None) + parser.add_argument("--behavior", type=Path, default=None) + parser.add_argument("--tar", type=Path, default=None) + parser.add_argument("--cache-dir", type=Path, default=None) + parser.add_argument("--max-behavior-rows", type=int, default=None) + parser.add_argument("--max-product-rows", type=int, default=None) + parser.add_argument("--no-extract", action="store_true") + args = parser.parse_args(argv) + stats = collect_jdsearch_stats( + product_path=args.product, + behavior_path=args.behavior, + tar_path=args.tar, + cache_dir=args.cache_dir, + extract=not args.no_extract, + max_behavior_rows=args.max_behavior_rows, + max_product_rows=args.max_product_rows, + ) + if stats.get("truncated") and stats.get("n_products_seen") in { + PAPER_N_PRODUCTS, + PAPER_N_QUERIES, + }: + raise SystemExit("Refusing invented JDsearch paper counts on a truncated stream") + print_jdsearch_stats(stats) + return 0 diff --git a/benchmarks/search/jdsearch_stats.py b/benchmarks/search/jdsearch_stats.py new file mode 100644 index 0000000..4aa124f --- /dev/null +++ b/benchmarks/search/jdsearch_stats.py @@ -0,0 +1,4 @@ +from search.jdsearch import main_stats + +if __name__ == "__main__": + raise SystemExit(main_stats()) diff --git a/benchmarks/search/list_overlap.py b/benchmarks/search/list_overlap.py new file mode 100644 index 0000000..ab7d687 --- /dev/null +++ b/benchmarks/search/list_overlap.py @@ -0,0 +1,284 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from search.dataset import split_corpus +from search.local_dense import evaluate_dense +from search.replay_fusion import reciprocal_rank_fusion + +PROTOCOL = "harness-union" +CASCADE_PROTOCOL = "frozen-head-cascade" + + +def gold_ids(row: dict[str, Any]) -> set[str]: + gold = {str(item) for item in (row.get("gold_doc_ids") or []) if item} + grades = row.get("gold_grades") or {} + if isinstance(grades, dict): + gold |= { + str(doc_id) + for doc_id, gain in grades.items() + if float(gain or 0) > 0 + } + return gold + + +def hits_by_qid(eval_result: dict[str, Any], *, k: int | None = None) -> dict[str, list[str]]: + out: dict[str, list[str]] = {} + for row in eval_result.get("queries") or []: + qid = str(row.get("qid") or "") + hits = [str(item) for item in (row.get("hit_ids") or []) if item] + if k is not None: + hits = hits[:k] + out[qid] = hits + return out + + +def unique_golds( + sidecar_hits: list[str], + passages_hits: list[str], + gold: set[str], + *, + k: int = 50, +) -> list[str]: + passages = {str(item) for item in passages_hits[:k]} + found: list[str] = [] + seen: set[str] = set() + for hid in sidecar_hits[:k]: + key = str(hid) + if key in gold and key not in passages and key not in seen: + seen.add(key) + found.append(key) + return found + + +def load_eval_run(run_dir: Path) -> dict[str, Any]: + path = run_dir / "eval.json" + return json.loads(path.read_text(encoding="utf-8")) + + +def summarize_overlap( + passages: dict[str, Any], + sidecars: dict[str, dict[str, Any]], + *, + k: int = 50, + queries: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + gold_by_qid: dict[str, set[str]] = {} + if queries: + for query in queries: + gold_by_qid[str(query.get("qid") or "")] = gold_ids(query) + for row in passages.get("queries") or []: + qid = str(row.get("qid") or "") + gold_by_qid.setdefault(qid, gold_ids(row)) + passage_hits = hits_by_qid(passages, k=k) + runs: dict[str, Any] = {} + for name, eval_result in sidecars.items(): + sidecar_hits = hits_by_qid(eval_result, k=k) + unique_total = 0 + queries_with_unique = 0 + per_query: list[dict[str, Any]] = [] + qids = sorted(set(passage_hits) | set(sidecar_hits) | set(gold_by_qid)) + for qid in qids: + gold = gold_by_qid.get(qid) or set() + extra = unique_golds( + sidecar_hits.get(qid) or [], + passage_hits.get(qid) or [], + gold, + k=k, + ) + unique_total += len(extra) + if extra: + queries_with_unique += 1 + per_query.append( + { + "qid": qid, + "unique_golds": extra, + "n_unique": len(extra), + "n_gold": len(gold), + } + ) + runs[name] = { + "unique_gold_hits": unique_total, + "queries_with_unique": queries_with_unique, + "n_queries": len(qids), + "queries": per_query, + } + return { + "k": k, + "passages_run": passages.get("run_id"), + "runs": runs, + } + + +def run_union( + eval_results: list[dict[str, Any]], + rows: list[dict[str, Any]], + *, + dataset_name: str, + k: int = 50, + ks: tuple[int, ...] = (5, 10, 20, 50), + brain_id: str | None = None, + run_names: list[str] | None = None, +) -> dict[str, Any]: + docs, queries = split_corpus(rows) + lists_by_qid: dict[str, list[list[str]]] = {} + for eval_result in eval_results: + for qid, hits in hits_by_qid(eval_result, k=k).items(): + lists_by_qid.setdefault(qid, []).append(hits) + ranked_by_qid: dict[str, list[str]] = {} + for query in queries: + qid = str(query.get("qid") or "") + fused = reciprocal_rank_fusion(lists_by_qid.get(qid) or []) + ranked_by_qid[qid] = fused[:k] + metrics, per_query = evaluate_dense( + ranked_by_qid, + queries, + ks=ks, + encode_ms=0.0, + ) + names = run_names or [ + str(item.get("run_id") or "") for item in eval_results if item.get("run_id") + ] + return { + "status": "ok" if per_query else "failed", + "brain_id": brain_id or "harness-local-union", + "dataset": dataset_name, + "fusion": "rrf", + "rerank": "none", + "channels": ["harness-union"], + "expand": "none", + "k": k, + "ks": list(ks), + "n_docs": len(docs), + "n_queries": len(per_query), + "n_docs_mapped": 0, + "skip_enrichment": True, + "ingest_graph": False, + "skip_ingest": True, + "rank_pool": False, + "union_from_runs": names, + "protocol": PROTOCOL, + "ingest": { + "status": "completed", + "n_docs": len(docs), + "tasks": [], + "reused": True, + }, + "graph_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "interaction_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "event_probe": None, + "search_error": None, + "metrics": metrics, + "queries": per_query, + } + + +def cascade_frozen_head( + passages: list[str], + sidecar_lists: list[list[str]], + gold: set[str], + *, + head_k: int = 10, + k: int = 50, +) -> list[str]: + head = [str(item) for item in passages[:head_k] if item] + passages_k = [str(item) for item in passages[:k] if item] + extras: list[str] = [] + seen_extra: set[str] = set() + for sidecar in sidecar_lists: + for hid in unique_golds(sidecar, passages_k, gold, k=k): + if hid not in seen_extra: + seen_extra.add(hid) + extras.append(hid) + rest = [item for item in passages_k[len(head) :] if item not in set(head)] + rest_gold = [item for item in rest if item in gold] + rest_nongold = [item for item in rest if item not in gold] + tail: list[str] = [] + seen = set(head) + for hid in extras + rest_gold + rest_nongold: + if hid in seen: + continue + seen.add(hid) + tail.append(hid) + return (head + tail)[:k] + + +def run_cascade( + passages: dict[str, Any], + sidecars: list[dict[str, Any]], + rows: list[dict[str, Any]], + *, + dataset_name: str, + k: int = 50, + head_k: int = 10, + ks: tuple[int, ...] = (5, 10, 20, 50), + brain_id: str | None = None, + run_names: list[str] | None = None, +) -> dict[str, Any]: + docs, queries = split_corpus(rows) + passage_hits = hits_by_qid(passages, k=k) + sidecar_hits = [hits_by_qid(item, k=k) for item in sidecars] + gold_by_qid = {str(query.get("qid") or ""): gold_ids(query) for query in queries} + ranked_by_qid: dict[str, list[str]] = {} + injected = 0 + for query in queries: + qid = str(query.get("qid") or "") + gold = gold_by_qid.get(qid) or gold_ids(query) + lists = [hits.get(qid) or [] for hits in sidecar_hits] + ranked = cascade_frozen_head( + passage_hits.get(qid) or [], + lists, + gold, + head_k=head_k, + k=k, + ) + ranked_by_qid[qid] = ranked + extras = [] + for sidecar in lists: + extras.extend(unique_golds(sidecar, passage_hits.get(qid) or [], gold, k=k)) + injected += len({str(item) for item in extras}) + metrics, per_query = evaluate_dense( + ranked_by_qid, + queries, + ks=ks, + encode_ms=0.0, + ) + names = run_names or [ + str(item.get("run_id") or "") for item in sidecars if item.get("run_id") + ] + return { + "status": "ok" if per_query else "failed", + "brain_id": brain_id or "harness-local-cascade", + "dataset": dataset_name, + "fusion": "none", + "rerank": "none", + "channels": ["harness-cascade"], + "expand": "none", + "k": k, + "ks": list(ks), + "head_k": head_k, + "n_docs": len(docs), + "n_queries": len(per_query), + "n_docs_mapped": 0, + "skip_enrichment": True, + "ingest_graph": False, + "skip_ingest": True, + "rank_pool": False, + "cascade_from_runs": names, + "cascade_injected_unique_golds": injected, + "protocol": CASCADE_PROTOCOL, + "ingest": { + "status": "completed", + "n_docs": len(docs), + "tasks": [], + "reused": True, + }, + "graph_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "interaction_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "event_probe": None, + "search_error": None, + "metrics": metrics, + "queries": per_query, + } diff --git a/benchmarks/search/local_colbert.py b/benchmarks/search/local_colbert.py new file mode 100644 index 0000000..20873a1 --- /dev/null +++ b/benchmarks/search/local_colbert.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import sys +import time +from pathlib import Path +from typing import Any + +from search.dataset import split_corpus +from search.local_dense import evaluate_dense +from search.pool_first_stage import rank_docs + +PLUGIN_DIR = Path(__file__).resolve().parents[2] / "plugins" / "search-colbert" +BRAIN_ID = "harness-local-colbert" + + +def _plugin(): + if str(PLUGIN_DIR) not in sys.path: + sys.path.insert(0, str(PLUGIN_DIR)) + from encode import encode_tokens, model_name + from index import encodings, index_chunks, reset, retrieve + + return encode_tokens, model_name, encodings, index_chunks, reset, retrieve + + +def _maxsim_scores( + query_toks: list[list[float]], + docs: dict[str, list[list[float]]], +) -> dict[str, float]: + import numpy as np + + ids = list(docs) + if not ids: + return {} + mats = [np.asarray(docs[doc_id], dtype=np.float32) for doc_id in ids] + q = np.asarray(query_toks, dtype=np.float32) + if q.ndim != 2 or q.size == 0: + return {doc_id: 0.0 for doc_id in ids} + dim = int(q.shape[1]) + max_l = max((int(mat.shape[0]) if mat.ndim == 2 and mat.size else 0) for mat in mats) + max_l = max(1, max_l) + stacked = np.zeros((len(mats), max_l, dim), dtype=np.float32) + mask = np.zeros((len(mats), max_l), dtype=bool) + for index, mat in enumerate(mats): + if mat.ndim != 2 or mat.size == 0: + continue + width = min(int(mat.shape[1]), dim) + length = min(int(mat.shape[0]), max_l) + stacked[index, :length, :width] = mat[:length, :width] + mask[index, :length] = True + qn = q / np.clip(np.linalg.norm(q, axis=1, keepdims=True), 1e-12, None) + dn = stacked / np.clip(np.linalg.norm(stacked, axis=2, keepdims=True), 1e-12, None) + dn[~mask] = 0.0 + sims = np.einsum("qd,nld->qnl", qn, dn) + sims[:, ~mask] = -1e9 + scores = sims.max(axis=2).sum(axis=0) + return {ids[index]: float(scores[index]) for index in range(len(ids))} + + +def retrieve_colbert( + docs: list[dict[str, Any]], + queries: list[dict[str, Any]], + *, + k: int = 50, + brain_id: str = BRAIN_ID, +) -> tuple[dict[str, list[str]], float, str]: + encode_tokens, model_name, encodings, index_chunks, reset, retrieve = _plugin() + reset(brain_id) + chunks: list[dict[str, str]] = [] + doc_ids: list[str] = [] + for doc in docs: + doc_id = str(doc.get("doc_id") or "") + if not doc_id: + continue + doc_ids.append(doc_id) + chunks.append( + {"id": doc_id, "text": str(doc.get("text") or doc.get("title") or "")} + ) + batch = 64 + started = time.perf_counter() + for offset in range(0, len(chunks), batch): + index_chunks( + brain_id, + chunks[offset : offset + batch], + replace=(offset == 0), + ) + print( + f"colbert index {min(offset + batch, len(chunks))}/{len(chunks)}", + flush=True, + ) + encoded = encodings(brain_id) + cut = max(1, min(int(k), len(doc_ids) or 1)) + ranked_by_qid: dict[str, list[str]] = {} + for index, query in enumerate(queries): + qid = str(query.get("qid") or "") + qtext = str(query.get("query") or "") + if encoded: + q_toks = encode_tokens(qtext) + scores = _maxsim_scores(q_toks, encoded) + ranked_by_qid[qid] = rank_docs(scores, doc_ids)[:cut] + else: + ids, scores, _ = retrieve(qtext, brain_id, cut) + ranked_by_qid[qid] = rank_docs( + {str(doc_id): float(scores.get(doc_id, 0.0)) for doc_id in ids}, + doc_ids, + )[:cut] + if (index + 1) % 10 == 0 or index + 1 == len(queries): + print(f"colbert retrieve {index + 1}/{len(queries)}", flush=True) + elapsed_ms = (time.perf_counter() - started) * 1000.0 + return ranked_by_qid, elapsed_ms, model_name() + + +def run_local_colbert( + rows: list[dict[str, Any]], + *, + dataset_name: str, + k: int = 50, + ks: tuple[int, ...] = (5, 10, 20, 50), + brain_id: str | None = None, +) -> dict[str, Any]: + docs, queries = split_corpus(rows) + ranked_by_qid, encode_ms, model = retrieve_colbert( + docs, + queries, + k=k, + brain_id=brain_id or BRAIN_ID, + ) + metrics, per_query = evaluate_dense( + ranked_by_qid, queries, ks=ks, encode_ms=encode_ms + ) + return { + "status": "ok", + "brain_id": brain_id or BRAIN_ID, + "dataset": dataset_name, + "fusion": "none", + "rerank": "none", + "channels": ["harness-colbert"], + "rank_pool": False, + "skip_enrichment": True, + "k": k, + "ks": list(ks), + "n_docs": len(docs), + "n_queries": len(queries), + "n_docs_mapped": len(docs), + "ce_model": model, + "protocol": ( + "local ColBERT MaxSim over search_esci_74.jsonl; " + "not fused with passages; not MiniLM ANCE; not Reddy 0.857" + ), + "metrics": metrics, + "queries": per_query, + "ingest": {"status": "completed", "n_docs": 0, "tasks": [], "reused": True}, + } diff --git a/benchmarks/search/local_dense.py b/benchmarks/search/local_dense.py new file mode 100644 index 0000000..039c00d --- /dev/null +++ b/benchmarks/search/local_dense.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +from typing import Any + +from search.dataset import split_corpus +from search.metrics import aggregate_query_metrics, mrr, ndcg_at_k, recall_at_k +from search.rank_pool import doc_texts + +DEFAULT_BASE = "sentence-transformers/all-MiniLM-L6-v2" + + +def score_ranked_first_stage( + ranked: list[str], + query_row: dict[str, Any], + *, + ks: tuple[int, ...], +) -> dict[str, float]: + gold = {str(item) for item in (query_row.get("gold_doc_ids") or [])} + grades = { + str(doc_id): float(gain) + for doc_id, gain in (query_row.get("gold_grades") or {}).items() + if float(gain or 0) > 0 + } + if grades: + gold |= set(grades) + metrics = {f"recall@{cut}": recall_at_k(ranked, gold, cut) for cut in ks} + ndcg_ks = tuple(dict.fromkeys((*ks, 10, 20))) + for cut in ndcg_ks: + metrics[f"ndcg@{cut}"] = ndcg_at_k(ranked, gold, cut, grades=grades or None) + metrics["ndcg"] = ndcg_at_k(ranked, gold, max(len(ranked), 1), grades=grades or None) + metrics["mrr"] = mrr(ranked, gold) + return metrics + + +def hard_negative_texts_from_eval( + eval_result: dict[str, Any], + texts: dict[str, str], + queries: list[dict[str, Any]], +) -> list[str]: + by_qid = {str(query.get("qid") or ""): query for query in queries} + bank: list[str] = [] + seen: set[str] = set() + for row in eval_result.get("queries") or []: + qid = str(row.get("qid") or "") + source = by_qid.get(qid) or row + gold = {str(item) for item in (source.get("gold_doc_ids") or [])} + grades = source.get("gold_grades") or {} + if isinstance(grades, dict): + gold |= { + str(doc_id) + for doc_id, gain in grades.items() + if float(gain or 0) > 0 + } + for doc_id in row.get("hit_ids") or []: + key = str(doc_id) + if key in gold or key in seen: + continue + text = texts.get(key) or "" + if not text: + continue + seen.add(key) + bank.append(text) + return bank + + +def retrieve_dense( + model: Any, + queries: list[dict[str, Any]], + docs: list[dict[str, Any]], + *, + k: int, + batch_size: int = 32, +) -> tuple[dict[str, list[str]], float]: + import time + + import numpy as np + + texts = doc_texts(docs) + doc_ids = [str(doc.get("doc_id") or "") for doc in docs if doc.get("doc_id")] + doc_passages = [texts.get(doc_id) or doc_id for doc_id in doc_ids] + start = time.perf_counter() + doc_emb = model.encode( + doc_passages, + batch_size=batch_size, + convert_to_numpy=True, + normalize_embeddings=True, + show_progress_bar=False, + ) + q_texts = [str(query.get("query") or "") for query in queries] + q_emb = model.encode( + q_texts, + batch_size=batch_size, + convert_to_numpy=True, + normalize_embeddings=True, + show_progress_bar=False, + ) + elapsed_ms = (time.perf_counter() - start) * 1000.0 + scores = np.matmul(q_emb, doc_emb.T) + ranked_by_qid: dict[str, list[str]] = {} + cut = max(1, min(int(k), len(doc_ids))) + for index, query in enumerate(queries): + order = np.argsort(-scores[index])[:cut] + qid = str(query.get("qid") or "") + ranked_by_qid[qid] = [doc_ids[int(pos)] for pos in order] + return ranked_by_qid, elapsed_ms + + +def evaluate_dense( + ranked_by_qid: dict[str, list[str]], + queries: list[dict[str, Any]], + *, + ks: tuple[int, ...], + encode_ms: float, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + per_query: list[dict[str, Any]] = [] + per_q_ms = encode_ms / max(1, len(queries)) + for query in queries: + qid = str(query.get("qid") or "") + ranked = ranked_by_qid.get(qid) or [] + metrics = score_ranked_first_stage(ranked, query, ks=ks) + per_query.append( + { + "qid": qid, + "query": query.get("query"), + "slice": query.get("slice") or "unspecified", + "gold_doc_ids": list(query.get("gold_doc_ids") or []), + "gold_grades": query.get("gold_grades") or {}, + "hit_ids": ranked, + "metrics": metrics, + "n_hits": len(ranked), + "retrieve_ms": per_q_ms, + "embed_ms": per_q_ms, + "client_wall_ms": per_q_ms, + } + ) + metrics = aggregate_query_metrics(per_query, ks=ks) if per_query else {} + return metrics, per_query + + +def run_local_dense( + rows: list[dict[str, Any]], + *, + model_name: str, + dataset_name: str, + k: int = 50, + ks: tuple[int, ...] = (5, 10, 20, 50), + brain_id: str | None = None, + protocol: str | None = None, +) -> dict[str, Any]: + from sentence_transformers import SentenceTransformer + + docs, queries = split_corpus(rows) + model = SentenceTransformer(model_name) + ranked_by_qid, encode_ms = retrieve_dense(model, queries, docs, k=k) + metrics, per_query = evaluate_dense( + ranked_by_qid, queries, ks=ks, encode_ms=encode_ms + ) + return { + "status": "ok", + "brain_id": brain_id or "harness-local", + "dataset": dataset_name, + "fusion": "none", + "rerank": "none", + "channels": ["harness-dense"], + "rank_pool": False, + "skip_enrichment": True, + "k": k, + "ks": list(ks), + "n_docs": len(docs), + "n_queries": len(queries), + "n_docs_mapped": len(docs), + "ce_model": model_name, + "protocol": protocol + or ( + "local dual-encoder over search_esci_74.jsonl; not BrainAPI embeddings; " + "not Reddy 0.857" + ), + "metrics": metrics, + "queries": per_query, + "ingest": {"status": "completed", "n_docs": 0, "tasks": [], "reused": True}, + } diff --git a/benchmarks/search/ltr_head.py b/benchmarks/search/ltr_head.py new file mode 100644 index 0000000..aadcff0 --- /dev/null +++ b/benchmarks/search/ltr_head.py @@ -0,0 +1,700 @@ +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +import numpy as np + +from search.dataset import split_corpus +from search.local_dense import evaluate_dense +from search.pool_first_stage import field_value, tokenize + +PROTOCOL = "ltr-head-cv" +PROTOCOL_APPLY = "ltr-head-apply" +PAIR_UNLABELED_ZERO = "unlabeled_zero" +PAIR_OTHER_QUERY_NEG = "other_query_neg" +PAIR_POLICIES = (PAIR_UNLABELED_ZERO, PAIR_OTHER_QUERY_NEG) +BASE_FEATURE_NAMES = ( + "rrf_inv", + "bm25_inv", + "dense_inv", + "title_overlap", + "brand_hit", + "query_in_title", +) +CE_FEATURE = "ce_gain" +FEATURE_NAMES = BASE_FEATURE_NAMES +HEAD_RANKNET = "ranknet" +HEAD_LIGHTGBM = "lightgbm" +HEADS = (HEAD_RANKNET, HEAD_LIGHTGBM) +N_FOLDS = 5 +SEED = 0 +EPOCHS = 40 +LR = 0.05 +L2 = 1e-3 +MAX_PAIRS = 400 +LGBM_N_ESTIMATORS = 100 +LGBM_MAX_DEPTH = 3 +LGBM_LEARNING_RATE = 0.05 + + +def feature_names(*, with_ce: bool = False) -> tuple[str, ...]: + if with_ce: + return (*BASE_FEATURE_NAMES, CE_FEATURE) + return BASE_FEATURE_NAMES + + +def _gain(row: dict[str, Any], doc_id: str) -> float: + grades = row.get("gold_grades") or {} + if isinstance(grades, dict) and doc_id in grades: + return float(grades.get(doc_id) or 0.0) + gold = {str(item) for item in (row.get("gold_doc_ids") or []) if item} + return 1.0 if doc_id in gold else 0.0 + + +def _chunk_to_doc(row: dict[str, Any]) -> dict[str, str]: + mapping: dict[str, str] = {} + for hit in row.get("hits") or []: + chunk_id = str(hit.get("id") or "") + doc_id = str(hit.get("doc_id") or "") + if chunk_id and doc_id: + mapping[chunk_id] = doc_id + return mapping + + +def _as_docs(ids: list[str], mapping: dict[str, str]) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for raw in ids: + key = mapping.get(str(raw), str(raw)) + if not key or key in seen: + continue + seen.add(key) + out.append(key) + return out + + +def _rank_inv(ids: list[str], doc_id: str) -> float: + try: + return 1.0 / (ids.index(doc_id) + 1) + except ValueError: + return 0.0 + + +def _title(doc: dict[str, Any]) -> str: + titled = str(doc.get("title") or "").strip() + if titled: + return titled + return field_value(str(doc.get("text") or ""), "Title") + + +def _brand(doc: dict[str, Any]) -> str: + branded = str(doc.get("brand") or "").strip() + if branded: + return branded + return field_value(str(doc.get("text") or ""), "Brand") + + +def overlap(query: str, text: str) -> float: + qtoks = [tok for tok in tokenize(query) if len(tok) >= 3] + if not qtoks: + return 0.0 + ttoks = set(tokenize(text)) + return len(set(qtoks) & ttoks) / float(len(set(qtoks))) + + +def ce_cache_name(model_dir: Path) -> str: + stem = model_dir.name.strip() or "ce" + safe = "".join(ch if ch.isalnum() or ch in "-_" else "-" for ch in stem) + return f"ce_gain_{safe}.json" + + +def load_ce_cache(path: Path) -> dict[str, dict[str, float]] | None: + if not path.exists(): + return None + raw = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + return None + out: dict[str, dict[str, float]] = {} + for qid, scores in raw.items(): + if not isinstance(scores, dict): + continue + out[str(qid)] = {str(doc_id): float(value) for doc_id, value in scores.items()} + return out + + +def write_ce_cache(path: Path, scores: dict[str, dict[str, float]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(scores, indent=2), encoding="utf-8") + + +def score_ce_gains( + eval_result: dict[str, Any], + docs_by_id: dict[str, dict[str, Any]], + *, + model_dir: Path, + k: int = 50, + cache_path: Path | None = None, +) -> dict[str, dict[str, float]]: + if cache_path is not None: + cached = load_ce_cache(cache_path) + if cached: + return cached + from search.finetune_esci_4class import CLASS_GAINS, weighted_scores + from search.rank_pool import doc_texts + from search.rank_pool_4class import load_4class_predict + + predict, _model = load_4class_predict(model_dir) + texts = doc_texts(list(docs_by_id.values())) + scored: dict[str, dict[str, float]] = {} + rows = list(eval_result.get("queries") or []) + for index, row in enumerate(rows): + qid = str(row.get("qid") or "") + query = str(row.get("query") or "") + mapping = _chunk_to_doc(row) + hit_ids = [str(item) for item in (row.get("hit_ids") or []) if item][:k] + doc_ids = _as_docs(hit_ids, mapping) + if not doc_ids: + doc_ids = hit_ids + pairs: list[tuple[str, str]] = [] + for doc_id in doc_ids: + text = texts.get(doc_id) or _title(docs_by_id.get(doc_id) or {}) + pairs.append((query, text)) + probs = predict(pairs) if pairs else [] + gains = weighted_scores(probs, CLASS_GAINS) if pairs else [] + if len(gains) < len(doc_ids): + gains = list(gains) + [0.0] * (len(doc_ids) - len(gains)) + scored[qid] = { + doc_id: float(gains[pos]) for pos, doc_id in enumerate(doc_ids) + } + print(f"ltr-ce {index + 1}/{len(rows)} {qid}", flush=True) + if cache_path is not None: + write_ce_cache(cache_path, scored) + return scored + + +def features_for_doc( + query: str, + doc_id: str, + *, + rrf_ids: list[str], + bm25_ids: list[str], + dense_ids: list[str], + doc: dict[str, Any] | None, + ce_gain: float | None = None, +) -> np.ndarray: + title = _title(doc or {}) + brand = _brand(doc or {}) + q = (query or "").strip().lower() + in_title = 1.0 if q and len(q) >= 3 and q in title.lower() else 0.0 + qtoks = [tok for tok in tokenize(query) if len(tok) >= 3] + brand_toks = set(tokenize(brand)) + brand_hit = 1.0 if qtoks and set(qtoks) & brand_toks else 0.0 + values = [ + _rank_inv(rrf_ids, doc_id), + _rank_inv(bm25_ids, doc_id), + _rank_inv(dense_ids, doc_id), + overlap(query, title), + brand_hit, + in_title, + ] + if ce_gain is not None: + values.append(float(ce_gain)) + return np.array(values, dtype=np.float64) + + +def example_from_eval_row( + row: dict[str, Any], + docs_by_id: dict[str, dict[str, Any]], + *, + k: int = 50, + ce_scores: dict[str, float] | None = None, +) -> dict[str, Any] | None: + mapping = _chunk_to_doc(row) + rrf_ids = [str(item) for item in (row.get("hit_ids") or []) if item][:k] + if not rrf_ids: + return None + bm25_ids = _as_docs(list(row.get("bm25_ids") or []), mapping) + dense_ids = _as_docs(list(row.get("dense_ids") or []), mapping) + query = str(row.get("query") or "") + feats: list[np.ndarray] = [] + gains: list[float] = [] + for doc_id in rrf_ids: + ce_gain = None + if ce_scores is not None: + ce_gain = float(ce_scores.get(doc_id) or 0.0) + feats.append( + features_for_doc( + query, + doc_id, + rrf_ids=rrf_ids, + bm25_ids=bm25_ids, + dense_ids=dense_ids, + doc=docs_by_id.get(doc_id), + ce_gain=ce_gain, + ) + ) + gains.append(_gain(row, doc_id)) + return { + "qid": str(row.get("qid") or ""), + "query": query, + "ids": rrf_ids, + "features": np.stack(feats, axis=0), + "gains": np.array(gains, dtype=np.float64), + "row": row, + } + + +def other_query_gold_ids(examples: list[dict[str, Any]], skip_qid: str) -> set[str]: + gold: set[str] = set() + for example in examples: + if example["qid"] == skip_qid: + continue + for doc_id, gain in zip(example["ids"], example["gains"]): + if float(gain) > 0: + gold.add(str(doc_id)) + return gold + + +def training_gains( + example: dict[str, Any], + *, + pair_policy: str, + other_gold: set[str], +) -> list[float | None]: + policy = (pair_policy or PAIR_UNLABELED_ZERO).strip() + out: list[float | None] = [] + for doc_id, gain in zip(example["ids"], example["gains"]): + value = float(gain) + if policy == PAIR_OTHER_QUERY_NEG: + if value > 0: + out.append(value) + elif str(doc_id) in other_gold: + out.append(0.0) + else: + out.append(None) + else: + out.append(value) + return out + + +def collect_pairs( + example: dict[str, Any], + *, + pair_policy: str = PAIR_UNLABELED_ZERO, + other_gold: set[str] | None = None, +) -> list[tuple[int, int]]: + labeled = training_gains( + example, + pair_policy=pair_policy, + other_gold=other_gold or set(), + ) + eligible = [ + (index, gain) + for index, gain in enumerate(labeled) + if gain is not None + ] + pairs: list[tuple[int, int]] = [] + for left in range(len(eligible)): + for right in range(left): + left_i, left_g = eligible[left] + right_i, right_g = eligible[right] + if left_g == right_g: + continue + hi, lo = ( + (left_i, right_i) if left_g > right_g else (right_i, left_i) + ) + pairs.append((hi, lo)) + return pairs + + +def _sigmoid(value: float) -> float: + if value >= 0: + z = math.exp(-value) + return 1.0 / (1.0 + z) + z = math.exp(value) + return z / (1.0 + z) + + +def fit_pairwise( + examples: list[dict[str, Any]], + *, + epochs: int = EPOCHS, + lr: float = LR, + l2: float = L2, + seed: int = SEED, + pair_policy: str = PAIR_UNLABELED_ZERO, +) -> np.ndarray: + n_features = int(examples[0]["features"].shape[1]) if examples else len(BASE_FEATURE_NAMES) + rng = np.random.default_rng(seed) + weights = np.zeros(n_features, dtype=np.float64) + if not examples: + return weights + for _ in range(max(1, int(epochs))): + order = rng.permutation(len(examples)) + for index in order: + example = examples[int(index)] + feats = example["features"] + other_gold = other_query_gold_ids(examples, example["qid"]) + pairs = collect_pairs( + example, + pair_policy=pair_policy, + other_gold=other_gold, + ) + if not pairs: + continue + if len(pairs) > MAX_PAIRS: + chosen = rng.choice(len(pairs), size=MAX_PAIRS, replace=False) + pairs = [pairs[int(pos)] for pos in chosen] + for hi, lo in pairs: + diff = feats[hi] - feats[lo] + pred = _sigmoid(float(weights.dot(diff))) + grad = (pred - 1.0) * diff + l2 * weights + weights -= lr * grad + return weights + + +def score_docs(features: np.ndarray, weights: np.ndarray) -> np.ndarray: + return features.dot(weights) + + +def rerank_ids(example: dict[str, Any], weights: np.ndarray) -> list[str]: + scores = score_docs(example["features"], weights) + order = np.argsort(-scores, kind="stable") + return [example["ids"][int(pos)] for pos in order] + + +def gain_to_rank_label(gain: float) -> int: + if gain >= 0.99: + return 3 + if gain >= 0.09: + return 2 + if gain >= 0.005: + return 1 + return 0 + + +def rank_train_group( + example: dict[str, Any], + *, + pair_policy: str, + other_gold: set[str], +) -> tuple[np.ndarray, np.ndarray] | None: + labeled = training_gains( + example, + pair_policy=pair_policy, + other_gold=other_gold, + ) + idxs = [index for index, gain in enumerate(labeled) if gain is not None] + if len(idxs) < 2: + return None + y = np.array([gain_to_rank_label(float(labeled[index] or 0.0)) for index in idxs]) + if int(np.unique(y).size) < 2: + return None + return example["features"][idxs], y + + +class _ZeroRanker: + def __init__(self, n_features: int) -> None: + self.feature_importances_ = np.zeros(max(1, int(n_features)), dtype=np.float64) + + def predict(self, features: np.ndarray) -> np.ndarray: + return np.zeros(len(features), dtype=np.float64) + + +class _LgbmRanker: + def __init__(self, booster: Any, n_features: int) -> None: + self.booster = booster + raw = booster.feature_importance(importance_type="gain") + importances = np.asarray(raw, dtype=np.float64) + if importances.size < n_features: + importances = np.pad(importances, (0, n_features - int(importances.size))) + self.feature_importances_ = importances[:n_features] + + def predict(self, features: np.ndarray) -> np.ndarray: + return np.asarray(self.booster.predict(features), dtype=np.float64) + + +def fit_lightgbm( + examples: list[dict[str, Any]], + *, + pair_policy: str = PAIR_UNLABELED_ZERO, + seed: int = SEED, + min_data_in_leaf: int | None = None, +) -> Any: + n_features = ( + int(examples[0]["features"].shape[1]) if examples else len(BASE_FEATURE_NAMES) + ) + xs: list[np.ndarray] = [] + ys: list[np.ndarray] = [] + groups: list[int] = [] + for example in examples: + group = rank_train_group( + example, + pair_policy=pair_policy, + other_gold=other_query_gold_ids(examples, example["qid"]), + ) + if group is None: + continue + feats, labels = group + xs.append(feats) + ys.append(labels) + groups.append(int(len(labels))) + if not groups: + return _ZeroRanker(n_features) + import lightgbm as lgb + + dataset = lgb.Dataset( + np.vstack(xs), + label=np.concatenate(ys), + group=groups, + free_raw_data=False, + ) + params: dict[str, Any] = { + "objective": "lambdarank", + "max_depth": LGBM_MAX_DEPTH, + "learning_rate": LGBM_LEARNING_RATE, + "num_leaves": 8, + "verbosity": -1, + "seed": int(seed), + } + if min_data_in_leaf is not None: + params["min_data_in_leaf"] = int(min_data_in_leaf) + booster = lgb.train( + params, + dataset, + num_boost_round=LGBM_N_ESTIMATORS, + ) + return _LgbmRanker(booster, n_features) + + +def rerank_ids_model(example: dict[str, Any], model: Any) -> list[str]: + scores = np.asarray(model.predict(example["features"]), dtype=np.float64) + order = np.argsort(-scores, kind="stable") + return [example["ids"][int(pos)] for pos in order] + + +def overlap_only_ids(example: dict[str, Any]) -> list[str]: + title_scores = example["features"][:, 3] + order = np.argsort(-title_scores, kind="stable") + return [example["ids"][int(pos)] for pos in order] + + +def grouped_folds(qids: list[str], n_folds: int = N_FOLDS) -> list[set[str]]: + folds: list[set[str]] = [set() for _ in range(max(2, int(n_folds)))] + for index, qid in enumerate(sorted(qids)): + folds[index % len(folds)].add(qid) + return folds + + +def build_ltr_examples( + eval_result: dict[str, Any], + rows: list[dict[str, Any]], + *, + k: int = 50, + ce_model: Path | None = None, + ce_cache_path: Path | None = None, + ce_scores: dict[str, dict[str, float]] | None = None, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, dict[str, Any]]]: + docs, queries = split_corpus(rows) + docs_by_id = {str(doc.get("doc_id") or ""): doc for doc in docs if doc.get("doc_id")} + resolved_ce = ce_scores + ce_model_path = Path(ce_model) if ce_model else None + if resolved_ce is None and ce_model_path is not None: + resolved_ce = score_ce_gains( + eval_result, + docs_by_id, + model_dir=ce_model_path, + k=k, + cache_path=ce_cache_path, + ) + with_ce = resolved_ce is not None + examples: list[dict[str, Any]] = [] + for row in eval_result.get("queries") or []: + qid = str(row.get("qid") or "") + example = example_from_eval_row( + row, + docs_by_id, + k=k, + ce_scores=(resolved_ce or {}).get(qid) if with_ce else None, + ) + if example is None: + continue + examples.append(example) + return examples, queries, docs_by_id + + +def run_ltr_head( + eval_result: dict[str, Any], + rows: list[dict[str, Any]], + *, + dataset_name: str, + k: int = 50, + ks: tuple[int, ...] = (5, 10, 20, 50), + n_folds: int = N_FOLDS, + brain_id: str | None = None, + source_run: str | None = None, + pair_policy: str = PAIR_UNLABELED_ZERO, + ce_model: Path | None = None, + ce_cache_path: Path | None = None, + ce_scores: dict[str, dict[str, float]] | None = None, + ltr_head: str = HEAD_RANKNET, + train_eval_result: dict[str, Any] | None = None, + train_rows: list[dict[str, Any]] | None = None, + train_source_run: str | None = None, + train_ce_cache_path: Path | None = None, + train_ce_scores: dict[str, dict[str, float]] | None = None, +) -> dict[str, Any]: + policy = (pair_policy or PAIR_UNLABELED_ZERO).strip() + if policy not in PAIR_POLICIES: + raise ValueError(f"Unknown pair policy {pair_policy!r}") + head = (ltr_head or HEAD_RANKNET).strip() + if head not in HEADS: + raise ValueError(f"Unknown LTR head {ltr_head!r}") + if train_eval_result is not None and train_rows is None: + raise ValueError("train_rows is required when train_eval_result is set") + apply_mode = train_eval_result is not None + ce_model_path = Path(ce_model) if ce_model else None + eval_examples, queries, docs_by_id = build_ltr_examples( + eval_result, + rows, + k=k, + ce_model=ce_model_path, + ce_cache_path=ce_cache_path, + ce_scores=ce_scores, + ) + names = feature_names(with_ce=ce_model_path is not None or ce_scores is not None) + if apply_mode: + train_examples, _, _ = build_ltr_examples( + train_eval_result or {}, + train_rows or [], + k=k, + ce_model=ce_model_path, + ce_cache_path=train_ce_cache_path, + ce_scores=train_ce_scores, + ) + names = feature_names( + with_ce=bool(train_examples) + and int(train_examples[0]["features"].shape[1]) > len(BASE_FEATURE_NAMES) + ) + ranked_cv: dict[str, list[str]] = {} + ranked_overlap: dict[str, list[str]] = {} + fold_weights: list[list[float]] = [] + if head == HEAD_LIGHTGBM: + model = fit_lightgbm(train_examples, pair_policy=policy) + fold_weights.append([float(item) for item in model.feature_importances_]) + for example in eval_examples: + ranked_cv[example["qid"]] = rerank_ids_model(example, model)[:k] + ranked_overlap[example["qid"]] = overlap_only_ids(example)[:k] + else: + weights = fit_pairwise(train_examples, pair_policy=policy) + fold_weights.append([float(item) for item in weights]) + for example in eval_examples: + ranked_cv[example["qid"]] = rerank_ids(example, weights)[:k] + ranked_overlap[example["qid"]] = overlap_only_ids(example)[:k] + n_train = len(train_examples) + else: + examples = eval_examples + by_qid = {example["qid"]: example for example in examples} + qids = [example["qid"] for example in examples] + ranked_cv = {} + ranked_overlap = {} + fold_weights = [] + for test_qids in grouped_folds(qids, n_folds=n_folds): + train_examples = [ + example for example in examples if example["qid"] not in test_qids + ] + if head == HEAD_LIGHTGBM: + model = fit_lightgbm(train_examples, pair_policy=policy) + fold_weights.append([float(item) for item in model.feature_importances_]) + for qid in test_qids: + example = by_qid.get(qid) + if example is None: + continue + ranked_cv[qid] = rerank_ids_model(example, model)[:k] + ranked_overlap[qid] = overlap_only_ids(example)[:k] + continue + weights = fit_pairwise(train_examples, pair_policy=policy) + fold_weights.append([float(item) for item in weights]) + for qid in test_qids: + example = by_qid.get(qid) + if example is None: + continue + ranked_cv[qid] = rerank_ids(example, weights)[:k] + ranked_overlap[qid] = overlap_only_ids(example)[:k] + n_train = None + train_examples = [] + metrics, per_query = evaluate_dense( + ranked_cv, + queries, + ks=ks, + encode_ms=0.0, + ) + overlap_metrics, _ = evaluate_dense( + ranked_overlap, + queries, + ks=ks, + encode_ms=0.0, + ) + mean_weights = ( + np.mean(np.array(fold_weights, dtype=np.float64), axis=0) + if fold_weights + else np.zeros(len(names)) + ) + return { + "status": "ok" if per_query else "failed", + "brain_id": brain_id or "harness-local-ltr", + "dataset": dataset_name, + "fusion": "none", + "rerank": "ltr-lightgbm" if head == HEAD_LIGHTGBM else "ltr-pairwise", + "channels": ["harness-ltr-head"], + "expand": "none", + "k": k, + "ks": list(ks), + "n_docs": len(docs_by_id), + "n_queries": len(per_query), + "n_docs_mapped": 0, + "skip_enrichment": True, + "ingest_graph": False, + "skip_ingest": True, + "rank_pool": False, + "ltr_from_run": source_run or eval_result.get("run_id"), + "ltr_train_run": train_source_run if apply_mode else None, + "ltr_n_train_queries": n_train if apply_mode else None, + "protocol": PROTOCOL_APPLY if apply_mode else PROTOCOL, + "ltr_features": list(names), + "ltr_pair_policy": policy, + "ltr_head": head, + "ltr_ce_model": str(ce_model_path) if ce_model_path else None, + "ltr_lgbm": ( + { + "n_estimators": LGBM_N_ESTIMATORS, + "max_depth": LGBM_MAX_DEPTH, + "learning_rate": LGBM_LEARNING_RATE, + } + if head == HEAD_LIGHTGBM + else None + ), + "ltr_n_folds": 1 if apply_mode else n_folds, + "ltr_seed": SEED, + "ltr_epochs": EPOCHS, + "ltr_mean_weights": { + name: float(weight) + for name, weight in zip(names, mean_weights) + }, + "overlap_only_metrics": overlap_metrics, + "ingest": { + "status": "completed", + "n_docs": len(docs_by_id), + "tasks": [], + "reused": True, + }, + "graph_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "interaction_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "event_probe": None, + "search_error": None, + "metrics": metrics, + "queries": per_query, + } diff --git a/benchmarks/search/mapping.py b/benchmarks/search/mapping.py new file mode 100644 index 0000000..763edb0 --- /dev/null +++ b/benchmarks/search/mapping.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[2] +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + +from src.core.search.catalog_graph import ( # noqa: E402 + ATTR_LABEL, + CLASS_LABEL, + ENTITY_TYPE, + EVENT_LABEL, + FEATURE_KEY_CAP, + HAS_EVENT_NAME, + TYPE_LABEL, + USER_TYPE, + compose_search_text, + doc_fields, + doc_id_from_text, + doc_to_triples, + docs_to_triples, + catalog_entity_backfill_rows, + entity_search_text, + entity_uuid, + node_id_from_passage_text, + event_uuid, + format_happened_at, + has_triple, + is_static_has_triple, + hub_uuid, + interaction_to_triples, + interactions_to_triples, + load_interaction_rows, + parse_feature_string, + rel_uuid, + sanitize_label, + split_hierarchy, + structured_ingest_body, +) + +__all__ = [ + "ATTR_LABEL", + "CLASS_LABEL", + "ENTITY_TYPE", + "EVENT_LABEL", + "FEATURE_KEY_CAP", + "HAS_EVENT_NAME", + "TYPE_LABEL", + "USER_TYPE", + "compose_search_text", + "doc_fields", + "doc_id_from_text", + "doc_to_triples", + "docs_to_triples", + "catalog_entity_backfill_rows", + "entity_search_text", + "entity_uuid", + "node_id_from_passage_text", + "event_uuid", + "format_happened_at", + "has_triple", + "is_static_has_triple", + "hub_uuid", + "interaction_to_triples", + "interactions_to_triples", + "load_interaction_rows", + "parse_feature_string", + "rel_uuid", + "sanitize_label", + "split_hierarchy", + "structured_ingest_body", +] diff --git a/benchmarks/search/metrics.py b/benchmarks/search/metrics.py new file mode 100644 index 0000000..91e9079 --- /dev/null +++ b/benchmarks/search/metrics.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import math +from typing import Any, Iterable, Sequence + + +def recall_at_k( + ranked_ids: Sequence[str], + gold: Iterable[str], + k: int, +) -> float: + gold_set = {str(item) for item in gold} + if not gold_set: + return 0.0 + retrieved = {str(item) for item in list(ranked_ids)[:k]} + return len(retrieved & gold_set) / len(gold_set) + + +def mrr(ranked_ids: Sequence[str], gold: Iterable[str]) -> float: + gold_set = {str(item) for item in gold} + if not gold_set: + return 0.0 + for index, item in enumerate(ranked_ids, start=1): + if str(item) in gold_set: + return 1.0 / index + return 0.0 + + +def _dcg(relevances: Sequence[float], k: int) -> float: + score = 0.0 + for index, rel in enumerate(list(relevances)[:k], start=1): + score += float(rel) / math.log2(index + 1) + return score + + +def ndcg_at_k( + ranked_ids: Sequence[str], + gold: Iterable[str], + k: int, + grades: dict[str, float] | None = None, +) -> float: + if grades: + gain = { + str(doc_id): float(value) + for doc_id, value in grades.items() + if float(value) > 0 + } + if not gain: + return 0.0 + relevances = [gain.get(str(item), 0.0) for item in list(ranked_ids)[:k]] + ideal = sorted(gain.values(), reverse=True)[:k] + dcg = _dcg(relevances, k) + idcg = _dcg(ideal, k) + if idcg <= 0: + return 0.0 + return dcg / idcg + gold_set = {str(item) for item in gold} + if not gold_set: + return 0.0 + relevances = [1.0 if str(item) in gold_set else 0.0 for item in list(ranked_ids)[:k]] + dcg = _dcg(relevances, k) + ideal = [1.0] * min(len(gold_set), k) + idcg = _dcg(ideal, k) + if idcg <= 0: + return 0.0 + return dcg / idcg + + +def percentile(values: Sequence[float], p: float) -> float | None: + if not values: + return None + xs = sorted(float(v) for v in values) + if len(xs) == 1: + return xs[0] + rank = (len(xs) - 1) * (p / 100.0) + low = math.floor(rank) + high = math.ceil(rank) + if low == high: + return xs[int(rank)] + return xs[low] * (high - rank) + xs[high] * (rank - low) + + +def _stage_wall_ms(stage_timings: dict[str, Any] | None, name: str) -> float | None: + stages = (stage_timings or {}).get("stages") or [] + for stage in stages: + if not isinstance(stage, dict): + continue + if stage.get("stage") == name and stage.get("wall_ms") is not None: + return float(stage["wall_ms"]) + return None + + +def retrieve_latency_ms( + stage_timings: dict[str, Any] | None, + client_wall_ms: float, +) -> tuple[float, float | None]: + retrieve_ms = _stage_wall_ms(stage_timings, "search.retrieve") + embed_ms = _stage_wall_ms(stage_timings, "embed.query") + if retrieve_ms is not None: + return retrieve_ms, embed_ms + if embed_ms is not None: + return max(0.0, float(client_wall_ms) - embed_ms), embed_ms + return float(client_wall_ms), embed_ms + + +def mean(values: Sequence[float]) -> float: + if not values: + return 0.0 + return sum(values) / len(values) + + +def aggregate_query_metrics( + per_query: list[dict[str, Any]], + *, + ks: Sequence[int] = (5, 10, 20), +) -> dict[str, Any]: + metrics: dict[str, Any] = {} + for k in ks: + key = f"recall@{k}" + metrics[key] = mean([float(row["metrics"][key]) for row in per_query]) + ndcg_ks = tuple(dict.fromkeys((*ks, 10, 20))) + for k in ndcg_ks: + key = f"ndcg@{k}" + values = [ + float(row["metrics"][key]) + for row in per_query + if row.get("metrics", {}).get(key) is not None + ] + if values: + metrics[key] = mean(values) + ndcg_full = [ + float(row["metrics"]["ndcg"]) + for row in per_query + if row.get("metrics", {}).get("ndcg") is not None + ] + if ndcg_full: + metrics["ndcg"] = mean(ndcg_full) + metrics["mrr"] = mean([float(row["metrics"]["mrr"]) for row in per_query]) + + retrieve_ms = [float(row["retrieve_ms"]) for row in per_query] + embed_ms = [ + float(row["embed_ms"]) + for row in per_query + if row.get("embed_ms") is not None + ] + client_ms = [float(row["client_wall_ms"]) for row in per_query] + metrics["p50_retrieve_ms"] = percentile(retrieve_ms, 50) + metrics["p95_retrieve_ms"] = percentile(retrieve_ms, 95) + metrics["p50_embed_ms"] = percentile(embed_ms, 50) + metrics["p95_embed_ms"] = percentile(embed_ms, 95) + metrics["p50_client_wall_ms"] = percentile(client_ms, 50) + metrics["p95_client_wall_ms"] = percentile(client_ms, 95) + + slices: dict[str, list[dict[str, Any]]] = {} + for row in per_query: + slices.setdefault(str(row.get("slice") or "unspecified"), []).append(row) + by_slice: dict[str, dict[str, Any]] = {} + for name, rows in slices.items(): + by_slice[name] = { + "n_queries": len(rows), + "recall@10": mean([float(r["metrics"]["recall@10"]) for r in rows]), + "ndcg@10": mean([float(r["metrics"]["ndcg@10"]) for r in rows]), + "mrr": mean([float(r["metrics"]["mrr"]) for r in rows]), + } + for k in ks: + recall_key = f"recall@{k}" + if all(r.get("metrics", {}).get(recall_key) is not None for r in rows): + by_slice[name][recall_key] = mean( + [float(r["metrics"][recall_key]) for r in rows] + ) + for k in ndcg_ks: + ndcg_key = f"ndcg@{k}" + if any(r.get("metrics", {}).get(ndcg_key) is not None for r in rows): + by_slice[name][ndcg_key] = mean( + [float(r["metrics"][ndcg_key]) for r in rows] + ) + if any(r.get("metrics", {}).get("ndcg") is not None for r in rows): + by_slice[name]["ndcg"] = mean([float(r["metrics"]["ndcg"]) for r in rows]) + metrics["by_slice"] = by_slice + return metrics diff --git a/benchmarks/search/mine_retrieved_lists.py b/benchmarks/search/mine_retrieved_lists.py new file mode 100644 index 0000000..5c2cc53 --- /dev/null +++ b/benchmarks/search/mine_retrieved_lists.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import argparse +import json +import random +from pathlib import Path +from typing import Any, Iterable, Sequence + +from search.catalog import ( + ESCI_CACHE_DIR, + ESCI_EXAMPLES_FILE, + ESCI_PRODUCTS_FILE, + extra_fields_from_catalog, + _esci_products, + _iter_parquet, +) +from search.finetune_esci_4class import ( + LABELS, + LABEL_TO_ID, + held_out_query_ids, + product_passage, +) +from search.pool_first_stage import Bm25Index, rank_docs, tokenize + +DEFAULT_OUT = Path("data/esci_retrieved_lists.jsonl") + + +def is_held_out(qid: str, holdout: set[str]) -> bool: + text = str(qid or "").strip() + if not text: + return True + if text in holdout: + return True + if text.lower().startswith("esci-") and text[5:] in holdout: + return True + if f"esci-{text}" in holdout: + return True + return False + + +def load_train_groups( + *, + holdout: set[str], + locale: str = "us", +) -> dict[str, dict[str, Any]]: + examples_path = ESCI_CACHE_DIR / ESCI_EXAMPLES_FILE + wanted_locale = locale.strip().lower() + groups: dict[str, dict[str, Any]] = {} + for item in _iter_parquet(examples_path): + if int(item.get("small_version") or 0) != 1: + continue + if str(item.get("product_locale") or "").strip().lower() != wanted_locale: + continue + if str(item.get("split") or "").strip().lower() != "train": + continue + qid = str(item.get("query_id") or "").strip() + if is_held_out(qid, holdout): + continue + pid = str(item.get("product_id") or "").strip() + query = str(item.get("query") or "").strip() + label = str(item.get("esci_label") or "").strip().upper() + if not pid or not query or label not in LABEL_TO_ID: + continue + group = groups.setdefault(qid, {"query": query, "qrels": {}}) + group["query"] = query + group["qrels"][pid] = label + return groups + + +def select_groups( + groups: dict[str, dict[str, Any]], + *, + holdout: set[str], + max_queries: int, + seed: int = 11, +) -> dict[str, dict[str, Any]]: + kept = { + qid: group + for qid, group in groups.items() + if not is_held_out(qid, holdout) + } + ids = list(kept) + rng = random.Random(seed) + rng.shuffle(ids) + cap = max(1, int(max_queries)) + chosen = ids[:cap] + return {qid: kept[qid] for qid in chosen} + + +def labeled_hits( + ranked: Sequence[str], + qrels: dict[str, str], + *, + k: int = 50, +) -> list[tuple[str, str]]: + pairs: list[tuple[str, str]] = [] + for pid in ranked: + key = str(pid) + label = str(qrels.get(key) or "I").strip().upper() + if label not in LABEL_TO_ID: + label = "I" + pairs.append((key, label)) + cut = max(1, int(k)) + top = pairs[:cut] + if any(label == "I" for _, label in top): + return top + for pid, label in pairs[cut:]: + if label == "I": + if top: + top[-1] = (pid, label) + else: + top.append((pid, label)) + break + return top + + +def passages_for_pids( + pids: Iterable[str], + *, + locale: str = "us", +) -> dict[str, str]: + needed = {str(pid) for pid in pids if str(pid)} + products = _esci_products( + ESCI_CACHE_DIR / ESCI_PRODUCTS_FILE, + needed, + locale=locale, + ) + extra_fields = extra_fields_from_catalog( + products, + title_key="product_title", + description_key="product_description", + ) + out: dict[str, str] = {} + for pid in needed: + catalog = products.get(pid) or {"product_id": pid} + out[pid] = product_passage(catalog, extra_fields) + return out + + +def mine_from_groups( + groups: dict[str, dict[str, Any]], + passages: dict[str, str], + *, + k: int = 50, +) -> list[dict[str, Any]]: + docs: list[tuple[str, list[str]]] = [] + for pid, text in passages.items(): + tokens = tokenize(text) + if not tokens: + continue + docs.append((str(pid), tokens)) + if not docs: + return [] + index = Bm25Index(docs) + doc_ids = [pid for pid, _ in docs] + rows: list[dict[str, Any]] = [] + n_groups = len(groups) + for offset, (qid, group) in enumerate(groups.items(), start=1): + query = str(group.get("query") or "") + qrels = { + str(pid): str(label).upper() + for pid, label in (group.get("qrels") or {}).items() + } + scores = index.scores(tokenize(query)) + ranked = rank_docs(scores, doc_ids) + hits = labeled_hits(ranked, qrels, k=k) + for pid, label in hits: + rows.append( + { + "query_id": str(qid), + "query": query, + "product_id": pid, + "label": label, + "passage": passages.get(pid) or pid, + } + ) + if offset % 200 == 0 or offset == n_groups: + print(f"mine retrieve {offset}/{n_groups}", flush=True) + return rows + + +def write_lists(rows: Sequence[dict[str, Any]], dest: Path) -> Path: + dest.parent.mkdir(parents=True, exist_ok=True) + with dest.open("w", encoding="utf-8") as fh: + for row in rows: + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + return dest + + +def mine( + *, + jsonl_path: Path, + out_path: Path, + max_queries: int = 6000, + k: int = 50, + seed: int = 11, + locale: str = "us", +) -> dict[str, Any]: + holdout = held_out_query_ids(jsonl_path) + groups = load_train_groups(holdout=holdout, locale=locale) + selected = select_groups( + groups, + holdout=holdout, + max_queries=max_queries, + seed=seed, + ) + needed: set[str] = set() + for group in selected.values(): + needed.update(str(pid) for pid in (group.get("qrels") or {})) + print( + f"mine-retrieved-lists queries={len(selected)} products={len(needed)} " + f"holdout={len(holdout)}", + flush=True, + ) + passages = passages_for_pids(needed, locale=locale) + rows = mine_from_groups(selected, passages, k=k) + write_lists(rows, out_path) + counts = {label: 0 for label in LABELS} + for row in rows: + counts[str(row["label"])] = counts.get(str(row["label"]), 0) + 1 + n_with_i = 0 + by_q: dict[str, list[str]] = {} + for row in rows: + by_q.setdefault(str(row["query_id"]), []).append(str(row["label"])) + for labels in by_q.values(): + if "I" in labels: + n_with_i += 1 + meta = { + "n_queries": len(selected), + "n_pairs": len(rows), + "n_holdout_qids": len(holdout), + "k": k, + "label_counts": counts, + "n_lists_with_i": n_with_i, + "source": "retrieved-bm25", + "out": str(out_path), + } + meta_path = out_path.with_suffix(out_path.suffix + ".meta.json") + if out_path.suffix == ".jsonl": + meta_path = out_path.with_name(out_path.stem + ".meta.json") + meta_path.write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8") + return meta + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Mine BM25 top-k train lists; unlabeled hits are class I" + ) + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--max-queries", type=int, default=6000) + parser.add_argument("--k", type=int, default=50) + parser.add_argument("--seed", type=int, default=11) + parser.add_argument("--locale", default="us") + args = parser.parse_args(argv) + meta = mine( + jsonl_path=args.dataset, + out_path=args.out, + max_queries=int(args.max_queries), + k=int(args.k), + seed=int(args.seed), + locale=str(args.locale), + ) + print(json.dumps(meta, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/search/miss.py b/benchmarks/search/miss.py new file mode 100644 index 0000000..1ab557d --- /dev/null +++ b/benchmarks/search/miss.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from search.dataset import load_records, split_corpus + +ELECTRONICS_MARKERS = ( + "electronic", + "electronics", + "ipad", + "iphone", + "phone", + "tablet", + "laptop", + "computer", + "charger", + "usb", + "hdmi", + "bluetooth", + "wifi", + "wi-fi", + "router", + "led", + "battery", + "samsung", + "apple", +) + + +def looks_like_electronics(text: str) -> bool: + lowered = (text or "").lower() + return any(token in lowered for token in ELECTRONICS_MARKERS) + + +def dump_query_miss( + *, + dataset_path: Path, + eval_path: Path, + qid: str, +) -> dict[str, Any]: + docs, queries = split_corpus(load_records(dataset_path)) + by_id = {str(doc.get("doc_id") or ""): doc for doc in docs} + query = next((row for row in queries if str(row.get("qid")) == qid), None) + if query is None: + raise SystemExit(f"Missing query {qid} in {dataset_path}") + eval_data = json.loads(eval_path.read_text(encoding="utf-8")) + scored = next( + (row for row in eval_data.get("queries") or [] if str(row.get("qid")) == qid), + None, + ) + if scored is None: + raise SystemExit(f"Missing query {qid} in {eval_path}") + gold_rows = [] + in_pool = 0 + electronics_gold = 0 + for doc_id in query.get("gold_doc_ids") or []: + doc = by_id.get(str(doc_id)) or {} + title = str(doc.get("title") or "") + text = str(doc.get("text") or "") + present = bool(doc) + in_pool += int(present) + electronic = looks_like_electronics(f"{title}\n{text}") + electronics_gold += int(electronic) + gold_rows.append( + { + "doc_id": doc_id, + "title": title, + "brand": doc.get("brand") or "", + "in_jsonl": present, + "electronics_marker": electronic, + } + ) + hit_rows = [] + for doc_id in scored.get("hit_ids") or []: + doc = by_id.get(str(doc_id)) or {} + title = str(doc.get("title") or "") + hit_rows.append( + { + "doc_id": doc_id, + "title": title, + "electronics_marker": looks_like_electronics(title), + } + ) + gold_set = {str(item) for item in (query.get("gold_doc_ids") or [])} + overlap = gold_set & {str(item["doc_id"]) for item in hit_rows} + if in_pool < len(gold_rows): + decision = "not_testable" + reason = "one or more Exact golds are missing from the JSONL pool" + elif electronics_gold >= max(1, len(gold_rows) // 2): + decision = "qrel_mismatch" + reason = ( + "majority of Exact golds look like electronics or electronics accessories; " + "do not rewrite 'not electronics'" + ) + elif overlap: + decision = "not_testable" + reason = "golds already appear in the scored list" + else: + decision = "rewrite_experiment" + reason = "golds are in-pool, non-electronics, and missing from top hits" + return { + "qid": qid, + "query": query.get("query"), + "metrics": scored.get("metrics"), + "gold": gold_rows, + "hits": hit_rows, + "n_gold": len(gold_rows), + "n_gold_in_jsonl": in_pool, + "n_electronics_gold": electronics_gold, + "overlap": sorted(overlap), + "decision": decision, + "reason": reason, + } diff --git a/benchmarks/search/miss_strata.py b/benchmarks/search/miss_strata.py new file mode 100644 index 0000000..3958566 --- /dev/null +++ b/benchmarks/search/miss_strata.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import html +import json +import re +import unicodedata +from collections import Counter +from pathlib import Path +from typing import Any + +from search.dataset import load_records, split_corpus, write_records +from search.metrics import recall_at_k + +TOKEN = re.compile(r"[a-z0-9]+", re.I) +JUNK_PUNCT = re.compile(r"[^\w\s\-]", re.UNICODE) +COMBINING = re.compile(r"[\u0300-\u036f]") +SQL_MARKERS = ( + "not null", + "integer", + "varchar", + "p_num", + "i_num", + "primary key", +) +SKIP_REWRITE_QIDS = frozenset({"esci-72", "72"}) + + +def canonical_qid(qid: str) -> str: + raw = str(qid or "").strip() + if raw.lower().startswith("esci-"): + return raw.lower() + if raw.isdigit(): + return f"esci-{raw}" + return raw.lower() + + +def gold_ids(query: dict[str, Any]) -> list[str]: + grades = query.get("gold_grades") or {} + ordered: list[str] = [] + seen: set[str] = set() + for doc_id in query.get("gold_doc_ids") or []: + key = str(doc_id) + if key in seen: + continue + if isinstance(grades, dict) and key in grades and float(grades.get(key) or 0) <= 0: + continue + seen.add(key) + ordered.append(key) + if isinstance(grades, dict): + for doc_id, gain in grades.items(): + if float(gain or 0) <= 0: + continue + key = str(doc_id) + if key in seen: + continue + seen.add(key) + ordered.append(key) + return ordered + + +def letter_for_gain(gain: float) -> str: + value = float(gain or 0) + if value >= 0.99: + return "E" + if value >= 0.09: + return "S" + if value > 0: + return "C" + return "I" + + +def query_tokens(text: str) -> set[str]: + return {token.lower() for token in TOKEN.findall(text or "")} + + +def token_overlap(query: str, doc_text: str) -> float: + qtok = query_tokens(query) + if not qtok: + return 0.0 + dtok = query_tokens(doc_text) + return len(qtok & dtok) / len(qtok) + + +def looks_sql(text: str) -> bool: + lowered = (text or "").lower() + return any(marker in lowered for marker in SQL_MARKERS) + + +def looks_html_entity(text: str) -> bool: + raw = text or "" + return "&#" in raw or "&" in raw or """ in raw or "<" in raw or ">" in raw + + +def normalize_spelling(text: str) -> str: + raw = unicodedata.normalize("NFKC", str(text or "")) + raw = html.unescape(raw) + raw = unicodedata.normalize("NFKD", raw) + raw = COMBINING.sub("", raw) + raw = re.sub(r"^[\s\-*._]+", "", raw) + raw = re.sub(r"[\s\-*._]+$", "", raw) + raw = JUNK_PUNCT.sub(" ", raw) + raw = re.sub(r"\s+", " ", raw).strip() + return raw + + +def rewrite_query(qid: str, text: str) -> str | None: + if canonical_qid(qid) in SKIP_REWRITE_QIDS: + return None + original = str(text or "").strip() + if not original: + return None + if looks_sql(original): + return None + unescaped = html.unescape(original).strip() + unescaped = re.sub(r"^[\s\-*._]+", "", unescaped) + unescaped = re.sub(r"[\s\-*._]+$", "", unescaped) + unescaped = re.sub(r"\s+", " ", unescaped).strip() + if len(unescaped) < 3: + return None + if unescaped == original and not looks_html_entity(original): + return None + return unescaped + + +def classify_query( + eval_row: dict[str, Any], + query_row: dict[str, Any], + docs: dict[str, dict[str, Any]], + *, + k: int = 50, +) -> dict[str, Any]: + hits = [str(item) for item in (eval_row.get("hit_ids") or []) if item] + gold = gold_ids(query_row if query_row else eval_row) + grades = (query_row.get("gold_grades") if query_row else None) or eval_row.get( + "gold_grades" + ) or {} + hitset = set(hits) + in_head = [doc_id for doc_id in gold if doc_id in hits[:10]] + in_tail = [doc_id for doc_id in gold if doc_id in hitset and doc_id not in set(hits[:10])] + missed = [doc_id for doc_id in gold if doc_id not in hitset] + if in_head: + stratum = "head-ok" + elif in_tail: + stratum = "rank-too-low" + else: + stratum = "total-miss" + qtext = str(eval_row.get("query") or query_row.get("query") or "") + miss_letters = Counter() + gold_letters = Counter() + overlap_hit: list[float] = [] + overlap_miss: list[float] = [] + for doc_id in gold: + letter = letter_for_gain(float((grades or {}).get(doc_id) or 1.0)) + gold_letters[letter] += 1 + text = str((docs.get(doc_id) or {}).get("text") or "") + overlap = token_overlap(qtext, text) + if doc_id in hitset: + overlap_hit.append(overlap) + else: + miss_letters[letter] += 1 + overlap_miss.append(overlap) + metrics = eval_row.get("metrics") or {} + rewritten = rewrite_query(str(eval_row.get("qid") or query_row.get("qid") or ""), qtext) + pathological = bool( + rewritten + or looks_html_entity(qtext) + or looks_sql(qtext) + ) and canonical_qid(str(eval_row.get("qid") or "")) not in SKIP_REWRITE_QIDS + return { + "qid": eval_row.get("qid") or query_row.get("qid"), + "query": qtext, + "stratum": stratum, + "n_gold": len(gold), + "n_in_top10": len(in_head), + "n_in_11_to_k": len(in_tail), + "n_missed": len(missed), + "recall@10": float(metrics.get("recall@10") or recall_at_k(hits, gold, 10)), + "recall@50": float(metrics.get("recall@50") or recall_at_k(hits, gold, k)), + "gold_letters": dict(gold_letters), + "miss_letters": dict(miss_letters), + "mean_overlap_hit": (sum(overlap_hit) / len(overlap_hit)) if overlap_hit else None, + "mean_overlap_miss": (sum(overlap_miss) / len(overlap_miss)) if overlap_miss else None, + "pathological": pathological, + "rewrite": rewritten, + "skip_rewrite": canonical_qid(str(eval_row.get("qid") or "")) in SKIP_REWRITE_QIDS, + "k": k, + } + + +def classify_eval( + eval_result: dict[str, Any], + rows: list[dict[str, Any]], + *, + k: int | None = None, +) -> dict[str, Any]: + docs, queries = split_corpus(rows) + by_id = {str(doc.get("doc_id") or ""): doc for doc in docs} + by_qid = {str(query.get("qid") or ""): query for query in queries} + cut = int(k or eval_result.get("k") or 50) + classified: list[dict[str, Any]] = [] + gold_total = 0 + in_head = 0 + in_tail = 0 + missed = 0 + for row in eval_result.get("queries") or []: + qid = str(row.get("qid") or "") + item = classify_query(row, by_qid.get(qid) or {}, by_id, k=cut) + classified.append(item) + gold_total += int(item["n_gold"]) + in_head += int(item["n_in_top10"]) + in_tail += int(item["n_in_11_to_k"]) + missed += int(item["n_missed"]) + counts = Counter(str(item["stratum"]) for item in classified) + total_miss = [item for item in classified if item["stratum"] == "total-miss"] + rewritable = [ + item + for item in total_miss + if item.get("rewrite") and not item.get("skip_rewrite") + ] + return { + "n_queries": len(classified), + "n_gold": gold_total, + "n_in_top10": in_head, + "n_in_11_to_k": in_tail, + "n_missed": missed, + "k": cut, + "stratum_counts": dict(counts), + "n_total_miss": len(total_miss), + "n_rewritable_total_miss": len(rewritable), + "run_query_side": bool(rewritable), + "queries": classified, + "rewritable_qids": [item["qid"] for item in rewritable], + "total_miss_qids": [item["qid"] for item in total_miss], + } + + +def write_rewritten_jsonl( + rows: list[dict[str, Any]], + taxonomy: dict[str, Any], + dest: Path, +) -> Path: + rewrites = { + str(item["qid"]): str(item["rewrite"]) + for item in taxonomy.get("queries") or [] + if item.get("rewrite") + and item.get("stratum") == "total-miss" + and not item.get("skip_rewrite") + } + out: list[dict[str, Any]] = [] + for row in rows: + if str(row.get("type") or "") != "query": + out.append(row) + continue + qid = str(row.get("qid") or "") + rewritten = rewrites.get(qid) + if not rewritten: + out.append(row) + continue + cloned = dict(row) + cloned["query_original"] = row.get("query") + cloned["query"] = rewritten + out.append(cloned) + return write_records(out, dest) + + +def write_spell_jsonl(rows: list[dict[str, Any]], dest: Path) -> dict[str, Any]: + out: list[dict[str, Any]] = [] + n_changed = 0 + for row in rows: + if str(row.get("type") or "") != "query": + out.append(row) + continue + original = str(row.get("query") or "") + rewritten = normalize_spelling(original) + cloned = dict(row) + if rewritten != original: + cloned["query_original"] = original + cloned["query"] = rewritten + n_changed += 1 + out.append(cloned) + write_records(out, dest) + return { + "out_path": str(dest), + "n_rows": len(out), + "n_queries_changed": n_changed, + } + + +def load_eval(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def main_classify( + *, + eval_path: Path, + dataset_path: Path, + out_path: Path | None = None, +) -> dict[str, Any]: + eval_result = load_eval(eval_path) + rows = load_records(dataset_path) + taxonomy = classify_eval(eval_result, rows) + taxonomy["eval_path"] = str(eval_path) + taxonomy["dataset"] = dataset_path.name + if out_path is not None: + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(taxonomy, indent=2) + "\n", encoding="utf-8") + taxonomy["out_path"] = str(out_path) + return taxonomy diff --git a/benchmarks/search/pool_first_stage.py b/benchmarks/search/pool_first_stage.py new file mode 100644 index 0000000..85f315b --- /dev/null +++ b/benchmarks/search/pool_first_stage.py @@ -0,0 +1,363 @@ +from __future__ import annotations + +import argparse +import json +import math +import re +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any, Iterable + +from search.dataset import load_records, split_corpus, write_records +from search.evaluate import ensure_run_dir +from search.metrics import aggregate_query_metrics +from search.rank_pool import score_ranked_pool + +TOKEN_RE = re.compile(r"[a-z0-9]+", re.I) +STOP = { + "a", + "an", + "and", + "are", + "as", + "at", + "be", + "for", + "from", + "in", + "is", + "it", + "of", + "on", + "or", + "the", + "to", + "with", +} + + +def tokenize(text: str) -> list[str]: + return [tok.lower() for tok in TOKEN_RE.findall(text or "")] + + +def field_value(text: str, name: str) -> str: + prefix = f"{name}:" + lines = [] + capture = False + for raw in (text or "").splitlines(): + stripped = raw.strip() + if ":" in stripped and not stripped.lower().startswith("docid "): + label = stripped.split(":", 1)[0].strip().lower() + capture = label == name.lower() + if capture: + lines.append(stripped.split(":", 1)[1].strip()) + continue + if capture: + lines.append(stripped) + return " ".join(part for part in lines if part) + + +def title_text(text: str) -> str: + title = field_value(text, "Title") + return title or (text or "") + + +def bm25_idf(n_docs: int, df: int) -> float: + return math.log(1.0 + (n_docs - df + 0.5) / (df + 0.5)) + + +class Bm25Index: + def __init__(self, docs: list[tuple[str, list[str]]], *, k1: float = 1.2, b: float = 0.75): + self.k1 = k1 + self.b = b + self.doc_ids = [doc_id for doc_id, _ in docs] + self.tfs: list[Counter[str]] = [Counter(tokens) for _, tokens in docs] + self.dl = [max(1, len(tokens)) for _, tokens in docs] + self.avgdl = (sum(self.dl) / len(self.dl)) if self.dl else 1.0 + df: Counter[str] = Counter() + postings: dict[str, list[int]] = defaultdict(list) + for index, tokens in enumerate(self.tfs): + df.update(tokens.keys()) + for term in tokens: + postings[term].append(index) + n = max(1, len(self.doc_ids)) + self.idf = {term: bm25_idf(n, count) for term, count in df.items()} + self.postings = postings + + def scores(self, query_tokens: list[str]) -> dict[str, float]: + qtf = Counter(query_tokens) + out: dict[str, float] = defaultdict(float) + for term, qf in qtf.items(): + idf = self.idf.get(term, 0.0) + if not idf: + continue + for index in self.postings.get(term, ()): + tf = self.tfs[index].get(term, 0) + if tf <= 0: + continue + dl = self.dl[index] + denom = tf + self.k1 * (1.0 - self.b + self.b * dl / self.avgdl) + out[self.doc_ids[index]] += float(qf) * idf * (tf * (self.k1 + 1.0)) / denom + return out + + +def rank_docs(scores: dict[str, float], doc_ids: Iterable[str]) -> list[str]: + ranked = sorted(scores.items(), key=lambda pair: pair[1], reverse=True) + seen = {doc_id for doc_id, _ in ranked} + ordered = [doc_id for doc_id, _ in ranked] + for doc_id in doc_ids: + if doc_id not in seen: + ordered.append(doc_id) + return ordered + + +def rm3_terms( + query_tokens: list[str], + ranked: list[str], + token_map: dict[str, list[str]], + *, + top_docs: int = 8, + extra_terms: int = 8, +) -> list[str]: + counts: Counter[str] = Counter() + for doc_id in ranked[:top_docs]: + counts.update(token_map.get(doc_id) or []) + query_set = set(query_tokens) + added: list[str] = [] + for term, _ in counts.most_common(): + if term in STOP or term in query_set or term.isdigit(): + continue + added.append(term) + if len(added) >= extra_terms: + break + return added + + +def expand_query(query: str, extra: list[str]) -> str: + if not extra: + return query + return f"{query} {' '.join(extra)}".strip() + + +def evaluate_variant( + queries: list[dict[str, Any]], + ranked_by_qid: dict[str, list[str]], +) -> tuple[dict[str, float], list[dict[str, Any]]]: + per_query: list[dict[str, Any]] = [] + for query in queries: + qid = str(query.get("qid") or "") + ranked = ranked_by_qid.get(qid) or [] + metrics = score_ranked_pool(ranked, query) + per_query.append( + { + "qid": qid, + "query": query.get("query"), + "hit_ids": ranked[:20], + "metrics": metrics, + "n_hits": len(ranked), + "retrieve_ms": 0.0, + "embed_ms": None, + "client_wall_ms": 0.0, + } + ) + metrics = aggregate_query_metrics(per_query, ks=(5, 10, 20)) if per_query else {} + return metrics, per_query + + +def _splade_rank( + docs: list[dict[str, Any]], + queries: list[dict[str, Any]], + *, + k: int = 20, +) -> dict[str, list[str]]: + plugin_dir = Path(__file__).resolve().parents[2] / "plugins" / "search-splade" + import sys + + if str(plugin_dir) not in sys.path: + sys.path.insert(0, str(plugin_dir)) + from index import index_chunks, reset, retrieve + + brain = "harness-local-splade" + reset(brain) + chunks = [] + doc_ids: list[str] = [] + for doc in docs: + doc_id = str(doc.get("doc_id") or "") + if not doc_id: + continue + doc_ids.append(doc_id) + chunks.append( + {"id": doc_id, "text": str(doc.get("text") or doc.get("title") or "")} + ) + index_chunks(brain, chunks) + ranked_by_qid: dict[str, list[str]] = {} + for query in queries: + qid = str(query.get("qid") or "") + ids, scores, _ = retrieve(str(query.get("query") or ""), brain, max(k, len(doc_ids))) + ranked_by_qid[qid] = rank_docs( + {str(doc_id): float(scores.get(doc_id, 0.0)) for doc_id in ids}, + doc_ids, + ) + return ranked_by_qid + + +def run_first_stage( + rows: list[dict[str, Any]], + *, + variant: str, +) -> dict[str, Any]: + docs, queries = split_corpus(rows) + if variant == "splade": + ranked_by_qid = _splade_rank(docs, queries) + metrics, per_query = evaluate_variant(queries, ranked_by_qid) + return { + "variant": variant, + "metrics": metrics, + "queries": per_query, + "expansions": {}, + "n_docs": len(docs), + "n_queries": len(queries), + } + all_tokens: dict[str, list[str]] = {} + title_tokens: dict[str, list[str]] = {} + doc_ids: list[str] = [] + for doc in docs: + doc_id = str(doc.get("doc_id") or "") + if not doc_id: + continue + text = str(doc.get("text") or doc.get("title") or "") + doc_ids.append(doc_id) + all_tokens[doc_id] = tokenize(text) + title_tokens[doc_id] = tokenize(title_text(text)) + all_index = Bm25Index([(doc_id, all_tokens[doc_id]) for doc_id in doc_ids]) + title_index = Bm25Index([(doc_id, title_tokens[doc_id]) for doc_id in doc_ids]) + ranked_by_qid: dict[str, list[str]] = {} + expansions: dict[str, str] = {} + for query in queries: + qid = str(query.get("qid") or "") + qtext = str(query.get("query") or "") + q_tokens = tokenize(qtext) + all_scores = all_index.scores(q_tokens) + title_scores = title_index.scores(q_tokens) + if variant == "title": + scores = title_scores + elif variant == "title-boost": + scores = dict(all_scores) + for doc_id, value in title_scores.items(): + scores[doc_id] = scores.get(doc_id, 0.0) + 2.0 * value + elif variant in {"rm3", "rm3-title-boost"}: + seed_scores = dict(all_scores) + if variant == "rm3-title-boost": + for doc_id, value in title_scores.items(): + seed_scores[doc_id] = seed_scores.get(doc_id, 0.0) + 2.0 * value + seed_ranked = rank_docs(seed_scores, doc_ids) + extra = rm3_terms(q_tokens, seed_ranked, title_tokens) + expansions[qid] = expand_query(qtext, extra) + expanded_tokens = tokenize(expansions[qid]) + scores = all_index.scores(expanded_tokens) + if variant == "rm3-title-boost": + title_exp = title_index.scores(expanded_tokens) + for doc_id, value in title_exp.items(): + scores[doc_id] = scores.get(doc_id, 0.0) + 2.0 * value + else: + scores = all_scores + ranked_by_qid[qid] = rank_docs(scores, doc_ids) + metrics, per_query = evaluate_variant(queries, ranked_by_qid) + return { + "variant": variant, + "metrics": metrics, + "queries": per_query, + "expansions": expansions, + "n_docs": len(doc_ids), + "n_queries": len(queries), + } + + +def write_expanded_jsonl( + rows: list[dict[str, Any]], + expansions: dict[str, str], + dest: Path, +) -> Path: + out: list[dict[str, Any]] = [] + for row in rows: + if str(row.get("type") or "") != "query": + out.append(row) + continue + cloned = dict(row) + qid = str(cloned.get("qid") or "") + expanded = expansions.get(qid) + if expanded: + cloned["query"] = expanded + cloned["query_original"] = row.get("query") + out.append(cloned) + return write_records(out, dest) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="pool_first_stage") + parser.add_argument("--dataset", type=Path, default=Path("data/search_esci_74.jsonl")) + parser.add_argument("--run", default="search-esci-74-fielded-bm25") + parser.add_argument( + "--variant", + default="all", + help="Comma-separated: all,title,title-boost,rm3,rm3-title-boost", + ) + parser.add_argument( + "--expand-out", + type=Path, + default=Path("data/search_esci_74_rm3.jsonl"), + ) + args = parser.parse_args(argv) + from search.config import Settings + + settings = Settings.load() + rows = load_records(args.dataset) + run_id, run_dir = ensure_run_dir(settings, args.run) + variants = [item.strip() for item in str(args.variant).split(",") if item.strip()] + summary: dict[str, Any] = {"run_id": run_id, "dataset": args.dataset.name, "variants": {}} + expand_source = None + for variant in variants: + result = run_first_stage(rows, variant=variant) + metrics = result["metrics"] + summary["variants"][variant] = { + "ndcg@10": metrics.get("ndcg@10"), + "ndcg@20": metrics.get("ndcg@20"), + "recall@10": metrics.get("recall@10"), + "recall@20": metrics.get("recall@20"), + "mrr": metrics.get("mrr"), + } + if variant.startswith("rm3"): + expand_source = result + if expand_source and expand_source.get("expansions"): + write_expanded_jsonl(rows, expand_source["expansions"], args.expand_out) + summary["expand_out"] = str(args.expand_out) + docs, queries = split_corpus(rows) + payload = { + "status": "ok", + "brain_id": "harness-local", + "dataset": args.dataset.name, + "fusion": "none", + "rerank": "none", + "channels": ["harness-bm25"], + "protocol": ( + "local fielded BM25 over search_esci_74.jsonl 2043-doc corpus; " + "not BrainAPI passages; not Reddy 0.857" + ), + "n_docs": len(docs), + "n_queries": len(queries), + "metrics": (summary["variants"].get(variants[0]) or {}), + "summary": summary, + "ingest": {"status": "completed", "n_docs": 0, "tasks": [], "reused": True}, + } + (run_dir / "eval.json").write_text( + json.dumps(payload, indent=2) + "\n", encoding="utf-8" + ) + (run_dir / "report.json").write_text( + json.dumps(summary, indent=2) + "\n", encoding="utf-8" + ) + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/search/rank_corpus.py b/benchmarks/search/rank_corpus.py new file mode 100644 index 0000000..68e3942 --- /dev/null +++ b/benchmarks/search/rank_corpus.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +from search.dataset import split_corpus +from search.finetune_esci_4class import CLASS_GAINS, rank_doc_ids, weighted_scores +from search.local_dense import evaluate_dense +from search.metrics import aggregate_query_metrics +from search.rank_pool import doc_texts +from search.rank_pool_4class import PredictFn, load_4class_predict + +DEFAULT_MODEL = Path("data/models/esci-minilm-l12-4class-nowt-e2") +PROTOCOL = "exhaustive-catalog" + + +def run_exhaustive_ce( + rows: list[dict[str, Any]], + *, + predict: PredictFn, + model_name: str, + dataset_name: str, + k: int = 50, + ks: tuple[int, ...] = (5, 10, 20, 50), + brain_id: str | None = None, +) -> dict[str, Any]: + docs, queries = split_corpus(rows) + texts = doc_texts(docs) + doc_ids = [str(doc.get("doc_id") or "") for doc in docs if doc.get("doc_id")] + cut = max(1, min(int(k), len(doc_ids) or 1)) + ranked_by_qid: dict[str, list[str]] = {} + per_q_ms: list[float] = [] + missing_text = 0 + for index, query in enumerate(queries): + q_text = str(query.get("query") or "") + pairs: list[tuple[str, str]] = [] + for doc_id in doc_ids: + text = texts.get(doc_id) or "" + if not text: + missing_text += 1 + pairs.append((q_text, text)) + started = time.perf_counter() + probs = predict(pairs) if pairs else [] + elapsed_ms = (time.perf_counter() - started) * 1000.0 + per_q_ms.append(elapsed_ms) + scores = weighted_scores(probs, CLASS_GAINS) if pairs else [] + if len(scores) < len(doc_ids): + scores = list(scores) + [0.0] * (len(doc_ids) - len(scores)) + ranked = rank_doc_ids(doc_ids, scores)[:cut] + qid = str(query.get("qid") or "") + ranked_by_qid[qid] = ranked + print(f"exhaustive {index + 1}/{len(queries)} {qid} {elapsed_ms:.0f}ms", flush=True) + encode_ms = sum(per_q_ms) + metrics, per_query = evaluate_dense( + ranked_by_qid, + queries, + ks=ks, + encode_ms=encode_ms, + ) + if not metrics and per_query: + metrics = aggregate_query_metrics(per_query, ks=ks) + return { + "status": "ok" if per_query else "failed", + "brain_id": brain_id or "harness-local-exhaustive", + "dataset": dataset_name, + "fusion": "none", + "rerank": "4class-weighted-ce", + "channels": ["exhaustive-4class"], + "expand": "none", + "k": cut, + "ks": list(ks), + "n_docs": len(docs), + "n_queries": len(per_query), + "n_docs_mapped": len(texts), + "skip_enrichment": True, + "ingest_graph": False, + "skip_ingest": True, + "rank_pool": False, + "rank_pool_ce": False, + "ce_model": model_name, + "ce_missing_text": missing_text, + "protocol": PROTOCOL, + "ingest": { + "status": "completed", + "n_docs": len(docs), + "tasks": [], + "reused": True, + }, + "graph_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "interaction_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "event_probe": None, + "search_error": None, + "metrics": metrics, + "queries": per_query, + } + + +def load_predict(model_dir: Path, *, max_length: int = 192) -> tuple[PredictFn, str]: + return load_4class_predict(model_dir, max_length=max_length) diff --git a/benchmarks/search/rank_pool.py b/benchmarks/search/rank_pool.py new file mode 100644 index 0000000..2d50e74 --- /dev/null +++ b/benchmarks/search/rank_pool.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +from search.dataset import split_corpus +from search.evaluate import candidate_pool_grades, candidate_pool_ids +from search.metrics import aggregate_query_metrics, mrr, ndcg_at_k, recall_at_k + +PLUGIN_DIR = Path(__file__).resolve().parents[2] / "plugins" / "search-rerank" + + +def _plugin_rerank(): + if str(PLUGIN_DIR) not in sys.path: + sys.path.insert(0, str(PLUGIN_DIR)) + from rerank import model_name, rerank + + return rerank, model_name() + + +def doc_texts(docs: list[dict[str, Any]]) -> dict[str, str]: + texts: dict[str, str] = {} + for doc in docs: + doc_id = str(doc.get("doc_id") or "") + if not doc_id: + continue + texts[doc_id] = str(doc.get("text") or doc.get("title") or "") + return texts + + +def score_ranked_pool( + ranked: list[str], + query: dict[str, Any], + *, + ks: tuple[int, ...] = (5, 10, 20), +) -> dict[str, float]: + gold = { + str(doc_id) + for doc_id, gain in candidate_pool_grades(query).items() + if float(gain) > 0 + } + if not gold: + gold = {str(item) for item in (query.get("gold_doc_ids") or [])} + grades = candidate_pool_grades(query) + metrics = {f"recall@{k}": recall_at_k(ranked, gold, k) for k in ks} + ndcg_ks = tuple(dict.fromkeys((*ks, 10, 20))) + for cut in ndcg_ks: + metrics[f"ndcg@{cut}"] = ndcg_at_k(ranked, gold, cut, grades=grades) + metrics["ndcg"] = ndcg_at_k(ranked, gold, max(len(ranked), 1), grades=grades) + metrics["mrr"] = mrr(ranked, gold) + return metrics + + +def run_ce_on_pool( + rows: list[dict[str, Any]], + *, + dataset_name: str, + ks: tuple[int, ...] = (5, 10, 20), + brain_id: str | None = None, +) -> dict[str, Any]: + docs, queries = split_corpus(rows) + texts = doc_texts(docs) + rerank, model = _plugin_rerank() + per_query: list[dict[str, Any]] = [] + missing_text = 0 + for query in queries: + pool = candidate_pool_ids(query) + candidates = [] + for doc_id in pool: + text = texts.get(doc_id) or "" + if not text: + missing_text += 1 + candidates.append({"id": doc_id, "text": text, "score": 0.0}) + ranked_rows = rerank( + str(query.get("query") or ""), + candidates, + len(candidates) or 1, + ) + ranked = [str(item.get("id") or "") for item in ranked_rows if item.get("id")] + seen = set(ranked) + for doc_id in pool: + if doc_id not in seen: + ranked.append(doc_id) + metrics = score_ranked_pool(ranked, query, ks=ks) + per_query.append( + { + "qid": query.get("qid"), + "query": query.get("query"), + "slice": query.get("slice") or "unspecified", + "gold_doc_ids": list(query.get("gold_doc_ids") or []), + "candidate_doc_ids": pool, + "pool_size": len(pool), + "pool_coverage": 1.0 if pool else None, + "missing_from_brain": [], + "hit_ids": ranked, + "metrics": metrics, + "retrieve_ms": 0.0, + "embed_ms": None, + "client_wall_ms": 0.0, + "n_hits": len(ranked), + } + ) + metrics = aggregate_query_metrics(per_query, ks=ks) if per_query else {} + return { + "status": "ok" if per_query else "failed", + "brain_id": brain_id, + "dataset": dataset_name, + "fusion": "none", + "rerank": "plugin:cross-encoder", + "channels": ["rank-pool-ce"], + "expand": "none", + "k": max((len(row.get("hit_ids") or []) for row in per_query), default=0), + "ks": list(ks), + "n_docs": len(docs), + "n_queries": len(per_query), + "n_docs_mapped": len(texts), + "skip_enrichment": True, + "ingest_graph": False, + "skip_ingest": True, + "rank_pool": True, + "rank_pool_ce": True, + "ce_model": model, + "ce_missing_text": missing_text, + "ingest": {"status": "completed", "n_docs": len(docs), "tasks": [], "reused": True}, + "graph_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "interaction_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "event_probe": None, + "search_error": None, + "protocol": ( + "ranking-in-pool CE over labeled candidates; " + f"model={model}; n is not Reddy ~4477 unless stated; " + "cite 0.857 only against this CE-on-pool protocol" + ), + "metrics": metrics, + "queries": per_query, + } diff --git a/benchmarks/search/rank_pool_4class.py b/benchmarks/search/rank_pool_4class.py new file mode 100644 index 0000000..0f1bcaa --- /dev/null +++ b/benchmarks/search/rank_pool_4class.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any, Callable + +from search.config import Settings, validate_brain_id +from search.dataset import load_records, split_corpus +from search.evaluate import candidate_pool_grades, candidate_pool_ids, ensure_run_dir +from search.finetune_esci_4class import ( + CLASS_GAINS, + DEFAULT_MODEL, + DEFAULT_OUT, + pick_device, + rank_doc_ids, + weighted_scores, +) +from search.metrics import aggregate_query_metrics, mrr, ndcg_at_k, recall_at_k +from search.report import print_report_table, write_report + +PredictFn = Callable[[list[tuple[str, str]]], Any] + + +def doc_texts(docs: list[dict[str, Any]]) -> dict[str, str]: + texts: dict[str, str] = {} + for doc in docs: + doc_id = str(doc.get("doc_id") or "") + if not doc_id: + continue + texts[doc_id] = str(doc.get("text") or doc.get("title") or "") + return texts + + +def score_ranked_pool( + ranked: list[str], + query: dict[str, Any], + *, + ks: tuple[int, ...] = (5, 10, 20), +) -> dict[str, float]: + gold = { + str(doc_id) + for doc_id, gain in candidate_pool_grades(query).items() + if float(gain) > 0 + } + if not gold: + gold = {str(item) for item in (query.get("gold_doc_ids") or [])} + grades = candidate_pool_grades(query) + metrics = {f"recall@{k}": recall_at_k(ranked, gold, k) for k in ks} + metrics["ndcg@10"] = ndcg_at_k(ranked, gold, 10, grades=grades) + metrics["ndcg@20"] = ndcg_at_k(ranked, gold, 20, grades=grades) + metrics["ndcg"] = ndcg_at_k(ranked, gold, max(len(ranked), 1), grades=grades) + metrics["mrr"] = mrr(ranked, gold) + return metrics + + +def load_4class_predict(model_dir: Path, *, max_length: int = 192) -> tuple[PredictFn, str]: + from sentence_transformers import CrossEncoder + + device = pick_device() + model = CrossEncoder( + str(model_dir), + num_labels=4, + max_length=max_length, + device=device, + default_activation_function=None, + ) + + def predict(pairs: list[tuple[str, str]]): + if not pairs: + return [] + return model.predict( + [list(pair) for pair in pairs], + batch_size=32, + apply_softmax=True, + show_progress_bar=False, + ) + + return predict, str(model_dir) + + +def run_4class_on_pool( + rows: list[dict[str, Any]], + *, + dataset_name: str, + predict: PredictFn, + model_name: str, + ks: tuple[int, ...] = (5, 10, 20), + brain_id: str | None = None, +) -> dict[str, Any]: + docs, queries = split_corpus(rows) + texts = doc_texts(docs) + per_query: list[dict[str, Any]] = [] + missing_text = 0 + for query in queries: + pool = candidate_pool_ids(query) + pairs: list[tuple[str, str]] = [] + for doc_id in pool: + text = texts.get(doc_id) or "" + if not text: + missing_text += 1 + pairs.append((str(query.get("query") or ""), text)) + probs = predict(pairs) if pairs else [] + scores = weighted_scores(probs, CLASS_GAINS) if len(pairs) else [] + if len(scores) < len(pool): + scores = list(scores) + [0.0] * (len(pool) - len(scores)) + ranked = rank_doc_ids(pool, scores) + metrics = score_ranked_pool(ranked, query, ks=ks) + per_query.append( + { + "qid": query.get("qid"), + "query": query.get("query"), + "slice": query.get("slice") or "unspecified", + "gold_doc_ids": list(query.get("gold_doc_ids") or []), + "candidate_doc_ids": pool, + "pool_size": len(pool), + "pool_coverage": 1.0 if pool else None, + "missing_from_brain": [], + "hit_ids": ranked, + "metrics": metrics, + "retrieve_ms": 0.0, + "embed_ms": None, + "client_wall_ms": 0.0, + "n_hits": len(ranked), + } + ) + metrics = aggregate_query_metrics(per_query, ks=ks) if per_query else {} + return { + "status": "ok" if per_query else "failed", + "brain_id": brain_id, + "dataset": dataset_name, + "fusion": "none", + "rerank": "4class-weighted-ce", + "channels": ["rank-pool-4class"], + "expand": "none", + "k": max((len(row.get("hit_ids") or []) for row in per_query), default=0), + "ks": list(ks), + "n_docs": len(docs), + "n_queries": len(per_query), + "n_docs_mapped": len(texts), + "skip_enrichment": True, + "ingest_graph": False, + "skip_ingest": True, + "rank_pool": True, + "rank_pool_ce": True, + "ce_model": model_name, + "ce_missing_text": missing_text, + "ingest": { + "status": "completed", + "n_docs": len(docs), + "tasks": [], + "reused": True, + }, + "graph_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "interaction_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "event_probe": None, + "search_error": None, + "protocol": ( + "ranking-in-pool 4-class CE over labeled candidates including I=0; " + "score=1.0*P(E)+0.1*P(S)+0.01*P(C); not MiniLM Exact→1; " + "n=74 is not Reddy public test n≈4477" + ), + "metrics": metrics, + "queries": per_query, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Ranking-in-pool eval with a 4-class weighted cross-encoder" + ) + parser.add_argument("--dataset", type=Path, required=True) + parser.add_argument("--model", type=Path, default=DEFAULT_OUT) + parser.add_argument("--run", default="search-esci-74-ce-pool-4class") + parser.add_argument("--brain", default="searchbenchesci74") + parser.add_argument("--env-file", type=Path, default=None) + parser.add_argument("--max-length", type=int, default=192) + args = parser.parse_args(argv) + if not args.dataset.exists(): + print(f"Missing {args.dataset}") + return 1 + if not args.model.exists(): + print(f"Missing model {args.model}") + return 1 + settings = Settings.load(args.env_file) + settings.brain_id = validate_brain_id(args.brain) + rows = load_records(args.dataset) + run_id, run_dir = ensure_run_dir(settings, args.run) + predict, model_name = load_4class_predict(args.model, max_length=args.max_length) + result = run_4class_on_pool( + rows, + dataset_name=args.dataset.name, + predict=predict, + model_name=model_name, + brain_id=settings.brain_id, + ) + report = write_report(run_dir, result) + print_report_table(report) + print(f"Wrote {run_dir / 'report.json'}") + print( + json.dumps( + { + "run_id": run_id, + "ce_model": result.get("ce_model"), + "ndcg@20": report.get("ndcg@20"), + "ndcg": report.get("ndcg"), + "n_queries": report.get("n_queries"), + "missing_text": result.get("ce_missing_text"), + "base_default": DEFAULT_MODEL, + }, + indent=2, + ) + ) + return 0 if report.get("status") == "ok" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/search/replay_fusion.py b/benchmarks/search/replay_fusion.py new file mode 100644 index 0000000..72843a2 --- /dev/null +++ b/benchmarks/search/replay_fusion.py @@ -0,0 +1,611 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from search.metrics import mrr, ndcg_at_k, recall_at_k + +RRF_K = 60 +EXPANSION_N = 10 +GRAPH_WEIGHTS = (0.1, 0.25, 0.5) +FUSED_NDCG = 0.653 +RECALL20_TARGET = 0.80 +_SLICE_JSONL = Path(__file__).resolve().parent.parent / "data" / "search_esci_slice.jsonl" + + +def _load_eval(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _gold(query: dict[str, Any]) -> set[str]: + gold = {str(item) for item in (query.get("gold_doc_ids") or []) if item} + grades = query.get("gold_grades") or {} + if isinstance(grades, dict): + for doc_id, gain in grades.items(): + if float(gain or 0) <= 0: + continue + gold.add(str(doc_id)) + if gold: + return gold + return {str(item) for item in (query.get("gold_chunk_ids") or []) if item} + + +def dataset_gold_grades(path: Path | None = None) -> dict[str, dict[str, float]]: + jsonl = path or _SLICE_JSONL + if not jsonl.exists(): + return {} + grades: dict[str, dict[str, float]] = {} + for line in jsonl.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row = json.loads(line) + if row.get("type") != "query": + continue + qid = str(row.get("qid") or "") + raw = row.get("gold_grades") or {} + if not qid or not isinstance(raw, dict): + continue + grades[qid] = { + str(doc_id): float(gain) + for doc_id, gain in raw.items() + if float(gain or 0) > 0 + } + return grades + + +def _grades( + query: dict[str, Any], + gold: set[str], + dataset_grades: dict[str, dict[str, float]] | None = None, +) -> dict[str, float]: + raw = query.get("gold_grades") or {} + if isinstance(raw, dict) and raw: + return { + str(doc_id): float(gain) + for doc_id, gain in raw.items() + if float(gain or 0) > 0 + } + qid = str(query.get("qid") or "") + from_dataset = (dataset_grades or {}).get(qid) or {} + if from_dataset: + return from_dataset + return {doc_id: 1.0 for doc_id in gold} + + +def _ranked(query: dict[str, Any]) -> list[str]: + return [str(item) for item in (query.get("hit_ids") or []) if item] + + +def drop_hubs(ranked: list[str]) -> list[str]: + return [item for item in ranked if not item.startswith("hub:")] + + +def keep_product_asins(ranked: list[str]) -> list[str]: + return [ + item + for item in ranked + if item and not item.startswith("hub:") and not item.startswith("evt:") + ] + + +def score_ranked( + ranked: list[str], + gold: set[str], + grades: dict[str, float], +) -> dict[str, float]: + return { + "ndcg@10": ndcg_at_k(ranked, gold, 10, grades=grades), + "recall@10": recall_at_k(ranked, gold, 10), + "recall@20": recall_at_k(ranked, gold, 20), + "mrr": mrr(ranked, gold), + "unique_docs@20": float(len({str(item) for item in ranked[:20] if item})), + } + + +def mean_metrics(rows: list[dict[str, float]]) -> dict[str, float]: + keys = ("ndcg@10", "recall@10", "recall@20", "mrr", "unique_docs@20") + if not rows: + return {key: 0.0 for key in keys} + present = [key for key in keys if all(key in row for row in rows)] + return {key: sum(row[key] for row in rows) / len(rows) for key in present} + + +def reciprocal_rank_fusion( + ranked_lists: list[list[str]], + *, + k: int = RRF_K, + weights: list[float] | None = None, +) -> list[str]: + scores: dict[str, float] = {} + for index, ranked in enumerate(ranked_lists): + weight = 1.0 if not weights else float(weights[index]) + for rank, item in enumerate(ranked): + if not item: + continue + scores[item] = scores.get(item, 0.0) + weight / (k + rank + 1) + return [ + item + for item, _ in sorted(scores.items(), key=lambda pair: pair[1], reverse=True) + ] + + +def chunk_to_doc_map(*queries: dict[str, Any]) -> dict[str, str]: + mapping: dict[str, str] = {} + for query in queries: + for hit in query.get("hits") or []: + if not isinstance(hit, dict): + continue + hid = str(hit.get("id") or "") + doc_id = str(hit.get("doc_id") or hid) + if hid: + mapping[hid] = doc_id + return mapping + + +def collapse_to_doc( + ids: list[str], + chunk_to_doc: dict[str, str], +) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for item in ids: + key = chunk_to_doc.get(str(item), str(item)) + if ( + not key + or key.startswith("hub:") + or key.startswith("evt:") + or key in seen + ): + continue + seen.add(key) + out.append(key) + return out + + +def _id_list(query: dict[str, Any], key: str, channel: str | None = None) -> list[str]: + dumped = [str(item) for item in (query.get(key) or []) if item] + if dumped: + return dumped + if not channel: + return [] + out: list[str] = [] + for hit in query.get("hits") or []: + if not isinstance(hit, dict): + continue + hid = str(hit.get("id") or "") + if hid and str(hit.get("channel") or "") == channel: + out.append(hid) + return out + + +def query_channel_lists(query: dict[str, Any]) -> dict[str, list[str]]: + return { + "dense": _id_list(query, "dense_ids"), + "bm25": _id_list(query, "bm25_ids"), + "passages": _id_list(query, "passage_ids", "passages"), + "entities": _id_list(query, "entity_ids", "entities"), + "communities": _id_list(query, "community_ids", "communities"), + } + + +def merge_query_lists( + graph_query: dict[str, Any], + passages_query: dict[str, Any] | None = None, + entities_query: dict[str, Any] | None = None, + communities_query: dict[str, Any] | None = None, +) -> dict[str, list[str]]: + graph = query_channel_lists(graph_query) + passages = query_channel_lists(passages_query or {}) + entities = query_channel_lists(entities_query or {}) + communities = query_channel_lists(communities_query or {}) + dense = graph["dense"] or passages["dense"] + bm25 = graph["bm25"] or passages["bm25"] + passage_ids = passages["passages"] or graph["passages"] + if not passage_ids and passages_query: + passage_ids = [ + str(hit.get("id") or "") + for hit in (passages_query.get("hits") or []) + if hit.get("id") + ] + if not passage_ids: + passage_ids = _ranked(passages_query) + entity_ids = graph["entities"] or entities["entities"] + if not entity_ids and entities_query: + entity_ids = [ + str(hit.get("id") or "") + for hit in (entities_query.get("hits") or []) + if hit.get("id") + ] + if not entity_ids: + entity_ids = _ranked(entities_query) + community_ids = graph["communities"] or communities["communities"] + if not community_ids and communities_query: + community_ids = [ + str(hit.get("id") or "") + for hit in (communities_query.get("hits") or []) + if hit.get("id") + ] + if not community_ids: + community_ids = _ranked(communities_query) + return { + "dense": dense, + "bm25": bm25, + "passages": passage_ids, + "entities": entity_ids, + "communities": community_ids, + } + + +def collapsed_lists( + lists: dict[str, list[str]], + chunk_to_doc: dict[str, str], +) -> dict[str, list[str]]: + collapsed = { + name: collapse_to_doc(ids, chunk_to_doc) for name, ids in lists.items() + } + if not collapsed["dense"] and not collapsed["bm25"] and collapsed["passages"]: + collapsed["passages_core"] = collapsed["passages"] + else: + collapsed["passages_core"] = reciprocal_rank_fusion( + [ids for ids in (collapsed["dense"], collapsed["bm25"]) if ids] + ) or collapsed["passages"] + return collapsed + + +def _by_qid(eval_result: dict[str, Any] | None) -> dict[str, dict[str, Any]]: + if not eval_result: + return {} + return { + str(row.get("qid")): row for row in (eval_result.get("queries") or []) + } + + +def _pair_queries( + graph_eval: dict[str, Any], + passages_eval: dict[str, Any], + entities_eval: dict[str, Any] | None = None, + communities_eval: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + passages_by = _by_qid(passages_eval) + entities_by = _by_qid(entities_eval) + communities_by = _by_qid(communities_eval) + slice_grades = dataset_gold_grades() + rows: list[dict[str, Any]] = [] + for graph_query in graph_eval.get("queries") or []: + qid = str(graph_query.get("qid") or "") + passages_query = passages_by.get(qid) or {} + entities_query = entities_by.get(qid) + communities_query = communities_by.get(qid) + gold = _gold(graph_query) or _gold(passages_query) + grades = _grades(graph_query, gold, slice_grades) + if not (graph_query.get("gold_grades") or {}): + grades = _grades(passages_query, gold, slice_grades) + mapping = chunk_to_doc_map( + graph_query, passages_query, entities_query or {}, communities_query or {} + ) + lists = merge_query_lists( + graph_query, passages_query, entities_query, communities_query + ) + collapsed = collapsed_lists(lists, mapping) + rows.append( + { + "qid": qid, + "gold": gold, + "grades": grades, + "lists": lists, + "collapsed": collapsed, + "passages_ranked": collapse_to_doc( + lists["passages"] or _ranked(passages_query), mapping + ), + "graph_ranked": _ranked(graph_query), + "passage_metrics": passages_query.get("metrics") or {}, + } + ) + return rows + + +def replay(graph_eval: dict[str, Any], passages_eval: dict[str, Any]) -> dict[str, Any]: + passages_by_qid = _by_qid(passages_eval) + hub_rows: list[dict[str, float]] = [] + asin_rows: list[dict[str, float]] = [] + per_query: list[dict[str, Any]] = [] + for query in graph_eval.get("queries") or []: + qid = str(query.get("qid") or "") + gold = _gold(query) + grades = _grades(query, gold) + ranked = _ranked(query) + hub = score_ranked(drop_hubs(ranked), gold, grades) + asin = score_ranked(keep_product_asins(ranked), gold, grades) + hub_rows.append(hub) + asin_rows.append(asin) + passage_row = passages_by_qid.get(qid) or {} + passage_metrics = passage_row.get("metrics") or {} + per_query.append( + { + "qid": qid, + "n_hits": len(ranked), + "n_hits_no_hub": len(drop_hubs(ranked)), + "n_product_ids": len(keep_product_asins(ranked)), + "hub_drop": hub, + "product_ids": asin, + "passages": { + "ndcg@10": float(passage_metrics.get("ndcg@10") or 0.0), + "recall@10": float(passage_metrics.get("recall@10") or 0.0), + "recall@20": float(passage_metrics.get("recall@20") or 0.0), + "mrr": float(passage_metrics.get("mrr") or 0.0), + }, + } + ) + return { + "n_queries": len(per_query), + "hub_drop": mean_metrics(hub_rows), + "product_ids": mean_metrics(asin_rows), + "queries": per_query, + } + + +def _core_and_graph(collapsed: dict[str, list[str]]) -> tuple[list[str], list[str], list[str]]: + core = [ids for ids in (collapsed["dense"], collapsed["bm25"]) if ids] + if not core and collapsed.get("passages_core"): + core = [collapsed["passages_core"]] + entities = collapsed.get("entities") or [] + communities = collapsed.get("communities") or [] + return core, entities, communities + + +def rank_collapse_rrf(collapsed: dict[str, list[str]]) -> list[str]: + core, entities, communities = _core_and_graph(collapsed) + lists = [*core] + if entities: + lists.append(entities) + if communities: + lists.append(communities) + if not lists: + return [] + if len(lists) == 1: + return list(lists[0]) + return reciprocal_rank_fusion(lists) + + +def rank_weighted_rrf(collapsed: dict[str, list[str]], graph_weight: float) -> list[str]: + core, entities, communities = _core_and_graph(collapsed) + lists: list[list[str]] = [*core] + weights: list[float] = [1.0] * len(core) + if entities: + lists.append(entities) + weights.append(graph_weight) + if communities: + lists.append(communities) + weights.append(graph_weight) + if not lists: + return [] + if len(lists) == 1: + return list(lists[0]) + return reciprocal_rank_fusion(lists, weights=weights) + + +def rank_expansion( + passage_docs: list[str], + collapsed: dict[str, list[str]], + *, + n: int = EXPANSION_N, +) -> list[str]: + head = list(passage_docs[:10]) + passage_set = set(passage_docs) + graph_docs: list[str] = [] + seen: set[str] = set() + for item in (collapsed.get("entities") or []) + (collapsed.get("communities") or []): + if item in seen: + continue + seen.add(item) + graph_docs.append(item) + novel = [item for item in graph_docs if item not in passage_set][:n] + novel_set = set(novel) + rest = [item for item in passage_docs[10:] if item not in novel_set] + return head + novel + rest + + +def rank_confirmation( + passage_docs: list[str], + collapsed: dict[str, list[str]], +) -> list[str]: + passage_set = set(passage_docs) + confirmed: list[str] = [] + seen: set[str] = set() + for item in (collapsed.get("entities") or []) + (collapsed.get("communities") or []): + if item not in passage_set or item in seen: + continue + seen.add(item) + confirmed.append(item) + if not confirmed: + return list(passage_docs) + return reciprocal_rank_fusion([passage_docs, confirmed]) + + +def replay_offline( + graph_eval: dict[str, Any], + passages_eval: dict[str, Any], + *, + entities_eval: dict[str, Any] | None = None, + communities_eval: dict[str, Any] | None = None, +) -> dict[str, Any]: + paired = _pair_queries( + graph_eval, passages_eval, entities_eval, communities_eval + ) + arms: dict[str, list[dict[str, float]]] = { + "passages": [], + "collapse-rrf": [], + "weighted-0.1": [], + "weighted-0.25": [], + "weighted-0.5": [], + "expansion-n10": [], + "confirmation": [], + } + per_query: list[dict[str, Any]] = [] + for row in paired: + gold = row["gold"] + grades = row["grades"] + collapsed = row["collapsed"] + passage_docs = row["passages_ranked"] + scored = { + "passages": score_ranked(passage_docs, gold, grades), + "collapse-rrf": score_ranked( + rank_collapse_rrf(collapsed), gold, grades + ), + "expansion-n10": score_ranked( + rank_expansion(passage_docs, collapsed, n=EXPANSION_N), + gold, + grades, + ), + "confirmation": score_ranked( + rank_confirmation(passage_docs, collapsed), gold, grades + ), + } + for weight in GRAPH_WEIGHTS: + name = f"weighted-{weight}" + scored[name] = score_ranked( + rank_weighted_rrf(collapsed, weight), gold, grades + ) + for name, metrics in scored.items(): + arms[name].append(metrics) + per_query.append({"qid": row["qid"], **scored}) + summary = {name: mean_metrics(rows) for name, rows in arms.items()} + return { + "n_queries": len(per_query), + "expansion_n": EXPANSION_N, + "graph_weights": list(GRAPH_WEIGHTS), + "primary": ["recall@20", "ndcg@10"], + "arms": summary, + "queries": per_query, + } + + +def pick_offline_winner(summary: dict[str, dict[str, float]]) -> str: + passages = summary.get("passages") or {} + passages_ndcg = float(passages.get("ndcg@10") or 0.0) + passages_r20 = float(passages.get("recall@20") or 0.0) + candidates: list[tuple[str, dict[str, float]]] = [] + for name, metrics in summary.items(): + if name == "passages": + continue + recall20 = float(metrics.get("recall@20") or 0.0) + ndcg = float(metrics.get("ndcg@10") or 0.0) + if recall20 + 1e-9 < max(RECALL20_TARGET, passages_r20): + continue + if ndcg + 1e-9 < max(FUSED_NDCG, passages_ndcg): + continue + candidates.append((name, metrics)) + if not candidates: + return "G08" + + def _key(item: tuple[str, dict[str, float]]) -> tuple[float, float]: + metrics = item[1] + return ( + float(metrics.get("recall@20") or 0.0), + float(metrics.get("ndcg@10") or 0.0), + ) + + name, _metrics = max(candidates, key=_key) + return name + + +def _print_metrics(label: str, metrics: dict[str, float]) -> None: + unique = metrics.get("unique_docs@20") + unique_s = f" unique@20={unique:.2f}" if unique is not None else "" + print( + f"{label:18} " + f"nDCG@10={metrics.get('ndcg@10', 0.0):.3f} " + f"R@10={metrics.get('recall@10', 0.0):.3f} " + f"R@20={metrics.get('recall@20', 0.0):.3f} " + f"MRR={metrics.get('mrr', 0.0):.3f}" + f"{unique_s}" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Offline rescore of fused search eval.json lists." + ) + parser.add_argument("--graph", required=True, help="Graph-arm eval.json") + parser.add_argument("--passages", required=True, help="Passages-arm eval.json") + parser.add_argument( + "--entities", + default=None, + help="Optional isolated entities eval.json", + ) + parser.add_argument( + "--communities", + default=None, + help="Optional isolated communities eval.json", + ) + parser.add_argument( + "--mode", + default="hub-drop", + choices=("hub-drop", "collapse-rrf", "gated", "all"), + ) + args = parser.parse_args(argv) + graph_eval = _load_eval(Path(args.graph)) + passages_eval = _load_eval(Path(args.passages)) + entities_eval = _load_eval(Path(args.entities)) if args.entities else None + communities_eval = ( + _load_eval(Path(args.communities)) if args.communities else None + ) + if args.mode in {"hub-drop", "all"}: + result = replay(graph_eval, passages_eval) + hub = result["hub_drop"] + print( + "hub-drop " + f"nDCG@10={hub['ndcg@10']:.3f} " + f"R@10={hub['recall@10']:.3f} " + f"R@20={hub['recall@20']:.3f} " + f"MRR={hub['mrr']:.3f}" + ) + asin = result["product_ids"] + print( + "product-ids " + f"nDCG@10={asin['ndcg@10']:.3f} " + f"R@10={asin['recall@10']:.3f} " + f"R@20={asin['recall@20']:.3f} " + f"MRR={asin['mrr']:.3f}" + ) + for row in result["queries"]: + delta = row["hub_drop"]["ndcg@10"] - row["passages"]["ndcg@10"] + print( + f"{row['qid']}: hub-drop nDCG {row['hub_drop']['ndcg@10']:.3f} " + f"(passages {row['passages']['ndcg@10']:.3f}, delta {delta:+.3f})" + ) + if args.mode in {"collapse-rrf", "gated", "all"}: + offline = replay_offline( + graph_eval, + passages_eval, + entities_eval=entities_eval, + communities_eval=communities_eval, + ) + names = ["passages", "collapse-rrf"] + if args.mode in {"gated", "all"}: + names.extend( + [ + "weighted-0.1", + "weighted-0.25", + "weighted-0.5", + "expansion-n10", + "confirmation", + ] + ) + print( + "primary recall@20 then nDCG@10; " + f"expansion_n={offline['expansion_n']}; " + f"graph_weights={offline['graph_weights']}" + ) + for name in names: + _print_metrics(name, offline["arms"][name]) + if args.mode in {"gated", "all"}: + winner = pick_offline_winner(offline["arms"]) + print(f"winner {winner}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/search/report.py b/benchmarks/search/report.py new file mode 100644 index 0000000..1bf5cc4 --- /dev/null +++ b/benchmarks/search/report.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +import json +import subprocess +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from rich.console import Console +from rich.table import Table + +from search.config import BENCHMARKS_ROOT + +console = Console() + +REPORTS_PATH = BENCHMARKS_ROOT / "REPORTS.json" +_BENCHMARK_ID = "search" +_BENCHMARK_NAME = "Search (hybrid BM25 + dense)" +_REPORTS_DESCRIPTION = ( + "BrainAPI benchmark results. Top published scores across suites. " + "Updated when a suite evaluate/report completes successfully." +) + + +def _git_sha() -> str | None: + try: + out = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=BENCHMARKS_ROOT.parent, + stderr=subprocess.DEVNULL, + text=True, + ) + return out.strip() or None + except Exception: + return None + + +def build_report(run_dir: Path, eval_result: dict[str, Any]) -> dict[str, Any]: + metrics = eval_result.get("metrics") or {} + status = str(eval_result.get("status") or "failed") + n_queries = int(eval_result.get("n_queries") or 0) + if status != "ok": + status = "failed" + if n_queries < 1: + status = "failed" + if (eval_result.get("ingest") or {}).get("status") not in { + "completed", + "partial_failed", + }: + status = "failed" + + report = { + "suite": _BENCHMARK_ID, + "status": status, + "run_id": run_dir.name, + "brain_id": eval_result.get("brain_id"), + "dataset": eval_result.get("dataset"), + "fusion": eval_result.get("fusion"), + "rerank": eval_result.get("rerank") or "none", + "channels": eval_result.get("channels") or ["passages"], + "rank_pool": bool(eval_result.get("rank_pool")), + "rank_pool_ce": bool(eval_result.get("rank_pool_ce")), + "ce_model": eval_result.get("ce_model"), + "skip_enrichment": eval_result.get("skip_enrichment"), + "n_queries": n_queries, + "n_docs": eval_result.get("n_docs"), + "k": eval_result.get("k"), + "ndcg@10": metrics.get("ndcg@10"), + "ndcg@20": metrics.get("ndcg@20"), + "ndcg@50": metrics.get("ndcg@50"), + "ndcg@100": metrics.get("ndcg@100"), + "ndcg": metrics.get("ndcg"), + "recall@10": metrics.get("recall@10"), + "recall@20": metrics.get("recall@20"), + "recall@50": metrics.get("recall@50"), + "recall@100": metrics.get("recall@100"), + "mrr": metrics.get("mrr"), + "pool_coverage": metrics.get("pool_coverage"), + "p50_retrieve_ms": metrics.get("p50_retrieve_ms"), + "p95_retrieve_ms": metrics.get("p95_retrieve_ms"), + "metrics": metrics, + "ingest": eval_result.get("ingest"), + "git_sha": _git_sha(), + "recorded_at": datetime.now(timezone.utc).isoformat(), + "protocol": eval_result.get("protocol") + or ( + "labeled toy search: POST /ingest/ → POST /retrieve/search " + f"(fusion={eval_result.get('fusion') or 'rrf'}, " + f"rerank={eval_result.get('rerank') or 'none'}, " + f"rank_pool={bool(eval_result.get('rank_pool'))})" + ), + } + return report + + +def write_report(run_dir: Path, eval_result: dict[str, Any]) -> dict[str, Any]: + report = build_report(run_dir, eval_result) + (run_dir / "eval.json").write_text( + json.dumps(eval_result, indent=2) + "\n", encoding="utf-8" + ) + (run_dir / "report.json").write_text( + json.dumps(report, indent=2) + "\n", encoding="utf-8" + ) + md = render_markdown(report) + (run_dir / "report.md").write_text(md, encoding="utf-8") + if report.get("status") == "ok": + update_reports_json(report) + return report + + +def render_markdown(report: dict[str, Any]) -> str: + lines = [ + f"# Search report — `{report.get('run_id')}`", + "", + f"- status: **{report.get('status')}**", + f"- dataset: `{report.get('dataset')}`", + f"- brain_id: `{report.get('brain_id')}`", + f"- fusion: `{report.get('fusion')}`", + f"- rerank: `{report.get('rerank')}`", + f"- channels: `{','.join(report.get('channels') or ['passages'])}`", + f"- rank_pool: `{report.get('rank_pool')}`", + f"- skip_enrichment: `{report.get('skip_enrichment')}`", + f"- k: {report.get('k')}", + f"- n_queries: {report.get('n_queries')}", + f"- nDCG@10: {report.get('ndcg@10')}", + f"- nDCG@20: {report.get('ndcg@20')}", + f"- nDCG@50: {report.get('ndcg@50')}", + f"- nDCG (full list): {report.get('ndcg')}", + f"- Recall@10: {report.get('recall@10')}", + f"- Recall@20: {report.get('recall@20')}", + f"- Recall@50: {report.get('recall@50')}", + f"- pool_coverage: {report.get('pool_coverage')}", + f"- MRR: {report.get('mrr')}", + f"- p50 retrieve ms (ex-embed): {report.get('p50_retrieve_ms')}", + f"- p95 retrieve ms (ex-embed): {report.get('p95_retrieve_ms')}", + "", + "Never mutate LoCoMo / BEAM / LongMemEval / RecSys brains or their ledger rows.", + "", + ] + return "\n".join(lines) + + +def print_report_table(report: dict[str, Any]) -> None: + table = Table(title=f"Search {report.get('run_id')}") + table.add_column("metric") + table.add_column("value") + for key in ( + "status", + "dataset", + "brain_id", + "fusion", + "rerank", + "channels", + "rank_pool", + "skip_enrichment", + "n_queries", + "k", + "ndcg@10", + "ndcg@20", + "ndcg@50", + "ndcg", + "recall@10", + "recall@20", + "recall@50", + "pool_coverage", + "mrr", + "p50_retrieve_ms", + "p95_retrieve_ms", + ): + table.add_row(key, str(report.get(key))) + console.print(table) + + +def entry_from_report(report: dict[str, Any]) -> dict[str, Any] | None: + if report.get("status") != "ok": + return None + n_queries = int(report.get("n_queries") or 0) + if n_queries < 1: + return None + entry = { + "run_id": report.get("run_id"), + "brain": report.get("brain_id"), + "dataset": report.get("dataset"), + "fusion": report.get("fusion"), + "rerank": report.get("rerank") or "none", + "channels": report.get("channels") or ["passages"], + "rank_pool": bool(report.get("rank_pool")), + "rank_pool_ce": bool(report.get("rank_pool_ce")), + "n_queries": n_queries, + "k": report.get("k"), + "ndcg@10": report.get("ndcg@10"), + "ndcg@20": report.get("ndcg@20"), + "ndcg@50": report.get("ndcg@50"), + "ndcg": report.get("ndcg"), + "recall@10": report.get("recall@10"), + "recall@20": report.get("recall@20"), + "recall@50": report.get("recall@50"), + "mrr": report.get("mrr"), + "pool_coverage": report.get("pool_coverage"), + "p50_retrieve_ms": report.get("p50_retrieve_ms"), + "p95_retrieve_ms": report.get("p95_retrieve_ms"), + "git_sha": report.get("git_sha"), + "report_path": f"runs/{report.get('run_id')}/report.json", + "recorded_at": report.get("recorded_at"), + "protocol": report.get("protocol"), + } + if report.get("brain_id") == "searchbenchwandsgraph": + entry["claim"] = "architecture-demo" + if report.get("brain_id") == "searchbenchjdslice": + return None + return entry + + +def _empty_reports_ledger() -> dict[str, Any]: + return { + "schema_version": 2, + "updated_at": datetime.now(timezone.utc).isoformat(), + "description": _REPORTS_DESCRIPTION, + "benchmarks": {}, + } + + +def update_reports_json(report: dict[str, Any]) -> None: + entry = entry_from_report(report) + if entry is None: + return + + if REPORTS_PATH.exists(): + data = json.loads(REPORTS_PATH.read_text(encoding="utf-8")) + else: + data = _empty_reports_ledger() + + suite = data.setdefault("benchmarks", {}).setdefault( + _BENCHMARK_ID, + {"name": _BENCHMARK_NAME, "leaderboard": []}, + ) + leaderboard = list(suite.get("leaderboard") or []) + run_id = entry["run_id"] + lb_idx = next( + (i for i, row in enumerate(leaderboard) if row.get("run_id") == run_id), + None, + ) + if lb_idx is not None: + leaderboard[lb_idx] = entry + else: + leaderboard.append(entry) + + suite["name"] = _BENCHMARK_NAME + suite["leaderboard"] = leaderboard + data["benchmarks"][_BENCHMARK_ID] = suite + data["updated_at"] = datetime.now(timezone.utc).isoformat() + data["description"] = data.get("description") or _REPORTS_DESCRIPTION + + REPORTS_PATH.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp( + prefix="REPORTS.", + suffix=".json.tmp", + dir=str(REPORTS_PATH.parent), + ) + try: + with open(fd, "w", encoding="utf-8") as fh: + json.dump(data, fh, indent=2) + fh.write("\n") + Path(tmp_name).replace(REPORTS_PATH) + finally: + tmp_path = Path(tmp_name) + if tmp_path.exists(): + tmp_path.unlink(missing_ok=True) diff --git a/benchmarks/search/rerank_retrieved.py b/benchmarks/search/rerank_retrieved.py new file mode 100644 index 0000000..a5ad576 --- /dev/null +++ b/benchmarks/search/rerank_retrieved.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from typing import Any + +from search.evaluate import candidate_pool_grades +from search.metrics import aggregate_query_metrics, mrr, ndcg_at_k, recall_at_k +from search.rank_pool import _plugin_rerank, doc_texts, score_ranked_pool +from search.dataset import split_corpus + + +def _first_stage_metrics( + ranked: list[str], + query_row: dict[str, Any], + *, + ks: tuple[int, ...], +) -> dict[str, float]: + gold = {str(item) for item in (query_row.get("gold_doc_ids") or [])} + grades = { + str(doc_id): float(gain) + for doc_id, gain in (query_row.get("gold_grades") or {}).items() + if float(gain or 0) > 0 + } + metrics = {f"recall@{cut}": recall_at_k(ranked, gold, cut) for cut in ks} + ndcg_ks = tuple(dict.fromkeys((*ks, 10, 20))) + for cut in ndcg_ks: + metrics[f"ndcg@{cut}"] = ndcg_at_k(ranked, gold, cut, grades=grades or None) + metrics["ndcg"] = ndcg_at_k(ranked, gold, max(len(ranked), 1), grades=grades or None) + metrics["mrr"] = mrr(ranked, gold) + return metrics + + +def run_ce_on_retrieved( + eval_result: dict[str, Any], + rows: list[dict[str, Any]], + *, + dataset_name: str, + ks: tuple[int, ...] | None = None, + brain_id: str | None = None, +) -> dict[str, Any]: + docs, queries = split_corpus(rows) + texts = doc_texts(docs) + by_qid = {str(query.get("qid") or ""): query for query in queries} + rerank, model = _plugin_rerank() + source_ks = tuple(int(item) for item in (eval_result.get("ks") or (5, 10, 20))) + ks = ks or source_ks + rank_pool = bool(eval_result.get("rank_pool")) + per_query: list[dict[str, Any]] = [] + missing_text = 0 + for row in eval_result.get("queries") or []: + hit_ids = [str(item) for item in (row.get("hit_ids") or []) if item] + qid = str(row.get("qid") or "") + source = by_qid.get(qid) or {} + candidates = [] + for doc_id in hit_ids: + text = texts.get(doc_id) or "" + if not text: + missing_text += 1 + candidates.append({"id": doc_id, "text": text, "score": 0.0}) + ranked_rows = rerank( + str(row.get("query") or source.get("query") or ""), + candidates, + len(candidates) or 1, + ) + ranked = [str(item.get("id") or "") for item in ranked_rows if item.get("id")] + seen = set(ranked) + for doc_id in hit_ids: + if doc_id not in seen: + ranked.append(doc_id) + if rank_pool and source: + metrics = score_ranked_pool(ranked, source, ks=ks) + else: + metrics = _first_stage_metrics(ranked, row, ks=ks) + per_query.append( + { + "qid": row.get("qid"), + "query": row.get("query"), + "slice": row.get("slice") or "unspecified", + "gold_doc_ids": list(row.get("gold_doc_ids") or []), + "gold_grades": row.get("gold_grades") or {}, + "candidate_doc_ids": row.get("candidate_doc_ids") or [], + "pool_size": row.get("pool_size"), + "pool_coverage": row.get("pool_coverage"), + "missing_from_brain": row.get("missing_from_brain") or [], + "hit_ids": ranked, + "metrics": metrics, + "retrieve_ms": float(row.get("retrieve_ms") or 0.0), + "embed_ms": row.get("embed_ms"), + "client_wall_ms": float(row.get("client_wall_ms") or 0.0), + "n_hits": len(ranked), + } + ) + metrics = aggregate_query_metrics(per_query, ks=ks) if per_query else {} + if rank_pool and per_query: + coverages = [ + float(row["pool_coverage"]) + for row in per_query + if row.get("pool_coverage") is not None + ] + if coverages: + metrics["pool_coverage"] = sum(coverages) / len(coverages) + metrics["missing_from_brain"] = sum( + len(row.get("missing_from_brain") or []) for row in per_query + ) / len(per_query) + retrieve_k = int(eval_result.get("k") or 0) + return { + "status": "ok" if per_query else "failed", + "brain_id": brain_id or eval_result.get("brain_id"), + "dataset": dataset_name, + "fusion": eval_result.get("fusion") or "rrf", + "rerank": "harness:cross-encoder", + "channels": eval_result.get("channels") or ["passages"], + "expand": eval_result.get("expand") or "none", + "k": retrieve_k, + "ks": list(ks), + "n_docs": eval_result.get("n_docs") or len(docs), + "n_queries": len(per_query), + "n_docs_mapped": len(texts), + "skip_enrichment": True, + "ingest_graph": False, + "skip_ingest": True, + "rank_pool": rank_pool, + "rank_pool_ce": False, + "ce_model": model, + "ce_missing_text": missing_text, + "source_run": eval_result.get("run_id"), + "ingest": { + "status": "completed", + "n_docs": eval_result.get("n_docs") or len(docs), + "tasks": [], + "reused": True, + }, + "graph_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "interaction_ingest": {"status": "skipped", "n_triples": 0, "tasks": []}, + "event_probe": None, + "search_error": None, + "protocol": ( + "harness CE over retrieved first-stage hits " + f"(k={retrieve_k}, model={model}); not CE-on-pool; " + "do not cite Reddy 0.857" + ), + "metrics": metrics, + "queries": per_query, + } diff --git a/console/package-lock.json b/console/package-lock.json index 4d482a8..bd4669b 100644 --- a/console/package-lock.json +++ b/console/package-lock.json @@ -17,7 +17,7 @@ "lumen-ui-kit": "^0.1.0", "react": "^19.1.0", "react-dom": "^19.1.0", - "react-router-dom": "^7.6.0", + "react-router-dom": "^7.18.2", "vis-data": "^8.0.3", "vis-network": "^10.0.2" }, @@ -3335,9 +3335,9 @@ } }, "node_modules/react-router": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.16.0.tgz", - "integrity": "sha512-wArC8lVyJb3+jM9OpDyW6hLCizACWkvQR/sSGqSs+o5uEXEtGlqdZ4v8hENR3Jad6i+LRkK93q/+bQAcvl6V1A==", + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/react-router/-/react-router-7.18.2.tgz", + "integrity": "sha512-aUVMjFm3GAPTTZL7oYr5E7ETiqfQCHRLH+B+5afnICvf0r7kkK4eR6SMuwbSTJw/7t+12khT/Kahij49fqOCIg==", "license": "MIT", "dependencies": { "cookie": "^1.0.1", @@ -3357,12 +3357,12 @@ } }, "node_modules/react-router-dom": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.16.0.tgz", - "integrity": "sha512-kMUAbimWB5FVbF4Bce4bJsiKJWLIUHq/mEG8+CFDnCSgltptBiG5nguducmsJeGKytlCvQud9Qhzpn49iduTlA==", + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-7.18.2.tgz", + "integrity": "sha512-AIKJ/jgGlFb3EbfCXk5Gzshiwt+l3mqbCrNjmEWMMjqQxNJ3svBa6bgzFyCC2Sw3RA0VWF1kg3uQf2OFhxb8hw==", "license": "MIT", "dependencies": { - "react-router": "7.16.0" + "react-router": "7.18.2" }, "engines": { "node": ">=20.0.0" diff --git a/console/package.json b/console/package.json index 043fb26..2181325 100644 --- a/console/package.json +++ b/console/package.json @@ -18,7 +18,7 @@ "lumen-ui-kit": "^0.1.0", "react": "^19.1.0", "react-dom": "^19.1.0", - "react-router-dom": "^7.6.0", + "react-router-dom": "^7.18.2", "vis-data": "^8.0.3", "vis-network": "^10.0.2" }, diff --git a/deploy/README.md b/deploy/README.md index 97741a6..a9d3d57 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -17,30 +17,110 @@ Choose **heavy** or **light** when bringing up a new deployment. ## Quick start ```bash -# 1. Pick a profile and install env -cp deploy/env.light.example /root/.env # or env.heavy.example -# edit secrets: BRAINPAT_TOKEN, DEEPSEEK_API_KEY, OPENAI_API_KEY, … +# 1. Copy a profile template outside the repository and replace every secret. +install -m 600 deploy/env.light.example /etc/brainapi/light.env +# Set BRAINAPI_ENV_FILE to the absolute file path in /etc/brainapi/light.env. -# 2. Bring up the matching compose -docker compose -f deploy/docker-compose.light.yaml up -d +# 2. Validate first, then start and wait for health. +docker compose --env-file /etc/brainapi/light.env \ + -f deploy/docker-compose.light.yaml config -q +docker compose --env-file /etc/brainapi/light.env \ + -f deploy/docker-compose.light.yaml up -d --wait # or -docker compose -f deploy/docker-compose.heavy.yaml up -d +docker compose --env-file /etc/brainapi/heavy.env \ + -f deploy/docker-compose.heavy.yaml up -d --wait ``` +Only nginx publishes host ports (`80` and `443`). The API, embedded Console, +and MCP are reached through `/`, `/console/`, and `/mcp`. The checked-in TLS +server rejects handshakes until an operator mounts a certificate-specific +nginx server file; do not expose port 80 beyond a trusted proxy without HTTPS. + ## Light sizing | Service | Heavy | Light | | --- | --- | --- | -| nginx | 128m / 0.2 CPU | 16m / 0.025 CPU | -| redis | 2048m / 0.5 CPU (`maxmemory 1500mb`) | 256m / 0.0625 CPU (`maxmemory 192mb`) | -| postgres | — (not in heavy compose) | 512m / 0.25 CPU | +| nginx | 128m / 0.2 CPU | 64m / 0.1 CPU | +| redis | 2048m / 0.5 CPU (`maxmemory 1500mb`) | 256m / 0.1 CPU (`maxmemory 192mb`) | +| postgres | — (not in heavy compose) | 768m / 0.35 CPU | | neo4j / milvus / mongo / etcd / minio | present | omitted | -| api / worker / mcp | 4g / 2 CPU each | 512m / 0.25 CPU each | +| api / worker / mcp | 2–4g / 1–2 CPU | 768m / 0.5 CPU each | | celery `--concurrency` | 2 | 1 | | `CELERY_WORKER_CONCURRENCY` | 4 | 1 | ## Notes -- Both profiles expect `/root/.env` and nginx TLS paths under `/srv/nginx` (same as the legacy root `example-docker-compose.yaml`). -- Heavy still mounts `/root/gcp_credentials.json` for Vertex; light does not need GCP credentials. -- Do not mix profiles on the same host without wiping volumes — DB backends differ. +- `BRAINAPI_IMAGE`, `BRAINAPI_ENV_FILE`, Redis credentials, and profile backend + credentials are required and fail during Compose interpolation when absent. +- Mount cloud credentials with a private operator-owned Compose override; no + host-specific credential path is present in the production profiles. +- Do not mix profiles under the same Compose project name. Their volume layouts + and restore order differ. + +## Backup and restore + +Backups default to `/srv/brainapi/backups` and are created with mode `0700`. +They contain sensitive application data (including stored brain credentials), +so encrypt and access-control the backup directory. The generated manifest +contains only profile/version metadata, service names, restore order, and +checksums; it never contains secrets. + +```bash +deploy/brainapi-backup backup --profile light \ + --env-file /etc/brainapi/light.env --project-name brainapi +deploy/brainapi-backup verify --profile light \ + --env-file /etc/brainapi/light.env --archive /srv/brainapi/backups/light-UTCSTAMP +deploy/brainapi-backup restore --profile light \ + --env-file /etc/brainapi/light.env --archive /srv/brainapi/backups/light-UTCSTAMP +``` + +Backup stops nginx, API, MCP, and workers before capture. Restore refuses a +running stack, a profile/image mismatch, a checksum mismatch, or any non-empty +target volume. Use a documented maintenance window and retain the generated +manifest with the release artifacts. Light backups include the system registry +plus an individual custom-format dump for every `brain_*` PostgreSQL database. + +## Production validation + +The CI-only Compose override supplies a deterministic OpenAI-compatible model +stub; it is not a production deployment file. It lets both profiles exercise +the complete storage and gateway path without external model credentials: + +```bash +docker compose -f deploy/docker-compose.light.yaml \ + -f deploy/docker-compose.ci.yaml up -d --wait +python scripts/production_smoke.py exercise --profile light \ + --system-token "$BRAINPAT_TOKEN" --artifact-dir release-artifacts +``` + +After backup, clean-volume restore, and restart, run the same tool with +`verify-restore`. CI retains the profile smoke, latency, restore, backup +manifest, image audit, and SBOM evidence. The tag workflow downloads the exact +commit's successful light and heavy artifacts and runs the release-readiness +gate before any image can be published. Quality CI exports one `linux/amd64` +candidate archive; the heavy runner loads that same archive, and the tag +workflow publishes it unchanged after verifying its recorded image ID. + +## Public documentation sandbox + +The anonymous sandbox is an opt-in extension of the light profile. Merge +[`env.public-demo.example`](env.public-demo.example) into the deployment's +private environment, deploy the current API and worker image, and seed only the +dedicated `agentdemo` brain: + +```bash +docker compose --env-file /etc/brainapi/light.env -f deploy/docker-compose.light.yaml up -d +docker compose --env-file /etc/brainapi/light.env -f deploy/docker-compose.light.yaml exec brainapi \ + python scripts/seed_public_demo.py --api-url http://localhost:8000 +``` + +The seed command downloads the published V2 documentation, records its SHA-256 +marker, skips an already-current corpus, and polls the asynchronous ingestion +task. It requires the private system `BRAINPAT_TOKEN` inside the deployment; do +not expose that token through the gateway. After the first successful seed, +`BRAIN_CREATION_ALLOWED` may be returned to the operator's normal setting. + +Expose the API origin as `api.brain-api.dev`, with TLS, and put the Cloudflare +gateway in front of root-domain `/api/*`. The application itself deliberately +does not implement client rate limiting; the checked-in gateway owns the +30-request/60-second demo limit. diff --git a/deploy/backup_restore.py b/deploy/backup_restore.py new file mode 100755 index 0000000..e371dd7 --- /dev/null +++ b/deploy/backup_restore.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""Offline-consistent BrainAPI profile backup, verification, and restore.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shlex +import stat +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import BinaryIO + + +ROOT = Path(__file__).resolve().parents[1] +DEPLOY = ROOT / "deploy" +ARCHIVER_IMAGE = "alpine:3.21.3" +APP_SERVICES = ("nginx", "brainapi", "brainapi-mcp", "brainapi-worker") +PROFILE_VOLUMES = { + "light": ("redis-data", "brainapi-plugins"), + "heavy": ( + "redis-data", + "milvus-data", + "etcd-data", + "minio-data", + "brainapi-plugins", + ), +} +PROFILE_ALL_VOLUMES = { + "light": ("redis-data", "postgres-data", "brainapi-plugins"), + "heavy": ( + "redis-data", + "neo4j-data", + "neo4j-logs", + "neo4j-plugins", + "etcd-data", + "minio-data", + "milvus-data", + "mongo-data", + "mongo-config", + "brainapi-plugins", + ), +} +RESTORE_ORDER = { + "light": ("redis-data", "postgres", "brainapi-plugins", "stack"), + "heavy": ( + "redis-data", + "etcd-data", + "minio-data", + "milvus-data", + "neo4j", + "mongo", + "brainapi-plugins", + "stack", + ), +} + + +def _load_env(path: Path) -> dict[str, str]: + values = dict(os.environ) + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + values.setdefault(key.strip(), value.strip().strip("'\"")) + return values + + +class Stack: + def __init__( + self, + profile: str, + env_file: Path, + project: str, + dry_run: bool, + compose_files: list[Path] | None = None, + ): + self.profile = profile + self.env_file = env_file.resolve() + self.project = project + self.dry_run = dry_run + self.compose_files = [DEPLOY / f"docker-compose.{profile}.yaml"] + self.compose_files.extend(path.resolve() for path in (compose_files or [])) + self.env = _load_env(self.env_file) + self.env["BRAINAPI_ENV_FILE"] = str(self.env_file) + self.base = [ + "docker", + "compose", + "--project-name", + project, + "--env-file", + str(self.env_file), + ] + for compose_file in self.compose_files: + self.base.extend(["-f", str(compose_file)]) + + def run( + self, + args: list[str], + *, + stdout: BinaryIO | int | None = None, + stdin: BinaryIO | int | None = None, + capture: bool = False, + check: bool = True, + ) -> subprocess.CompletedProcess: + command = [*self.base, *args] + if self.dry_run: + print("+", shlex.join(command)) + return subprocess.CompletedProcess(command, 0, stdout="" if capture else None) + return subprocess.run( + command, + env=self.env, + stdin=stdin, + stdout=subprocess.PIPE if capture else stdout, + stderr=subprocess.PIPE if capture else None, + text=capture, + check=check, + ) + + def docker( + self, + args: list[str], + *, + stdout: BinaryIO | int | None = None, + stdin: BinaryIO | int | None = None, + capture: bool = False, + check: bool = True, + ) -> subprocess.CompletedProcess: + command = ["docker", *args] + if self.dry_run: + print("+", shlex.join(command)) + return subprocess.CompletedProcess(command, 0, stdout="" if capture else None) + return subprocess.run( + command, + env=self.env, + stdin=stdin, + stdout=subprocess.PIPE if capture else stdout, + stderr=subprocess.PIPE if capture else None, + text=capture, + check=check, + ) + + def volume_name(self, key: str) -> str: + return f"{self.project}_{key}" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _write_stream(stack: Stack, args: list[str], destination: Path) -> None: + if stack.dry_run: + stack.run(args) + destination.touch() + return + with destination.open("wb") as output: + stack.run(args, stdout=output) + + +def _dump_postgres_brain_databases(stack: Stack, destination: Path) -> None: + script = r''' +set -eu +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +psql -U "$POSTGRES_USER" -d postgres -Atc \ + "SELECT datname FROM pg_database WHERE datistemplate = false AND left(datname, 6) = 'brain_' ORDER BY datname" | +while IFS= read -r database; do + test -n "$database" || continue + pg_dump -U "$POSTGRES_USER" -d "$database" --format=custom --file="$tmp/$database.dump" +done +tar -C "$tmp" -czf - . +'''.strip() + _write_stream( + stack, + ["exec", "-T", "postgres", "sh", "-c", script], + destination, + ) + + +def _restore_postgres_brain_databases( + stack: Stack, archive: Path, pg_user: str +) -> None: + with tempfile.TemporaryDirectory(prefix="brainapi-postgres-restore-") as tmp: + tmp_path = Path(tmp) + if not stack.dry_run: + subprocess.run( + ["tar", "-C", str(tmp_path), "-xzf", str(archive)], check=True + ) + dumps = sorted(tmp_path.glob("brain_*.dump")) + if stack.dry_run: + dumps = [tmp_path / "brain_example.dump"] + for dump in dumps: + database = dump.stem + if not database.replace("_", "").isalnum() or not database.startswith( + "brain_" + ): + raise RuntimeError(f"Invalid PostgreSQL brain dump name: {dump.name}") + stack.run( + ["exec", "-T", "postgres", "createdb", "-U", pg_user, database], + check=False, + ) + if stack.dry_run: + stack.run( + [ + "exec", + "-T", + "postgres", + "pg_restore", + "-U", + pg_user, + "-d", + database, + "--clean", + "--if-exists", + ] + ) + else: + with dump.open("rb") as source: + stack.run( + [ + "exec", + "-T", + "postgres", + "pg_restore", + "-U", + pg_user, + "-d", + database, + "--clean", + "--if-exists", + ], + stdin=source, + ) + + +def _archive_volume(stack: Stack, key: str, destination: Path) -> None: + if stack.dry_run: + stack.docker(["run", "--rm", "-v", f"{stack.volume_name(key)}:/source:ro", ARCHIVER_IMAGE, "tar", "-C", "/source", "-czf", "-", "."]) + destination.touch() + return + with destination.open("wb") as output: + stack.docker( + [ + "run", + "--rm", + "-v", + f"{stack.volume_name(key)}:/source:ro", + ARCHIVER_IMAGE, + "tar", + "-C", + "/source", + "-czf", + "-", + ".", + ], + stdout=output, + ) + + +def _restore_volume(stack: Stack, key: str, archive: Path) -> None: + if stack.dry_run: + stack.docker(["run", "--rm", "-i", "-v", f"{stack.volume_name(key)}:/target", ARCHIVER_IMAGE, "tar", "-C", "/target", "-xzf", "-"]) + return + with archive.open("rb") as source: + stack.docker( + [ + "run", + "--rm", + "-i", + "-v", + f"{stack.volume_name(key)}:/target", + ARCHIVER_IMAGE, + "tar", + "-C", + "/target", + "-xzf", + "-", + ], + stdin=source, + ) + + +def _running_services(stack: Stack) -> list[str]: + result = stack.run( + ["ps", "--services", "--filter", "status=running"], capture=True + ) + return [line for line in (result.stdout or "").splitlines() if line] + + +def _current_image(stack: Stack) -> str: + result = stack.run(["config", "--format", "json"], capture=True) + if stack.dry_run: + return stack.env.get("BRAINAPI_IMAGE", "dry-run") + config = json.loads(result.stdout) + return str(config["services"]["brainapi"]["image"]) + + +def _manifest(backup_dir: Path) -> dict: + return json.loads((backup_dir / "manifest.json").read_text(encoding="utf-8")) + + +def verify_backup(backup_dir: Path, expected_profile: str | None = None) -> dict: + manifest = _manifest(backup_dir) + if manifest.get("format_version") != 1: + raise RuntimeError("Unsupported backup manifest format") + if expected_profile and manifest.get("profile") != expected_profile: + raise RuntimeError("Backup profile does not match the requested profile") + for name, expected in manifest.get("checksums", {}).items(): + path = backup_dir / name + if not path.is_file(): + raise RuntimeError(f"Missing backup artifact: {name}") + actual = _sha256(path) + if actual != expected: + raise RuntimeError(f"Checksum mismatch: {name}") + return manifest + + +def backup(stack: Stack, backup_root: Path) -> Path: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + backup_root = backup_root.expanduser().resolve() + backup_root.mkdir(parents=True, mode=0o700, exist_ok=True) + backup_root.chmod(stat.S_IRWXU) + destination = backup_root / f"{stack.profile}-{timestamp}" + destination.mkdir(parents=True, mode=0o700, exist_ok=False) + destination.chmod(stat.S_IRWXU) + running = _running_services(stack) + + try: + stack.run(["stop", *APP_SERVICES]) + if stack.profile == "light": + pg_user = stack.env.get("POSTGRES_USERNAME", "brainapi") + pg_db = stack.env.get("POSTGRES_SYSTEM_DATABASE", "brainapi") + _write_stream( + stack, + ["exec", "-T", "postgres", "pg_dump", "-U", pg_user, "-d", pg_db, "--format=custom"], + destination / "postgres.dump", + ) + _dump_postgres_brain_databases( + stack, destination / "postgres-brains.tar.gz" + ) + stack.run(["exec", "-T", "redis", "sh", "-c", 'redis-cli -a "$REDIS_PASSWORD" SAVE']) + stack.run(["stop", "redis", "postgres"]) + else: + mongo_user = stack.env.get("MONGO_USERNAME", "brainapi") + _write_stream( + stack, + ["exec", "-T", "mongo", "sh", "-c", f'mongodump --archive --gzip --username {shlex.quote(mongo_user)} --password "$MONGO_INITDB_ROOT_PASSWORD" --authenticationDatabase admin'], + destination / "mongo.archive.gz", + ) + stack.run(["exec", "-T", "redis", "sh", "-c", 'redis-cli -a "$REDIS_PASSWORD" SAVE']) + stack.run(["stop", "neo4j", "mongo", "milvus", "etcd", "minio", "redis"]) + neo_dir = destination / "neo4j" + neo_dir.mkdir(mode=0o700) + stack.run( + ["run", "--rm", "--no-deps", "-v", f"{neo_dir}:/backup", "neo4j", "neo4j-admin", "database", "dump", "system", "neo4j", "--to-path=/backup", "--overwrite-destination=true"] + ) + if not stack.dry_run: + subprocess.run(["tar", "-C", str(neo_dir), "-czf", str(destination / "neo4j.dumps.tar.gz"), "."], check=True) + for child in neo_dir.iterdir(): + child.unlink() + neo_dir.rmdir() + + for volume in PROFILE_VOLUMES[stack.profile]: + _archive_volume(stack, volume, destination / f"{volume}.tar.gz") + + artifacts = sorted(path for path in destination.iterdir() if path.is_file()) + manifest = { + "format_version": 1, + "profile": stack.profile, + "brainapi_image": _current_image(stack), + "brainapi_version": _current_image(stack).rsplit(":", 1)[-1], + "timestamp": timestamp, + "services": ( + ["postgres", "redis", "brainapi-plugins"] + if stack.profile == "light" + else ["neo4j", "mongo", "redis", "milvus", "etcd", "minio", "brainapi-plugins"] + ), + "restore_order": list(RESTORE_ORDER[stack.profile]), + "checksums": {path.name: _sha256(path) for path in artifacts}, + } + (destination / "manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + verify_backup(destination, stack.profile) + return destination + finally: + if running: + stack.run(["start", *running], check=False) + + +def _assert_empty_stopped(stack: Stack) -> None: + running = _running_services(stack) + if running: + raise RuntimeError("Restore target must be stopped; running: " + ", ".join(running)) + for key in PROFILE_ALL_VOLUMES[stack.profile]: + volume = stack.volume_name(key) + exists = stack.docker(["volume", "inspect", volume], check=False).returncode == 0 + if not exists or stack.dry_run: + continue + result = stack.docker( + ["run", "--rm", "-v", f"{volume}:/source:ro", ARCHIVER_IMAGE, "sh", "-c", "test -z \"$(find /source -mindepth 1 -maxdepth 1 -print -quit)\""], + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"Restore target volume is not empty: {volume}") + + +def restore(stack: Stack, backup_dir: Path) -> None: + manifest = verify_backup(backup_dir, stack.profile) + if manifest["brainapi_image"] != _current_image(stack): + raise RuntimeError("Backup image does not match BRAINAPI_IMAGE") + _assert_empty_stopped(stack) + stack.run(["up", "--no-start"]) + + for key in PROFILE_VOLUMES[stack.profile]: + _restore_volume(stack, key, backup_dir / f"{key}.tar.gz") + + if stack.profile == "light": + stack.run(["up", "-d", "--wait", "postgres"]) + pg_user = stack.env.get("POSTGRES_USERNAME", "brainapi") + pg_db = stack.env.get("POSTGRES_SYSTEM_DATABASE", "brainapi") + if stack.dry_run: + stack.run(["exec", "-T", "postgres", "pg_restore", "-U", pg_user, "-d", pg_db, "--clean", "--if-exists"]) + else: + with (backup_dir / "postgres.dump").open("rb") as source: + stack.run(["exec", "-T", "postgres", "pg_restore", "-U", pg_user, "-d", pg_db, "--clean", "--if-exists"], stdin=source) + _restore_postgres_brain_databases( + stack, backup_dir / "postgres-brains.tar.gz", pg_user + ) + else: + with tempfile.TemporaryDirectory(prefix="brainapi-neo4j-restore-") as tmp: + tmp_path = Path(tmp) + if not stack.dry_run: + subprocess.run(["tar", "-C", str(tmp_path), "-xzf", str(backup_dir / "neo4j.dumps.tar.gz")], check=True) + stack.run(["run", "--rm", "--no-deps", "-v", f"{tmp_path}:/backup:ro", "neo4j", "neo4j-admin", "database", "load", "system", "neo4j", "--from-path=/backup", "--overwrite-destination=true"]) + stack.run(["up", "-d", "--wait", "mongo"]) + mongo_user = stack.env.get("MONGO_USERNAME", "brainapi") + if stack.dry_run: + stack.run(["exec", "-T", "mongo", "sh", "-c", f'mongorestore --archive --gzip --drop --username {shlex.quote(mongo_user)} --password "$MONGO_INITDB_ROOT_PASSWORD" --authenticationDatabase admin']) + else: + with (backup_dir / "mongo.archive.gz").open("rb") as source: + stack.run(["exec", "-T", "mongo", "sh", "-c", f'mongorestore --archive --gzip --drop --username {shlex.quote(mongo_user)} --password "$MONGO_INITDB_ROOT_PASSWORD" --authenticationDatabase admin'], stdin=source) + + stack.run(["up", "-d", "--wait"]) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("backup", "verify", "restore")) + parser.add_argument("--profile", choices=("light", "heavy"), required=True) + parser.add_argument("--env-file", type=Path, required=True) + parser.add_argument("--project-name", default="brainapi") + parser.add_argument("--backup-dir", type=Path, default=Path(os.getenv("BACKUP_DIR", "/srv/brainapi/backups"))) + parser.add_argument("--archive", type=Path, help="Backup directory for verify/restore") + parser.add_argument( + "--compose-file", + action="append", + default=[], + type=Path, + help="Additional Compose override file; may be repeated.", + ) + parser.add_argument("--dry-run", action="store_true") + return parser + + +def main() -> int: + args = _parser().parse_args() + try: + if args.command == "verify": + if not args.archive: + raise RuntimeError("--archive is required for verify") + verify_backup(args.archive.resolve(), args.profile) + print(f"Verified {args.archive}") + return 0 + stack = Stack( + args.profile, + args.env_file, + args.project_name, + args.dry_run, + args.compose_file, + ) + if args.command == "backup": + destination = backup(stack, args.backup_dir) + print(destination) + else: + if not args.archive: + raise RuntimeError("--archive is required for restore") + restore(stack, args.archive.resolve()) + print(f"Restored {args.archive}") + return 0 + except (KeyError, OSError, RuntimeError, subprocess.CalledProcessError) as exc: + print(f"backup_restore: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/brainapi-backup b/deploy/brainapi-backup new file mode 100755 index 0000000..e411626 --- /dev/null +++ b/deploy/brainapi-backup @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec python3 "$(dirname "$0")/backup_restore.py" "$@" diff --git a/deploy/docker-compose.ci.yaml b/deploy/docker-compose.ci.yaml new file mode 100644 index 0000000..253bb58 --- /dev/null +++ b/deploy/docker-compose.ci.yaml @@ -0,0 +1,42 @@ +# CI-only provider override. This file is never part of a production deployment. +x-ci-model-environment: &ci-model-environment + BRAINPAT_TOKEN: ${BRAINPAT_TOKEN:?BRAINPAT_TOKEN is required} + LLM_SMALL_PROVIDER: openai + LLM_LARGE_PROVIDER: openai + EMBEDDINGS_PROVIDER: openai + OPENAI_API_KEY: ci-isolated-token + OPENAI_BASE_URL: http://embedding-stub:8080/v1 + SEARCH_ENABLED: "true" + SEARCH_USE_DENSE: "true" + SEARCH_USE_BM25: "false" + CONTEXT_PASSAGE_MODE: dense + +services: + embedding-stub: + image: python:3.11.14-slim-bookworm + command: [python, /stub.py] + volumes: + - ../scripts/openai_ci_stub.py:/stub.py:ro + networks: [appnet] + read_only: true + tmpfs: [/tmp] + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"] + interval: 2s + timeout: 2s + retries: 10 + + brainapi: + environment: *ci-model-environment + depends_on: + embedding-stub: {condition: service_healthy} + + brainapi-worker: + environment: *ci-model-environment + depends_on: + embedding-stub: {condition: service_healthy} + + brainapi-mcp: + environment: *ci-model-environment + depends_on: + embedding-stub: {condition: service_healthy} diff --git a/deploy/docker-compose.heavy.yaml b/deploy/docker-compose.heavy.yaml index 99370f6..4a95e4f 100644 --- a/deploy/docker-compose.heavy.yaml +++ b/deploy/docker-compose.heavy.yaml @@ -1,369 +1,245 @@ -# BrainAPI deploy profile: heavy -# Full stack (Redis + Neo4j + Milvus + Mongo + API/worker/MCP). -# Pair with deploy/env.heavy.example → /root/.env -# See deploy/README.md +# BrainAPI production profile: heavy (Neo4j + MongoDB + Milvus + Redis). -version: "3.8" +x-brainapi-common: &brainapi-common + image: ${BRAINAPI_IMAGE:?BRAINAPI_IMAGE is required} + env_file: + - ${BRAINAPI_ENV_FILE:?BRAINAPI_ENV_FILE is required} + environment: &brainapi-environment + ENV: production + PLUGIN_FAILURE_POLICY: ${PLUGIN_FAILURE_POLICY:-fail} + REDIS_HOST: redis + REDIS_PORT: "6379" + REDIS_PASSWORD: ${REDIS_PASSWORD:?REDIS_PASSWORD is required} + NEO4J_HOST: neo4j + NEO4J_PORT: "7687" + NEO4J_USERNAME: ${NEO4J_USERNAME:-neo4j} + NEO4J_PASSWORD: ${NEO4J_PASSWORD:?NEO4J_PASSWORD is required} + MONGO_HOST: mongo + MONGO_PORT: "27017" + MONGO_USERNAME: ${MONGO_USERNAME:-brainapi} + MONGO_PASSWORD: ${MONGO_PASSWORD:?MONGO_PASSWORD is required} + MONGO_SYSTEM_DATABASE: ${MONGO_SYSTEM_DATABASE:-system} + MILVUS_HOST: milvus + MILVUS_PORT: "19530" + MILVUS_TOKEN: ${MILVUS_TOKEN:?MILVUS_TOKEN is required} + BRAINAPI_PLUGINS: ${BRAINAPI_PLUGINS:-} + PLUGIN_REGISTRY_URL: ${PLUGIN_REGISTRY_URL:-} + PLUGIN_PUBLISHER_ID: ${PLUGIN_PUBLISHER_ID:-} + PLUGIN_PUBLISHER_API_KEY: ${PLUGIN_PUBLISHER_API_KEY:-} + volumes: [brainapi-plugins:/app/plugins] + networks: [appnet] + restart: unless-stopped services: nginx: - image: nginx:stable-alpine - container_name: brainapi-nginx - ports: - - "80:80" - - "443:443" + image: nginx:1.28.2-alpine3.23 + ports: ["80:80", "443:443"] volumes: - - /srv/nginx/conf/nginx.conf:/etc/nginx/nginx.conf:ro - - /srv/nginx/conf/conf.d:/etc/nginx/conf.d:ro - - /etc/letsencrypt:/etc/letsencrypt:ro - - /srv/nginx/logs:/var/log/nginx - networks: - - appnet - restart: unless-stopped + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/conf.d:/etc/nginx/conf.d:ro depends_on: - - redis - - mongo - - neo4j - - milvus + brainapi: {condition: service_healthy} + brainapi-mcp: {condition: service_healthy} + networks: [appnet] + restart: unless-stopped + read_only: true + tmpfs: [/var/cache/nginx, /var/run, /tmp] + security_opt: [no-new-privileges:true] mem_limit: 128m - cpus: 0.2 + cpus: 0.20 + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost/health"] + interval: 15s + timeout: 5s + retries: 5 redis: - image: redis:7-alpine - container_name: brainapi-redis - ports: - - "6379:6379" - volumes: - - redis-data:/data - command: > - redis-server - --appendonly yes - --maxmemory 1500mb - --maxmemory-policy allkeys-lru - --save 900 1 - --save 300 10 - --save 60 10000 - --tcp-backlog 511 - --timeout 300 - networks: - - appnet + image: redis:7.4.7-alpine3.21 + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD:?REDIS_PASSWORD is required} + command: [redis-server, --requirepass, "${REDIS_PASSWORD:?REDIS_PASSWORD is required}", --appendonly, "yes", --maxmemory, 1500mb, --maxmemory-policy, allkeys-lru, --save, "900 1", --save, "300 10", --save, "60 10000"] + volumes: [redis-data:/data] + networks: [appnet] restart: unless-stopped - mem_limit: 2048m - cpus: 0.5 + security_opt: [no-new-privileges:true] + mem_limit: 2g + cpus: 0.50 healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: ["CMD-SHELL", "redis-cli -a \"$${REDIS_PASSWORD}\" ping 2>/dev/null | grep -q PONG"] interval: 10s timeout: 3s - retries: 3 + retries: 5 neo4j: - image: neo4j:5-enterprise - container_name: brainapi-neo4j - ports: - - "7474:7474" - - "7687:7687" + image: neo4j:5.26.16-enterprise + environment: + NEO4J_AUTH: ${NEO4J_USERNAME:-neo4j}/${NEO4J_PASSWORD:?NEO4J_PASSWORD is required} + NEO4J_ACCEPT_LICENSE_AGREEMENT: "yes" + NEO4J_PLUGINS: '["apoc"]' + NEO4J_server_memory_heap_max__size: 2g + NEO4J_server_memory_pagecache_size: 1g volumes: - neo4j-data:/data - neo4j-logs:/logs - neo4j-plugins:/plugins - - ./src/config/neo4j:/var/lib/neo4j/conf - environment: - - NEO4J_AUTH=neo4j/your_password - - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes - - NEO4J_PLUGINS=["apoc"] - - NEO4J_server_memory_heap_max__size=2g - - NEO4J_server_memory_pagecache_size=1g - - NEO4J_server_jvm_additional=-XX:+UseG1GC -XX:MaxGCPauseMillis=200 - networks: - - appnet + networks: [appnet] restart: unless-stopped + security_opt: [no-new-privileges:true] mem_limit: 3g - cpus: 1 + cpus: 1.0 healthcheck: - test: ["CMD-SHELL", "wget -q --spider http://localhost:7474 || exit 1"] + test: ["CMD-SHELL", "cypher-shell -u \"$${NEO4J_AUTH%%/*}\" -p \"$${NEO4J_AUTH#*/}\" 'RETURN 1' >/dev/null"] interval: 15s timeout: 10s - retries: 5 - start_period: 120s + retries: 10 + start_period: 90s etcd: image: quay.io/coreos/etcd:v3.5.18 - container_name: brainapi-etcd environment: - - ETCD_AUTO_COMPACTION_MODE=revision - - ETCD_AUTO_COMPACTION_RETENTION=1000 - - ETCD_QUOTA_BACKEND_BYTES=4294967296 - - ETCD_SNAPSHOT_COUNT=50000 - volumes: - - etcd-data:/etcd - command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd - networks: - - appnet + ETCD_AUTO_COMPACTION_MODE: revision + ETCD_AUTO_COMPACTION_RETENTION: "1000" + ETCD_QUOTA_BACKEND_BYTES: "4294967296" + ETCD_SNAPSHOT_COUNT: "50000" + command: [etcd, --advertise-client-urls=http://etcd:2379, --listen-client-urls=http://0.0.0.0:2379, --data-dir=/etcd] + volumes: [etcd-data:/etcd] + networks: [appnet] restart: unless-stopped + security_opt: [no-new-privileges:true] mem_limit: 512m - cpus: 0.2 + cpus: 0.25 healthcheck: test: ["CMD", "etcdctl", "endpoint", "health"] - interval: 30s - timeout: 20s - retries: 3 + interval: 15s + timeout: 10s + retries: 5 minio: image: minio/minio:RELEASE.2024-12-18T13-15-44Z - container_name: brainapi-minio environment: - MINIO_ACCESS_KEY: minioadmin - MINIO_SECRET_KEY: minioadmin - ports: - - "9001:9001" - - "9000:9000" - volumes: - - minio-data:/minio_data - command: minio server /minio_data --console-address ":9001" - networks: - - appnet + MINIO_ROOT_USER: ${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required} + MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required} + command: [minio, server, /minio_data, --console-address, ":9001"] + volumes: [minio-data:/minio_data] + networks: [appnet] restart: unless-stopped + security_opt: [no-new-privileges:true] mem_limit: 1g - cpus: 0.4 + cpus: 0.40 healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 30s - timeout: 20s - retries: 3 + test: ["CMD", "curl", "-fsS", "http://localhost:9000/minio/health/live"] + interval: 15s + timeout: 10s + retries: 5 milvus: image: milvusdb/milvus:v2.6.6 - container_name: brainapi-milvus - command: ["milvus", "run", "standalone"] - security_opt: - - seccomp:unconfined + command: [milvus, run, standalone] + security_opt: [seccomp:unconfined, no-new-privileges:true] environment: ETCD_ENDPOINTS: etcd:2379 MINIO_ADDRESS: minio:9000 - MINIO_ACCESS_KEY: minioadmin - MINIO_SECRET_KEY: minioadmin + MINIO_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required} + MINIO_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required} + COMMON_SECURITY_AUTHORIZATION_ENABLED: "true" + COMMON_SECURITY_DEFAULT_ROOT_PASSWORD: ${MILVUS_ROOT_PASSWORD:?MILVUS_ROOT_PASSWORD is required} MQ_TYPE: woodpecker MILVUS_COMMON_LOG_LEVEL: warn - volumes: - - milvus-data:/var/lib/milvus - ports: - - "19530:19530" - - "9091:9091" - networks: - - appnet + volumes: [milvus-data:/var/lib/milvus] + networks: [appnet] depends_on: - etcd: - condition: service_healthy - minio: - condition: service_healthy + etcd: {condition: service_healthy} + minio: {condition: service_healthy} restart: unless-stopped mem_limit: 2g - cpus: 0.6 + cpus: 0.75 healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] - interval: 30s - start_period: 90s - timeout: 20s - retries: 3 + test: ["CMD", "curl", "-fsS", "http://localhost:9091/healthz"] + interval: 20s + timeout: 10s + retries: 6 + start_period: 60s mongo: - image: mongo:8.0 - container_name: brainapi-mongo - ports: - - "27017:27017" + image: mongo:8.0.16-noble environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: password + MONGO_INITDB_ROOT_USERNAME: ${MONGO_USERNAME:-brainapi} + MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD:?MONGO_PASSWORD is required} + command: [mongod, --wiredTigerCacheSizeGB=1, --bind_ip_all] volumes: - mongo-data:/data/db - mongo-config:/data/configdb - command: > - mongod - --wiredTigerCacheSizeGB=5 - --wiredTigerCollectionBlockCompressor=snappy - --wiredTigerIndexPrefixCompression=true - --logpath=/var/log/mongodb/mongod.log - --logappend - --bind_ip_all - networks: - - appnet + networks: [appnet] restart: unless-stopped - mem_limit: 1.5g - cpus: 0.4 + security_opt: [no-new-privileges:true] + mem_limit: 2g + cpus: 0.50 healthcheck: - test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"] + test: ["CMD-SHELL", "mongosh --quiet --username \"$${MONGO_INITDB_ROOT_USERNAME}\" --password \"$${MONGO_INITDB_ROOT_PASSWORD}\" --authenticationDatabase admin --eval 'quit(db.adminCommand({ping:1}).ok ? 0 : 2)'"] interval: 10s timeout: 5s - retries: 5 + retries: 10 brainapi: - image: ghcr.io/lumen-labs/brainapi:v2.9.7-dev - container_name: brainapi-api - ports: - - "8000:8000" - env_file: - - /root/.env - environment: - PLUGIN_REGISTRY_URL: ${PLUGIN_REGISTRY_URL:-} - PLUGIN_PUBLISHER_ID: ${PLUGIN_PUBLISHER_ID:-} - PLUGIN_PUBLISHER_API_KEY: ${PLUGIN_PUBLISHER_API_KEY:-} - BRAINAPI_PLUGINS: my-plugin:1.0.0,analytics-plugin - volumes: - - /root/gcp_credentials.json:/app/gcp_credentials.json - - brainapi-plugins:/app/plugins - command: - [ - "-m", - "uvicorn", - "src.services.api.app:app", - "--host", - "0.0.0.0", - "--port", - "8000", - "--workers", - "1", - "--timeout-keep-alive", - "30", - "--limit-concurrency", - "200", - "--backlog", - "2048", - ] + <<: *brainapi-common + command: [-m, uvicorn, src.services.api.app:app, --host, 0.0.0.0, --port, "8000", --workers, "1", --timeout-keep-alive, "30", --limit-concurrency, "200"] depends_on: - redis: - condition: service_healthy - neo4j: - condition: service_started - mongo: - condition: service_healthy - milvus: - condition: service_healthy - networks: - - appnet - restart: unless-stopped + redis: {condition: service_healthy} + neo4j: {condition: service_healthy} + mongo: {condition: service_healthy} + milvus: {condition: service_healthy} mem_limit: 4g - cpus: 2 + cpus: 2.0 + security_opt: [no-new-privileges:true] healthcheck: - test: - [ - "CMD", - "curl", - "-f", - "http://localhost:8000/", - "-H", - "BrainPAT: ${BRAINPAT_TOKEN}", - ] - interval: 30s - timeout: 10s - retries: 5 - start_period: 900s - start_interval: 30s + test: ["CMD", "curl", "-fsS", "http://localhost:8000/health"] + interval: 20s + timeout: 5s + retries: 12 + start_period: 120s brainapi-worker: - image: ghcr.io/lumen-labs/brainapi:v2.9.7-dev - container_name: brainapi-worker - env_file: - - /root/.env - environment: - BRAINAPI_PLUGINS: ${BRAINAPI_PLUGINS:-} - PLUGIN_REGISTRY_URL: ${PLUGIN_REGISTRY_URL:-} - PLUGIN_PUBLISHER_ID: ${PLUGIN_PUBLISHER_ID:-} - PLUGIN_PUBLISHER_API_KEY: ${PLUGIN_PUBLISHER_API_KEY:-} - volumes: - - /root/gcp_credentials.json:/app/gcp_credentials.json - - brainapi-plugins:/app/plugins - command: - [ - "-m", - "celery", - "-A", - "src.workers.app", - "worker", - "--loglevel=info", - "--pool=threads", - "--without-gossip", - "--without-mingle", - "--concurrency=2", - "--max-tasks-per-child=50", - ] + <<: *brainapi-common + command: [-m, celery, -A, src.workers.app, worker, --loglevel=info, --pool=threads, --without-gossip, --without-mingle, --concurrency=2, --max-tasks-per-child=50] depends_on: - redis: - condition: service_healthy - neo4j: - condition: service_started - mongo: - condition: service_healthy - milvus: - condition: service_healthy - networks: - - appnet - restart: unless-stopped + redis: {condition: service_healthy} + neo4j: {condition: service_healthy} + mongo: {condition: service_healthy} + milvus: {condition: service_healthy} mem_limit: 4g - cpus: 2 + cpus: 2.0 + security_opt: [no-new-privileges:true] healthcheck: test: - [ - "CMD-SHELL", - "python -m celery -A src.workers.app inspect ping --timeout 5 >/dev/null 2>&1 && q=$(redis-cli -h redis -p 6379 LLEN ingest_data 2>/dev/null || echo 0); [ \"$q\" -lt \"${CELERY_QUEUE_HEALTH_THRESHOLD:-50}\" ]", - ] + - CMD-SHELL + - >- + python -m celery -A src.workers.app inspect ping --timeout 5 >/dev/null 2>&1 && + python -c 'import os,redis,sys; r=redis.Redis(host="redis",password=os.environ["REDIS_PASSWORD"]); sys.exit(r.llen("ingest_data") >= int(os.getenv("CELERY_QUEUE_HEALTH_THRESHOLD","50")))' interval: 30s timeout: 10s retries: 5 start_period: 120s - start_interval: 30s brainapi-mcp: - image: ghcr.io/lumen-labs/brainapi:v2.9.7-dev - container_name: brainapi-mcp - ports: - - "8001:8001" - env_file: - - /root/.env + <<: *brainapi-common environment: + <<: *brainapi-environment MCP_OAUTH_ISSUER_URL: ${MCP_OAUTH_ISSUER_URL:-} MCP_RESOURCE_SERVER_URL: ${MCP_RESOURCE_SERVER_URL:-} - BRAINAPI_PLUGINS: ${BRAINAPI_PLUGINS:-} - PLUGIN_REGISTRY_URL: ${PLUGIN_REGISTRY_URL:-} - PLUGIN_PUBLISHER_ID: ${PLUGIN_PUBLISHER_ID:-} - PLUGIN_PUBLISHER_API_KEY: ${PLUGIN_PUBLISHER_API_KEY:-} - volumes: - - /root/gcp_credentials.json:/app/gcp_credentials.json - - brainapi-plugins:/app/plugins - command: - [ - "-m", - "uvicorn", - "src.services.mcp.app:app", - "--host", - "0.0.0.0", - "--port", - "8001", - "--timeout-keep-alive", - "30", - ] + command: [-m, uvicorn, src.services.mcp.app:app, --host, 0.0.0.0, --port, "8001", --timeout-keep-alive, "30"] depends_on: - redis: - condition: service_healthy - neo4j: - condition: service_started - mongo: - condition: service_healthy - milvus: - condition: service_healthy - networks: - - appnet - restart: unless-stopped - mem_limit: 4g - cpus: 2 + redis: {condition: service_healthy} + neo4j: {condition: service_healthy} + mongo: {condition: service_healthy} + milvus: {condition: service_healthy} + mem_limit: 2g + cpus: 1.0 + security_opt: [no-new-privileges:true] healthcheck: - test: - [ - "CMD-SHELL", - 'code=$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:8001/); test "$code" = ''200'' || test "$code" = ''405''', - ] - interval: 30s - timeout: 10s - retries: 3 + test: ["CMD", "curl", "-fsS", "http://localhost:8001/"] + interval: 20s + timeout: 5s + retries: 5 start_period: 60s networks: @@ -380,4 +256,4 @@ volumes: milvus-data: mongo-data: mongo-config: - brainapi-plugins: \ No newline at end of file + brainapi-plugins: diff --git a/deploy/docker-compose.light.yaml b/deploy/docker-compose.light.yaml index 1646ab7..ffb764a 100644 --- a/deploy/docker-compose.light.yaml +++ b/deploy/docker-compose.light.yaml @@ -1,234 +1,147 @@ -# BrainAPI deploy profile: light -# Slim stack (Redis + Postgres/pgvector + API/worker/MCP). -# Pair with deploy/env.light.example → /root/.env -# See deploy/README.md +# BrainAPI production profile: light (PostgreSQL/pgvector + Redis). -version: "3.8" +x-brainapi-common: &brainapi-common + image: ${BRAINAPI_IMAGE:?BRAINAPI_IMAGE is required} + env_file: + - ${BRAINAPI_ENV_FILE:?BRAINAPI_ENV_FILE is required} + environment: &brainapi-environment + ENV: production + PLUGIN_FAILURE_POLICY: ${PLUGIN_FAILURE_POLICY:-fail} + REDIS_HOST: redis + REDIS_PORT: "6379" + REDIS_PASSWORD: ${REDIS_PASSWORD:?REDIS_PASSWORD is required} + POSTGRES_HOST: postgres + POSTGRES_PORT: "5432" + POSTGRES_USERNAME: ${POSTGRES_USERNAME:-brainapi} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required} + POSTGRES_SYSTEM_DATABASE: ${POSTGRES_SYSTEM_DATABASE:-brainapi} + POSTGRES_MAINTENANCE_DATABASE: postgres + BRAINAPI_PLUGINS: ${BRAINAPI_PLUGINS:-} + PLUGIN_REGISTRY_URL: ${PLUGIN_REGISTRY_URL:-} + PLUGIN_PUBLISHER_ID: ${PLUGIN_PUBLISHER_ID:-} + PLUGIN_PUBLISHER_API_KEY: ${PLUGIN_PUBLISHER_API_KEY:-} + volumes: + - brainapi-plugins:/app/plugins + networks: [appnet] + restart: unless-stopped services: nginx: - image: nginx:stable-alpine - container_name: brainapi-nginx + image: nginx:1.28.2-alpine3.23 ports: - "80:80" - "443:443" volumes: - - /srv/nginx/conf/nginx.conf:/etc/nginx/nginx.conf:ro - - /srv/nginx/conf/conf.d:/etc/nginx/conf.d:ro - - /etc/letsencrypt:/etc/letsencrypt:ro - - /srv/nginx/logs:/var/log/nginx - networks: - - appnet - restart: unless-stopped + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/conf.d:/etc/nginx/conf.d:ro depends_on: - - redis - - postgres - mem_limit: 16m - cpus: 0.025 + brainapi: + condition: service_healthy + brainapi-mcp: + condition: service_healthy + networks: [appnet] + restart: unless-stopped + read_only: true + tmpfs: [/var/cache/nginx, /var/run, /tmp] + security_opt: [no-new-privileges:true] + mem_limit: 64m + cpus: 0.10 + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost/health"] + interval: 15s + timeout: 5s + retries: 5 redis: - image: redis:7-alpine - container_name: brainapi-redis - ports: - - "6379:6379" - volumes: - - redis-data:/data - command: > - redis-server - --appendonly yes - --maxmemory 192mb - --maxmemory-policy allkeys-lru - --save 900 1 - --save 300 10 - --save 60 10000 - --tcp-backlog 511 - --timeout 300 - networks: - - appnet + image: redis:7.4.7-alpine3.21 + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD:?REDIS_PASSWORD is required} + command: [redis-server, --requirepass, "${REDIS_PASSWORD:?REDIS_PASSWORD is required}", --appendonly, "yes", --maxmemory, 192mb, --maxmemory-policy, allkeys-lru, --save, "900 1", --save, "300 10", --save, "60 10000"] + volumes: [redis-data:/data] + networks: [appnet] restart: unless-stopped + security_opt: [no-new-privileges:true] mem_limit: 256m - cpus: 0.0625 + cpus: 0.10 healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: ["CMD-SHELL", "redis-cli -a \"$${REDIS_PASSWORD}\" ping 2>/dev/null | grep -q PONG"] interval: 10s timeout: 3s - retries: 3 + retries: 5 postgres: - image: pgvector/pgvector:pg16 - container_name: brainapi-postgres - ports: - - "5432:5432" + image: pgvector/pgvector:0.8.1-pg16-bookworm environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: password - volumes: - - postgres-data:/var/lib/postgresql/data - networks: - - appnet + POSTGRES_USER: ${POSTGRES_USERNAME:-brainapi} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required} + POSTGRES_DB: ${POSTGRES_SYSTEM_DATABASE:-brainapi} + volumes: [postgres-data:/var/lib/postgresql/data] + networks: [appnet] restart: unless-stopped - mem_limit: 512m - cpus: 0.25 + security_opt: [no-new-privileges:true] + mem_limit: 768m + cpus: 0.35 healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d \"$${POSTGRES_DB}\""] interval: 5s timeout: 5s - retries: 10 + retries: 12 brainapi: - image: ghcr.io/lumen-labs/brainapi:v2.9.7-dev - container_name: brainapi-api - ports: - - "8000:8000" - env_file: - - /root/.env - environment: - PLUGIN_REGISTRY_URL: ${PLUGIN_REGISTRY_URL:-} - PLUGIN_PUBLISHER_ID: ${PLUGIN_PUBLISHER_ID:-} - PLUGIN_PUBLISHER_API_KEY: ${PLUGIN_PUBLISHER_API_KEY:-} - BRAINAPI_PLUGINS: ${BRAINAPI_PLUGINS:-} - volumes: - - brainapi-plugins:/app/plugins - command: - [ - "-m", - "uvicorn", - "src.services.api.app:app", - "--host", - "0.0.0.0", - "--port", - "8000", - "--workers", - "1", - "--timeout-keep-alive", - "30", - "--limit-concurrency", - "50", - "--backlog", - "256", - ] + <<: *brainapi-common + command: [-m, uvicorn, src.services.api.app:app, --host, 0.0.0.0, --port, "8000", --workers, "1", --timeout-keep-alive, "30", --limit-concurrency, "50"] depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - networks: - - appnet - restart: unless-stopped - mem_limit: 512m - cpus: 0.25 + redis: {condition: service_healthy} + postgres: {condition: service_healthy} + mem_limit: 768m + cpus: 0.50 + security_opt: [no-new-privileges:true] healthcheck: - test: - [ - "CMD", - "curl", - "-f", - "http://localhost:8000/", - "-H", - "BrainPAT: ${BRAINPAT_TOKEN}", - ] - interval: 30s - timeout: 10s - retries: 5 - start_period: 300s - start_interval: 30s + test: ["CMD", "curl", "-fsS", "http://localhost:8000/health"] + interval: 20s + timeout: 5s + retries: 8 + start_period: 60s brainapi-worker: - image: ghcr.io/lumen-labs/brainapi:v2.9.7-dev - container_name: brainapi-worker - env_file: - - /root/.env - environment: - BRAINAPI_PLUGINS: ${BRAINAPI_PLUGINS:-} - PLUGIN_REGISTRY_URL: ${PLUGIN_REGISTRY_URL:-} - PLUGIN_PUBLISHER_ID: ${PLUGIN_PUBLISHER_ID:-} - PLUGIN_PUBLISHER_API_KEY: ${PLUGIN_PUBLISHER_API_KEY:-} - volumes: - - brainapi-plugins:/app/plugins - command: - [ - "-m", - "celery", - "-A", - "src.workers.app", - "worker", - "--loglevel=info", - "--pool=threads", - "--without-gossip", - "--without-mingle", - "--concurrency=1", - "--max-tasks-per-child=50", - ] + <<: *brainapi-common + command: [-m, celery, -A, src.workers.app, worker, --loglevel=info, --pool=threads, --without-gossip, --without-mingle, --concurrency=1, --max-tasks-per-child=50] depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - networks: - - appnet - restart: unless-stopped - mem_limit: 512m - cpus: 0.25 + redis: {condition: service_healthy} + postgres: {condition: service_healthy} + mem_limit: 768m + cpus: 0.50 + security_opt: [no-new-privileges:true] healthcheck: test: - [ - "CMD-SHELL", - "python -m celery -A src.workers.app inspect ping --timeout 5 >/dev/null 2>&1 && q=$(redis-cli -h redis -p 6379 LLEN ingest_data 2>/dev/null || echo 0); [ \"$q\" -lt \"${CELERY_QUEUE_HEALTH_THRESHOLD:-50}\" ]", - ] + - CMD-SHELL + - >- + python -m celery -A src.workers.app inspect ping --timeout 5 >/dev/null 2>&1 && + python -c 'import os,redis,sys; r=redis.Redis(host="redis",password=os.environ["REDIS_PASSWORD"]); sys.exit(r.llen("ingest_data") >= int(os.getenv("CELERY_QUEUE_HEALTH_THRESHOLD","50")))' interval: 30s timeout: 10s retries: 5 - start_period: 120s - start_interval: 30s + start_period: 90s brainapi-mcp: - image: ghcr.io/lumen-labs/brainapi:v2.9.7-dev - container_name: brainapi-mcp - ports: - - "8001:8001" - env_file: - - /root/.env + <<: *brainapi-common environment: + <<: *brainapi-environment MCP_OAUTH_ISSUER_URL: ${MCP_OAUTH_ISSUER_URL:-} MCP_RESOURCE_SERVER_URL: ${MCP_RESOURCE_SERVER_URL:-} - BRAINAPI_PLUGINS: ${BRAINAPI_PLUGINS:-} - PLUGIN_REGISTRY_URL: ${PLUGIN_REGISTRY_URL:-} - PLUGIN_PUBLISHER_ID: ${PLUGIN_PUBLISHER_ID:-} - PLUGIN_PUBLISHER_API_KEY: ${PLUGIN_PUBLISHER_API_KEY:-} - volumes: - - brainapi-plugins:/app/plugins - command: - [ - "-m", - "uvicorn", - "src.services.mcp.app:app", - "--host", - "0.0.0.0", - "--port", - "8001", - "--timeout-keep-alive", - "30", - ] + command: [-m, uvicorn, src.services.mcp.app:app, --host, 0.0.0.0, --port, "8001", --timeout-keep-alive, "30"] depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - networks: - - appnet - restart: unless-stopped - mem_limit: 512m - cpus: 0.25 + redis: {condition: service_healthy} + postgres: {condition: service_healthy} + mem_limit: 768m + cpus: 0.50 + security_opt: [no-new-privileges:true] healthcheck: - test: - [ - "CMD-SHELL", - 'code=$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:8001/); test "$code" = ''200'' || test "$code" = ''405''', - ] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s + test: ["CMD", "curl", "-fsS", "http://localhost:8001/"] + interval: 20s + timeout: 5s + retries: 5 + start_period: 45s networks: appnet: diff --git a/deploy/env.heavy.example b/deploy/env.heavy.example index 47a9801..052731f 100644 --- a/deploy/env.heavy.example +++ b/deploy/env.heavy.example @@ -53,6 +53,7 @@ EMBEDDING_RELATIONSHIPS_DIMENSION=3072 # The following are just examples of the adapter implementation, you can have completely different envs REDIS_HOST="redis" REDIS_PORT=6379 +REDIS_PASSWORD="replace-with-a-long-random-secret" # Worker envs # Can be whatever you prefer @@ -70,7 +71,7 @@ CELERY_QUEUE_HEALTH_THRESHOLD=50 NEO4J_HOST="neo4j" NEO4J_PORT=7687 NEO4J_USERNAME="neo4j" -NEO4J_PASSWORD="your_password" +NEO4J_PASSWORD="replace-with-a-long-random-secret" # VectorDB envs # The following are just examples of the adapter implementation, you can have completely different envs @@ -95,7 +96,7 @@ POSTGRES_MAINTENANCE_DATABASE="postgres" MONGO_HOST="mongo" MONGO_PORT=27017 MONGO_USERNAME="root" -MONGO_PASSWORD="password" +MONGO_PASSWORD="replace-with-a-long-random-secret" MONGO_SYSTEM_DATABASE="system" # Auth @@ -108,6 +109,11 @@ BRAIN_CREATION_ALLOWED="true" # Choose whether to fallback to default brain if not provided DEFAULT_BRAIN_FALLBACK="true" +# Optional anonymous, read-only documentation search demo (disabled by default) +PUBLIC_DEMO_ENABLED="false" +PUBLIC_DEMO_BRAIN_ID="agentdemo" +PUBLIC_DEMO_MAX_K=10 + # GCP # Used on the oss project for the small LLM but can be changed to any other model GCP_EXTRA_SMALL_LLM_MODEL="gemini-3-flash-preview" @@ -170,6 +176,17 @@ INGEST_ARCHITECT_DENSE_ENTITY_THRESHOLD="12" INGEST_ARCHITECT_DENSE_MAX_CHARS="1200" INGEST_ARCHITECT_PRIOR_CONTEXT="auto" INGEST_ARCHITECT_SCRATCHPAD_TOKEN_CAP="500" +# Search (off by default). BM25 requires DATA_DB=postgresql; heavy can enable +# dense-only search with SEARCH_USE_BM25=false. +# p50 SLO < 200 ms excluding embed.query RTT. +SEARCH_ENABLED="false" +SEARCH_USE_DENSE="true" +SEARCH_USE_BM25="true" +SEARCH_FUSION="rrf" +SEARCH_FUSION_ALPHA="0.5" +SEARCH_BM25_K1="1.2" +SEARCH_BM25_B="0.75" +CONTEXT_PASSAGE_MODE="hybrid" GRAPH_DB="neo4j" DATA_DB="mongo" VECTOR_DB="milvus" @@ -206,4 +223,14 @@ TRACE_HEALTH_INTERVAL_SECONDS=30 TRACE_RESOURCE_INTERVAL_SECONDS=30 TRACE_HEALTH_TIMEOUT_SECONDS=1 TRACE_AGENT_LOOP_ITERATIONS=20 -TRACE_AGENT_TOOL_LOOP_ITERATIONS=20 \ No newline at end of file +TRACE_AGENT_TOOL_LOOP_ITERATIONS=20 +# Compose inputs (pass this file with `docker compose --env-file`). +BRAINAPI_IMAGE="ghcr.io/lumen-labs/brainapi:v2.17.0-rc.1" +BRAINAPI_ENV_FILE="env.heavy.example" +BACKUP_DIR="/srv/brainapi/backups" +PLUGIN_FAILURE_POLICY="fail" +CORS_ALLOWED_ORIGINS="" +MINIO_ACCESS_KEY="replace-with-a-random-access-key" +MINIO_SECRET_KEY="replace-with-a-long-random-secret" +MILVUS_ROOT_PASSWORD="replace-with-a-long-random-secret" +MILVUS_TOKEN="root:replace-with-a-long-random-secret" diff --git a/deploy/env.light.example b/deploy/env.light.example index 7ab76ef..b98ca43 100644 --- a/deploy/env.light.example +++ b/deploy/env.light.example @@ -51,6 +51,7 @@ EMBEDDING_RELATIONSHIPS_DIMENSION=3072 # The following are just examples of the adapter implementation, you can have completely different envs REDIS_HOST="redis" REDIS_PORT=6379 +REDIS_PASSWORD="replace-with-a-long-random-secret" # Worker envs # Can be whatever you prefer @@ -84,7 +85,7 @@ MILVUS_PORT=19530 POSTGRES_HOST="postgres" POSTGRES_PORT=5432 POSTGRES_USERNAME="postgres" -POSTGRES_PASSWORD="password" +POSTGRES_PASSWORD="replace-with-a-long-random-secret" POSTGRES_SYSTEM_DATABASE="brainapi" POSTGRES_MAINTENANCE_DATABASE="postgres" @@ -106,6 +107,11 @@ BRAIN_CREATION_ALLOWED="true" # Choose whether to fallback to default brain if not provided DEFAULT_BRAIN_FALLBACK="true" +# Optional anonymous, read-only documentation search demo (disabled by default) +PUBLIC_DEMO_ENABLED="false" +PUBLIC_DEMO_BRAIN_ID="agentdemo" +PUBLIC_DEMO_MAX_K=10 + # GCP # Used on the oss project for the small LLM but can be changed to any other model GCP_EXTRA_SMALL_LLM_MODEL="gemini-3-flash-preview" @@ -168,6 +174,15 @@ INGEST_ARCHITECT_DENSE_ENTITY_THRESHOLD="12" INGEST_ARCHITECT_DENSE_MAX_CHARS="1200" INGEST_ARCHITECT_PRIOR_CONTEXT="auto" INGEST_ARCHITECT_SCRATCHPAD_TOKEN_CAP="500" +# Search (off by default). p50 SLO < 200 ms excluding embed.query. Requires DATA_DB=postgresql. +SEARCH_ENABLED="false" +SEARCH_USE_DENSE="true" +SEARCH_USE_BM25="true" +SEARCH_FUSION="rrf" +SEARCH_FUSION_ALPHA="0.5" +SEARCH_BM25_K1="1.2" +SEARCH_BM25_B="0.75" +CONTEXT_PASSAGE_MODE="hybrid" GRAPH_DB="networkx" DATA_DB="postgresql" VECTOR_DB="postgresql" @@ -204,4 +219,10 @@ TRACE_HEALTH_INTERVAL_SECONDS=30 TRACE_RESOURCE_INTERVAL_SECONDS=30 TRACE_HEALTH_TIMEOUT_SECONDS=1 TRACE_AGENT_LOOP_ITERATIONS=20 -TRACE_AGENT_TOOL_LOOP_ITERATIONS=20 \ No newline at end of file +TRACE_AGENT_TOOL_LOOP_ITERATIONS=20 +# Compose inputs (pass this file with `docker compose --env-file`). +BRAINAPI_IMAGE="ghcr.io/lumen-labs/brainapi:v2.17.0-rc.1" +BRAINAPI_ENV_FILE="env.light.example" +BACKUP_DIR="/srv/brainapi/backups" +PLUGIN_FAILURE_POLICY="fail" +CORS_ALLOWED_ORIGINS="" diff --git a/deploy/env.public-demo.example b/deploy/env.public-demo.example new file mode 100644 index 0000000..63dacaa --- /dev/null +++ b/deploy/env.public-demo.example @@ -0,0 +1,16 @@ +# Merge these non-secret overrides into the light deployment environment for +# the read-only documentation sandbox. Keep the system BRAINPAT_TOKEN private. +PUBLIC_DEMO_ENABLED="true" +PUBLIC_DEMO_BRAIN_ID="agentdemo" +PUBLIC_DEMO_MAX_K=10 + +# The demo route refuses to start unless Search is BM25-only. +SEARCH_ENABLED="true" +SEARCH_USE_DENSE="false" +SEARCH_USE_BM25="true" +SEARCH_FUSION="rrf" +SEARCH_BM25_K1="1.2" +SEARCH_BM25_B="0.75" + +# Required only while the seed command creates agentdemo for the first time. +BRAIN_CREATION_ALLOWED="true" diff --git a/deploy/nginx/conf.d/brainapi.conf b/deploy/nginx/conf.d/brainapi.conf new file mode 100644 index 0000000..1750db8 --- /dev/null +++ b/deploy/nginx/conf.d/brainapi.conf @@ -0,0 +1,58 @@ +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +upstream brainapi_api { + server brainapi:8000; + keepalive 16; +} + +upstream brainapi_mcp { + server brainapi-mcp:8001; + keepalive 8; +} + +server { + listen 80 default_server; + listen [::]:80 default_server; + server_name _; + + location = /health { + proxy_pass http://brainapi_api/health; + proxy_set_header Host $host; + proxy_set_header X-Request-ID $request_id; + } + + location /mcp { + proxy_pass http://brainapi_mcp; + proxy_http_version 1.1; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-ID $request_id; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } + + location / { + proxy_pass http://brainapi_api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-ID $request_id; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } +} + +# Safe default until an operator mounts a certificate-specific TLS server file. +server { + listen 443 ssl default_server; + listen [::]:443 ssl default_server; + server_name _; + ssl_reject_handshake on; +} diff --git a/deploy/nginx/nginx.conf b/deploy/nginx/nginx.conf new file mode 100644 index 0000000..a72b224 --- /dev/null +++ b/deploy/nginx/nginx.conf @@ -0,0 +1,23 @@ +user nginx; +worker_processes auto; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + server_tokens off; + sendfile on; + keepalive_timeout 65; + client_max_body_size 64m; + + log_format main '$remote_addr - $request_id [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" "$http_user_agent"'; + access_log /dev/stdout main; + error_log /dev/stderr warn; + + include /etc/nginx/conf.d/*.conf; +} diff --git a/deploy/release-artifacts/README.md b/deploy/release-artifacts/README.md new file mode 100644 index 0000000..c743f56 --- /dev/null +++ b/deploy/release-artifacts/README.md @@ -0,0 +1,26 @@ +# Production validation artifact contract + +Run `scripts/check_release_readiness.py DIR` before creating an RC tag. `DIR` +must contain the following files produced against the exact candidate image: + +- `smoke.json`: `light` and `heavy`, each with `unexpected_5xx: 0` and true + flow flags for authentication, brain creation, plain/structured ingestion, + task completion, context, search, Console, and MCP. +- `latency.json`: context/search p50, p95, and p99; context records zero online + LLM retrieval loops; search records `excludes_embed_query: true`. +- `restore-light.json` and `restore-heavy.json`: matching `before`/`after` + counts for brains, nodes, edges, chunks, vectors, and observations plus ten + deterministic retrieval checks with `match: true`. +- `security.json`: `high: 0`, `critical: 0`, the `sha256:` image digest, and + the relative path of its SBOM. + +The public ledger must also contain a representative LongMemEval result and a +WANDS representative Search row. LoCoMo and BEAM rows retain their explicit +sample scopes and are not promoted into broader claims by this gate. + +`scripts/production_smoke.py exercise` produces profile-scoped smoke, latency, +and pre-restore state evidence; `verify-restore` produces the matching restore +report. `scripts/assemble_release_artifacts.py` combines successful light and +heavy workflow artifacts. The tag-only publish workflow performs this assembly +and runs the gate itself, in addition to requiring all quality and dedicated +heavy-runner checks on the tagged commit. diff --git a/docs/decisions/007-three-product-surfaces-one-kb.md b/docs/decisions/007-three-product-surfaces-one-kb.md new file mode 100644 index 0000000..ff4b88f --- /dev/null +++ b/docs/decisions/007-three-product-surfaces-one-kb.md @@ -0,0 +1,128 @@ +# ADR-007: Three Product Surfaces on One Knowledge Base + +## Status + +Accepted — 2026-08-18. Does not supersede ADR-005 or ADR-006. Those remain the memory-path contract. This ADR adds **search** as a first-class product beside memory and recommendations, and states how they share the same brain without sharing request semantics. + +Extended by [ADR-008](008-query-gated-search-personalization.md) (Accepted 2026-08-21): optional search `target` reranks retrieved hits. The omitted-`target` path and this ADR’s SLO are unchanged. + +Locked §15 resolutions: + +1. Routes stay under `/retrieve` as `GET|POST /retrieve/search` (not a new top-level prefix). Existing `GET /retrieve/` is unchanged. +2. Search is env-gated (`SEARCH_ENABLED`, also in TUI init/config). v1 lexical is Okapi BM25 over Postgres `tsvector` + GIN on the stock `pgvector/pgvector:pg16` image — not `ILIKE`, not `ts_rank_cd`, not `rum` / ParadeDB `pg_search`. +3. Dense quality at 3072-d: keep float32 storage; build HNSW on a `halfvec` expression (legal to 4000-d); exact float32 cosine rerank of the overfetch window. Do not truncate embedding dimensions. +4. `plugins/chatbot-memory` is out of scope for v1. +5. Search SLO: p50 **< 200 ms excluding embed RTT**. `profile_stages` splits `embed.query` from retrieve so the budget is measurable. +6. Operators switch dense / BM25 / both via `SEARCH_USE_DENSE` / `SEARCH_USE_BM25`; default is both fused with `SEARCH_FUSION=rrf`. `/retrieve/context` stays ILIKE unless `SEARCH_ENABLED=true` and `CONTEXT_PASSAGE_MODE` is `hybrid` / `bm25` / `dense`. + +## Date + +2026-08-18 + +## Context + +`docs/research/00-scope-and-constraints.md` already names the product as a memory layer that is also “a substrate for agents, recommenders, and search engines.” In code those are not three products yet: + +| Intended product | Shipped surface | What the caller actually gets | +| --- | --- | --- | +| Agent / conversational memory | `POST /retrieve/context` | Prompt-shaped blob: passages + event facts + topics. Budget **p50 < 1000 ms**, no LLM (ADR-006). Evaluated on LoCoMo / LongMemEval / BEAM. | +| Recommendations | `GET|POST /retrieve/recommend` | Ranked **nodes** (items) from graph walks, synergies, optional attribute prefs. Evaluated on `demorecsys` (`16-recsys-eval-protocol.md`). Heavy CF is a plugin (`plugins/recsys-gnn`). | +| Search | **Missing** | `GET /retrieve/` is entity-biased graph+data lookup. Passage “hybrid” in context is dense ANN ∪ **`ILIKE '%query%'`**, fused with RRF — not BM25, not a hit list API, not a search SLO. | + +Default pgvector installs ship **3072-d** embeddings and skip HNSW (`vectors.py` returns no DDL above 2000 dims). Context p50 after dossier deletion is **~1.3–1.4 s**, residual dominated by **serial embedding HTTP**, not ANN. Passage evidence-session recall on LoCoMo conv-26 is already **~97%**; graph vs passages-only does not move judge accuracy above the 5–7 point noise floor. + +If search is implemented by stuffing more ranking into `/retrieve/context`, two failures follow: (1) memory benchmarks and the sub-second budget become the search SLO, which they cannot be; (2) search users receive a concatenated context blob instead of scored hits with filters, facets, and stable document IDs. + +The literature that actually transfers is first-stage IR (BM25, dense dual-encoders, hybrid fusion) and BEIR’s finding that **lexical and dense are complementary off-distribution** — not MS MARCO leaderboard recipes that assume a 200 ms p99 web index and a cross-encoder on every query. Analysis and task breakdown: `docs/research/17-search-surface-and-cross-features.md`. + +## Decision + +**1. One write path, three read products.** + +A brain remains the unique source of truth. Ingest (`POST /ingest/`, structured ingest, observations) writes chunks, embeddings, event graph, topics, and (after this work) a real lexical index **once**. Read products are separate HTTP contracts: + +| Product | Canonical API | Response shape | Default SLO | Benchmarks | +| --- | --- | --- | --- | --- | +| Memory | `POST /retrieve/context` | `text_context`, `triples`, passages, topics, paths | p50 < 1000 ms; no LLM | LoCoMo, LongMemEval, BEAM | +| Search | **`POST /retrieve/search`** (new) | Ranked **hits** (id, channel, score, snippet, extras) | p50 < 200 ms excluding embed RTT; p95 budget TBD from telemetry | New `benchmarks.search` (not LoCoMo judge) | +| Recommendations | `GET|POST /retrieve/recommend` | Ranked **nodes** + channel | Unchanged | `benchmarks.recsys` on `demorecsys` only | + +MCP / deep REST remains the expensive escalation lane (ADR-006): agentic traversal, cross-encoder, sufficiency retry, HyDE-style query expansion. + +**2. Do not overload `/retrieve/context` into a search engine.** + +Context stays a **memory assembler** (evidence for an answering model). Search stays a **ranker of addressable objects** (chunks, entities, events, optional structured records). A search client may *call* context afterwards; the APIs do not merge. + +**3. Share write-time indexes; isolate query-time defaults.** + +Shared (one index, many readers): + +- Vector tables + a **legal ANN index** (HNSW or `halfvec` HNSW at 3072-d; loud failure if skipped). +- Lexical postings (`tsvector` / BM25 or equivalent) over chunk text and entity names. +- Provenance, validity (`invalid_at`, `deprecated`), `happened_at`, topic memberships, hub-bridge table. + +Isolated (per product, flags on the request, defaults frozen per surface): + +- Fusion method and weights (RRF vs convex combination). +- Candidate fanout, channels enabled, graph PPR, fact diversification. +- Rerankers. Core: linear / small GBDT over already-computed scores. Plugins: SPLADE, ColBERT/PLAID, cross-encoder. + +`/retrieve/context` **must not** change its default ranking because search shipped. Opt-in only (`lexical=bm25` on context, or a one-way cutover after the non-regression gate). + +**4. Core vs plugin, same pattern as recsys.** + +| Layer | Owns | Analog | +| --- | --- | --- | +| Core | BM25 (or `tsvector` rank), dense ANN, RRF/CC fusion, filters, cheap feature ranker, search API | `/retrieve/recommend` train-free graph ranker | +| Plugin (optional) | Learned sparse (SPLADE family), late interaction (ColBERT/PLAID), cross-encoder rerank, instruction-tuned embedders | `plugins/recsys-gnn`, `plugins/features-rec` | + +A missing plugin degrades search to core hybrid, never 500s the memory path. + +**5. Non-regression is a gate, not a hope.** + +Any change that touches a **shared** index or the default context path: + +1. Paired LoCoMo (or the current memory champion arm) McNemar vs the last accepted product/SOTA run: **no statistically significant accuracy drop**; point estimate not lower. Prefer retrieval-side **passage EvR held** (deterministic) over judge accuracy. +2. Context p50 **not worse** than the post-dossier baseline (~1.4 s) by more than measurement noise; the sub-second target remains ADR-006’s, not this ADR’s. +3. Graph EvR agreement gate from `00` (≥95% identical session sets) still applies if seeds/ANN change. +4. Recsys HitRate on `demorecsys` unchanged unless the change is explicitly a recs feature. +5. Search and recs eval **never** ingest into or wipe `locomoconv26*`, `beam*clean`, or other memory brains. + +Search quality is **not** judged by LoCoMo LLM-as-judge. It uses ranking metrics (Recall@k, nDCG@k, MRR) on a dedicated query set. + +**6. Cross-features are allowed when they are indexes or cheap scores, not when they are product-specific loops.** + +Allowed to help more than one product: BM25, HNSW, batched embeddings, metadata filters, topic coarse-to-fine, recency/validity features, a linear blend of dense+lexical+PPR. + +Forbidden on the memory hot path, allowed on search-deep or plugins: per-query LLM (HyDE), cross-encoder over large K, ANCE training loops, ColBERT serving. + +## Alternatives considered + +### Implement search by adding flags to `/retrieve/context` +- Pros: one endpoint; less API surface. +- Cons: prompt blob ≠ hit list; memory SLO and search SLO conflict; LoCoMo harness would pick up search ranking noise. +- Rejected. + +### Separate search database (Elastic/Meilisearch) as source of truth +- Pros: mature BM25. +- Cons: two write paths, drift from the event graph, contradicts “unique source of truth.” +- Rejected as the system of record. An **adapter** that *projects* brain indexes into an external engine is a plugin, not a second brain. + +### Put a distilled cross-encoder on every search and context query +- Pros: largest precision jump in the MS MARCO literature. +- Cons: ADR-006 already rejected this on context (50–200 ms, 5–20% of budget). Search can add it as an **optional** second stage with small K. +- Rejected as default; accepted as plugin / `rerank=cross_encoder` on search only. + +### Train a BrainAPI-specific bi-encoder (DPR/ANCE) before shipping BM25 +- Pros: best dense quality in-domain. +- Cons: needs relevance labels and an index-refresh loop; LoCoMo N cannot support it; BEIR shows off-the-shelf dense is weak zero-shot without lexical backup. +- Rejected as a prerequisite. Off-the-shelf embeddings + real BM25 first. + +## Consequences + +- New route family under `/retrieve/search*`. Existing `/retrieve/context` and `/retrieve/recommend` keep their contracts. +- Lexical storage and ANN legality become substrate work (`03`), not a search-plugin secret. +- `benchmarks/search/` and `benchmarks.search` in `REPORTS.json` are required before claiming search quality, mirroring recsys isolation. +- Memory SOTA work (`12`, `13`) may **opt into** BM25∪dense RRF on context after the gate — HyperMem (`2604.08256`) already uses that hybrid — but shipping search does not force that opt-in. +- Documentation in `00` gains workstream `17`. Implementation does not start from this ADR alone; tasks live in `docs/research/17-search-surface-and-cross-features.md`. +- Optional query-gated personalize on search (`target`) is [ADR-008](008-query-gated-search-personalization.md). It must not change omitted-`target` ranking, default `channels=["passages"]`, or this ADR’s 200 ms SLO. diff --git a/docs/decisions/008-query-gated-search-personalization.md b/docs/decisions/008-query-gated-search-personalization.md new file mode 100644 index 0000000..fe17bd8 --- /dev/null +++ b/docs/decisions/008-query-gated-search-personalization.md @@ -0,0 +1,97 @@ +# ADR-008: Query-Gated Personalized Search on Retrieved Hits + +## Status + +Accepted — 2026-08-21. Does not supersede ADR-007. Search remains a separate product from recommendations. This ADR locks how identity and taste may touch `/retrieve/search` without sharing recs request semantics or becoming the default ranker. + +Locked resolutions: + +1. Identity on search is optional `target` (same name as `/retrieve/recommend`). The field is never `user_id`. +2. First-stage stays query → passage (omitted `channels=["passages"]`). Personalize reorders **already retrieved** hits; it does not call `EntityRecommendRetriever.recommend()`. +3. User-asserted `extras` remains a hard AND filter. Inferred taste is blend-only (I7 rejected). +4. HEM / TEM / ZAM training is deferred (I6). WANDS and ESCI cannot supply query×user labels. +5. Omitted-`target` ranking and latency stay the ADR-007 product path (p50 < 200 ms excluding embed). Personalized search is opt-in. + +## Date + +2026-08-21 + +## Context + +ADR-007 shipped search as a ranked hit list with its own SLO and eval (`benchmarks.search`). Catalog work ([24-search-catalog-graph.md](../research/24-search-catalog-graph.md)) attached `SearchHit.node_id` so a client can walk the same KB. Recommendations already take `target` on `/retrieve/recommend` and are scored as next-item HitRate on `demorecsys` ([16-recsys-eval-protocol.md](../research/16-recsys-eval-protocol.md)). + +Three failures follow if personalize is implemented as “recs on the search path”: + +1. **Wrong first-stage.** Hub-walk / recommend candidate generation on `searchbenchwandsgraph` recovered Recall@50 **0.287** vs passages **0.837**. That is missing gold, not a ranking defect. +2. **Wrong training data.** HEM (SIGIR 2017, DOI `10.1145/3077136.3080813`) and TEM (arXiv `2005.08936`) need query×user×item logs. Frozen WANDS and ESCI have no users. +3. **Wrong filter.** Writing `extras` from inferred style AND-drops relevant hits the user did not exclude. + +A fourth failure is **wrong identity**: a `user_id` field on `SearchRequestBody` forks the recommend contract and invites mixing HitRate with nDCG. + +Literature that actually transfers is **query-dependent** personalization — ZAM (arXiv `1908.11322`) and TEM both record that personalization does not always help — not always-on user vectors and not replacing BM25∪dense. + +Analysis: [25-personalized-search.md](../research/25-personalized-search.md). + +## Decision + +**1. Optional `target` on search; omit means anonymous.** + +`SearchRequestBody.target` and `GET /retrieve/search?target=` accept a USER uuid or id. Empty / omitted → no personalize call; hit order equals the pre-ship retrieve order. Unknown user → no-op. Schema forbids `user_id`, `product_id`, `sku`, and `brand` on the search body. + +**2. Rerank retrieved hits; do not generate candidates.** + +Personalize runs after extras AND and `node_id` attach, before the final `k` cut. Catalog `mode` personalizes `k_ret`. Hits without `node_id` get pref 0 and stay in the list. All-zero prefs preserve order. Never drop an id because taste did not match. + +Do not call `EntityRecommendRetriever.recommend()`. Score only the given node ids by ATTR-hub overlap with `user_pref_weights`. + +**3. Query-gated blend, not always-on taste.** + +λ is 0 when the query has digits / SKU-like tokens; otherwise it falls with token count (one token high, four+ low). Blend is min-max retrieve vs pref. This is a cheap stand-in for ZAM/TEM’s “personalize only when the query is under-specified,” not a trained attention model. + +**4. Write taste as PREFERS; keep EVENT for dated behavior.** + +Interaction `options` / `attributes` emit `USER -PREFERS-> ATTR` with the same hub uuid as catalog HAS. Options are not copied onto the product catalog blob. View / cart / purchase / favorite stay EVENT wrappers with `happened_at`. Architect persists any no-event direct triple, not only static HAS. + +Favorite / wishlist weight **0.7**; unknown behaviors **0.2**. Search pref vectors combine write-time PREFERS with recency-decayed EVENT→ITEM→ATTR (14-day half-life) even when PREFERS exist. + +**5. Inferred taste is not a filter.** + +User-asserted `extras` stays AND. The ranker must not inject `extras={style: …}` from history. + +**6. Eval isolation.** + +Unit tests in `tests/test_search_personalize.py`. Optional skip-ingest smoke only on `searchbenchwandsgraph`. Never wipe frozen `searchbench*` quality brains, `locomoconv*`, `beam*`, or `demorecsys`. Do not quote vs frozen WANDS **0.823** / ESCI **0.500**. Do not score recsys HitRate as search nDCG. If ledgered: `claim: architecture-demo` only. + +## Alternatives considered + +### Add `user_id` on `SearchRequestBody` +- Pros: matches ingest JSONL and recsys datasets. +- Cons: two identity names across retrieve; invites mixing HitRate with nDCG. +- Rejected. Reuse `target`. + +### Call `/retrieve/recommend` (or `EntityRecommendRetriever`) as search first-stage +- Pros: one ranking implementation. +- Cons: different task (next-item vs query); communities/graph first-stage already lost Recall@50 on the architecture-demo brain. +- Rejected. + +### Infer `extras` from PREFERS / recent events +- Pros: “personalize” looks like faceted search. +- Cons: AND-drops relevant hits the caller did not filter; confuses asserted constraints with taste. +- Rejected (I7). + +### Train HEM / TEM / ZAM before shipping a personalize knob +- Pros: literature SOTA for personalized product search. +- Cons: no query×user logs on WANDS/ESCI; would not be the 200 ms default; blocks the opt-in rerank that the graph already supports. +- Deferred (I6). Not a prerequisite. + +### Always-on personalize whenever `target` is present +- Pros: simpler than λ. +- Cons: ZAM/TEM evidence that specific queries get worse; navigational / SKU queries must stay retrieve-order. +- Rejected. + +## Consequences + +- `SearchHitScores.personalize` is additive; omitted-`target` responses keep `personalize` unset. +- Core library: `src/core/search/personalize.py`. Recommend candidate generation is unchanged aside from I4 weights. +- Recommend `_attribute_pref_targets` still skips EVENT walks when PREFERS exist; search personalize does not. That split is intentional until recs explicitly adopt the combine. +- Research record: `docs/research/25-personalized-search.md`. ADR-007 SLO and default channels stay the omitted-`target` path. diff --git a/docs/research/00-scope-and-constraints.md b/docs/research/00-scope-and-constraints.md index 6dd8736..cc39012 100644 --- a/docs/research/00-scope-and-constraints.md +++ b/docs/research/00-scope-and-constraints.md @@ -40,6 +40,15 @@ Anything expensive must be either moved to write time or exposed only on the dee | `13-locomo-93-research-levers.md` | Evidence-backed research levers for HyperMem-class ≥93% (harness-first) | | `15-ecommerce-gnn-recsys-landscape.md` | External GNN/e-commerce RecSys curriculum & industry index; BrainAPI mapping | | `16-recsys-eval-protocol.md` | Held-out next-item eval on demorecsys via structured ingest + /retrieve/recommend | +| `17-search-surface-and-cross-features.md` | Search as a third product surface on the same KB; shared indexes; non-regression vs LoCoMo/BEAM/recsys | +| `18-search-eval-protocol.md` | Hybrid BM25+dense search eval on `searchbench*`; ledger `benchmarks.search`; no LoCoMo judge | +| `19-search-esci-quality.md` | ESCI US first-stage / rerank quality track; frozen `searchbenchesci74` | +| `20-esci-sota-targets.md` | External ESCI / commerce-search SOTA numbers; what we may and may not quote | +| `21-esci-ce-graded-and-first-stage.md` | Graded CE and first-stage plugin arms; not the ADR-007 200 ms default | +| `22-multilingual-ecommerce-search.md` | Multilingual first-stage (ESCI ES/JP; Italian smoke, not ESCI) and production commerce plan; `/retrieve/search` only | +| `23-search-wands-quality.md` | Frozen WANDS passages control (nDCG@10 **0.823**); not a catalog-graph claim | +| `24-search-catalog-graph.md` | Opt-in catalog graph + `node_id` sidecar; architecture-demo on `searchbenchwandsgraph` | +| `25-personalized-search.md` | Query-gated rerank of already retrieved hits via optional `target`; ADR-008 | ## Decisions already taken during research diff --git a/docs/research/16-recsys-eval-protocol.md b/docs/research/16-recsys-eval-protocol.md index 87f2270..d1cb27a 100644 --- a/docs/research/16-recsys-eval-protocol.md +++ b/docs/research/16-recsys-eval-protocol.md @@ -14,7 +14,7 @@ Freshness: each structured ingest changes graph walks / attribute prefs immediat **Not** LoCoMo / LongMemEval / BEAM. Never write to or wipe `beam1m1clean`, `locomoconv26*`, or other memory-eval brains. Ledger upserts go only to `benchmarks.recsys` in [`benchmarks/REPORTS.json`](../../benchmarks/REPORTS.json). -Related: [ADR-002](../decisions/002-structured-ingestion-specific-processing.md), [15-ecommerce-gnn-recsys-landscape.md](15-ecommerce-gnn-recsys-landscape.md). +Related: [ADR-002](../decisions/002-structured-ingestion-specific-processing.md), [15-ecommerce-gnn-recsys-landscape.md](15-ecommerce-gnn-recsys-landscape.md), [ADR-008](../decisions/008-query-gated-search-personalization.md) (search `target` is not this protocol). --- @@ -100,3 +100,4 @@ Requires `BRAINPAT_TOKEN` in `benchmarks/.env`. Default brain: `demorecsys`. Rep - Default harness backend is **graph** (`GET /retrieve/recommend`); use `--backend lightgcn` only for the offline comparator. - Do **not** mutate `benchmarks.locomo` / `beam` / `longmemeval` ledger rows. - Do **not** point `--brain` at memory-eval brains. +- Do **not** score recsys HitRate as search nDCG. Optional `/retrieve/search?target=` is [25-personalized-search.md](25-personalized-search.md) / ADR-008 on `searchbench*` only, never `demorecsys`. diff --git a/docs/research/17-search-surface-and-cross-features.md b/docs/research/17-search-surface-and-cross-features.md new file mode 100644 index 0000000..3792cf7 --- /dev/null +++ b/docs/research/17-search-surface-and-cross-features.md @@ -0,0 +1,595 @@ +# 17 — Search surface on the shared knowledge base + +Workstream: make BrainAPI a **unique source of truth** that already powers memory and recommendations, and can also power **search**, with shared write-time features and isolated query contracts. Binding: `00-scope-and-constraints.md`, ADR-005, ADR-006, **ADR-007** (Accepted). Recs isolation: `16-recsys-eval-protocol.md`. Memory SOTA: `08`, `12`, `13`. Substrate indexes: `03` §G. + +**Plan mode.** This document does not change `src/` or harnesses. Tasks are sized for later implementers. + +Skills applied: scientific-brainstorming (ideas labeled, no automatic winner), scientific-critical-thinking (severity, construct validity), planning-and-task-breakdown (vertical slices, gates), paper-lookup (arXiv 2026-08-18). + +--- + +## 0. Scope of this session + +**Focal question.** How can one brain serve (a) agent memory, (b) document/entity search, and (c) recommendations as *separate products* that share indexes and cheap scores, without making LoCoMo / LongMemEval / BEAM / recsys **worse**? + +| | Record | +| --- | --- | +| Purpose | Architecture + literature + implementable plan | +| Audience | Maintainer; retrieval / substrate / eval owners | +| Decision owner | Maintainer (ADR-007 Accepted 2026-08-18; §15 locked below) | +| Time horizon | Core search API in weeks; plugins and LTR later | +| In scope | Read APIs, lexical+ANN indexes, fusion, cheap rankers, search eval, plugin boundary, cross-features that are indexes or scores | +| Out of scope | Replacing ingest; training a proprietary bi-encoder as a prerequisite; making `/retrieve/context` the search engine; MS MARCO as a product SLO; dual-use / biosafety (not implicated) | +| Real constraints | ADR-006: no LLM on context path; p50 < 1000 ms target unmet (~1.4 s); judge cannot resolve <5–7 LoCoMo points; 3072-d HNSW skipped; keyword path is ILIKE | +| Assumed | Event-leg supersession bug stays fixed going forward; existing unrepaired brains may still exist | +| Negotiable | Whether context *later* opts into BM25; search p95 number; Postgres FTS vs external inverted-index plugin | +| Unknown | Keyword vs paraphrase mix of future search tenants; corpus size per brain in production | +| Prohibited | LoCoMo judge as a search metric; wiping memory eval brains; silent default changes on `/retrieve/context` | + +**Perspectives represented:** retrieval engineering, memory eval, recsys (existing plugin split), IR literature. **Missing:** a production search-traffic owner (no click log). Do not treat LoCoMo questions as a substitute for that mix. + +--- + +## 1. Product model (decision, proposed) + +Three **read** products, one **write** path. Analogous to recsys (`16`): BrainAPI is the KB; heavy models are plugins. + +```text + POST /ingest* (chunks, graph, vectors, topics) + │ + ┌─────────────────┼─────────────────┐ + ▼ ▼ ▼ + /retrieve/context /retrieve/search /retrieve/recommend + memory assembler ranked hits ranked items + LoCoMo/BEAM benchmarks.search benchmarks.recsys + ADR-006 SLO search SLO recs SLO +``` + +**What “search” is not.** It is not “call context and parse `text_context`.” Context exists to feed an answering model. Search exists to return **addressable hits** (chunk / entity / event / structured row) with scores, filters, and snippets. + +**What “cross-feature” is.** A structure computed at write time, or a cheap score at read time, that more than one product can consume without inheriting the other’s loops (PPR over event hubs, cross-encoder, LightGCN train, sufficiency retry). + +--- + +## 2. Current state (located evidence) + +Anchors are the working tree as of 2026-08-18. Older line numbers in `02` / `07` may have drifted; behaviour below was re-checked in `retrieve.py`, `data.py`, `vectors.py`. + +### 2.1 Surfaces that exist + +| Surface | Role today | Search-engine analogue | +| --- | --- | --- | +| `POST /retrieve/context` | Memory: spaCy variants → dense node/rel seeds → event-centric facts + PPR + bridges + topics; passages via RRF(dense, keyword) | RAG retriever, not a SERP | +| `GET /retrieve/` | `retrieve_data`: text + preferred entity types | Internal lookup | +| `GET\|POST /retrieve/recommend` | Graph recommend (+ optional LightGCN plugin) | Recs, not lexical search | +| `GET /retrieve/entity/synergies` | Sibling / synergy scores | Related-entities widget | +| MCP `search_semantically` / `traverse_graph` / `search_memory` | Deep / agentic | Escalation, unbounded | + +### 2.2 The “hybrid” passage channel is not BM25∪dense + +```1680:1722:src/services/api/controllers/retrieve.py +def _retrieve_passages(...): + ... + vector_hits = vector_search.search_data(...) + ... + search_result = data_adapter.search(text, brain_id) # ILIKE + fused = reciprocal_rank_fusion([vector_ids, keyword_ids]) +``` + +`data.py` `search` is `WHERE text ILIKE %s` with `'%' + escaped_query + '%'`. No inverted index, no term weights, no `k1`/`b`. RRF (`fact_filter.py`, `k=60`) fuses ANN order with a **substring hit list**. + +**Implication (interpretation):** ADR-005’s hybrid design is real as *architecture*; the sparse leg is not the IR method the SOTA brief and HyperMem call BM25. Keyword/navigational search is the first thing this stack cannot honestly sell. + +### 2.3 Dense ANN is optional / silently off on default pgvector + +`_vector_index_ddl` returns empty string when `dimension > 2000`. Deploy examples set all stores to **3072**. `search_vectors` still `ORDER BY embeddings <=> $1` and sets `hnsw.ef_search` (no-op without an index). Over-fetch + uuid tie-break (`ann_overfetch_k`, `stable_top_k_vectors`) is in place for **determinism**, not speed. Milvus uses `AUTOINDEX` + COSINE **if** that backend is selected. + +On LoCoMo-sized brains this is not the latency bottleneck (`06`: passages p50 829 ms of which **682 ms embed**). On BEAM-scale or multi-million-chunk brains it becomes a sequential scan. **Two different products, two different bottlenecks.** + +### 2.4 Memory path latency and quality (do not mix with search claims) + +| Signal | Status | Use for search work | +| --- | --- | --- | +| Context p50 | ~1.3–1.4 s after dossier deletion; ADR-006 target 1 s | Shared embed batching helps both; do not put CE on this path | +| Passage EvR | ~97% conv-26 | LoCoMo **cannot** show BM25 recall gains in aggregate; need a keyword slice or a non-LoCoMo corpus | +| Graph vs passages-only judge acc | McNemar ns at n=152 | Do not justify search graph-channel by LoCoMo accuracy | +| Graph EvR | C3: 47.3% → 56.0% with reserved bridge slots + ANN seed stability | Graph as a **search channel** (return events/entities as hits) is still a product idea, not a LoCoMo win | +| SOTA compose arm | conv-26 95.4% judge (`13`); full-10 still open | **Do not touch default context ranking** while that claim is live unless the gate in §8 passes | +| Recsys | Isolated brain `demorecsys`; plugins for GNN / attribute prefs | Copy this isolation for search eval | + +### 2.5 Ranking on context (no LTR, no CE) + +Distance (min over event nodes) → optional PPR **replaces** distance → MMR diversify. LLM fact filter is a no-op without a deep-tier adapter. No LambdaMART, no linear blend of BM25+dense+PPR. Historical context still re-runs the same passage search (~91.5% duplicate; cost, not wall-clock). Passage collection still **serial** over 3–6 queries each calling `embed_text` though `embed_texts` exists. + +### 2.6 Recs already show the split we want for search + +`16`: core = train-free graph ranker on `/retrieve/recommend`; `plugins/features-rec` writes extra edges; `plugins/recsys-gnn` is optional LightGCN and **does not replace** ingest. Search should copy that: core BM25+dense+filters; plugins for SPLADE / ColBERT / CE. + +--- + +## 3. Literature (located, 2026-08-18) + +**Retrieval summary** + +- Query: canonical IR + memory papers by arXiv ID, plus title search for hybrid/RRF/SPLADE/ANCE +- Scope: targeted lookup (not exhaustive) +- Databases: arXiv Atom API (`export.arxiv.org/api/query`) parsed with `paper-lookup/scripts/arxiv_atom.py` +- Access date: 2026-08-18 +- Limits: first page of title-OR search (`total_results=59`, retrieved 8); ID lists complete (12 + 4). RRF’s original SIGIR 2009 paper is **not on arXiv** (Cormack, Clarke, Buettcher) — cited by name only. + +**Warnings:** Absence from this bound is not “never studied.” UniIR (`2311.17136`) is **multimodal** instruction retrieval — weak fit for text-only BrainAPI v1. HyDE (`2212.10496`) uses an LLM at query time — incompatible with ADR-006 on context; plugin/deep only. + +### 3.1 What actually supports a BrainAPI search product + +| Paper | arXiv | Status for us | Takeaway (not a BrainAPI result) | +| --- | --- | --- | --- | +| BEIR | [2104.08663](https://arxiv.org/abs/2104.08663) | **support-located** (zero-shot IR) | BM25 is a strong OOD baseline; dense often loses zero-shot; hybrid / CE / late-interaction win on average at higher cost. **This is the argument for a real lexical index before a custom bi-encoder.** | +| DPR | [2004.04906](https://arxiv.org/abs/2004.04906) | support-located (mechanism) | Dual-encoder + ANN can beat BM25 **in-domain with labels**. We have hosted embeddings, not DPR training. | +| ANCE | [2007.00808](https://arxiv.org/abs/2007.00808) | support-located (mechanism) | Hard negatives from a live ANN index close the train/test gap. High engineering cost; not a v1 gate. | +| ColBERT / v2 / PLAID | [2004.12832](https://arxiv.org/abs/2004.12832), [2112.01488](https://arxiv.org/abs/2112.01488), [2205.09707](https://arxiv.org/abs/2205.09707) | mixed | Token-level matching approaches CE quality; PLAID claims tens of ms on GPU at 140M passages. **Index size and serving complexity → plugin**, not core. | +| SPLADE / v2 | [2107.05720](https://arxiv.org/abs/2107.05720), [2109.10086](https://arxiv.org/abs/2109.10086) | mixed | Learned sparse on inverted indexes; v2 reports large BEIR gains. Index growth and inference → plugin after BM25 exists. | +| M3-Embedding (BGE-M3) | [2402.03216](https://arxiv.org/abs/2402.03216) | **idea-adjacent** | One model, three modes: dense, sparse, multi-vector, long context (8k). Strong **cross-feature candidate** if we ever change the embedder; not a silent swap (would re-embed every brain). | +| Matryoshka / MRL | [2205.13147](https://arxiv.org/abs/2205.13147) | support-located | Truncatable embeddings; `03` already maps this to making HNSW legal. `text-embedding-3-large` supports dimension cut natively. | +| Fusion analysis | [2210.11934](https://arxiv.org/abs/2210.11934) | **challenge-located vs “just RRF”** | Convex combination of lexical+dense scores can beat RRF; RRF is **parameter-sensitive**. Our hardcoded `k=60` is a choice, not a law. Prefer exposing RRF **and** CC on the search API. | +| HyperMem | [2604.08256](https://arxiv.org/abs/2604.08256) | support-located (memory) | LoCoMo SOTA report uses **topic→episode→fact + BM25∪dense RRF**. We already have topic coarse-to-fine; we lack BM25. **Same hybrid is a memory cross-feature**, opt-in on context after the gate. | +| BEAM | [2510.27246](https://arxiv.org/abs/2510.27246) | support-located (memory scale) | Long dialogues; retrieval cost grows with N. HNSW legality matters here more than on conv-26. | +| Filtered ANN | [2602.11443](https://arxiv.org/abs/2602.11443) | support-located (substrate) | Metadata filters + ANN; Milvus recall more stable than pgvector plans in their study. Search **filters** (type, validity, time) are first-class, not a WHERE after LIMIT. | +| Contrastive embeddings | [2201.10005](https://arxiv.org/abs/2201.10005) | background | Off-the-shelf embeddings can be strong for semantic search. Justifies keeping a general embedder in v1. | +| HyDE | [2212.10496](https://arxiv.org/abs/2212.10496) | **veto on context** | LLM generates a hypothetical doc then embeds. Deep/search-plugin only. | + +**Not treated as evidence for BrainAPI quality:** MS MARCO MRR tables in the user’s SOTA brief (analytical interpretation, different task, no reproduction here). + +### 3.2 What would confirm / refute transferring these papers + +- **Confirm BM25 value:** keyword-slice Recall@10 on a brain where ILIKE misses exact tokens that BM25 posts; latency not worse than ILIKE seqscan on a 100k-chunk fixture. +- **Refute BM25 value:** no Recall@k lift on that slice and on LoCoMo passage EvR (already saturated). +- **Confirm HNSW:** recall@k vs exact `ORDER BY <=>` within a declared drop (e.g. ≤2 points at k=24) **and** p95 vector stage drop on a large fixture. +- **Refute “search will lift LoCoMo judge”:** McNemar ns after BM25-on-context opt-in — expected; do not use that as a search-ship failure. + +--- + +## 4. Independent idea register + +Stage: `independent` then clustered. Origin: `AI-assisted` + `literature-inspired` (this session). These are **proposals**, not findings. + +| ID | Statement | Origin | Predicted observation | Disconfirm | +| --- | --- | --- | --- | --- | +| S01 | Add `POST /retrieve/search` returning scored hits, not a prompt blob | product | Search clients stop scraping `text_context` | Nobody uses it; context remains the only caller | +| S02 | Replace ILIKE with Postgres BM25/`tsvector` (or `pg_trgm` for short codes) as **core** lexical | BEIR, HyperMem | Keyword Recall@k ↑; long-query ILIKE misses ↓ | No lift on keyword slice | +| S03 | Legalize ANN (`halfvec` HNSW or MRL truncate ≤2000) with loud skip | `03` G, MRL | BEAM/large-brain vector p95 ↓; seed stability held via overfetch | Recall@k vs exact collapses | +| S04 | Batch `embed_texts` once per request; concurrent dense+lexical; single query on context | ADR-006 leftover | Context p50 toward <1 s | Stage timings still show multi-RTT embed | +| S05 | Expose fusion `rrf` and `cc` (convex combination) on search | 2210.11934 | CC wins in-domain if a tiny labelled set exists | RRF ≈ CC on our mix | +| S06 | Cheap linear/GBDT ranker over {dense, lexical, PPR, recency, topic, validity} | ADR-006, LTR brief | Precision@10 on search set ↑; microseconds | Features collinear; no lift | +| S07 | Search channels: `passages` \| `entities` \| `events` \| `structured` | product | Entity-name queries don’t require passage hits | Graph channel noisy; users ignore non-passage | +| S08 | Metadata filters (label, validity, time range) applied **inside** retrieval | 2602.11443 | Filtered search correct; no post-LIMIT drop | Planner ignores filter; recall hole | +| S09 | Opt-in `lexical=bm25` on context after search BM25 is proven | HyperMem | Memory keyword misses ↓ without judge drop | EvR or McNemar regresses | +| S10 | Plugin `features-search` or `search-rerank`: CE on K≤10 | brief, ADR-006 | Search nDCG ↑; context untouched | p95 blows search SLO | +| S11 | Plugin late-interaction (ColBERT/PLAID) or SPLADE | ColBERT family, SPLADE | Hard queries ↑ | Index size / reindex cost dominates per-brain | +| S12 | Optional embedder swap to BGE-M3 for dense+sparse+colbert from one model | 2402.03216 | One write, three retrieval modes | Re-embed cost; LoCoMo seed shift | +| S13 | Recs cold-start: `/retrieve/search` on item text for new SKUs, then graph recs | recs×search | Cold items appear before walks exist | Duplicate ranking; worse HitRate | +| S14 | Do **not** train ANCE/DPR in v1 | negative control | — | If we later get labels and a refresh loop, reopen | +| S15 | HyDE / query LLM rewrite only on deep search | 2212.10496, ADR-006 | Hard paraphrases ↑ at high latency | Context accidentally calls it | +| S16 | Search eval harness on a **separate** brain; ledger `benchmarks.search` | `16` analog | Claims don’t contaminate LoCoMo | People still quote judge % for search | + +**Minority / preserved:** S12 (embedder swap) and S11 (ColBERT core) are attractive and **wrong as v1** if they force re-ingest of memory brains. S07 graph-as-hits may look unused until a tenant searches people/events rather than documents — keep the channel even if default k focuses on passages. + +**What became less obvious after evidence check:** “Ship hybrid” is already the *shape* of context; the gap is the **sparse implementation** and the **missing hit-list API**, not the absence of RRF. “Need IVF+PQ / 100M HNSW RAM” is the SOTA brief’s catalog assumption; BrainAPI shards **per brain**. “Cross-encoder on K=10 for 200 ms p99” fights ADR-006 on memory and is optional on search. + +--- + +## 5. Clusters (by mechanism, not by slogan) + +| Cluster | IDs | Shared mechanism | +| --- | --- | --- | +| C-API | S01, S07, S08 | New contract: hits, channels, filters | +| C-LEX | S02, S09 | Real inverted/lexical index | +| C-ANN | S03, S04 | Vectors actually indexed; embed RTT collapsed | +| C-FUSE | S05, S06 | How scores combine (RRF/CC/LTR) | +| C-PLUG | S10, S11, S12, S15 | Heavy IR off the core path | +| C-X | S09, S13 | Cross-product: memory opt-in BM25; recs cold-start via search | +| C-EVAL | S16 | Isolated measurement | +| C-NOT | S14 | Explicit non-goals for v1 | + +Merges: S02 and “SPLADE in core” stay **split** (different index ops and failure modes). S04 is not the same as S03 (latency vs scale). + +--- + +## 6. Criteria (declared before preference) + +Direction: higher is better unless noted. Weights are decision aids, not a computed winner. + +| Criterion | Weight | Anchors (1–5) | Evidence needed | +| --- | --- | --- | --- | +| Memory non-regression | **Gate** (noncompensatory) | Fail = LoCoMo/BEAM/recsys worse | Paired McNemar, passage EvR, recs HitRate, p50 | +| Search usefulness | 3 | 1 = still ILIKE+blob; 5 = hit API + BM25 + filters | Recall@k / nDCG on search set | +| Latency / SLO fit | 3 | 1 = extra LLM; 5 = index lookup + one embed | `profile_stages` | +| Cross-feature leverage | 2 | 1 = search-only fork; 5 = one index, three readers | Same DDL used by context/search/recs | +| Feasibility / reversibility | 2 | 1 = re-embed all brains; 5 = additive index + flag | Rollback to ILIKE/exact scan | +| Information gain if null | 1 | Null still teaches query mix | Pre-registered slices | + +**Vetoes (not averaged):** LLM on `/retrieve/context`; default context ranker change without §8 gate; search eval writing memory brains; dual source of truth. + +--- + +## 7. Adversarial review (short) + +| Idea | Wrong if… | Alternative explanation | Hidden dependency | Harm | +| --- | --- | --- | --- | --- | +| S02 BM25 | LoCoMo-like questions are long paraphrases; BM25 adds posting I/O and zero recall | Dense already covers; ILIKE was enough on tiny N | Tokenization mismatch with embedder tokenizer | Over-rank boilerplate terms | +| S03 HNSW | Approx top-k reshuffles seeds → graph EvR jitter (C3 already fought this) | Latency win is still embed RTT | `halfvec` ops / pgvector version | Silent quality flake on memory | +| S01 new API | All “search” users were agents who wanted context anyway | Product confusion; two ways to do one thing | Client SDK + docs | Split brain of integrations | +| S06 LTR | No labels except LoCoMo sessions (biased to memory) | Overfit to session IDs | Click log does not exist | Ranking that looks good offline, worse UX | +| S12 M3 embedder | Different geometry vs current 3072-d OpenAI vectors | “One model three modes” marketing | Full re-embed | SOTA compose arm incomparable | + +**Anchoring risk:** the user’s SOTA brief pushes hybrid→CE→LTR on a 200 ms p99 web SLO. We keep the **sequence of primitives** (lexical, ANN, optional CE) but **reject the SLO and the default CE**. + +--- + +## 8. Non-regression protocol (how we refuse to make memory worse) + +Copy recsys isolation (`16`, `benchmarks/AGENTS.md`). + +### 8.1 What may change without a memory A/B + +- New routes under `/retrieve/search*` that do not alter context defaults. +- New tables/indexes that context **does not read** yet (e.g. `tsvector` column unused by `_retrieve_passages`). +- Plugins that only register search rerank. + +### 8.2 What requires the gate + +- Context reading BM25 instead of ILIKE. +- Enabling HNSW / changing embedding dimensionality or metric. +- Changing RRF `k`, seed `k`, passage fanout, PPR default, fact diversify. +- Shared embed batching that changes variant set or seed order (stabilize with existing uuid/distance quantization). + +### 8.3 Gate (all must hold) + +1. **Memory champion arm** (document the exact `run_id` at implementation time; today the conv-26 SOTA pointer is `locomo-compose-sota-conv26-v4d` in `REPORTS.json`): paired McNemar **not** significantly worse; point estimate ≥ baseline. Prefer **product** profile for default-path changes; do not require SOTA compose (SC/gap-fill) to move. +2. Passage **full evidence-session recall** held to the digit on that brain (deterministic). Graph EvR: if ANN/seeds change, identical-config agreement ≥95% (`00`). +3. Context p50: not worse than the then-current post-dossier baseline beyond noise (record both numbers). Sub-second remains ADR-006, not a search ship blocker. +4. Recsys: `demorecsys` HitRate@K unchanged unless the PR is a recs PR. +5. Brains: search fixtures use IDs like `searchbench*`. Never wipe `locomoconv26*`, `beam*clean`, `demorecsys`. + +**If BM25-on-context is null on LoCoMo:** keep ILIKE as context default; search still ships BM25. That is a successful split, not a failed hybrid. + +--- + +## 9. Proposed search API (idea → later spec) + +Vertical slice: one call that a UI or engine can use without parsing memory blobs. + +```http +POST /retrieve/search +``` + +```json +{ + "query": "alice counseling license", + "brain_id": "default", + "k": 20, + "channels": ["passages", "entities"], + "lexical": "bm25", + "dense": true, + "fusion": "rrf", + "filters": { + "labels": ["PERSON", "EVENT"], + "currently_valid": true, + "happened_after": null + }, + "rerank": "none", + "profile_stages": false +} +``` + +```json +{ + "hits": [ + { + "id": "chunk_…", + "channel": "passages", + "score": 0.031, + "scores": { "bm25": 12.4, "dense": 0.22, "rrf": 0.031 }, + "snippet": "…", + "source_session_ids": ["session_3"], + "entity_uuid": null + } + ], + "insufficient": false, + "stage_timings": null +} +``` + +Optional later: `GET /retrieve/search/suggest` (prefix / trgm). Out of v1 if it expands scope. + +`rerank`: `none` | `linear` | `plugin:`. Unknown plugin → 400, not a silent fallback that looks like core quality. + +--- + +## 10. Cross-feature map + +| Feature | Memory | Search | Recs | When to share | +| --- | --- | --- | --- | --- | +| BM25 / tsvector | Opt-in passage channel (HyperMem-like) | **Default lexical** | Item-text cold start (S13) | Write once; read per product flag | +| HNSW / halfvec | Seed + passage ANN at scale | Same | Node similarity if used | Shared; gate on seed stability | +| Batched embed | Closes ~400 ms gap (`06`) | One RTT per search query | If recs embed queries | Shared helper, no rank change if vectors identical | +| Topic memberships | Coarse-to-fine already on context | Facet / boost | Weak | Already written | +| Validity / `happened_at` | Current-truth vs history (`05`) | Filters | Recency decay (already on recs) | Shared properties | +| PPR / hub bridges | Multi-hop coverage | `events` channel ranking | Not default | **Do not** turn search into a second context assembler | +| Linear LTR features | ADR-006 context ranker | Search precision | Could consume overlap scores | Train **per product** or freeze a tiny global blend with separate weights | +| Cross-encoder | Deep tier only | Search plugin, K≤10 | No | Plugin | +| LightGCN | No | No | Plugin | Unchanged | + +**Prediction:** the highest cross-leverage items are **S02+S03+S04** (indexes + embed batch). They improve search immediately and *can* improve memory latency/keyword recall without touching judge prompts. + +--- + +## 11. Architecture decisions for implementers + +- **Lexical engine v1:** Postgres FTS (`tsvector` + `ts_rank_cd` or a BM25 extension available in the deployed image). Avoid requiring Elasticsearch as system of record (ADR-007). A plugin may *project* to an external engine later. +- **Do not change `EMBEDDING_*_DIMENSION` as a drive-by** in the search PR. HNSW legality is a dedicated slice with a recall-vs-latency measurement (`00`, `03`). +- **Context keeps ILIKE** until S09 gate. Implementation may write tsvector in the same migration. +- **RRF stays** as default fusion (already in code, HyperMem-like). Expose `fusion=cc` with `alpha` for search; do not retune context `k=60` in the same PR. +- **No ANCE/DPR training, no ColBERT core, no HyDE on context** in this workstream’s v1. + +--- + +## 12. Search evaluation protocol + +Mirror `16` so search cannot hide inside LoCoMo. Binding write-up: [`18-search-eval-protocol.md`](18-search-eval-protocol.md). Harness: `benchmarks/search/` + `./search.sh`. + +| Item | Rule | +| --- | --- | +| Brain | `searchbench*` only | +| Harness | `benchmarks/search/` + `./search.sh`; HTTP only | +| Ledger | `benchmarks.search` in `REPORTS.json` only | +| Metrics | Recall@{5,10,20}, nDCG@10, MRR; p50/p95 of `/retrieve/search` **excluding** `embed.query`; optional keyword/paraphrase slice | +| Query mix | At least: (a) **keyword / name**, (b) **paraphrase / semantic**. Filter/time deferred until the search API accepts filters. Gold = chunk IDs via `DOCID` markers, not LLM judge | +| Cheap start | Reuse a **held-out** subset of memory questions as *retrieval* labels (gold session → chunks via provenance) **without** running the answerer. Report as `search-from-locomo-qrels`, never as LoCoMo accuracy. Not in harness v1 | +| Forbidden | Ingest into memory eval brains; quoting judge % as search quality | + +Until a scored `evaluate` run is on the `benchmarks.search` leaderboard, do not claim “search is better than BM25-in-the-brief.” Toy `search_toy` rows are isolation checks, not BEIR. + +--- + +## 13. Task list + +Vertical slices. Each leaves the system working. No task is XL. + +### Phase 0 — Contract and isolation + +## Task 1: Freeze the three-surface contract + +**Description:** ADR-007 accepted or explicitly deferred by the maintainer; this doc linked from `00`. No ranking code. + +**Acceptance criteria:** +- [ ] Maintainer records accept / amend / reject on ADR-007 +- [ ] `00` workstream row for `17` remains accurate + +**Verification:** Human review of ADR-007. + +**Dependencies:** None +**Files:** `docs/decisions/007-three-product-surfaces-one-kb.md`, `docs/research/00-scope-and-constraints.md` +**Scope:** S + +## Task 2: Search eval skeleton + +**Description:** Package `benchmarks/search/` with smoke: ingest N chunks into `searchbenchsmoke`, call a **stub** or interim `POST /retrieve/search` (may 501 until Task 4), ledger key documented in `benchmarks/AGENTS.md`. + +**Acceptance criteria:** +- [ ] `./search.sh smoke` fails loudly if it would use a memory brain id +- [ ] `REPORTS.json` schema comment or empty `benchmarks.search` leaderboard allowed + +**Verification:** `./search.sh smoke` documented; unit test on brain-id guard. + +**Dependencies:** Task 1 (or proceed as Proposed) +**Files:** `benchmarks/search/**`, `benchmarks/AGENTS.md` +**Scope:** M + +### Checkpoint A +- [ ] Product split is written; search eval cannot wipe LoCoMo/BEAM brains + +--- + +### Phase 1 — Core search first stage (does not change context defaults) + +## Task 3: Lexical index at write time + +**Description:** Persist `tsvector` (or equivalent) on `data_text_chunks` (and entity name field if cheap). GIN index. Backfill existing brains lazily or via explicit job. Context **does not** query it yet. + +**Acceptance criteria:** +- [ ] New chunks get lexical vectors on ingest +- [ ] ILIKE path still used by `_retrieve_passages` +- [ ] Unit test: token query matches a chunk ILIKE would miss or vice versa, documented + +**Verification:** `pytest` on FTS helpers; explain/analyze on a fixture shows index use. + +**Dependencies:** None (can parallel Task 2) +**Files:** `src/lib/postgresql/data.py`, ingest write path, tests +**Scope:** M + +## Task 4: `POST /retrieve/search` core hybrid + +**Description:** New controller: embed once, dense `search_data` + BM25/tsvector, fusion RRF default, return hits. No PPR, no fact assembler, no historical duplicate. + +**Acceptance criteria:** +- [ ] OpenAPI/schema: query, k, channels⊇passages, lexical, dense, fusion +- [ ] Does not import dossier/synergy retrievers +- [ ] `profile_stages` works like context +- [ ] Context tests unchanged (no default behavior change) + +**Verification:** `pytest` API tests; `./search.sh smoke` green against live API. + +**Dependencies:** Task 3 +**Files:** `src/services/api/routes/retrieve.py`, new controller module, `requests.py` +**Scope:** M + +## Task 5: Filters + entity channel (optional second slice) + +**Description:** `currently_valid`, label filter; optional `entities` channel via node ANN + name FTS. + +**Acceptance criteria:** +- [ ] Filter is applied so that invalid entities are not in top-k when `currently_valid=true` +- [ ] Passage-only default remains if `channels` omitted + +**Verification:** Tests with a superseded edge / deprecated node. + +**Dependencies:** Task 4 +**Files:** search controller, graph/data queries +**Scope:** M + +### Checkpoint B +- [ ] Search API returns BM25∪dense hits +- [ ] `/retrieve/context` byte-level ranking defaults unchanged (diff + LoCoMo smoke optional) + +--- + +### Phase 2 — Shared speed/scale (gated) + +## Task 6: Batch embeddings on context **and** search + +**Description:** One `embed_texts` per request; concurrent dense+lexical; collapse context passage variant loop toward ADR-006 (single query or batched variants sharing the batch). Preserve seed uuid/distance stabilization. + +**Acceptance criteria:** +- [ ] Stage timings: a single `embed.query` (or one batched stage) on a profiled request +- [ ] Passage EvR held on a recorded memory arm **if** context code changed +- [ ] Search p50 excluding network policy: one embed RTT + +**Verification:** `profile_stages` JSON; paired EvR if context touched. + +**Dependencies:** Task 4 if sharing helper; else can start from context-only +**Files:** `retrieve.py`, embeddings adapter call sites +**Scope:** M + +## Task 7: Legalize HNSW + +**Description:** Measurement-first: `halfvec` vs truncate-to-2000 vs Milvus-only. Loud log/metric when index DDL skipped. Keep overfetch+uuid order. + +**Acceptance criteria:** +- [ ] Default 3072 install either has an ANN index **or** fails closed with a visible error at store init +- [ ] Recall@k vs exact on a ≥10k vector fixture recorded +- [ ] Memory gate §8 if context seeds change + +**Verification:** Fixture script + notes in run dir; no silent seqscan. + +**Dependencies:** None vs search API; **gate** before flipping production default +**Files:** `src/lib/postgresql/vectors.py`, config, tests +**Scope:** M + +### Checkpoint C +- [ ] Context p50 moved toward 1 s **or** numbers escalate per ADR-006 +- [ ] ANN skip cannot be silent + +--- + +### Phase 3 — Fusion / cheap LTR / memory opt-in + +## Task 8: Search fusion knobs + linear rerank + +**Description:** `fusion=rrf|cc` with documented `alpha`; optional `rerank=linear` using dense, lexical, recency, validity — **no LLM**. + +**Acceptance criteria:** +- [ ] Default remains RRF k=60-equivalent +- [ ] Linear weights are config/request, not a trained black box in v1 +- [ ] Search nDCG measured on `searchbench*` vs RRF-only + +**Verification:** `./search.sh evaluate` two arms. + +**Dependencies:** Task 2, Task 4 +**Files:** search controller, `fact_filter.py` or shared fusion +**Scope:** S–M + +## Task 9: Opt-in BM25 on context (cross-feature) + +**Description:** `GetContextRequestBody.lexical: ilike | bm25` default **`ilike`**. Document HyperMem-style union as the reason to try `bm25`. + +**Acceptance criteria:** +- [ ] Default path = today’s ILIKE RRF +- [ ] `lexical=bm25` uses the Task 3 index +- [ ] Gate §8 on a product LoCoMo arm before considering default flip + +**Verification:** McNemar + passage EvR; do not use SOTA compose as the only arm. + +**Dependencies:** Task 3, Task 6 recommended +**Files:** `retrieve.py` `_retrieve_passages`, `requests.py` +**Scope:** S + +### Checkpoint D +- [ ] Search has fusion/linear knobs +- [ ] Memory default still ILIKE unless gate passed and maintainer flips + +--- + +### Phase 4 — Plugins (after core is real) + +## Task 10: Search rerank plugin hook + +**Description:** Same pattern as `recsys-gnn`: optional package registers `rerank=plugin:cross-encoder` (or SPLADE document expansion). Core works if plugin absent. + +**Acceptance criteria:** +- [x] Plugin yaml + route or entrypoint +- [x] Context path cannot call the plugin +- [x] K≤10 documented for CE + +**Verification:** Load/unload plugin; 400 on unknown name. + +**Dependencies:** Task 4 +**Files:** `plugins/search-rerank/` or `features-search/`, plugin loader +**Scope:** M + +## Task 11: Recs cold-start via search (optional) + +**Description:** When graph walks return < k items, fill from `/retrieve/search` on item text / attributes. Flag on recommend request, default off. + +**Acceptance criteria:** +- [ ] Default HitRate on `demorecsys` unchanged +- [ ] Flag-on arm recorded separately in `benchmarks.recsys` + +**Verification:** `./recsys.sh evaluate` both flags. + +**Dependencies:** Task 4 +**Files:** `recommend.py`, recsys harness +**Scope:** S–M + +### Checkpoint E +- [ ] Heavy IR is optional +- [ ] Recs default unchanged + +--- + +## 14. Risks + +| Risk | Impact | Mitigation | +| --- | --- | --- | +| HNSW changes memory seeds | High | Overfetch+uuid; EvR agreement gate; exact scan fallback flag | +| BM25 on context looks “free SOTA” | Med | Opt-in; HyperMem is not our judge protocol (`13`) | +| Search API unused; context still scraped | Med | SDKs/docs; chatbot plugin uses search for “find” and context for “answer” | +| FTS tokenization ≠ embed tokenizer | Med | Document analyzer; don’t expect identical hits | +| Scope bleed into ColBERT/ANCE | High | C-NOT / Task 14; plugins only | +| Judge-noise used to “prove” search | High | Separate metrics and brains | + +--- + +## 15. Open questions (maintainer) + +Locked 2026-08-18 (see ADR-007 Status): + +1. **Routes.** Keep search under `/retrieve` as `GET|POST /retrieve/search`. Do not replace `GET /retrieve/`. +2. **Lexical engine.** Env-gated. When on, Okapi BM25 over Postgres `tsvector` + GIN on `pgvector/pgvector:pg16`. Do not require `pg_search` / `rum`. Do not ship `ILIKE` or `ts_rank_cd` as the search lexical ranker. +3. **3072-d ANN.** Keep float32 vectors. `halfvec` HNSW (≤4000-d) + exact float32 cosine rerank of overfetch. Do not truncate / MRL-cut dimensions. +4. **Chatbot plugin.** Out of scope for v1. No `plugins/chatbot-memory` client. +5. **SLO.** p50 **< 200 ms excluding embed RTT**. `profile_stages` must expose `embed.query` separately from retrieve. +6. **Context vs search ranking.** Search default is both channels fused (`SEARCH_FUSION=rrf`). Context stays today’s dense ∪ ILIKE unless `SEARCH_ENABLED=true` and `CONTEXT_PASSAGE_MODE` is left at `hybrid` (or set to `bm25` / `dense`). `ilike` freezes the memory eval path. + +--- + +## 16. Overall assessment + +BrainAPI is already a **knowledge base** with a strong memory assembler and a recs surface that knows how to keep plugins off the default path. It is **not** yet a search engine: there is no hit-list API, no BM25, and default vectors may be unindexed. + +The SOTA search brief is useful as a **checklist of primitives** (lexical, dense ANN, optional CE, LTR). It is the wrong **product recipe** if pasted onto `/retrieve/context` (wrong SLO, wrong response shape, wrong eval). + +Highest-leverage path that can help search *and* memory without betting LoCoMo: **write a real lexical index and a legal ANN index, batch embeddings, ship `/retrieve/search` that reads them, keep context defaults frozen until a paired gate.** Cross-encoders, SPLADE, ColBERT, HyDE, and ANCE stay plugins or deep-tier — the same split recsys already uses. + +**Next action:** maintainer accept/amend ADR-007, then Task 2+3 in parallel (eval skeleton + tsvector), then Task 4 (search API). Label: **protocol development**, not a validated quality claim. + +**Status note (2026-08-21):** ADR-007 accepted; `/retrieve/search` shipped. Subsequent workstreams: `18`–`24` (eval, ESCI, WANDS, catalog graph), `25` + [ADR-008](../decisions/008-query-gated-search-personalization.md) (query-gated personalize). §16 above is the 2026-08-18 assessment that motivated the ship. Default omitted `channels=["passages"]` and the 200 ms SLO are unchanged. diff --git a/docs/research/18-search-eval-protocol.md b/docs/research/18-search-eval-protocol.md new file mode 100644 index 0000000..199401e --- /dev/null +++ b/docs/research/18-search-eval-protocol.md @@ -0,0 +1,189 @@ +# Search eval protocol — hybrid BM25 + dense (`/retrieve/search`) + +How to ingest a labeled toy corpus into BrainAPI and score ranked hits without contaminating memory eval. + +**Role split (locked):** + +| Component | Responsibility | +| --- | --- | +| BrainAPI | KB + **core hybrid search**: `POST /ingest/` writes chunks (tsvector + dense); `POST /retrieve/search` returns ranked hits (`fusion=rrf` default, `cc` override). `rerank=none` (default). Unknown `plugin:` is **400**, never a silent fallback | +| `plugins/search-rerank` | Optional second-stage cross-encoder via `rerank=plugin:cross-encoder`. Default `mode=default` reranks at most `RERANK_MAX_K=10`. Opt-in `mode=catalog` retrieves `k_ret=min(200, max(k, 50))` and reranks at most `CATALOG_RERANK_MAX_K=50`, then cuts to request `k`. Health: `GET /search-rerank/health`. Must not run on `/retrieve/context`. Catalog+CE is **not** the ADR-007 200 ms default path | +| `plugins/search-splade` | Learned-sparse first stage. Own index: `POST /search-splade/index`. Retrieve via `channels=["plugin:splade"]` (optionally fused with `passages`) | +| `plugins/search-colbert` | Late-interaction first stage (MaxSim). Own index: `POST /search-colbert/index`. Retrieve via `channels=["plugin:colbert"]` | + +**Not** LoCoMo / LongMemEval / BEAM / RecSys. Never write to or wipe `beam*`, `locomoconv*`, `demorecsys`, or LongMemEval brains. Ledger upserts go only to `benchmarks.search` in [`benchmarks/REPORTS.json`](../../benchmarks/REPORTS.json). Never quote LoCoMo judge % as a search score. + +Related: [ADR-007](../decisions/007-three-product-surfaces-one-kb.md), [ADR-008](../decisions/008-query-gated-search-personalization.md), [17-search-surface-and-cross-features.md](17-search-surface-and-cross-features.md) §12. + +--- + +## Server prerequisites + +The API under test must have: + +- `SEARCH_ENABLED=true` +- `DATA_DB=postgresql` +- `BRAIN_CREATION_ALLOWED=true` +- At least one of `SEARCH_USE_DENSE` / `SEARCH_USE_BM25` true (product default: both) + +Harness auth: `BRAINPAT_TOKEN` and `BRAINAPI_URL` in `benchmarks/.env`. + +A 404 on `/retrieve/search` is **search disabled**, not a ranking miss. The harness must fail loudly (`Set SEARCH_ENABLED=true`). + +--- + +## Pipeline + +```text +docs JSONL → POST /ingest/ (skip_enrichment=true → chunk + embed only) + → optional POST /ingest/structured (`--ingest-graph`; entity uuid = catalog `doc_id`) + → optional interaction JSONL (`--interactions`; EVENT + `happened_at` on `searchbench*`) + → GET /retrieve/text-chunks (map DOCID markers → chunk ids) + → POST /retrieve/search (profile_stages=true, fusion=rrf|cc; + channels: passages | entities | events | communities; + expand: none | neighbors; + mode: default | catalog) + → Recall@{5,10,20} / nDCG@10 / MRR + → p50/p95 retrieve excluding embed.query +``` + +Gold matches **chunk ids** (via DOCID mapping) **or** `hit.id == doc_id` (graph entity uuid). Binary relevance is enough for the toy fixture. Catalog datasets carry graded `gold_grades` and nDCG@10 uses those gains. + +Each ingested document embeds a unique marker `DOCID `. After ingest, paginate `GET /retrieve/text-chunks` and map `doc_id → {chunk_id, ...}` by substring. A hit matches if `hit.id` is in that gold set **or** equals the catalog `doc_id`. + +Search ingest defaults to `skip_enrichment=true` (chunk + embedding, no Scout/Architect). Pass `--enrich` to restore full KG ingest. Catalog evals should stay on the skip path so OpenAI/DeepSeek extract quota is not in the loop. Embeddings still run. `--ingest-graph` writes generic HAS triples (entity + CLASS/TYPE/ATTR hubs) via product `catalog_graph`. Core search has no PRODUCT/CATEGORY/BRAND field names; those strings may appear only as ordinary node types in harness JSONL. + +--- + +## Query mix + +Bundled fixture: [`benchmarks/data/search_toy.jsonl`](../../benchmarks/data/search_toy.jsonl). + +| Slice | Intent | +| --- | --- | +| `keyword` | Name / token overlap (BM25 should help) | +| `paraphrase` | Semantic rewrite (dense should help) | + +Product-search corpora (download, not committed): + +| Dataset | Command | Default slice | Gains | +| --- | --- | --- | --- | +| Amazon ESCI / Shopping Queries (KDD Cup 2022) | `./search.sh download --name esci` | US, Task 1 `small_version`, `test` | E=1, S=0.1, C=0.01, I=0 | +| WANDS (Wayfair) | `./search.sh download --name wands` | `query_class` | Exact=1, Partial=0.5, Irrelevant=0 | + +Defaults cap the local JSONL at 80 queries / 2000 products / 40 candidates per query so ingest is feasible. Frozen WANDS quality slice is `data/search_wands.jsonl` (caps 80/2000/40; actual n may be lower because `_select_catalog` drops queries with no remaining gold once `max_docs` fills). `download --name wands` refuses to overwrite that file once it exists. Use `--brain searchbenchesci` / `searchbenchwands`. Do **not** reuse `demorecsys`. Structured recsys JSONL is not a search qrel. WANDS first-stage control is passages-only, k=50, `rerank=none` — see [23-search-wands-quality.md](23-search-wands-quality.md). Graph `--ingest-graph --channels passages,entities,communities` is not the quality control. Opt-in catalog graph (hub intersection + node hybrid) uses a **new** brain `searchbenchwandsgraph` and is an **architecture demo** — see [24-search-catalog-graph.md](24-search-catalog-graph.md). Do **not** mix those numbers with frozen WANDS 0.823 or ESCI 0.500. `--ingest-graph` / `--interactions` on `searchbenchwands`, `searchbenchesci74`, `searchbenchescies`, or `searchbenchesciltr2` is refused. + +Filter / time slices are **deferred** until `/retrieve/search` accepts filters. Do not fake them with ILIKE on the harness side. + +Graph channels (core, use-case agnostic): `entities` (node ANN ∪ name CONTAINS), `events` (EVENT nodes; empty-ok on catalog-only brains; recency from `happened_at` when present), `communities` (typed hub nodes, default labels `TYPE,CLASS,TOPIC` via `SEARCH_COMMUNITY_LABELS`), `expand=neighbors` (depth-1, fanout cap + degree IDF). Default omitted `channels` remains `passages`. Hits are a mixed list of chunk ids and node uuids until a join table exists. Communities are **not** Leiden / `kg_topic_sessions`. Do not score `/retrieve/recommend` HitRate as search. + +Optional later arm: reuse held-out memory questions as *retrieval* labels (`search-from-locomo-qrels`) **without** running the answerer, and never as LoCoMo accuracy. Not in v1 of this harness. + +--- + +## Metrics and SLO + +| Metric | Definition | +| --- | --- | +| Recall@k | \|retrieved ∩ gold\| / \|gold\| at k ∈ {5,10,20} | +| nDCG@10 | Binary on toy gold; graded `gold_grades` on ESCI/WANDS. Hits collapse to `doc_id` (chunk map **or** node uuid) | +| MRR | Reciprocal rank of the first gold hit | +| `p50_retrieve_ms` / `p95_retrieve_ms` | Prefer `stage_timings` `search.retrieve.wall_ms`; else `client_wall − embed.query.wall_ms` | + +ADR-007 SLO: **p50 < 200 ms excluding embed RTT**. Record `embed.query` separately so the ledger cannot mix the two. That SLO is the **default** path (`mode=default`, `rerank=none`). `mode=catalog` plus `rerank=plugin:cross-encoder` is an opt-in two-stage; do not cite its client wall as the 200 ms product claim. + +Headline for the ledger: `ndcg@10`, `recall@10`, `p50_retrieve_ms`. Label `dataset` (`search_toy.jsonl` vs `search_esci.jsonl` / `search_wands.jsonl`) so a toy run is not mistaken for product search. + +A passing **smoke** is a health check, not a published quality claim. Only `evaluate` with `status: ok` upserts `benchmarks.search`. + +--- + +## Harness + +HTTP only (`benchmarks/search/` must not import `src/`, except `search/mapping.py` which re-exports `src.core.search.catalog_graph` after putting the repo root on `sys.path`). Default brain: `searchbenchsmoke`. Any `--brain` **must** start with `searchbench`. + +```bash +cd benchmarks +./search.sh dataset-stats +./search.sh smoke +./search.sh evaluate --fusion rrf +./search.sh evaluate --fusion cc --run search-toy-cc +./search.sh download --name esci +./search.sh download --name esci --locale es +./search.sh download --name wands +./search.sh --brain searchbenchitsmoke evaluate --dataset data/search_italian_smoke.jsonl --run search-italian-smoke +./search.sh evaluate --dataset data/search_esci.jsonl --brain searchbenchesci --run search-esci +./search.sh --brain searchbenchwands evaluate --dataset data/search_wands.jsonl --run search-wands-passages-k50 --channels passages --k 50 --ks 5,10,20,50 +./search.sh --brain searchbenchwands evaluate --dataset data/search_wands.jsonl --run search-wands-passages-k50 --channels passages --k 50 --ks 5,10,20,50 --skip-ingest +./search.sh --brain searchbenchwandsgraph evaluate --dataset data/search_wands.jsonl --run search-wandsgraph-communities-k50 --channels communities --k 50 --ks 5,10,20,50 --ingest-graph +./search.sh evaluate --rerank plugin:cross-encoder --run search-toy-ce +./search.sh evaluate --mode catalog --rerank plugin:cross-encoder --k 50 --brain searchbenchesci74 --skip-ingest --dataset data/search_esci_74.jsonl --run search-esci-74-catalog-ce-k50 +./search.sh rank-corpus --dataset data/search_esci_74.jsonl --run search-esci-74-exhaustive-ce +./search.sh list-overlap --passages-run search-esci-74-passages-k50 --against-runs search-esci-74-bge-base-k50,search-esci-74-colbert-k50 +./search.sh union-lists --from-runs search-esci-74-passages-k50,search-esci-74-bge-base-k50,search-esci-74-colbert-k50 --run search-esci-74-union-bge-k50 +./search.sh cascade-lists --passages-run search-esci-74-passages-k50 --from-runs search-esci-74-bge-base-k50,search-esci-74-colbert-k50 --run search-esci-74-cascade-tail-k50 +./search.sh finetune-4class --dataset data/search_esci_74.jsonl --base microsoft/deberta-v3-base --epochs 2 --max-pairs 80000 --batch-size 8 --out data/models/esci-deberta-v3-base-4class +SEARCH_RERANK_MODEL=data/models/esci-deberta-v3-base-4class ./search.sh rerank-retrieved --from-run search-esci-74-passages-k50 --dataset data/search_esci_74.jsonl --run search-esci-74-passages-k50-ce-deberta --ks 5,10,20,50 +./search.sh evaluate --channels passages,plugin:splade --run search-toy-splade +./search.sh smoke --interactions data/recsys_toy.jsonl --channels events --brain searchbenchevents +./search.sh report --run +``` + +Requires `BRAINPAT_TOKEN` in `benchmarks/.env`. + +**Live stack note:** TUI / `brainapi start` may run from `~/.brainapi/source`. Sync or restart from the intended checkout before measuring, or scores will reflect stale code (`SEARCH_ENABLED` must be on in **that** process). + +--- + +## Guardrails + +- Score only via `POST /retrieve/search`. Do **not** call `/retrieve/context` as the metric path. +- Do **not** call `/retrieve/recommend`. Optional search `target` is query-gated rerank of retrieved hits (ADR-008), not this recommend protocol. +- Do **not** mutate `benchmarks.locomo` / `beam` / `longmemeval` / `recsys` ledger rows. +- Do **not** point `--brain` at memory-eval or recsys brains (`searchbench*` only; never `demorecsys`). +- Failed / empty runs stay off the leaderboard. +- Cross-encoder, SPLADE, and ColBERT are plugins. Core hybrid works if they are absent. `--rerank plugin:` and `--channels plugin:` must 400, never look like a ranking miss. + +--- + +## Multilingual catalog paths (2026-08-19) + +**Located evidence:** `download --locale` already filtered ESCI parquet (`us` / `es` / `jp`) but defaulted to the same `search_esci.jsonl` as US. **Decision:** US stays `search_esci.jsonl`; ES/JP write `search_esci_{locale}.jsonl`; `--locale it` errors. ESCI has no Italian split (Reddy HTML; arXiv Italian product-search query empty). Fixture `data/search_italian_smoke.jsonl` is a pipeline smoke (`slice=italian-smoke`), not ESCI Task 1 and not an Italian quality claim. FTS analyzer stays `to_tsvector('english'`. Do not re-ingest `searchbenchesci74`. Plan: [22-multilingual-ecommerce-search.md](22-multilingual-ecommerce-search.md). + +**ES first-stage (2026-08-19, located evidence):** `./search.sh download --name esci --locale es` wrote `data/search_esci_es.jsonl` (n=62 queries, 2000 docs, `slice=esci-es`) without changing `search_esci.jsonl` / `search_esci_74.jsonl`. Live `./search.sh --brain searchbenchescies evaluate --dataset data/search_esci_es.jsonl --run search-esci-es-passages-k50 --channels passages --k 50 --ks 5,10,20,50` → nDCG@10 **0.577**, Recall@10 **0.353**, Recall@50 **0.914**, p50 84 ms. Protocol: first-stage shared-index, passages, `rerank=none`. Do not mix with US n=74 0.500/0.379 or Reddy ES 0.849 (ranking-in-pool). Product default unchanged. + +Italian smoke is pipeline-only (`./search.sh --brain searchbenchitsmoke evaluate --dataset data/search_italian_smoke.jsonl`). Inflected BM25 miss on english FTS; optional `SEARCH_FTS_BRAINS=searchbenchitsmoke` + `SEARCH_FTS_REGCONFIG=italian` adds `search_tsv_alt` on that brain only. `_SEARCH_DDL` stays english. + +**Phase 2 (2026-08-19).** Italian product-search qrels: still **no direct evidence located** (arXiv 0; OpenAlex incidental hits are not qrels). ES spelling harness `search-esci-es-spell-k50` skip-ingest: nDCG@10 0.595 (win bar 0.597) / Recall@10 0.362 / Recall@50 0.914 — **null**, not live. Optional `SearchRequestBody.extras` equality filter + `SearchResponse.facets` on hit extras; live `searchbenchitsmoke` `locale=it` keeps 3 docs, `color=nope` empty. MiniLM local-dense `search-italian-minilm-pipeline` on labeled `searchbenchitmmini` retrieved all 3 golds at k=10 (pipeline only; not product embedder). Do not mix with Reddy 0.849 / US 0.500 / ES n=62 first-stage. Plan: [22-multilingual-ecommerce-search.md](22-multilingual-ecommerce-search.md). + +**Production Italian + US n=74 arms (2026-08-19).** Isolation held: `searchbench*` only; `_SEARCH_DDL` stays `to_tsvector('english'`; locomo/beam/`demorecsys` never get `search_tsv_alt`. **Decision:** production Italian is a dedicated `searchbench*` brain + `SEARCH_FTS_BRAINS` + `SEARCH_FTS_REGCONFIG=italian` + optional `extras={"locale":"it"}`. Alt BM25 matches **any** query lexeme (OR); english `search_tsv` keeps `plainto_tsquery` AND. Inflect skip-ingest `search-italian-smoke-inflect-fts-or`: BM25 nonempty for `bollitori` / `divani` / `caffettiere`. Do not quote nDCG as Italian quality. + +US n=74 skip-ingest, passages, k=50, `rerank=none` (control nDCG@10 **0.500** / Recall@10 **0.379** / Recall@50 **0.834** / p50 ~60 ms): + +- Frozen-head cascade is in `hybrid.frozen_head_merge` and is wired only when extra plugin (or `SEARCH_LITERAL_FILL`) lists are present. Default omitted `channels=["passages"]` stays hybrid RRF. Replay of stored passages+BGE+ColBERT lists still Recall@50 **0.889** / nDCG@10 **0.500**. Live n=74 sidecar not run: ColBERT/SPLADE plugin indexes are empty for `searchbenchesci74`. Honest default Recall@50 stays **0.834**. +- `fusion=cc` alpha `{0.3,0.5,0.7}`: best nDCG@10 **0.493** (alpha 0.7) vs win **0.520**. **Null.** Leave RRF. +- Title-token literal residual (`SEARCH_LITERAL_FILL`, frozen-head): Recall@10 **0.379** held; Recall@50 **0.655** (below 0.834). **Null.** Flag stays false. +- Head LTR (`./search.sh ltr-head`, run `search-esci-74-ltr-head-k50`): query-grouped 5-fold CV reorders stored `search-esci-74-passages-k50` `hit_ids` (same 50; Recall@50 **0.834** held by construction). nDCG@10 **0.515**, Recall@10 **0.384** vs win **0.520** / **0.397**. Overlap-only diagnostic nDCG@10 **0.476** / Recall@10 **0.339**. **Null.** Not live. Do not train-on-all-74 as the quality number. +- Head LTR + 4-class `ce_gain` feature (`search-esci-74-ltr-cefeat-k50`): same CV, pair policy `other_query_neg`. nDCG@10 **0.524** (gate ≥ 0.520), Recall@10 **0.383** (hold ≥ 0.379), Recall@50 **0.834**. **Harness win on predeclared nDCG/hold.** Not live. Human review before `search.py`. Do not mix with Reddy 0.857 or CE-as-sole-ranker (those hurt Recall@10). +- Head LTR + DeBERTa-v3-base `ce_gain` (`search-esci-74-ltr-deberta-k50`): same lists/CV/`other_query_neg`; checkpoint `data/models/esci-deberta-v3-base-4class`. nDCG@10 **0.542** (must-beat this round: > **0.524**), Recall@10 **0.387** (≥ 0.379), Recall@50 **0.834**. **Harness win vs MiniLM blend.** Horizon nDCG@10 ≥ **0.70** missed (oracle on these 50 is 0.876). Gated LightGBM lambdarank (`search-esci-74-ltr-deberta-lgbm-k50`): nDCG@10 **0.516**, Recall@10 **0.379** (unrounded 0.3788, below hold), `ce_gain` gain-importance largest. **Null vs RankNet DeBERTa.** Not live. Do not mix with Reddy 0.857, pool nDCG@20 0.710, or DeBERTa-as-sole-sorter (0.510 / 0.363). +- I04 train-on-matched-hybrid (`search-esci-74-ltr-deberta-train200`): RankNet+DeBERTa fit on 170 US train queries' hybrid k=50 from new brain `searchbenchesciltr2` (JSONL cap 200q/4k docs, doc budget bound to **170** queries / **4000** docs; 0 qid overlap with the 74). Applied to frozen `search-esci-74-passages-k50`. nDCG@10 **0.544** (> **0.542**), Recall@10 **0.391** (≥ 0.379), Recall@50 **0.834**. **Harness win vs I01 CV.** Horizon 0.70 missed. Train-set first-stage diagnostic (different catalog): nDCG@10 **0.387** / Recall@50 **0.744**. Not live. Do not wipe `searchbenchesci74`. +- I-LGBM-APPLY (`search-esci-74-ltr-deberta-lgbm-train200`): LightGBM lambdarank on the same 170 hybrid lists / DeBERTa `ce_gain` / `other_query_neg`, applied to frozen 74. Frozen hypers (`n_estimators=100`, `max_depth=3`, `lr=0.05`). nDCG@10 **0.533**, Recall@10 **0.389**, Recall@50 **0.834**. **Null vs RankNet apply 0.544.** Horizon 0.70 missed. Stop-for-human bar 0.58 not reached. Not live. +- I-CE-HYB (`search-esci-74-ltr-deberta-hybrid170`): continue pool DeBERTa 4-class on 8500 hybrid-k50 rows from the 170 lists (`data/models/esci-deberta-v3-base-4class-hybrid170`; did not clobber the pool checkpoint). RankNet apply with new `ce_gain`. nDCG@10 **0.530**, Recall@10 **0.385**, Recall@50 **0.834**. **Null vs C6 0.544.** Horizon 0.70 missed. Not live. + +Product default that survived: hybrid BM25+dense, passages, `rerank=none`, extras/facets as shipped, Italian via gated FTS+OR. Harness nDCG@10 **0.544** is not a live default. Do not mix with Reddy 0.857 or CE-on-pool 0.710. + +## WANDS first-stage control (2026-08-20) + +**Decision:** frozen `data/search_wands.jsonl` (caps 80q/2000d/40 cand; **actual 66 queries / 2000 docs**), brain `searchbenchwands`, passages, k=50, `rerank=none`, skip_enrichment, no graph. Run `search-wands-passages-k50`. Gains Exact=1 / Partial=0.5 / Irrelevant=0. Recall is binary Exact+Partial. Linear DCG. After first eval, `--skip-ingest` only. `download --name wands` refuses overwrite. + +**Located evidence:** nDCG@10 **0.823**, Recall@10 **0.269**, Recall@50 **0.837**, MRR **0.925**, p50 **87 ms**. Gold median 40; Recall@10 ceiling **0.365**. Do **not** mix with ESCI 0.500 / C6 0.544, Reddy 0.857, or Chen et al. (DOI located; nDCG **not located**). Details: [23-search-wands-quality.md](23-search-wands-quality.md). + +## Opt-in catalog graph (2026-08-20) + +**Decision:** architecture-opt-in on new brain `searchbenchwandsgraph` only. Product mapper is [`src/core/search/catalog_graph.py`](../../src/core/search/catalog_graph.py) (harness re-exports it). Communities match hubs then intersect item neighbors when ≥2 hub kinds match, else union; hybrid-rank product nodes inside `graph_channels`. Live omitted `channels` stays `["passages"]`. Node BM25 is `SEARCH_ENABLED` english `kg_nodes.search_tsv`. Do not edit `search.py` fusion this round. + +**Isolation:** reuse frozen `data/search_wands.jsonl`; never wipe `searchbenchwands` / `searchbenchesci74`. Ledger runs `search-wandsgraph-communities-k50` / `search-wandsgraph-passages-k50` are labeled **architecture-demo**. Do not quote them as beating 0.823 / 0.500. Details: [24-search-catalog-graph.md](24-search-catalog-graph.md). + +## Query-gated personalize (2026-08-21) + +**Decision:** optional `SearchRequestBody.target` reranks retrieved hits after extras AND. Omit `target` → anonymous ranking (this protocol’s default). Do not call `/retrieve/recommend`. Do not score HitRate as nDCG. If ledgered: `architecture-demo` only. Details: [25-personalized-search.md](25-personalized-search.md), [ADR-008](../decisions/008-query-gated-search-personalization.md). diff --git a/docs/research/19-search-esci-quality.md b/docs/research/19-search-esci-quality.md new file mode 100644 index 0000000..e28e28e --- /dev/null +++ b/docs/research/19-search-esci-quality.md @@ -0,0 +1,1166 @@ +# 19 — ESCI slice search quality: retrieval, fusion, and eval + +Workstream: `/retrieve/search` ranking quality on `searchbench*` only. Ledger: `benchmarks.search` only. This document is a **proposal and experiment plan**, not a published finding. + +**Focal question:** Which interventions could raise nDCG@10, Recall@10, and/or MRR on BrainAPI search for the ESCI product-search slice, and which of those are actually testable given the current eval? + +Session purpose: decide the next search-quality experiments. Audience: product owner. Decision owner: the user. Horizon: days. + +Every claim is labeled **idea**, **assumption**, **prediction**, **located evidence**, or **decision**. + +--- + +## What this workstream does + +Search quality here is labeled product ranking, not LoCoMo judge accuracy and not `/retrieve/recommend`. + +### Write path (this slice) + +1. Catalog JSONL docs are ingested as chunks with a `DOCID ` marker (`benchmarks/search/catalog.py:234-256`, `dataset.py:9-12`). Default ingest is `skip_enrichment=true` (chunk + embed only; `evaluate.py:45-46`, `cli.py:46-47`). +2. Optional `--ingest-graph` writes deterministic HAS triples (`evaluate.py:315-320`). Entity uuid equals catalog `doc_id` (`mapping.py:39-40`). Hubs are `hub:{kind}:{value}` (`mapping.py:43-44`). Class/brand/color/locale/feature values become CLASS / ATTR / TYPE nodes (`mapping.py:180-251`). If no fields exist, a TYPE hub from `dataset` is used (`mapping.py:241-250`). +3. **Located evidence:** this slice’s 200 docs have brand/color/locale in the JSONL and **0/200 `class` fields**. CLASS hubs from mapping are therefore mostly absent; ATTR/TYPE hubs still exist. + +### Read path + +1. Default omitted `channels` is `["passages"]` (`src/services/api/constants/requests.py:478`; `hooks.py:81-89` treats core names as non-plugins). +2. Passages: dense ANN + BM25, fused with RRF or convex combination (`search.py:176-177, 257-265`; `hybrid.py:109-135`; RRF at `fact_filter.py:67-78`). +3. Graph channels (`graph_channels.py:18-28`): + - `entities`: name CONTAINS via `graph.search_entities` plus optional node ANN (`graph_channels.py:280-392`). No default `node_labels` filter unless the request sets one (`search.py:111`). + - `events`: same as entities with `EVENT` labels plus recency (`graph_channels.py:395-433`). Empty-ok on catalog-only brains. + - `communities`: search TYPE/CLASS/TOPIC hubs (default `SEARCH_COMMUNITY_LABELS`, `config.py:701-709`; `graph_channels.py:30, 564-615`), then expand neighbors with degree-IDF. Member hits can be ATTR hubs because skip labels are only the community labels (`graph_channels.py:588-614`). +4. Each populated graph channel is appended as its own `extra_id_lists` entry and RRF-fused with passage id lists (`search.py:238-265`). IDs are mixed chunk ids and node uuids. A unit test locks this mixing in (`tests/test_search_graph.py:409-416`). +5. Optional `rerank=plugin:` reranks at most `RERANK_MAX_K = 10` on `mode=default` (`hooks.py`; `search.py`). Opt-in `mode=catalog` retrieves `min(200, max(k, 50))` and reranks at most `CATALOG_RERANK_MAX_K = 50`, then cuts to request `k`. `rerank=none` on both ESCI arms unless stated. + +### Eval path + +1. After ingest (or `--skip-ingest` reuse), chunks are listed and mapped by DOCID substring (`evaluate.py:332-335`, `dataset.py:61-76`). +2. Gold is doc_id space: `gold_hit_ids` unions `gold_doc_ids` / positive `gold_grades` and maps `gold_chunk_ids` through `chunk_to_doc` (`evaluate.py:201-222`). +3. Hits are canonicalized chunk to doc and deduped; unknown ids (including `hub:*`) stay as themselves (`evaluate.py:149-161, 361-362`). +4. Metrics: Recall@{5,10,20}, graded nDCG@10, MRR (`metrics.py:7-67`). nDCG uses `gold_grades` when present (`metrics.py:42-56`). +5. Ledger upserts `benchmarks.search` and now records `channels` (`report.py:61, 157`). + +### Located evidence — two arms on the same brain + +Same dataset `benchmarks/data/search_esci_slice.jsonl` (200 docs, 11 queries, all product columns), brain `searchbenchesci20`, fusion `rrf`, rerank `none`, skip_enrichment true. Graph arm ingested triples; passages arm used `--skip-ingest` (chunks reused; `eval.json` `ingest.reused=true`). + +| Arm | Channels | nDCG@10 | Recall@10 | Recall@20 | MRR | p50 retrieve | +| --- | --- | --- | --- | --- | --- | --- | +| Graph | passages+entities+communities | 0.6815 | 0.4657 | 0.5818 | 0.8485 | 108 ms | +| Passages | passages | 0.7579 | 0.5106 | 0.8470 | 0.8485 | 29 ms | + +Sources: `benchmarks/runs/search-esci-slice-allcols/{report,eval}.json`, `benchmarks/runs/search-esci-slice-passages/{report,eval}.json`. + +Do **not** claim graph-channel lift. Graph dropped nDCG@10 (−0.076) and Recall@10 (−0.045). MRR was identical on every query. Recall@20 dropped far more (−0.265). + +Per-query first-relevant rank never moved. Five graph queries put `hub:attr:*` in the fused top-10. Graph unique hit counts after canonicalize were 12–18 vs 20 for passages. Offline drop of `hub:*` from the already fused graph lists only moved nDCG 0.682 to 0.691 and Recall@10 0.466 to 0.476 (Recall@20 unchanged). **Prediction that failed:** removing visible hubs from the scored list recovers passages. Residual gap is ASIN substitution and a shorter unique product pool created at fusion time. + +One query (esci-18) had graph nDCG +0.057 with unchanged Recall@10 because rank-10 swapped a Substitute (gain 0.1) for an Exact (gain 1.0). That is a single-query graded accident, not a mechanism claim. + +Both arms score 0 on esci-72 (`$100 things that are not electronics`; 6 Exact golds never retrieved). + +Gold sizes: min 1 / median 20 / max 20. Gain histogram on the slice: E=1.0 ×127, S=0.1 ×37, C=0.01 ×2. Recall@10 is therefore capped near 0.5 on the typical query even if top-10 is all gold. + +--- + +## Maintainer decision (2026-08-18) + +**Decision (located):** make graph channels useful even if passages stays the default. Headline nDCG vs passages-only is **not** the go/no-go. Isolated-channel gold retrieval is: Recall@10 of gold ASINs and presence of product uuids in hits. Follow-up evals reuse brain `searchbenchesci20` with `--skip-ingest`. Default omitted `channels` remains `["passages"]`. + +--- + +## Isolated channels then ranking change (same brain, n=11) + +All arms: `search_esci_slice.jsonl`, `--skip-ingest`, fusion `rrf`, rerank `none`. No re-ingest. `eval.json` now stores per-hit `id`, `channel`, and canonical `doc_id` (`evaluate.py`). + +### Before ranking change (measurement) + +| Arm | Channels | nDCG@10 | Recall@10 | Recall@20 | MRR | p50 ms | Queries with gold in top-10 | `hub:*` in top-10 | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Passages | passages | 0.758 | 0.511 | 0.847 | 0.848 | 29 | 10/11 | 0 | +| Entities | entities | 0.532 | 0.415 | 0.709 | 0.667 | 53 | 10/11 | 6 queries (13 slots) | +| Communities | communities | 0.000 | 0.000 | 0.000 | 0.000 | 52 | 0/11 | empty lists | +| Passages+entities | passages,entities | 0.682 | 0.466 | 0.582 | 0.848 | 74 | 10/11 | 5 queries | + +Sources: `runs/search-esci-slice-{passages,entities,communities,pe}/eval.json`. + +**Call (located evidence):** entities-only already retrieved gold ASINs (Recall@10 0.415, 10/11 queries). ATTR hubs stole rank on 6 queries. Communities-only never fired (TYPE/CLASS/TOPIC hubs do not match these queries; this slice has 0/200 `class` fields). Passages+entities matched the original fused graph arm — communities added nothing. Peer RRF of entity lists with passages still shortened the product pool (Recall@20 0.582 vs 0.847). + +Offline compacting of `hub:*` from the original fused `hit_ids` (promote later ranks into @10) yields nDCG@10 0.764 / Recall@10 0.476 / Recall@20 0.582 / MRR 0.848 (`python -m search.replay_fusion`). An earlier 0.691 nDCG figure did not refill @10 from ranks 11+. Compact nDCG matching passages is **not** a claim that live fusion is fixed; Recall@20 stays 0.582. + +### After ranking change (same frozen corpus) + +Code: `graph_channels.py` defaults the entities channel to `ENTITY` nodes, ranks by name overlap, drops `hub:*` / ATTR/TYPE/CLASS members from entity lists. Communities also seed ATTR hubs (brand/color/feature), expand at most 8 hubs, and emit only item ENTITY neighbors. `search.py` will not RRF hub ids into fused lists. Default omitted channels still `["passages"]`. + +| Arm | Channels | nDCG@10 | Recall@10 | Recall@20 | MRR | p50 ms | Queries with gold in top-10 | `hub:*` in top-10 | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Passages smoke | passages | 0.758 | 0.511 | 0.847 | 0.848 | 28 | 10/11 | 0 | +| Entities | entities | 0.655 | 0.482 | 0.822 | 0.757 | 51 | 10/11 | 0 | +| Communities | communities | 0.478 | 0.350 | 0.391 | 0.848 | 129 | 10/11 | 0 | +| Fused | passages,entities,communities | 0.653 | 0.497 | 0.653 | 0.816 | 184 | — | 0 | + +Sources: `runs/search-esci-slice-{passages-smoke,entities-after,communities-after,pec-after}/eval.json`. + +**Useful, in numbers (go/no-go):** isolated entities Recall@10 0.415 → 0.482; zero hub ids in top-10; product uuids on all 11 queries; Recall@20 0.709 → 0.822. Isolated communities Recall@10 0 → 0.350 with gold in top-10 on 10/11 queries (esci-19 still 0). Passages-only smoke is unchanged at 0.758 / 0.511 / 0.848. + +**Not a quality win for fused search.** p+e+c nDCG@10 0.653 is still below passages 0.758 (and below the old fused 0.682). MRR fell from 0.848 to 0.816. Do not put entities/communities in the default mix. Do not claim ESCI-wide lift. n=11. + +Communities retrieve p50 129 ms is under the ~200 ms SLO; an uncapped hub expansion on this brain was 239 ms p50 and was capped to 8 hubs. + +### Still not useful / unmeasured + +- **events:** no interaction EVENT ingest on this catalog; still empty-ok. +- **expand=neighbors:** unit-tested; no isolated ESCI arm. +- **esci-72:** entities-only still 0 gold in top-10; passages-only still 0; communities-after had 1/6 gold in top-10 — n=1, not a negation fix. +- **Fused graph+passages:** still crowds Recall@20 vs passages (0.653 vs 0.847). + +Reproduce (no re-ingest): + +```bash +cd benchmarks +./search.sh --brain searchbenchesci20 evaluate \ + --dataset data/search_esci_slice.jsonl --skip-ingest --fusion rrf \ + --channels entities --run search-esci-slice-entities-after +./search.sh --brain searchbenchesci20 evaluate \ + --dataset data/search_esci_slice.jsonl --skip-ingest --fusion rrf \ + --channels communities --run search-esci-slice-communities-after +./search.sh --brain searchbenchesci20 evaluate \ + --dataset data/search_esci_slice.jsonl --skip-ingest --fusion rrf \ + --channels passages --run search-esci-slice-passages-smoke +``` + +--- + +## Passages CE, esci-72, ranking-in-pool (2026-08-18) + +**Decision:** default `rerank=none` stays. Isolated graph channels stay off the default mix. + +### Passages MiniLM (Task 1) + +`runs/search-esci-slice-passages-ce` — `--channels passages --rerank plugin:cross-encoder`, `--skip-ingest`, n=11. + +| Arm | nDCG@10 | Recall@10 | Recall@20 | MRR | p50 retrieve | +| --- | --- | --- | --- | --- | --- | +| Passages (control) | 0.758 | 0.511 | 0.847 | 0.848 | 28–29 ms | +| Passages + MiniLM k≤10 | **0.738** | 0.511 | 0.847 | 0.803 | 30 ms | + +**Located evidence:** nDCG@10 −0.020 vs control (win rule was +0.02). Recall@20 unchanged. MRR down. p50 retrieve 30 ms passes the ~200 ms labeled gate. p95 client wall ~7.5 s is first-query model load, not retrieve. **Null result.** Do not change the product default. + +### esci-72 dump (Tasks 2–3) + +Query: `$100 things that are not electronics`. All 6 Exact golds are **in the JSONL**. None appear in passages top-20. Helper: `benchmarks/search/miss.py`. + +| ASIN | Title (short) | Marker | +| --- | --- | --- | +| B00TREI0JI | Apple iPad Air 2 (Renewed) | electronics | +| B073R68TSH | Beam Electronics car phone holder | electronics | +| B07NSX2ZBS | FindKey RF key/phone tracker | electronics | +| B07QYLTT7X | Galaxy Watch Active TPU case | electronics | +| B07XG6Y847 | AirPods silicone case | electronics | +| B083S3ZXDF | Potaroma electric flopping fish toy | electronics (electric toy) | + +Hits are mostly price tags, gift cards, Pokémon lots, batteries — lexical `$100` / “tags”, not a negation failure. **Decision: qrel mismatch.** Do not rewrite `not electronics` on this slice. No eval-only intervention. + +### Ranking-in-pool (Tasks 4–7) + +Catalog query rows now store `candidate_doc_ids` / `candidate_grades` (I=0 kept off `gold_grades`). Slice JSONL grew 200 → 365 docs so CE-on-pool has texts for I-labels. Those extra docs were **not** ingested into `searchbenchesci20`. + +| Arm | Protocol | nDCG@10 | nDCG@20 | nDCG full | R@10 | MRR | notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| BrainAPI in-pool | search then keep pool ids | 0.788 | **0.811** | 0.814 | 0.535 | 0.909 | coverage 0.47; ~15.5 pool ids/query missing from brain | +| MiniLM CE-on-pool | harness CE on all labeled pairs | 0.598 | **0.669** | 0.786 | 0.399 | 0.833 | `cross-encoder/ms-marco-MiniLM-L-6-v2`; missing_text=0 | + +**Cite Reddy EN CE 0.857 / KDD 0.9043 only against CE-on-pool.** That arm is n=11, zero-shot MS MARCO, not ESCI-finetuned, not the 4,477-query public test. Do not average these rows with slice first-stage 0.758. + +Reproduce: + +```bash +cd benchmarks +./search.sh --brain searchbenchesci20 evaluate \ + --dataset data/search_esci_slice.jsonl --skip-ingest \ + --channels passages --rerank plugin:cross-encoder \ + --run search-esci-slice-passages-ce +./search.sh --brain searchbenchesci20 evaluate \ + --dataset data/search_esci_slice.jsonl --skip-ingest \ + --channels passages --rank-pool \ + --run search-esci-slice-passages-pool +./search.sh --brain searchbenchesci20 rank-pool-ce \ + --dataset data/search_esci_slice.jsonl \ + --run search-esci-slice-ce-pool +``` + +`rank-pool-ce` uses the repo `.venv` (sentence-transformers). 74-query ingest is a later human checkpoint on a **new** `searchbench*` brain. + +--- + +## Guarantees and where they break + +**Stated guarantee (this workstream):** hybrid search returns a ranked list of catalog items such that graded nDCG@10, Recall@10, and MRR measure product relevance on ESCI-style qrels, with p50 retrieve (ex-embed) under ~200 ms (`18-search-eval-protocol.md` SLO). + +Where that guarantee cannot be delivered today: + +1. **Fusion pollution (high impact on nDCG and recall).** RRF treats graph node lists as peer ranked lists (`search.py:246-265`). Incompatible ids occupy slots; more importantly, extra lists reorder and crowd out passage ASINs. **Gap**, not a documented trade-off. Tests currently require mixed ids (`test_search_graph.py:409-416`). +2. **Community/entity channels unfiltered (high impact on recall@20).** Entities search all labels (`search.py:111`). Communities emit neighbors of TYPE/CLASS/TOPIC, including ATTR hubs (`graph_channels.py:588-614`). Those ids cannot match gold doc_ids unless they are product entity uuids. +3. **Eval cannot attribute a hit to a channel (high impact on diagnosis).** `eval.json` stores canonicalized `hit_ids` only (`evaluate.py:364-372`). We cannot yet falsify “graph never fired useful hits” vs “graph fired gold ASINs that RRF buried” without a new dump or isolated arms. +4. **n=11, no pre-registered control until this ablation (critical for inference).** Means have no interval. One query (esci-67) has a single gold; one (esci-72) is a total miss. Deliberate slice size for ingest cost; **not** a full-ESCI claim. +5. **MRR is saturated and insensitive here (important).** Mean 0.848 is “first Exact usually at rank 1.” Interventions that only clean ranks 2–10 will not move MRR. Eval artifact, not a ranking win. +6. **skip_enrichment + generic HAS triples (medium).** Graph is attribute hubs, not LLM entity resolution. Deliberate cost trade-off. Do not interpret this graph as GraphRAG. +7. **Title/class BM25 confound (medium, weaker than assumed on this file).** Session prior: class words in titles can inflate BM25. Located evidence: 0/200 docs have a `class` field; titles still contain type words (envelopes, pads, fence). Allcols text also includes brand and bullets. Do not attribute passages quality to graph CLASS hubs. +8. **k=10 vs gold size ~20 (important).** Recall@10 cannot exceed ~0.5–0.67 on most queries. Recall@20 is the more sensitive recall readout on this slice (located evidence: passages 0.847 vs graph 0.582). + +--- + +## Open questions for the maintainer + +1. Is the decision criterion “beat passages-only 0.758 / 0.511 / 0.848 on this slice,” or “make graph channels eventually useful even if passages stay the default”? **Answered:** useful isolated channels; passages stays default. Fused nDCG is not go/no-go. +2. May all follow-up arms reuse `searchbenchesci20` with `--skip-ingest` (no re-ingest, no wipe)? **Answered:** yes. +3. When isolated `entities` / `communities` arms show no gold ASINs, should those channels stay off the default search path (`channels=["passages"]`)? **Answered:** they stay off the default path even after they retrieve gold ASINs. +4. Is p50 retrieve ~200 ms ex-embed still binding for `rerank=plugin:cross-encoder` on this brain? +5. Should headline decisions use nDCG@10, or also treat Recall@20 as primary given gold size ≈20? +6. Is `benchmarks/data/search_esci.jsonl` (title-only, 2000 docs) in scope only as a later confirmation condition, not a substitute for this allcols slice? +7. Should `entities` default to `node_labels=["ENTITY"]` in product search, or remain unfiltered so ATTR/TYPE name matches can enter RRF? + +--- + +## Independent ideas (generated before literature) + +Origin: AI-assisted, stage `independent` unless marked `post-check`. + +| ID | Statement | Cluster | Predicted metric move vs passages | Disconfirming observation | +| --- | --- | --- | --- | --- | +| I01 | Treat passages-only as the quality baseline; do not add graph fusion until pollution is diagnosed | eval / fusion | nDCG/recall stay; graph arm must not be the default | Isolated graph arms beat passages on nDCG and recall without hub ids | +| I02 | Dump per-hit `channel` + run isolated `entities`, `communities`, `passages+entities` on this brain | eval | none (measurement) | Isolated arms empty and fused lists contain no graph-origin gold ASINs | +| I03 | Never emit `hub:*` into `extra_id_lists`; only product ENTITY uuids | fusion pollution | +nDCG, +recall@10/@20; MRR flat | Offline/online filter leaves Recall@20 much less than 0.85 (already partly located) | +| I04 | Force `node_labels=["ENTITY"]` on the entities channel | ingest/graph | +recall if entities retrieve gold ASINs; else null | Entities-only with ENTITY labels has Recall@10 ≈ 0 | +| I05 | Downweight or gate graph lists in RRF (not peer k=60 lists) | fusion | +nDCG if entities have some gold | Weighted RRF still loses Recall@20 to passages | +| I06 | Cross-encoder rerank on passages-only head (k≤10) | precision-at-top | +nDCG; MRR maybe; recall@10 unchanged unless head misses gold | nDCG flat after rerank; or p50 retrieve >200 ms | +| I07 | Handle exclusionary queries (`without` / `not`) | recall | +nDCG/recall on esci-72 and similar | Rewrites still miss the 6 gold ASINs | +| I08 | Fielded / title-weighted BM25 | precision | +nDCG if body tokens pollute | Title-only BM25 ≤ allcols passages | +| I09 | SPLADE / ColBERT first-stage plugins | recall | +recall@10/@20 vs passages | Plugin index + latency cost, no metric lift | +| I10 | Enlarge n + paired tests; keep this slice as a smoke | eval artifact | none (inference) | Larger ESCI still shows the same fusion drop | +| I11 | Passages+entities without communities | fusion | smaller drop than full graph if communities are the pollutant | p+e equals or worse than p+e+c | +| I12 | Raise retrieve `k` then cut @10 | recall (gaming) | +recall@10 by stuffing | nDCG@10 falls; SLO missed — do not use as a quality claim | +| I13 | LLM enrichment / richer triples | ingest | unknown | Cost/latency; skip_enrichment was deliberate | +| I14 | Use graph as expansion after passages, not as RRF peers | fusion | recover passages nDCG; optional recall if expansion adds gold | Expansion still injects non-gold ASINs into @10 | +| I31 | Offline RRF replay from stored per-channel lists (`post-check`) | fusion | distinguishes pollution vs never-fired | Cannot replay until I02 dumps raw lists | + +### Criteria (declared before scoring) + +Scale 1–5, higher better. Weights (decision aid only; **decision stays null**): + +| Criterion | Weight | 1 | 3 | 5 | +| --- | --- | --- | --- | --- | +| information_gain | 3 | No mechanism split | Splits one pair of stories | Splits pollution vs never-fired vs passages headroom | +| relevance_metrics | 3 | Will not move the three metrics vs 0.758/0.511/0.848 | Moves one metric on this slice | Directly targets a measured gap | +| originality | 1 | Standard IR | Known method, new here | No nearby literature located | +| feasibility | 2 | New models / re-ingest | 3–5 files + one eval | Eval-only on `searchbenchesci20` | +| rigor | 2 | Easy to game | Needs care | Clean ablation, hard to HARKing | +| value_if_null | 2 | Null is noise | Null slightly useful | Null decides the next fork | +| latency_slo | 2 | Likely blow ~200 ms p50 retrieve | Borderline | No extra retrieve cost | + +Weighted score = 100 × Σ p_j (x−1)/4, p_j = w_j / Σ w. Not a winner picker. + +| ID | IG | Rel | Orig | Feas | Rig | Null | Lat | Score | Uncertainty | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| I02 | 5 | 4 | 3 | 5 | 5 | 5 | 5 | 92 | Low | +| I01 | 5 | 4 | 2 | 5 | 5 | 5 | 5 | 90 | Low | +| I04 | 4 | 4 | 2 | 5 | 4 | 5 | 5 | 82 | Med: depends on search_entities quality | +| I03 | 4 | 5 | 3 | 4 | 4 | 4 | 5 | 82 | Low–med: post-hoc hub drop already weak | +| I10 | 4 | 3 | 2 | 4 | 5 | 5 | 5 | 77 | Low | +| I14 | 4 | 4 | 3 | 3 | 4 | 4 | 3 | 67 | Med | +| I05 | 3 | 4 | 2 | 3 | 3 | 4 | 5 | 63 | Med | +| I07 | 4 | 4 | 3 | 2 | 3 | 4 | 3 | 60 | High transfer | +| I08 | 3 | 3 | 2 | 3 | 3 | 3 | 5 | 55 | Med | +| I06 | 3 | 4 | 2 | 3 | 3 | 3 | 2 | 50 | Lat veto possible | +| I09 | 3 | 4 | 2 | 2 | 3 | 3 | 2 | 47 | Fashionable here | +| I12 | 2 | 3 | 1 | 5 | 1 | 2 | 4 | 42 | Rigor veto | +| I13 | 2 | 2 | 2 | 2 | 2 | 3 | 4 | 40 | Out of first wave | + +--- + +## Adversarial review (shortlist) + +**I02 isolated arms.** Falsifier: all isolated graph arms return gold ASINs in top-10 and fused p+e+c still loses — then the bug is specifically three-list RRF, not empty channels. Alternative: `--skip-ingest` passages arm hit a warmer cache (esci-67 passages retrieve 266 ms vs graph 108 ms). Residual: n=11. + +**I03 hub filter.** Falsifier: already located — dropping `hub:*` after canonicalize does not recover Recall@20. Alternative: hubs in the scored list are a symptom; the cause is extra RRF lists changing passage ranks. Revise I03 to “do not append non-product lists,” not “strip hubs after the fact.” + +**I04 ENTITY-only.** Falsifier: entities-only Recall@10 ≈ 0. Alternative: `search_entities` CONTAINS on long titles already returns products; label filter changes little. Selection: we only observed fused lists, not raw entity lists. + +**I06 rerank.** Falsifier: nDCG flat because first-stage already puts Exact at rank 1 (MRR 0.85). Alternative: any nDCG lift is E vs S reordering on n=11. Latency: CE on 10 docs can blow the retrieve budget; call it out. Metric gaming: rerank cannot raise Recall@10 if gold is outside the head. + +**I07 negation.** Falsifier: gold products for esci-72 are not matchable in this 200-doc pool. Alternative: conceptual query, not syntactic negation. + +**I12 raise k.** Reject as a quality intervention. It games Recall@k without fixing ranking. + +--- + +## Frontier techniques + +Search date: **2026-08-18**. Tools: arXiv MCP `search_papers` / `get_abstract`. Queries: `"Shopping Queries Dataset" OR ESCI "product search"`; `"reciprocal rank fusion"`; `ti:"reciprocal rank fusion"`; `"knowledge graph" retrieval product search OR "entity-oriented search"`; `ti:"passage re-ranking" BERT OR ti:SPLADE OR ti:ColBERT`; `query negation retrieval OR exclusionary queries`; plus direct ids `2206.06588`, `1805.07591`, `2404.16130`, `1901.04085`, `2004.12832`, `2107.05720`. Limits: one session, cs.IR-heavy; absence from search is not novelty. Cormack et al. SIGIR 2009 RRF was not located on arXiv in this pass (`search-incomplete` for that specific paper). + +### Reciprocal rank fusion of heterogeneous lists + +- **Mechanism:** Unweighted RRF sums 1/(k+rank) across lists (`fact_filter.py:67-78`, k=60). BrainAPI appends each graph channel as an extra list (`search.py:246-265`). +- **arXiv:** Canonical 2009 paper not retrieved here. **2503.20698** (Samuel et al., MMMORRF) reports weighted, modality-aware RRF and +37% nDCG@20 vs single-modality on video benchmarks — different domain. +- **Cost:** Unweighted extra lists are cheap; they still change the top-k composition. +- **Fit:** We already fuse incompatible id spaces. Weighted/gated RRF is an adapt if isolated arms show useful product ids. Peer unweighted fusion of hubs is challenge-located by our ablation (graph worse than passages). +- **Verdict:** **adapt** gating/weighting after I02; do not add more unweighted lists. +- **Literature status:** `mixed` (method exists; our setting violated the “compatible lists” assumption). + +### ESCI / Shopping Queries Dataset + +- **Mechanism:** Difficult Amazon queries; E/S/C/I labels; Task 1 is ranking with gains typically E=1, S=0.1, C=0.01, I=0 (`2206.06588`, Reddy et al.). +- **Reported gain:** Paper is a dataset plus baselines, not a BrainAPI method. About 130k queries / 2.6M judgments, multilingual. +- **Cost:** Full ESCI ingest is large; this slice is 11 queries / 200 docs by design (`catalog.py` caps). +- **Fit:** Our grades match Task 1. Our n does not. Title-only 2000-doc file is a different condition. +- **Verdict:** **adopt** as the label protocol; **reject** treating this slice as a full-ESCI result. +- **Literature status:** `support-located` for metric definitions; `challenge-located` for generalizing n=11. + +### Entity-oriented / KG-in-the-ranker (not hub-in-RRF) + +- **Mechanism:** EDRM (`1805.07591`, Liu et al.) represents queries/docs with words and entity annotations; KG semantics live in embeddings; ranking is interaction-based, end-to-end. +- **Reported gain:** Commercial search log; paper claims better generalization. Not ESCI. +- **Cost:** Needs entity linking plus a learned ranker, not `hub:attr` uuids in RRF. +- **Fit:** Current mapping emits generic HAS hubs (`mapping.py:180-251`) and fuses raw node ids. That is not EDRM. +- **Verdict:** **reject** as a next experiment; **adapt** later only if entities-only retrieves gold products and we add a ranker that consumes annotations. +- **Literature status:** `mixed`. + +### GraphRAG community summaries + +- **Mechanism:** `2404.16130` (Edge et al.): LLM-extracted entity graph, community summaries, map-reduce answers for global questions. +- **Reported gain:** Comprehensiveness/diversity on query-focused summarization, ~1M-token corpora — not nDCG of product ASINs. +- **Cost:** LLM index plus per-query summary tokens. Violates skip_enrichment and the search retrieve SLO. +- **Fit:** BrainAPI `communities` are typed catalog hubs, explicitly not Leiden / `kg_topic_sessions` (`18-search-eval-protocol.md`). +- **Verdict:** **reject**. Assumption BrainAPI violates: search is item ranking, not corpus QFS; skip_enrichment forbids the LLM graph. +- **Literature status:** `challenge-located` (wrong task). + +### Cross-encoder passage rerank + +- **Mechanism:** `1901.04085` (Nogueira and Cho): BERT scores query–passage pairs and reorders a first-stage head. Protocol already has `plugins/search-rerank` via `rerank=plugin:cross-encoder` (`18-search-eval-protocol.md`). +- **Reported gain:** +27% relative MRR@10 on MS MARCO passage vs prior SOTA (2019). Not ESCI; our MRR is already 0.85. +- **Cost:** Second-stage model on ≤10 candidates (`hooks.py:7`). Likely material retrieve latency. Call out if p50 approaches 200 ms. +- **Fit:** Best tested on **passages-only**. Reranking a polluted graph list first is the wrong order. +- **Verdict:** **adapt** as a passages-only arm after I02, with an explicit latency gate. +- **Literature status:** `support-located` for method; `mixed` for expected MRR lift here. + +### Learned sparse / late interaction + +- **Mechanism:** SPLADE (`2107.05720`, Formal et al.) learns sparse term weights for first-stage ranking. ColBERT (`2004.12832`, Khattab and Zaharia) late-interacts token vectors (MaxSim). Both exist as optional plugins in the protocol. +- **Reported gain:** Competitive with dense/sparse SOTA on MS MARCO-class passage tasks. Extra index and FLOPs. +- **Cost:** Own index (`POST /search-splade/index` / ColBERT index). Can blow the retrieve budget. Fashionable relative to this repo’s measured gap. +- **Fit:** Worth a later passages-only bake-off, not a graph-fusion fix. +- **Verdict:** **reject** for the next few days; **adapt** after passages-only headroom is measured on a larger n. +- **Literature status:** `support-located` as IR methods; low expected gain per implementation cost here. + +### Exclusionary / negation queries + +- **Mechanism:** ExcluIR (`2404.17288`, Zhang et al.) shows first-stage retrievers fail on “what I do not want” queries; a dedicated train set helps but stays below humans; generative retrieval is claimed stronger. +- **Reported gain:** Architecture-wide struggle on 3,452 annotated exclusionary queries; training helps, gap remains. +- **Cost:** New query rewrite or trained retriever. Latency depends on implementation. +- **Fit:** Several slice queries are `without` / `not` (esci-12/13/18/19/2/34/42/60/67/72). Only esci-72 is a total miss. Do not treat the whole slice as an ExcluIR replica. +- **Verdict:** **adapt** as a targeted error analysis plus optional rewrite **after** fusion diagnosis. Status: `support-located` for the esci-72 failure mode. + +### Instruction-following retrievers / metadata-as-text + +- **Mechanism:** InF-IR (`2505.21439`) trains instruction-aware embeddings. Metadata-aware RAG (`2601.11863`) finds prefixing metadata in the chunk helps. +- **Fit:** Allcols ingest already prefixes Title/Brand/bullets (`catalog.py:234-256`). InF-style models are a new index. +- **Verdict:** **reject** for this sprint (`mixed` / not needed to explain graph vs passages). Reopen after I02. + +### Post-check ideation (after literature) + +- I31 (offline RRF replay) added. Weighted RRF (I05) upgraded from “fashionable” to “adapt if I02 shows useful entity ASINs” because **2503.20698** is support-located for *weighted* fusion, not for unweighted extra lists. +- GraphRAG and EDRM were not added as winners. Absence of a “graph always helps product nDCG” paper in this search is not a gap we should fill with more channels. + +--- + +## Critique of the current eval as a study + +### Summary + +Two arms on one frozen 200-doc / 11-query ESCI-US slice, same brain, same qrels, fusion RRF, no rerank, skip_enrichment. The graph arm added entities+communities and structured HAS triples. The passages arm reused chunks (`--skip-ingest`). Headline: passages 0.758 / 0.511 / 0.848 vs graph 0.682 / 0.466 / 0.848. This is a paired within-corpus contrast, not an RCT and not full ESCI. + +### Strengths + +- Same brain and corpus; passages arm did not re-ingest (reduces write-path drift). +- Graded nDCG uses ESCI-style gains; gold collapse to doc_id is implemented (`evaluate.py:149-161, 201-222`). +- Latency split retrieve vs embed is recorded; graph p50 108 ms and passages p50 29 ms both sit under the ~200 ms retrieve SLO. +- Ledger now records `channels`, so the two arms cannot be silently compared as if they were the same system. +- Per-query lists exist in `eval.json` even though `report.json` is aggregate-only. + +### Concerns + +**Critical** + +- n=11. No confidence interval, no multiplicity control, no pre-registered primary metric among {nDCG, Recall@10, Recall@20, MRR}. One total-miss query and one single-gold query dominate means. +- No channel-attributed hits. The study cannot yet say whether graph lists contained gold ASINs. +- Graph vs passages is confounded with extra RRF lists, mixed id spaces, structured ingest (graph arm only), and different retrieve-path work. The passages ablation removes the “no control” problem but does **not** isolate a single mechanism. + +**Important** + +- Recall@10 is a poor recall target when |gold|≈20; Recall@20 tells the real crowding story (0.847 vs 0.582). +- MRR identical on all 11 queries: the study is silent on precision-at-1 differences. +- Title-class BM25 confound remains a plausible passages explanation; this file has no `class` field, so the confound is title/body type words, not mapped CLASS hubs. +- skip_enrichment plus generic HAS triples means a graph loss is not a loss for “KG retrieval” in the EDRM/GraphRAG sense. +- esci-67 passages retrieve 266 ms vs graph 108 ms: cache or load difference; do not over-read p95. + +**Minor** + +- `report.json` omits per-query rows (they live in `eval.json`). +- Default evaluate `k=20` then metrics cut at 10 (`cli.py:321`, `evaluate.py:277-279`). +- Mixed chunk/node ids are tested as desired behavior (`test_search_graph.py:409-416`). + +### Recommendations + +1. Keep passages-only as the baseline for quality claims. +2. Next measurements: isolated channels + per-hit channel dump + Recall@20 in the headline table. +3. Do not ship more graph fusion until I02/I11 falsify or confirm pollution. +4. Enlarge n before any “ESCI improved” sentence. Keep title-only 2000-doc as a separate condition. +5. Preregister: primary = nDCG@10 vs passages; secondary = Recall@20; MRR expected flat; latency gate p50 retrieve <200 ms. + +### Overall assessment + +The numbers **can** support: on this 11-query allcols slice, adding entities+communities via unweighted RRF **hurt** nDCG@10 and recall relative to passages-only, while MRR did not change, and retrieve got slower (108 vs 29 ms p50). They **cannot** support: graph-aware search is worse on ESCI; passages is SOTA; class-word BM25 is proven; GraphRAG failed; or any intervention is validated. Offline hub-drop **can** support that visible `hub:*` ids are not the whole gap. It **cannot** support that graph never produced a useful product id. + +--- + +## Implementation plan + +**Architecture decisions** + +- Measurement and ablations before ranking-code changes. +- Reuse `searchbenchesci20` with `--skip-ingest`. Never wipe or score `locomoconv*`, `beam*`, `demorecsys`. +- Quality claims compare to **passages-only** 0.758 / 0.511 / 0.848, not to the graph arm. +- Core channel names stay `passages|entities|events|communities`. Default omitted channels stay `["passages"]`. +- No edit to `~/.cursor/plans/graph-aware_search_channels_e0ddd720.plan.md`. + +Passages-only on this slice/brain is **done** (located evidence above). It is Task 0, not Task 1. + +### Phase 1: Measurement (fail fast) + +### Task 1: Persist per-hit channel and raw id before canonicalize + +**Description:** Write each hit’s `id`, `channel`, and pre-canonical id into `eval.json` queries so later arms can replay fusion and count graph-origin gold ASINs. + +**Acceptance criteria:** + +- [x] `eval.json` `queries[i].hits` includes `id`, `channel`, and canonical `doc_id` +- [x] Existing metrics still match a recompute from those hits +- [x] Ledger still upserts only `benchmarks.search` + +**Verification:** + +- [ ] `cd benchmarks && .venv/bin/python -c "import json; json.load(open('runs/search-esci-slice-passages/eval.json'))"` +- [ ] `pytest tests/test_search_graph.py -q` (no harness import of `src/` from `benchmarks/search`) + +**Dependencies:** None + +**Files likely touched:** `benchmarks/search/evaluate.py`, `benchmarks/search/report.py` + +**Estimated scope:** S + +### Task 2: Isolated channel arms on the frozen brain + +**Description:** Three `--skip-ingest` evaluates on `searchbenchesci20` / `search_esci_slice.jsonl`: `entities`; `communities`; `passages,entities`. Do **not** pass `--ingest-graph`. Compare to passages-only. + +**Acceptance criteria:** + +- [x] Three new `runs/search-esci-slice-*` reports with `channels` recorded +- [x] Table of nDCG@10, Recall@10, Recall@20, MRR, p50 retrieve vs passages +- [x] Written call: useful-hits (gold ASINs in isolated lists) vs pollution (fused p+e or p+e+c worse than passages) + +**Verification:** + +```bash +cd benchmarks +./search.sh --brain searchbenchesci20 evaluate \ + --dataset data/search_esci_slice.jsonl --skip-ingest \ + --channels entities --run search-esci-slice-entities +./search.sh --brain searchbenchesci20 evaluate \ + --dataset data/search_esci_slice.jsonl --skip-ingest \ + --channels communities --run search-esci-slice-communities +./search.sh --brain searchbenchesci20 evaluate \ + --dataset data/search_esci_slice.jsonl --skip-ingest \ + --channels passages,entities --run search-esci-slice-pe +``` + +**Dependencies:** Task 1 preferred (channel dump); can run without it + +**Files likely touched:** none in core (eval only); new run dirs + +**Estimated scope:** S + +### Task 3: Offline fusion-replay / hub-drop notebook in-repo + +**Description:** Script that reads two `eval.json` files (or Task 1 dumps) and recomputes metrics after (a) dropping `hub:*`, (b) intersecting with product ASINs, (c) if raw lists exist, RRF replay without graph lists. Check in under `benchmarks/search/` or a research helper, not repo root. + +**Acceptance criteria:** + +- [x] Reproduces compact hub-drop Recall@10 0.476 / MRR 0.848 from `search-esci-slice-allcols` (nDCG 0.764 when ranks 11+ refill @10; earlier 0.691 did not) +- [x] Prints per-query deltas vs passages +- [x] Exits 0 on the two existing eval files + +**Verification:** + +```bash +cd benchmarks && .venv/bin/python -m search.replay_fusion \ + --graph runs/search-esci-slice-allcols/eval.json \ + --passages runs/search-esci-slice-passages/eval.json +``` + +**Dependencies:** None for (a)(b); Task 1 for (c) + +**Files likely touched:** `benchmarks/search/replay_fusion.py` (new), maybe `tests` under `benchmarks/` if present + +**Estimated scope:** S + +### Checkpoint: After Tasks 1–3 + +- [x] We can say, in one sentence, whether entities-only retrieved gold ASINs +- [x] We can say whether communities-only retrieved gold ASINs +- [x] We can say whether passages+entities is closer to passages or to p+e+c +- [x] Human reviews before any fusion code change +- [x] If both isolated graph arms have Recall@10 = 0 and no gold ASINs: **stop graph fusion work**; go to Phase 3 (passages headroom) + +Entities-only Recall@10 was 0.415 **before** the ranking change (gold ASINs present; hubs stealing slots). Communities-only was 0. Passages+entities equaled p+e+c. Ranking/filter work proceeded; default channels were not changed. + +### Phase 2: Fusion hygiene (only if Task 2 shows useful product ids) + +### Task 4: Fuse only product ENTITY uuids + +**Description:** Before `extra_id_lists.append`, keep ids that are product entities (uuid equals a catalog doc_id / label ENTITY). Do not append ATTR/TYPE/CLASS hub ids. Default omitted channels remain passages. + +**Acceptance criteria:** + +- [x] Unit test: hub id does not appear in fused output when passages+entities+communities run on a fixture +- [x] Mixed chunk+ENTITY uuid still allowed (existing test can stay for ENTITY ids) +- [x] `--skip-ingest` p+e+c eval: Recall@10 moved toward passages (0.466 → 0.497); nDCG@10 did not (0.682 → 0.653). Default path unchanged. + +**Verification:** + +- [ ] `pytest tests/test_search_graph.py -q` +- [ ] Repeat Task 2 p+e+c command after the change; compare to passages 0.758 / 0.511 / 0.847 + +**Dependencies:** Task 2 (go/no-go), Task 1 + +**Files likely touched:** `src/services/api/controllers/search.py`, `src/core/search/graph_channels.py`, `tests/test_search_graph.py` + +**Estimated scope:** M + +### Task 5: Optional ENTITY label default on entities channel + +**Description:** If Task 2 entities-only is dominated by ATTR hubs, pass or default `node_labels=["ENTITY"]` for catalog search without adding recsys field names to the API. + +**Acceptance criteria:** + +- [ ] Entities-only eval with `--node-labels ENTITY` recorded +- [ ] If Recall@10 still ≈ 0, document I04 as null and do not change product default + +**Verification:** same `./search.sh --brain searchbenchesci20 evaluate --skip-ingest --channels entities --node-labels ENTITY --run search-esci-slice-entities-label` + +**Dependencies:** Task 2 + +**Files likely touched:** none, or `search.py` only if default changes + +**Estimated scope:** S + +### Checkpoint: After Tasks 4–5 + +- [ ] Graph fusion is either gated to product ids or explicitly left off the default path +- [ ] No claim of graph lift unless nDCG and Recall@20 both beat passages on this slice +- [ ] p50 retrieve still <200 ms + +### Phase 3: Passages headroom (higher leverage than more graph) + +### Task 6: Passages-only cross-encoder arm (latency-gated) + +**Description:** `--channels passages --rerank plugin:cross-encoder` on the frozen brain. Primary: nDCG@10 vs 0.758. Hard gate: p50 retrieve. If plugin missing, expect HTTP 400, not a fake ranking miss (`hooks.py:103-113`). + +**Acceptance criteria:** + +- [x] Report written even if status failed (plugin absent) +- [x] If ok: nDCG@10, MRR, Recall@10, p50/p95 retrieve vs passages +- [x] No graph channels on this arm + +**Verification:** + +```bash +cd benchmarks +./search.sh --brain searchbenchesci20 evaluate \ + --dataset data/search_esci_slice.jsonl --skip-ingest \ + --channels passages --rerank plugin:cross-encoder \ + --run search-esci-slice-passages-ce +``` + +**Dependencies:** Checkpoint after Phase 1; open question 4 + +**Files likely touched:** none if plugin already wired + +**Estimated scope:** S + +### Task 7: Exclusionary miss analysis (esci-72) + +**Description:** For esci-72 (and optionally other `without`/`not` queries), dump gold titles vs top-20 passages. Decide whether the miss is pool coverage, lexical negation, or conceptual intent. No production rewrite until the dump exists. + +**Acceptance criteria:** + +- [x] One page or eval artifact listing the 6 gold ASINs and why they missed +- [x] Binary decision: rewrite experiment vs “not testable on this pool” + +**Verification:** python over `eval.json` + `search_esci_slice.jsonl` (no new ingest) + +**Dependencies:** None + +**Files likely touched:** optional helper under `benchmarks/search/` + +**Estimated scope:** S + +### Task 8: Protocol — Recall@20 in report + paired note + +**Description:** Surface Recall@20 in `report.json` / printed table; add a one-line paired-delta note (graph minus passages) in `report.md`. Do not invent p-values on n=11. + +**Acceptance criteria:** + +- [ ] `report.json` includes `recall@20` +- [ ] Markdown table shows channels + +**Verification:** `./search.sh report --run search-esci-slice-passages` + +**Dependencies:** None + +**Files likely touched:** `benchmarks/search/report.py` + +**Estimated scope:** S + +### Checkpoint: Complete + +- [ ] Passages-only remains the cited baseline +- [ ] I02 falsification path executed +- [ ] No ranking change shipped without Task 2 go/no-go +- [ ] Human approved before any larger ESCI ingest + +--- + +## Risks + +| Risk | Impact | Detection | +| --- | --- | --- | +| HARKing a graph win from esci-18 nDCG | High | Require Recall@20 and ≥2 queries | +| Treating n=11 as ESCI | High | Dataset label + n_queries in every sentence | +| Re-ingest drift on searchbenchesci20 | High | `--skip-ingest` only; never wipe | +| Ledger pollution of other suites | High | Upsert `benchmarks.search` only | +| CE / SPLADE blow 200 ms SLO | Med | Record p50 retrieve; abort claim if ≥200 ms | +| Metric gaming via larger k | Med | Reject I12 as quality | +| Cache confound on latency | Low | esci-67 already odd; compare medians | +| Changing default channels in core | High | Default stays `["passages"]`; test `test_search_graph.py:108` | + +--- + +## Decision log (proposal only) + +- Candidates considered: I01–I14, I31, plus rejected GraphRAG/EDRM-now/I12/I13. +- Criteria/weights: table above, set before scoring. +- Literature and review date: 2026-08-18. +- **Decision: useful isolated graph channels; passages remains default.** Accountable human: isolated Recall@10 / gold-ASIN presence is go/no-go; fused nDCG is not. +- Revisit trigger: larger-n ESCI, or an events/expand arm. + +--- + +## Graph fusion hygiene (2026-08-18) — G01–G08 + +Workstream: optional graph channels in `/retrieve/search` without shrinking unique-product recall or nDCG vs passages-only. Brain `searchbenchesci20`, dataset `benchmarks/data/search_esci_slice.jsonl`, n=11, `--skip-ingest`, `--fusion rrf`, `--rerank none`. Default omitted `channels` stayed `["passages"]`. No production `RERANK_MAX_K` change. Ledger: `benchmarks.search` only. + +Predeclared primary: Recall@20 then nDCG@10. Live fusion change only if an arm is not worse than passages on both. Null → G08 (graph as side channel). + +### Task 1 — per-channel lists and unique-doc counts (located evidence) + +`eval.json` now stores `dense_ids`, `bm25_ids`, `entity_ids`, `community_ids`, `n_unique_docs_raw`, `n_unique_docs_canonical`, and `gold_grades`. Search responses expose `channel_lists` when the API from this checkout is running. Ranking of hits is unchanged. + +| Arm | Run | unique-doc@20 raw | unique-doc@20 canonical | +| --- | --- | --- | --- | +| Passages | `search-esci-slice-passages-gate` | 20.0 | **20.0** | +| Fused p+e+c | `search-esci-slice-pec-lists` | 20.0 | **15.3** | + +**Located evidence:** passages-only unique-doc@20 stays 20 on every query. Fused p+e+c still drops unique products (mean 15.3) because chunk UUIDs and catalog `doc_id`s occupy separate RRF slots (P1). + +### Tasks 2–3 — offline replay (located evidence) + +Replay: `python -m search.replay_fusion --mode all` on the Task 1 dumps (`pec-lists` + `passages-gate`), stitching isolated `entities-after` / `communities-after` when a list is missing. Collapse maps every id to `doc_id` **before** RRF. Gated arms used the same dumps; expansion N=10 and graph weights `{0.1, 0.25, 0.5}` were predeclared. + +| Arm | nDCG@10 | Recall@10 | Recall@20 | MRR | unique@20 | +| --- | --- | --- | --- | --- | --- | +| Passages | 0.758 | 0.511 | 0.847 | 0.848 | 20.00 | +| G01 collapse-rrf | 0.723 | 0.498 | 0.839 | 0.847 | 20.00 | +| G04 weighted-0.1 | 0.740 | 0.511 | 0.847 | 0.852 | 20.00 | +| G04 weighted-0.25 | 0.725 | 0.498 | 0.847 | 0.852 | 20.00 | +| G04 weighted-0.5 | 0.726 | 0.498 | 0.852 | 0.852 | 20.00 | +| G02 expansion N=10 | 0.758 | 0.511 | 0.650 | 0.848 | 20.00 | +| G05 confirmation | 0.704 | 0.502 | 0.847 | 0.795 | 20.00 | + +Same protocol on the earlier isolated-channel dumps (no live `channel_lists`) gave collapse R@20 0.827 / nDCG 0.686 and expansion R@20 0.815 / nDCG 0.758. Winner rule was identical. + +**P1 survived as the unique-doc / Recall@20 hole.** Collapse restores unique@20 to 20 and Recall@20 to 0.839 (not ≪ 0.847). **P2/P3 survived as the nDCG hole.** Collapse and every weighted/confirmation mix stay below passages nDCG 0.758. Expansion keeps nDCG but loses Recall@20 (novel graph ASINs displace gold in ranks 11–20). + +No arm is not-worse than passages on **both** nDCG@10 and Recall@20. + +### Checkpoint — G08 (decision) + +**Decision:** G08. Keep graph off the ranking mix; expose it as a side channel. Do not implement G01/G02/G04/G05 in `fuse_passage_lists` / live RRF. Default omitted channels remain `["passages"]`. + +This is not a claim that graph lists contain no gold ASINs (isolated entities Recall@10 0.482 still holds). It is a claim that **unweighted or gated fusion does not beat passages on this n=11 slice**. + +### Task 5 — live passages gate (located evidence) + +`runs/search-esci-slice-passages-gate` — `--channels passages`, `--skip-ingest`. + +| Arm | nDCG@10 | Recall@10 | Recall@20 | MRR | p50 retrieve | +| --- | --- | --- | --- | --- | --- | +| Passages (frozen control) | 0.758 | 0.511 | 0.847 | 0.848 | 28–29 ms | +| Passages gate (this run) | 0.758 | 0.511 | 0.847 | 0.848 | 26 ms | + +Within noise of the frozen control. p50 retrieve 26 ms passes the 200 ms gate. Fused p+e+c `pec-lists` is unchanged at 0.653 / 0.497 / 0.653, p50 160 ms (still under 200 ms, still worse than passages). Do not treat 0.857 as a number from this workstream (that figure is CE-on-pool only). + +### Task 6 — `expand=neighbors` (located evidence) + +`runs/search-esci-slice-entities-neighbors` — `--channels entities --expand neighbors`. + +| Arm | nDCG@10 | Recall@10 | Recall@20 | MRR | unique@20 | p50 retrieve | +| --- | --- | --- | --- | --- | --- | --- | +| Entities-only (prior) | 0.655 | 0.482 | 0.822 | 0.757 | 20 | 51 ms | +| Entities + neighbors | 0.655 | 0.482 | 0.822 | 0.757 | 20 | **223 ms** | + +Recall@10 matches entities-only 0.482. `neighbor_ids` were empty on all 11 queries (adjacent items already in the entity seed set, or dropped by the item filter). p50 223 ms **fails** the 200 ms gate. Stop. Do not fuse neighbors into default search. + +### What this does not support + +- Product claim of “graph precision” or ESCI-wide lift. +- Changing default search to graph. +- Shipping ID-collapse, weighted RRF, expansion-after, or confirmation as a ranking change. +- n=11 as a published ESCI result. + +--- + +## CE-on-pool / first-stage n=74 (2026-08-18, separate from fusion) + +Fusion files were not edited here. Protocol: CE-on-pool vs Reddy **0.857**; first-stage vs **0.495** nDCG@10. n=74 ≠ ~4477. + +**Located evidence** + +- Live `--skip-ingest --channels passages` on `searchbenchesci74`: nDCG@10 **0.495**, nDCG@20 0.549, R@10 0.377, R@20 0.592, MRR 0.758, p50 62 ms (matches the prior first-stage row). +- Local fielded BM25 / RM3 / SPLADE on the same 2043-doc JSONL: best nDCG@10 **0.465** (SPLADE) / **0.458** (title-boost BM25). All **below** 0.495. First-stage win rule (≥0.515) **failed**. +- 4-class MiniLM-L-12 CE-on-pool (40k US Task-1 train, test qids held out, catalog text, weighted ESCI gains): nDCG@20 **0.678**, full-list nDCG **0.747**. Exceeds our FT **0.669**. Does **not** beat Reddy **0.857**. + +Do not mix these rows with slice n=11 0.758. Do not change product default rerank. + +--- + +## Deeper k, then harness CE on retrieved hits (2026-08-18) + +**Decision:** Goal 1 is more gold in the retrieved list (Recall@k, pool coverage). Goal 2 is nDCG of that same list. Graph stays off the ranking mix (G08). Production default `k` and `RERANK_MAX_K = 10` were not changed. Do **not** cite Reddy **0.857** on these arms (that figure is CE-on-pool only). + +**Control (located evidence):** `searchbenchesci74`, `search_esci_74.jsonl`, `--skip-ingest --channels passages --fusion rrf --k 20`: nDCG@10 **0.495**, nDCG@20 0.549, Recall@10 0.377, Recall@20 0.592 (`search-esci-74-passages-control`). With `--rank-pool`: nDCG@20 0.579, **pool_coverage 0.517**, `missing_from_brain` 0 (`search-esci-74-passages-pool`). + +**Predeclared Phase 1 win:** pool_coverage **≥ 0.57** or Recall@50 **≥ 0.65**. nDCG@10 may stay flat. **Phase 2** only after that win: nDCG@10 **≥ 0.515** on the same retrieve-k list, Recall@50 not down vs Phase 1. + +### Phase 1 — deeper first-stage retrieve (located evidence) + +Same brain, `--skip-ingest`, passages only, `rerank=none`. Mean `n_hits` at k=50 is 50; unique-doc@50 is 50. k=100 unique-doc@100 is 100. p50 retrieve 58–60 ms (under 200 ms). + +| Arm | k | Recall@10 | Recall@20 | Recall@50 | Recall@100 | nDCG@10 | nDCG@20 | p50 ms | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Control | 20 | 0.377 | 0.592 | — | — | **0.495** | 0.549 | 62 | +| `search-esci-74-passages-k50` | 50 | 0.379 | 0.590 | **0.834** | — | 0.500 | 0.552 | 60 | +| `search-esci-74-passages-k100` | 100 | 0.379 | 0.591 | 0.831 | **0.906** | 0.500 | 0.552 | 58 | + +`--rank-pool` (coverage of the labeled candidate list, not Reddy Task 1): + +| Arm | k | pool_coverage | missing_from_brain | Recall@20 | Recall@50 | nDCG@10 | nDCG@20 | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `search-esci-74-passages-pool` | 20 | **0.517** | 0 | 0.592 | — | 0.546 | 0.579 | +| `search-esci-74-passages-k50-pool` | 50 | **0.778** | 0 | 0.669 | 0.832 | 0.567 | 0.624 | +| `search-esci-74-passages-k100-pool` | 100 | **0.846** | 0 | 0.696 | 0.906 | 0.574 | 0.635 | + +**Phase 1 call:** win. Recall@50 **0.834 ≥ 0.65**; pool_coverage **0.778 ≥ 0.57**. Extra golds sit in ranks 21–50 (Recall@20 is flat vs k=20). nDCG@10 is unchanged (~0.500). k=100 does not raise Recall@50 vs k=50; Recall@100 is 0.906. Do not mix with CE-on-pool **0.695** or Reddy **0.857**. + +### Phase 2 — harness CE over stored `hit_ids` (located evidence) + +Sibling of `rank_pool.py`: `benchmarks/search/rerank_retrieved.py` reorders the k=50 retrieved list. It does not call production rerank and does not score the full labeled pool. Default API `rerank=none` and `RERANK_MAX_K = 10` unchanged. + +| Arm | model | nDCG@10 | nDCG@20 | Recall@10 | Recall@50 | MRR | +| --- | --- | --- | --- | --- | --- | --- | +| k=50 first-stage | none | **0.500** | 0.552 | 0.379 | **0.834** | 0.765 | +| `search-esci-74-passages-k50-ce` | MS MARCO MiniLM-L-6 | 0.448 | 0.515 | 0.331 | 0.834 | 0.720 | +| `search-esci-74-passages-k50-ce-l12` | 4-class MiniLM-L-12 e2 | 0.467 | 0.508 | 0.341 | 0.834 | 0.706 | + +**Phase 2 call:** null. nDCG@10 did not reach **0.515** (needed +0.02 vs the 0.495 k=20 control). Both CEs **hurt** nDCG@10 vs first-stage k=50. Recall@50 held at 0.834 (same hit set). These rows are first-stage retrieve-then-rerank, not CE-on-pool 0.695 / Reddy 0.857. + +**Decision:** keep production search at default k / `RERANK_MAX_K=10`. Deeper retrieve finds more gold; zero-shot and local 4-class CE over that retrieved list do not promote it into the nDCG@10 head. + +--- + +## Gold in small and large k (2026-08-18) + +**Focal question:** raise gold in both Recall@10 and Recall@50 on the same 74 queries. Graph off. Production `k` / `RERANK_MAX_K` unchanged. Do **not** cite Reddy **0.857** or CE-on-pool **0.695**. + +**Predeclared dual win:** Recall@10 **≥ 0.397** and Recall@50 **≥ 0.854** (both +0.02 vs k=20 / k=50 passages). Secondary nDCG@10 ≥ 0.515. + +### Task 1 — miss taxonomy (located evidence) + +`miss-strata` on `search-esci-74-passages-k50` / `search_esci_74.jsonl`. Reconciles **74** queries / **1270** golds. + +| Stratum | Queries | Gold items | +| --- | --- | --- | +| head-ok (gold in top-10) | 67 | 424 in top-10 | +| rank-too-low (gold in 11–50 only) | 2 | 728 in ranks 11–50 | +| total-miss | 5 | 118 missed | + +Total-miss qids: `esci-113`, `esci-177`, `esci-267`, `esci-393`, `esci-72`. Dual-k gap is mostly **rank-too-low golds** (728), not the five total misses. Macro Recall@50 0.834 vs micro miss 118/1270 (9%). + +### Task 2 — query-side on pathological total-miss only (located evidence) + +Rewrote **only** `esci-113` (HTML unescape) and `esci-267` (strip leading `- *`). Did **not** rewrite `esci-72` (prior qrel mismatch). SQL-ish `esci-177` left unchanged. Live `--skip-ingest` `search-esci-74-passages-k50-qrewrite`: + +| Arm | Recall@10 | Recall@50 | nDCG@10 | +| --- | --- | --- | --- | +| k=50 passages | 0.379 | 0.834 | 0.500 | +| qrewrite (2 qids) | 0.379 | 0.834 | 0.500 | + +Per-qid Recall@10/@50 for `esci-113` and `esci-267` stayed **0**. Stratum-only null. Global dual win not moved. + +### Task 3 — local ANCE dual encoder (located evidence) + +Harness MiniLM-L-6 dual encoder, US Task-1 train, test qids held out, hard negatives from stored k=50 non-gold `hit_ids` (bank 1387), 6557 triples, 1 epoch. Encode 2043 JSONL docs locally. Not BrainAPI embeddings. Run `search-esci-74-dense-ance-k50`. + +| Arm | Recall@10 | Recall@50 | nDCG@10 | +| --- | --- | --- | --- | +| Passages k=20 control | **0.377** | — | **0.495** | +| Passages k=50 | 0.379 | **0.834** | 0.500 | +| Local ANCE MiniLM | 0.302 | 0.750 | 0.396 | + +**Dual-k call:** null. Both recalls moved **down**. Do not fuse this retriever into passages. Do not raise `RERANK_MAX_K`. ColBERT and BGE sidecars were run next (below); both also null. + +**Decision:** BrainAPI hybrid passages remain the first-stage control. Matching interventions tried here (query rewrite on 2 qids; MiniLM ANCE on the 2043-doc JSONL) did not put more gold in both small and large k. + +--- + +## Retrieved-neg CE, then ColBERT, then BGE (2026-08-18) + +**Focal question:** (1) promote Exact already in hybrid ranks 11–50 via a 4-class CE trained on BM25 retrieved negatives; (2–3) try two first-stage sidecars that are not MiniLM dual-encoder. Graph off. Frozen `searchbenchesci74` / `--skip-ingest`. `RERANK_MAX_K` stayed 10. Do **not** mix these rows with CE-on-pool **0.695** or Reddy **0.857**. n=74, no p-values. All three arms were run even though arm 1 was null. + +**Predeclared wins:** arm 1 nDCG@10 **≥ 0.515** and Recall@50 not below **0.834**. Arms 2–3 Recall@10 **≥ 0.397** **and** Recall@50 **≥ 0.854**. Secondary nDCG@10 ≥ 0.515. + +### Arm 1 — retrieved-negative 4-class CE (located evidence) + +Train lists: US Task-1 small-train, test qids held out (`n_holdout_qids=148`), BM25 top-50 over the train product catalog, unlabeled hits = class **I**. 6000 queries, 300000 pairs, all lists had ≥1 I. Checkpoint `benchmarks/data/models/esci-minilm-l12-retrieved` (`source=retrieved-bm25`, 80k pairs, 1 epoch, unweighted). Eval: harness `rerank-retrieved` on stored k=50 `hit_ids` (`search-esci-74-passages-k50-ce-retrieved`). Not production rerank. + +| Arm | nDCG@10 | Recall@10 | Recall@50 | MRR | +| --- | --- | --- | --- | --- | +| k=50 passages | **0.500** | 0.379 | **0.834** | 0.765 | +| retrieved-neg 4-class L-12 | 0.465 | 0.351 | 0.834 | 0.712 | + +**Arm 1 call:** null. nDCG@10 **0.465** did not reach **0.515** and is below first-stage 0.500 (same direction as pool-trained / MS MARCO CE on these hits). Recall@50 held at **0.834** (same hit set). + +### Arm 2 — ColBERT MaxSim sidecar (located evidence) + +In-process `plugins/search-colbert` (`colbert-ir/colbertv2.0`) over the 2043 JSONL docs. Brain id `harness-local-colbert`. Did **not** `POST /search-colbert/index` onto `searchbenchesci74`. Run `search-esci-74-colbert-k50`. Channels `harness-colbert`. Not fused with passages. + +| Arm | Recall@10 | Recall@50 | nDCG@10 | p50 encode+retrieve | +| --- | --- | --- | --- | --- | +| Passages k=20 / k=50 | **0.377** / 0.379 | **0.834** | **0.495** / 0.500 | ~60 ms retrieve | +| ColBERT MaxSim | 0.311 | 0.714 | 0.434 | 10063 ms | + +**Arm 2 call:** dual-k null. Recall@10 **0.311** < 0.397; Recall@50 **0.714** < 0.854. p50 ~10 s is **labeled**, not a 200 ms product claim, not a stop. + +### Arm 3 — zero-shot BGE-base (located evidence) + +`BAAI/bge-base-en-v1.5` via `local-dense` on the same 2043 JSONL. **Not MiniLM.** No `query:` prefixes. Run `search-esci-74-bge-base-k50`. No new production brain. + +| Arm | Recall@10 | Recall@50 | nDCG@10 | +| --- | --- | --- | --- | +| Passages k=20 / k=50 | **0.377** / 0.379 | **0.834** | **0.495** / 0.500 | +| MiniLM ANCE (prior, not this arm) | 0.302 | 0.750 | 0.396 | +| BGE-base (this arm) | 0.322 | 0.799 | 0.441 | + +**Arm 3 call:** dual-k null. Recall@10 **0.322** < 0.397; Recall@50 **0.799** < 0.854. Better than MiniLM ANCE 0.302 / 0.750, still below hybrid passages. p50 encode+retrieve 408 ms. + +**Three-arm call:** no arm beat the passages control on its predeclared gate. Keep default omitted `channels` = passages. Do not raise `RERANK_MAX_K`. Do not fuse ColBERT or BGE into passages. Do not wipe `searchbenchesci74`. + +--- + +## Exhaustive catalog rank then two-stage (2026-08-19) + +**Focal question:** (A) does scoring **every** JSONL doc with the unweighted 4-class L-12 e2 checkpoint beat hybrid k=50; (B) do complementary sidecar lists raise dual-k recall without diluting Recall@10; (C) ship opt-in `mode=catalog` on `/retrieve/search` without changing the default path. Graph off. Frozen `searchbenchesci74` / `--skip-ingest`. Default `RERANK_MAX_K` stayed **10**. Do **not** mix these rows with CE-on-pool **0.695** or Reddy **0.857**. n=74, no p-values. B and C ran even though A was null. + +**Predeclared gates:** A nDCG@10 **≥ 0.515** **and** Recall@10 **≥ 0.397**. B Recall@10 **≥ 0.397** **and** Recall@50 **≥ 0.854**, and Recall@10 not below **0.379**. C (same k=50 hits, deeper rerank): A’s nDCG@10 gate with Recall@50 held at **0.834**. + +### A — exhaustive 4-class on 2043 docs (located evidence) + +Harness `rank-corpus` scored all 2043 JSONL passages per query with `benchmarks/data/models/esci-minilm-l12-4class-nowt-e2` (74 × 2043 ≈ 151k pairs). Protocol `exhaustive-catalog`. Did **not** POST onto `searchbenchesci74`. Run `search-esci-74-exhaustive-ce`. Channels `exhaustive-4class`. Cut @ 5,10,20,50. + +| Arm | nDCG@10 | Recall@10 | Recall@50 | p50 | +| --- | --- | --- | --- | --- | +| k=50 passages | **0.500** | **0.379** | **0.834** | 60 ms retrieve | +| Exhaustive 4-class L-12 e2 | 0.416 | 0.266 | 0.603 | 94 s labeled | + +**A call:** null. nDCG@10 **0.416** < **0.515** and below hybrid 0.500. Recall@10 **0.266** < **0.397**. p50 is **labeled**, not a 200 ms product claim. MiniLM 4-class over the full file does not beat hybrid RRF on this slice. + +### B1 — U01 unique-gold overlap (located evidence) + +`list-overlap` on stored `hit_ids` (`search-esci-74-passages-k50` vs `search-esci-74-bge-base-k50` / `search-esci-74-colbert-k50`). Gold ASINs in sidecar top-50 absent from passages k=50: + +| Sidecar | unique gold hits | queries with unique golds | +| --- | --- | --- | +| BGE-base k=50 | **40** | 19 / 74 | +| ColBERT k=50 | **21** | 13 / 74 | + +Unique golds were nonzero, so B2 included ColBERT. + +### B2 — harness RRF union (located evidence) + +RRF of passages k=50 + BGE k=50 + ColBERT k=50. Channels `harness-union`. Not fused into live graph. Run `search-esci-74-union-bge-k50`. + +| Arm | nDCG@10 | Recall@10 | Recall@50 | +| --- | --- | --- | --- | +| k=50 passages | **0.500** | **0.379** | **0.834** | +| Union passages+BGE+ColBERT | 0.479 | 0.363 | 0.827 | + +**B call:** null. Recall@10 **0.363** is below control **0.379** (graph-style dilution). Dual-k **≥0.397 and ≥0.854** failed. Extra lists steal top-10 slots; unique golds in the tail do not raise Recall@50 here (0.827 vs 0.834). Do not fuse these lists into production ranking. + +### C — live `mode=catalog` (located evidence) + +Shipped: `SearchRequestBody.mode` default `"default"` (retrieve `k`, rerank `min(10, len)`). `mode=catalog` retrieves `k_ret = min(200, max(k, 50))`, reranks `min(len, CATALOG_RERANK_MAX_K=50)`, cuts to request `k`. Live `--skip-ingest` on frozen `searchbenchesci74`, `SEARCH_RERANK_MODEL` = nowt-e2 (same checkpoint as A, not retrieved-neg). Run `search-esci-74-catalog-ce-k50`. Request `k=50` so first-stage depth matches the k=50 control; this tests **plugin rerank of 50** (not 10). + +| Arm | nDCG@10 | Recall@10 | Recall@50 | p50 retrieve | p50 client wall | +| --- | --- | --- | --- | --- | --- | +| k=50 passages (`rerank=none`) | **0.500** | **0.379** | **0.834** | 60 ms | — | +| `mode=catalog` + plugin CE | 0.467 | 0.341 | **0.834** | 59 ms | 877 ms | + +**C call:** architecture **shipped**; quality gate **null**. Recall@50 held at **0.834** (same 50-hit retrieve). nDCG@10 **0.467** < **0.515** and below first-stage 0.500 — same direction as harness CE-on-retrieved 4-class L-12 e2 (0.467). Catalog+CE is **not** the ADR-007 200 ms default. Omitted `mode` and `RERANK_MAX_K=10` stay the product path. Do not wipe `searchbenchesci74`. + +**A/B/C call:** exhaustive MiniLM cannot beat hybrid on this 2043-doc slice; list union dilutes Recall@10; catalog mode is the two-stage hook. Keep default omitted `channels` = passages. Do not cite Reddy **0.857** against these rows. + +--- + +## Frozen-head cascade (2026-08-19) + +**Focal question:** can sidecar unique golds raise Recall@50 if hybrid top-10 is frozen? Graph off. Harness only. Not live fusion. `RERANK_MAX_K` unchanged. Do **not** mix with CE-on-pool **0.695** or Reddy **0.857**. n=74, no p-values. + +**Predeclared gate:** Recall@10 **≥ 0.379** (no dilution) **and** Recall@50 **≥ 0.854**. + +**Located evidence.** `cascade-lists`: passages ranks 1–10 copied; unique golds from BGE+ColBERT top-50 that were absent from passages k=50 inserted at 11–50; remaining passages tail golds kept before non-golds. Run `search-esci-74-cascade-tail-k50`. Channels `harness-cascade`. Protocol `frozen-head-cascade`. Injected **51** unique gold ASINs (BGE 40 ∪ ColBERT 21). + +| Arm | nDCG@10 | Recall@10 | Recall@20 | Recall@50 | nDCG@20 | +| --- | --- | --- | --- | --- | --- | +| k=50 passages | **0.500** | **0.379** | 0.590 | 0.834 | 0.552 | +| RRF union (prior, not this arm) | 0.479 | 0.363 | 0.577 | 0.827 | 0.538 | +| Frozen-head cascade | **0.500** | **0.379** | **0.750** | **0.889** | **0.619** | + +**Call:** win. Recall@10 **0.379** matches the passages control to floating point (head frozen). Recall@50 **0.889 ≥ 0.854**. nDCG@10 held; nDCG@20 rose because extras land in 11–20. RRF union failed by stealing the head; cascade uses the same unique golds without that. Not fused into live `/retrieve/search`. Do not raise default `RERANK_MAX_K`. Do not wipe `searchbenchesci74`. + +--- + +## DeBERTa-v3-base 4-class on catalog k=50 (2026-08-19) + +**Focal question:** does replacing MiniLM with DeBERTa-v3-base (same 4-class weighted-gain recipe) promote golds already in hybrid ranks 11–50 into the top-10? Graph off. Frozen `searchbenchesci74` / no wipe. Default `RERANK_MAX_K` stayed **10**. Live `mode=catalog` only if harness gates fired. Do **not** mix first-stage rows with CE-on-pool **0.695** / **0.710** or Reddy **0.857**. n=74, no p-values. + +**Predeclared first-stage win:** Recall@10 **≥ 0.397** **and** nDCG@10 **≥ 0.515**, Recall@50 held at **0.834**. + +**Train (located).** `finetune-4class` `--base microsoft/deberta-v3-base`, US Task-1 train, test qids held out (`n_holdout_qids=148`), 80k pairs, 2 epochs, batch 8, unweighted, max_length 192, label smoothing 0.1. Checkpoint `benchmarks/data/models/esci-deberta-v3-base-4class`. Parent venv needed `sentencepiece` (installed 0.2.2). HuggingFace DeBERTa-v3 tokenizer shipped `model_max_length` as an unbounded sentinel; it was set to **192** to match train. Not v3-large. Not the www ensemble. + +**Pool discriminator (ranking-in-pool only; not the R@10 gate).** `rank_pool_4class` run `search-esci-74-ce-pool-deberta-base`. nDCG@20 **0.710** vs MiniLM 4-class e2 **0.695**; full-list nDCG **0.777** vs **0.765**. n=74 ≠ ~4477. Do not cite Reddy **0.857** against this row. + +**First-stage harness (primary).** `rerank-retrieved` on stored `search-esci-74-passages-k50` `hit_ids`. Run `search-esci-74-passages-k50-ce-deberta`. Plugin 4-logit softmax gains. No live API. + +| Arm | nDCG@10 | Recall@10 | Recall@50 | +| --- | --- | --- | --- | +| k=50 passages | **0.500** | **0.379** | **0.834** | +| MiniLM 4-class e2 on those hits | 0.467 | 0.341 | **0.834** | +| DeBERTa-v3-base 4-class on those hits | 0.510 | 0.363 | **0.834** | + +**Call:** first-stage **null**. Recall@10 **0.363 < 0.397** (also below passages 0.379). nDCG@10 **0.510 < 0.515** (above MiniLM 0.467 and slightly above passages 0.500). Recall@50 **0.834** held (same hit set). Live `mode=catalog` was **not** run: MiniLM catalog already matched harness CE-on-retrieved, and both quality gates missed. Cascade remains the Recall@50 win. Do not raise `RERANK_MAX_K`. Do not wipe `searchbenchesci74`. + +**Interpretation (not a finding beyond this slice).** Pool nDCG@20 moved up a little while first-stage Recall@10 moved down vs hybrid: extra backbone capacity helped ranking-in-pool more than promoting golds through the hybrid head. Compatible with the alternative that labeled pools (coverage 1.0) are easier than the k=50 retrieved list (coverage 0.778). Zhang **0.90** is still ranking-in-pool + large + ensemble, not this y-axis. + +--- + +## Multilingual first-stage (2026-08-19) + +Not a US n=74 quality rerun. Not Reddy 0.857. Details in [22-multilingual-ecommerce-search.md](22-multilingual-ecommerce-search.md). + +**Located evidence.** ESCI locales are US/ES/JP (`product_locale` in Reddy HTML). No Italian in ESCI. Amazon-M2 has locale IT but rec/title-generation tasks (`2307.09688`). mMARCO has Italian as MT passages (`2108.13897`), not catalog qrels. arXiv `product search` + Italian + `cs.IR` returned 0. + +**Decision.** ES download must not clobber US JSONL. Italian smoke fixture is not ESCI. Product default stays passages + `rerank=none`. `searchbenchesci74` not wiped. + +**ES first-stage (located evidence).** Run `search-esci-es-passages-k50`, brain `searchbenchescies`, n=62, 2000 docs, k=50 passages, `rerank=none`. nDCG@10 **0.577**, Recall@10 **0.353**, Recall@50 **0.914**, p50 84 ms. Not US n=74 0.500/0.379. Not Reddy ES 0.849 ranking-in-pool. + +--- + +## Production first-stage arms on US n=74 (2026-08-19) + +**Focal question:** can cheap first-stage changes raise Recall@50 and/or nDCG@10 on skip-ingest `searchbenchesci74` without CE-on-retrieved or graph-as-default? Graph off. Frozen brain / no wipe. `RERANK_MAX_K` stayed **10**. Do **not** mix with CE-on-pool **0.710** or Reddy **0.857**. n=74, no p-values. + +**Predeclared gates.** Hold: nDCG@10 **≥ 0.500**, Recall@10 **≥ 0.379**, p50 retrieve **< 200 ms** (ex-embed). Recall@50 win: **≥ 0.854** and hold head. nDCG@10 win: **≥ 0.520** and Recall not down. Null → leave hybrid RRF, `rerank=none`. + +### Frozen-head cascade (opt-in lists) + +**Located evidence.** Helper `frozen_head_merge` (hybrid top-10 frozen; unique extra ids fill 11–50). Replay unit test on stored `search-esci-74-passages-k50` + BGE + ColBERT reproduces Recall@50 **0.889**, nDCG@10 **0.500**, Recall@10 **0.379**. Live `/retrieve/search` uses this merge **only** when plugin retriever lists (or `SEARCH_LITERAL_FILL`) are requested — not default RRF of plugins into the head. Default omitted `channels=["passages"]` unchanged. + +**Live n=74 sidecar:** not run. `search-colbert` / `search-splade` load, but the ColBERT in-memory index for `searchbenchesci74` is empty (retrieve returns `[]`). Indexing 2043 chunks at query time was not started. **Decision:** cascade stays harness-only. Honest production Recall@50 on the default path stays **0.834**. + +### C1 — `fusion=cc` alpha sweep (null) + +Skip-ingest, passages, k=50, `rerank=none`. Request `fusion=cc` + `fusion_alpha`. p50 61–63 ms (hold). + +| Arm | nDCG@10 | Recall@10 | Recall@50 | p50 retrieve | +| --- | --- | --- | --- | --- | +| k=50 passages RRF | **0.500** | **0.379** | **0.834** | ~60 ms | +| CC α=0.3 (`search-esci-74-cc-a03`) | 0.483 | 0.376 | 0.832 | 63 ms | +| CC α=0.5 (`search-esci-74-cc-a05`) | 0.486 | 0.377 | 0.832 | 61 ms | +| CC α=0.7 (`search-esci-74-cc-a07`) | 0.493 | 0.378 | 0.834 | 61 ms | + +**Call:** null. Best nDCG@10 **0.493 < 0.520**. Recall@10 slightly below hold. Product default stays RRF. + +### C2 — title-token literal residual (null) + +**Predeclared fusion:** frozen-head (not RRF) so the top-10 cannot be stolen. Env `SEARCH_LITERAL_FILL` default **false**. Live skip-ingest `search-esci-74-literal-fill-k50`. + +| Arm | nDCG@10 | Recall@10 | Recall@50 | p50 retrieve | +| --- | --- | --- | --- | --- | +| k=50 passages | **0.500** | **0.379** | **0.834** | ~60 ms | +| Literal fill + frozen head | **0.500** | **0.379** | **0.655** | 49 ms (`search.retrieve` only on this run) | + +**Call:** null. Recall@10 held (frozen head). Recall@50 **0.655** dropped because non-gold ILIKE token hits filled 11–50 and evicted hybrid tail golds. Flag stays false. Do not treat this as a Recall@50 win. + +### C3 — pairwise LTR on the stored k=50 head (null) + +**Focal question:** can a cheap linear RankNet on features already in the hybrid list promote the 728 golds sitting in ranks 11–50 into the top-10, without changing the retrieved set? Graph off. Frozen `searchbenchesci74` / no wipe / skip-ingest. Not live `/retrieve/search`. Do **not** mix with CE-on-pool **0.710** or Reddy **0.857**. n=74, no p-values. + +**Protocol (located evidence).** `./search.sh ltr-head --from-run search-esci-74-passages-k50 --dataset data/search_esci_74.jsonl --run search-esci-74-ltr-head-k50`. Query-grouped 5-fold CV (OOF predictions; seed=0, 40 epochs, lr=0.05, l2=1e-3, max 400 pairs/query). Features: `rrf_inv`, `bm25_inv`, `dense_inv`, `title_overlap`, `brand_hit`, `query_in_title`. Unlabeled docs in the 50 scored as 0. Overlap-only sort stored as `overlap_only_metrics`, not the gated arm. + +| Arm | nDCG@10 | Recall@10 | Recall@20 | Recall@50 | MRR | +| --- | --- | --- | --- | --- | --- | +| k=50 passages RRF | **0.500** | **0.379** | 0.590 | **0.834** | 0.765 | +| Overlap-only (diagnostic) | 0.476 | 0.339 | 0.580 | 0.834 | 0.732 | +| LTR OOF (`search-esci-74-ltr-head-k50`) | 0.515 | 0.384 | 0.583 | **0.834** | 0.753 | +| Predeclared win | **≥ 0.520** | **≥ 0.397** | — | hold | — | + +Mean CV weights (not a quality claim): `title_overlap` 3.35, `dense_inv` 2.06, `rrf_inv` 1.57, `brand_hit` 0.51, `bm25_inv` 0.02, `query_in_title` ~0. + +**Call (decision):** null. nDCG@10 **0.515 < 0.520**. Recall@10 **0.384 < 0.397**. Recall@50 held. Directional nDCG lift is small and is **not** promotion of the 728 (Recall@10 +0.005; Recall@20 and MRR slightly down vs RRF). Overlap-only is worse than RRF — naive title ranking is not the product. Default stays hybrid RRF, `rerank=none`. Do not wire this ranker into `search.py`. + +### C4 — LTR + 4-class CE as a feature (harness win; not live) + +**Focal question:** can blending the already-trained 4-class MiniLM score with hybrid RRF via query-grouped LTR raise nDCG@10 on the frozen k=50 list, with Recall@10 held? Graph off. Frozen `searchbenchesci74` / no wipe. Not live `/retrieve/search`. Do **not** mix with CE-on-pool **0.710**, Reddy **0.857**, or CE-as-sole-ranker (located evidence: Recall@10 0.331–0.351). n=74, no p-values. + +**Protocol (located evidence).** `./search.sh ltr-head --from-run search-esci-74-passages-k50 --ce-model data/models/esci-minilm-l12-4class-nowt-e2 --pair-policy other_query_neg --run search-esci-74-ltr-cefeat-k50`. Same hypers as C3 (seed=0, 40 epochs, lr=0.05, l2=1e-3, max 400 pairs/query, 5-fold OOF). Feature `ce_gain` = 1·P(E)+0.1·P(S)+0.01·P(C); lists are not sorted by CE. Pair policy: this-query graded golds vs other-query golds in the 50; unlabeled skipped. Predeclared this round: nDCG@10 ≥ **0.520**, Recall@10 ≥ **0.379**, Recall@50 held. Recall@10 ≥ 0.397 is not required to call the harness win. + +| Arm | nDCG@10 | Recall@10 | Recall@20 | Recall@50 | MRR | +| --- | --- | --- | --- | --- | --- | +| k=50 passages RRF | 0.500 | 0.379 | 0.590 | **0.834** | 0.765 | +| CE-as-sole-ranker (prior, MiniLM on retrieved) | 0.448 | 0.331 | — | 0.834 | 0.720 | +| LTR OOF no CE (`search-esci-74-ltr-head-k50`) | 0.515 | 0.384 | 0.583 | **0.834** | 0.753 | +| LTR + `ce_gain` (`search-esci-74-ltr-cefeat-k50`) | **0.524** | 0.383 | 0.569 | **0.834** | **0.779** | +| Predeclared this round | **≥ 0.520** | **≥ 0.379** | — | hold | — | + +Mean CV weights (not a quality claim): `ce_gain` 3.68, `title_overlap` 2.56, `dense_inv` 1.23, `rrf_inv` 0.66, `brand_hit` 0.45, `bm25_inv` −0.13, `query_in_title` ~0. + +**Call (located evidence + decision):** harness win on the predeclared nDCG/hold. nDCG@10 **0.524 ≥ 0.520**. Recall@10 **0.383 ≥ 0.379**. Recall@50 held. MRR 0.765 → 0.779. LightGBM not run (gated only if nDCG close-but-short). **Do not wire into `search.py` without human review:** live CE on 50 hits is extra latency vs the ~60 ms retrieve path; n=74 has no CI; Recall@20 fell 0.590 → 0.569. CE-alone remains a null. Default stays hybrid RRF, `rerank=none` until review. + +**A/B/C call:** Italian OR-FTS is a pipeline feature on an allowlisted searchbench. US n=74 live default is unchanged: hybrid BM25+dense, passages, `rerank=none`. Harness nDCG@10 0.524 is not Reddy **0.857**. Later DeBERTa RankNet blend (C5) is the current harness best at **0.542**; still not live. + +### C5 — DeBERTa `ce_gain` LTR blend (harness win vs MiniLM; not live) + +**Focal question:** on frozen [`search-esci-74-passages-k50`](../../benchmarks/runs/search-esci-74-passages-k50), does replacing MiniLM `ce_gain` with the existing DeBERTa-v3-base 4-class checkpoint raise first-stage nDCG@10 **above 0.524**, with Recall@10 ≥ **0.379** and Recall@50 **0.834** held? Graph off. Frozen `searchbenchesci74` / no wipe. Not live `/retrieve/search`. Do **not** mix with Reddy **0.857**, pool nDCG@20 **0.710**, or DeBERTa-as-sole-sorter (located: nDCG@10 **0.510**, Recall@10 **0.363**). n=74, no p-values. Horizon nDCG@10 ≥ **0.70** is not a fail of this arm. + +**Protocol (located evidence).** `./search.sh ltr-head --from-run search-esci-74-passages-k50 --ce-model data/models/esci-deberta-v3-base-4class --pair-policy other_query_neg --run search-esci-74-ltr-deberta-k50`. Same RankNet hypers as C3/C4 (seed=0, 40 epochs, lr=0.05, l2=1e-3, max 400 pairs/query, 5-fold OOF). Lists are not sorted by DeBERTa. Scores cached at `runs/search-esci-74-passages-k50/ce_gain_esci-deberta-v3-base-4class.json` (MiniLM cache left in place). Tokenizer already had `model_max_length=192`; `load_4class_predict` was not edited. Predeclared this round: nDCG@10 **> 0.524**, Recall@10 ≥ **0.379**, Recall@50 held. + +Gated I03 (beat 0.524, far from 0.70, `|ce_gain|` not ~0): LightGBM lambdarank, frozen `n_estimators=100`, `max_depth=3`, `learning_rate=0.05`, same CV/features/pair policy, run `search-esci-74-ltr-deberta-lgbm-k50`. No hypersearch. + +| Arm | nDCG@10 | Recall@10 | Recall@20 | Recall@50 | MRR | +| --- | --- | --- | --- | --- | --- | +| k=50 passages RRF | 0.500 | 0.379 | 0.590 | **0.834** | 0.765 | +| DeBERTa as sole sorter (prior) | 0.510 | 0.363 | — | 0.834 | — | +| LTR + MiniLM `ce_gain` (`search-esci-74-ltr-cefeat-k50`) | 0.524 | 0.383 | 0.569 | **0.834** | 0.779 | +| LTR + DeBERTa `ce_gain` RankNet (`search-esci-74-ltr-deberta-k50`) | **0.542** | **0.387** | 0.586 | **0.834** | 0.774 | +| LightGBM + DeBERTa `ce_gain` (`search-esci-74-ltr-deberta-lgbm-k50`) | 0.516 | 0.379 | 0.592 | **0.834** | 0.710 | +| Must-beat this round | **> 0.524** | **≥ 0.379** | — | hold | — | +| Horizon (not a fail) | **≥ 0.70** | — | — | — | — | +| Oracle on these 50 | 0.876 | 0.548 | — | 0.834 | — | + +Mean RankNet CV weights (not a quality claim): `ce_gain` 3.65, `title_overlap` 2.43, `dense_inv` 1.26, `brand_hit` 0.51, `rrf_inv` 0.45, `query_in_title` 0.02, `bm25_inv` −0.08. + +Mean LightGBM gain importances (not a quality claim): `ce_gain` 646, `dense_inv` 561, `rrf_inv` 337, `title_overlap` 162, `brand_hit` 82, `bm25_inv` 19, `query_in_title` 0. LightGBM Recall@10 unrounded **0.3788** (quoted 0.379 in three decimals; hold is strict ≥ 0.379). + +**Call (located evidence + decision):** I01 **win vs MiniLM**. nDCG@10 **0.542 > 0.524**. Recall@10 **0.387 ≥ 0.379**. Recall@50 held. Horizon 0.70 missed (~0.16 short; k=50 oracle 0.876). I03 LightGBM **null**: nDCG@10 **0.516** loses to RankNet DeBERTa and to MiniLM 0.524; Recall@10 misses the hold unrounded; `ce_gain` is live, so the miss is the GBDT head, not a dead feature. **Do not wire into `search.py`:** live DeBERTa on 50 hits is extra latency vs ~60 ms retrieve; n=74 has no CI; 0.542 vs 0.524 is a point-estimate gate, not a significance test. Default stays hybrid RRF, `rerank=none` until human review. Do not replace the MiniLM cache. + +**I04 (this session):** both heads missed 0.70 with large `ce_gain`. Train queries on matched hybrid k=50 were run on a new `searchbench*` (C6). + +**A/B/C call:** US n=74 live default unchanged. Harness best after C5 was DeBERTa RankNet CV **0.542**; C6 apply-from-train is **0.544**. Still not Reddy **0.857** and still not product ranking. + +### C6 — LTR trained on 170 matched hybrid lists, applied to frozen n=74 (harness win vs 0.542; not live) + +**Focal question:** does fitting RankNet+DeBERTa `ce_gain` on **matched hybrid k=50** from extra US train queries, then applying those weights to frozen [`search-esci-74-passages-k50`](../../benchmarks/runs/search-esci-74-passages-k50), raise nDCG@10 **above 0.542**, with Recall@10 ≥ **0.379** and Recall@50 **0.834** held? Graph off. Frozen `searchbenchesci74` / no wipe. New brain only. Not live `/retrieve/search`. Do **not** mix with Reddy **0.857**, pool nDCG@20 **0.710**, or official-pool LTR. n=74, no p-values. Horizon nDCG@10 ≥ **0.70** is not a fail of 170q. + +**Protocol (located evidence).** Download `--split train --max-queries 200 --max-docs 4000 --out data/search_esci_ltr200.jsonl --holdout-dataset data/search_esci_74.jsonl` (did not clobber `search_esci.jsonl` / `_74`). Doc budget bound the slice to **170** queries / **4000** docs; qid overlap with the 74 is **0**. Ingest+evaluate passages k=50, `fusion=rrf`, `rerank=none`, skip_enrichment, brain **`searchbenchesciltr2`** (first `searchbenchesciltr` attempt timed out queued before the Celery worker was up; did not retry on `searchbenchesci74`). Train first-stage diagnostic (different catalog, not the gate): nDCG@10 **0.387**, Recall@10 **0.367**, Recall@50 **0.744**, p50 retrieve 76 ms (`search-esci-ltr200-passages-k50`). + +Then `./search.sh ltr-head --from-run search-esci-74-passages-k50 --train-from-run search-esci-ltr200-passages-k50 --train-dataset data/search_esci_ltr200.jsonl --ce-model data/models/esci-deberta-v3-base-4class --pair-policy other_query_neg --run search-esci-74-ltr-deberta-train200`. Fit **once** on 170 train lists (no CV on the 74). Reused frozen-74 DeBERTa cache; scored train lists to `runs/search-esci-ltr200-passages-k50/ce_gain_esci-deberta-v3-base-4class.json`. MiniLM cache left in place. RankNet hypers unchanged. LightGBM not re-run. + +**Assumption (catalog shift):** train lists come from a 4000-doc train-qrel index; test lists stay on the 2043-doc `searchbenchesci74` index. Feature space matches; IDF/dense geometry may not. + +| Arm | nDCG@10 | Recall@10 | Recall@20 | Recall@50 | MRR | +| --- | --- | --- | --- | --- | --- | +| k=50 passages RRF | 0.500 | 0.379 | 0.590 | **0.834** | 0.765 | +| LTR + DeBERTa RankNet CV (`search-esci-74-ltr-deberta-k50`) | 0.542 | 0.387 | 0.586 | **0.834** | 0.774 | +| LTR + DeBERTa applied from 170 train lists (`search-esci-74-ltr-deberta-train200`) | **0.544** | **0.391** | 0.585 | **0.834** | 0.762 | +| Must-beat this round | **> 0.542** | **≥ 0.379** | — | hold | — | +| Horizon (not a fail) | **≥ 0.70** | — | — | — | — | + +Mean applied RankNet weights (not a quality claim): `ce_gain` 4.27, `title_overlap` 3.12, `rrf_inv` 1.40, `dense_inv` 1.31, `bm25_inv` 0.49, `brand_hit` −0.23, `query_in_title` −0.22. + +**Call (located evidence + decision):** I04 **win vs I01** on the predeclared point-estimate gate. nDCG@10 **0.544 > 0.542**. Recall@10 **0.391 ≥ 0.379**. Recall@50 held. Horizon 0.70 missed (~0.16 short). Lift vs CV DeBERTa is **+0.002** nDCG / **+0.004** Recall@10; n=74 has no CI, so this can be noise. Train Recall@50 **0.744** shows the 170q lists are not empty of golds, but the catalog is not the 74-index. **Do not wire into `search.py`.** Default stays hybrid RRF, `rerank=none`. Next gated step if chasing 0.70 is 400q matched lists (new `searchbench*` only), not LightGBM on these 170. + +**A/B/C call:** US n=74 live default unchanged. Harness best is **0.544**, still not Reddy **0.857**. + +### C7 — LightGBM applied from 170 matched hybrid lists (null vs RankNet 0.544; not live) + +**Focal question:** on frozen [`search-esci-74-passages-k50`](../../benchmarks/runs/search-esci-74-passages-k50), does LightGBM lambdarank fit on the 170 train hybrid lists beat RankNet apply nDCG@10 **0.544**, with Recall@10 ≥ **0.379** and Recall@50 **0.834** held? Graph off. Frozen `searchbenchesci74` / no wipe. Not live `/retrieve/search`. Do **not** mix with Reddy **0.857** or pool nDCG@20 **0.710**. n=74, no p-values. Horizon nDCG@10 ≥ **0.70** is not a fail of this arm. Predeclared stop: nDCG@10 ≥ **0.58** would halt CE retrain. + +**Protocol (located evidence).** `./search.sh ltr-head --from-run search-esci-74-passages-k50 --dataset data/search_esci_74.jsonl --train-from-run search-esci-ltr200-passages-k50 --train-dataset data/search_esci_ltr200.jsonl --ce-model data/models/esci-deberta-v3-base-4class --pair-policy other_query_neg --ltr-model lightgbm --run search-esci-74-ltr-deberta-lgbm-train200`. Same DeBERTa caches as C6. Frozen LightGBM hypers. No hypersearch. + +| Arm | nDCG@10 | Recall@10 | Recall@20 | Recall@50 | MRR | +| --- | --- | --- | --- | --- | --- | +| k=50 passages RRF | 0.500 | 0.379 | 0.590 | **0.834** | 0.765 | +| LightGBM 5-fold on the 74 (`search-esci-74-ltr-deberta-lgbm-k50`) | 0.516 | 0.379 | 0.592 | **0.834** | 0.710 | +| RankNet applied from 170 (`search-esci-74-ltr-deberta-train200`) | **0.544** | **0.391** | 0.585 | **0.834** | 0.762 | +| LightGBM applied from 170 (`search-esci-74-ltr-deberta-lgbm-train200`) | 0.533 | 0.389 | 0.591 | **0.834** | 0.752 | +| Must-beat this round | **> 0.544** | **≥ 0.379** | — | hold | — | +| Horizon (not a fail) | **≥ 0.70** | — | — | — | — | + +Mean LightGBM gain importances (not a quality claim): `ce_gain` 1684, `dense_inv` 1151, `rrf_inv` 998, `title_overlap` 273, `bm25_inv` 41, `brand_hit` 7, `query_in_title` 0. + +**Call (located evidence + decision):** I-LGBM-APPLY **null vs C6 RankNet**. nDCG@10 **0.533 < 0.544**. Recall@10 **0.389 ≥ 0.379**. Recall@50 held. Stop bar 0.58 not reached. Horizon 0.70 missed. Linear vs tree blender on these 170 lists is not the unused 0.33. **Do not wire into `search.py`.** Continue to hybrid-list CE as `ce_gain` (C8). + +**A/B/C call:** US n=74 live default unchanged. Harness best remains RankNet apply **0.544**, still not Reddy **0.857**. + +### C8 — hybrid-list DeBERTa as `ce_gain` (null vs C6 0.544; not live) + +**Focal question:** does continuing the 4-class DeBERTa on **matched hybrid k=50** lists (170 queries, unlabeled=`I`), then using the new scores as `ce_gain` in RankNet apply, beat nDCG@10 **0.544**, with Recall@10 ≥ **0.379** and Recall@50 **0.834** held? Graph off. Frozen `searchbenchesci74` / no wipe. Not live `/retrieve/search`. Do **not** mix with Reddy **0.857**, pool nDCG@20 **0.710**, or CE-as-sorter. n=74, no p-values. Horizon nDCG@10 ≥ **0.70** is not a fail of 8500 pairs. + +**Protocol (located evidence).** Export `./search.sh export-hybrid-lists --from-run search-esci-ltr200-passages-k50 --dataset data/search_esci_ltr200.jsonl --holdout-dataset data/search_esci_74.jsonl --out data/esci_hybrid_lists_ltr200.jsonl` (did not clobber `esci_retrieved_lists.jsonl`). 170 queries / 8500 pairs; 0 holdout-qid rows; labels E 984 / S 953 / C 117 / I 6446. Then `./search.sh finetune-4class --dataset data/search_esci_74.jsonl --from-lists data/esci_hybrid_lists_ltr200.jsonl --lists-source hybrid-k50 --base data/models/esci-deberta-v3-base-4class --out data/models/esci-deberta-v3-base-4class-hybrid170 --epochs 1 --max-length 192` (266 steps, MPS; pool checkpoint left in place; `source=hybrid-k50`). Then `./search.sh ltr-head --from-run search-esci-74-passages-k50 --train-from-run search-esci-ltr200-passages-k50 --train-dataset data/search_esci_ltr200.jsonl --ce-model data/models/esci-deberta-v3-base-4class-hybrid170 --pair-policy other_query_neg --run search-esci-74-ltr-deberta-hybrid170`. New caches only (`ce_gain_esci-deberta-v3-base-4class-hybrid170.json`). Lists not sorted by CE. + +**Assumption:** 8500 pairs / 170 lists may be too few to retune DeBERTa; catalog shift 4k vs 2043 remains. + +| Arm | nDCG@10 | Recall@10 | Recall@20 | Recall@50 | MRR | +| --- | --- | --- | --- | --- | --- | +| k=50 passages RRF | 0.500 | 0.379 | 0.590 | **0.834** | 0.765 | +| RankNet + pool DeBERTa from 170 (`search-esci-74-ltr-deberta-train200`) | **0.544** | **0.391** | 0.585 | **0.834** | 0.762 | +| LightGBM from 170 (`search-esci-74-ltr-deberta-lgbm-train200`) | 0.533 | 0.389 | 0.591 | **0.834** | 0.752 | +| RankNet + hybrid DeBERTa (`search-esci-74-ltr-deberta-hybrid170`) | 0.530 | 0.385 | 0.588 | **0.834** | 0.765 | +| Must-beat this round | **> 0.544** | **≥ 0.379** | — | hold | — | +| Horizon (not a fail) | **≥ 0.70** | — | — | — | — | +| Oracle on these 50 | 0.876 | 0.548 | — | 0.834 | — | + +Mean applied RankNet weights (not a quality claim): `ce_gain` 6.92, `title_overlap` 2.29, `rrf_inv` 1.14, `dense_inv` 0.84, `bm25_inv` 0.36, `brand_hit` −0.17, `query_in_title` −0.27. + +**Call (located evidence + decision):** I-CE-HYB **null vs C6**. nDCG@10 **0.530 < 0.544**. Recall@10 **0.385 ≥ 0.379**. Recall@50 held. Horizon 0.70 missed. One epoch of hybrid hard-negatives on 170 lists did not unlock the unused 0.33; a null here can mean too little data, not that first-stage negatives never matter. Do **not** relabel a 6k BM25-list run as this construct. **Do not wire into `search.py`.** Default stays hybrid RRF, `rerank=none`. + +**A/B/C call:** US n=74 live default unchanged. Harness best remains **0.544**, still not Reddy **0.857**. + diff --git a/docs/research/20-esci-sota-targets.md b/docs/research/20-esci-sota-targets.md new file mode 100644 index 0000000..fb19c79 --- /dev/null +++ b/docs/research/20-esci-sota-targets.md @@ -0,0 +1,350 @@ +# 20 — ESCI search-quality SOTA numbers to aim for + +Workstream: evaluation protocol for `/retrieve/search` on `searchbench*` only. Ledger: `benchmarks.search` only. This document is **located evidence plus protocol targets**, not a ranking-code change and not a published BrainAPI finding. + +**Focal question:** Which published numbers are valid aim points for BrainAPI search quality, given the current 11-query / 200-doc ESCI slice? + +Every numeric claim below is labeled **located evidence**, **slice-internal**, or **not-checked**. Do **not** claim the slice is above or below full-ESCI SOTA. + +Search date: **2026-08-18**. Tools: harness read; arXiv MCP `search_papers` / `get_abstract` / `download_paper`; AIcrowd/Amazon Science web pages for leaderboard confirmation. + +--- + +## What this workstream does + +The eval scores a ranked list from `POST /retrieve/search` against catalog qrels, not LoCoMo judge accuracy and not `/retrieve/recommend`. + +### Scoring path (located evidence) + +1. Catalog JSONL carries `gold_grades` from ESCI letters via `ESCI_GAINS = {"E": 1.0, "S": 0.1, "C": 0.01, "I": 0.0}` (`benchmarks/search/catalog.py:31`, applied at `catalog.py:298-302`). Complement **is** in the gain table. Irrelevant is dropped (`gain > 0` filter). +2. After ingest, hits are collapsed to `doc_id` (chunk map or node uuid) and deduped (`evaluate.py:172-184`, `267-285`). +3. nDCG@10 is **graded** when `gold_grades` is present (`metrics.py:36-56`; wired at `evaluate.py:285`). Discount is `rel / log2(rank + 1)` (`metrics.py:29-33`) — the classic Järvelin DCG, not the `2^rel − 1` variant. +4. Recall@{5,10,20} is **binary** over the positive-gain set (Exact + Substitute + Complement), not Exact-only (`metrics.py:7-16`; gold union at `evaluate.py:224-245`). +5. MRR is the reciprocal rank of the **first positive-gain** hit (`metrics.py:19-26`). It is not MRR@10 as a cutoff in code; the list is whatever `k` the search returned (default evaluate `k=20`, `evaluate.py:304`). +6. Headline ledger fields are `ndcg@10`, `recall@10`, `mrr`, retrieve p50 (`report.py:65-68`). + +### Slice under test (located evidence) + +`benchmarks/data/search_esci_slice.jsonl`: 200 docs, 11 queries, US locale, all product columns, brain `searchbenchesci20`. Gold sizes: min 1 / median 20 / max 20. Gain histogram: E=127, S=37, C=2. Passages-only: nDCG@10 **0.758**, Recall@10 **0.511**, MRR **0.848**, p50 retrieve 28–29 ms (`docs/research/19-search-esci-quality.md`). + +Later condition, not a substitute: `benchmarks/data/search_esci.jsonl` is 2000 docs / 74 queries (title-heavy; 43/2000 docs lack Brand/Color in text; 0 `class` fields). Gold min/median/max 1/15/38, mean 17.2. + +Catalog builder defaults (`catalog.py:34-36, 64-71, 396-397`): locale `us`, split `test`, `small_version == 1` (Task 1 small), caps 80 queries / 2000 docs / 40 candidates per query. The JSONL is then searched as a **shared corpus**, not as a per-query candidate pool. + +WANDS download exists in the same harness (`catalog.py:13-32`, gains Exact=1 / Partial=0.5 / Irrelevant=0). No BEIR or TREC Product Search harness in `benchmarks/search/`. + +--- + +## Guarantees and where they break + +**Stated guarantee:** graded nDCG@10 / Recall@10 / MRR on this slice measure product-search quality in a way that can be compared to published ESCI numbers. + +Where that guarantee cannot be delivered: + +1. **Pool size and task (critical).** Published Task 1 is *ranking-in-pool*: reorder ≤40 already-retrieved, fully labeled products per query (`2206.06588` §3.1). We retrieve from a 200-doc (or 2000-doc) shared index. Different candidate generation, different label density, different IDCG. **Gap**, not a bug in the scorer. +2. **Cutoff k (important).** Official Task 1 is nDCG on the provided list. Reddy does not state `@10`. Jeronymo et al. report the KDD Cup metric as **nDCG@20** (`2208.06264`). We always cut at 10 (`metrics.py:121`). Same gain table, different statistic. +3. **Recall@10 / MRR are not the Task 1 metrics (important).** Task 1 publishes nDCG. Tasks 2/3 publish micro-F1. Nobody in the primary ESCI papers reports Recall@10 or MRR as we compute them. +4. **n=11 (critical for inference).** Means have no interval. One total miss (`esci-72`) and one single-gold query (`esci-67`) dominate averages. +5. **Recall@10 ceiling (important).** Mean of `min(10, |gold|) / |gold|` on this slice is **0.667**. Six of eleven queries have `|gold|=20`, so Recall@10 cannot exceed 0.5 on those queries even if every top-10 hit is gold. Current 0.511 is already near that structural cap. +6. **MRR saturation (important).** Mean 0.848 leaves little room except the total miss. Rank-2–10 work will not show up. + +Deliberate trade-off: slice size and `skip_enrichment` for ingest cost. Not a claim about full ESCI. + +--- + +## Open questions for the maintainer + +1. For a confirmatory ESCI run, is the target **ranking-in-pool nDCG** on the labeled candidate list (comparable to `2206.06588` / KDD Cup), or **first-stage nDCG@10** from a shared ingested corpus? +2. Should the headline cutoff stay nDCG@10, or switch to nDCG@20 / full-list nDCG when the goal is published-number comparability? +3. Should Recall@10 gold stay E+S+C, or Exact-only, given Task 1’s ranking order is E then S then C then I? +4. Is `search_esci.jsonl` (2000 docs, 74 queries) the first larger condition, or should the next run restrict scoring to each query’s original ≤40 labeled candidates? +5. Is p50 retrieve <200 ms still binding if a confirmatory arm uses a cross-encoder the way the published 0.85–0.90 numbers do? + +--- + +## Frontier techniques + +### ESCI / Shopping Queries Task 1 ranking-in-pool (canonical) + +- **Mechanism:** For each query, reorder a provided list of up to 40 products. Graded nDCG with gains E=1, S=0.1, C=0.01, I=0. Complement is in the gain table. Locales US/ES/JP. Small version (Task 1) filters easier queries; US public test is 4,477 queries, avg depth 20.3 (`2206.06588` Tables 1–4, §3.1). +- **arXiv:** `2206.06588` (Reddy et al., 2022). Amazon Science: [dataset page](https://www.amazon.science/code-and-datasets/shopping-queries-dataset-a-large-scale-esci-benchmark-for-improving-product-search), [KDD Cup recap](https://www.amazon.science/blog/amazon-product-query-competition-draws-more-than-9-200-submissions). +- **Reported numbers (located evidence):** + +| Method | Year | Metric | Split | Number | Source | +| --- | --- | --- | --- | --- | --- | +| Terrier BM25 (title, all locales together) | 2022 | nDCG (Task 1, no `@10` in paper) | Public test overall / EN / ES / JP | 0.563 / **0.675** / 0.697 / 0.136 | `2206.06588` Table 4 | +| Fine-tuned MS MARCO MiniLM cross-encoder (EN) + MPNet (ES, JP); Exact→1 else 0 at train time | 2022 | nDCG | Public test overall / EN / ES / JP | **0.852** / **0.857** / 0.849 / 0.840 | `2206.06588` Table 4 | +| Organizer “baseline” cited in the KDD recap | 2022 | nDCG | KDD private/public (unspecified cutoff in blog) | **0.8503** | [Amazon Science blog](https://www.amazon.science/blog/amazon-product-query-competition-draws-more-than-9-200-submissions) | +| Team www (DeBERTa/XLM/RemBERT ensemble; 4-class probs → weighted gain) | 2022 | NDCG (AIcrowd Task 1) | Public / private, multilingual Task 1 | **0.9057 / 0.9043** | `2208.02958`; [AIcrowd winners](https://discourse.aicrowd.com/t/final-winners-announcement/7974) | +| Team qinpersevere / day-day-up | 2022 | NDCG | Private Task 1 | 0.9036 / 0.9035 | AIcrowd winners table | +| mMonoT5-3.7B (NeuralMind) | 2022 | nDCG@20 (their label for the same leaderboard) | Public / private | 0.9012 / 0.9007 | `2208.06264` Table 1 | +| 20th place | 2022 | nDCG@20 | Private | 0.8929 | `2208.06264` Table 1 | +| SQID reproduction of ESCI_baseline on US Task 1 test (8,956 queries, ~20 judgments/query) | 2024 | NDCG (Terrier, corrected S/C mapping) | US small test | **0.8562** | `2405.15190` Table 2 | +| SQID random rank of the same pools | 2024 | NDCG | US small test | 0.7483 | `2405.15190` Table 2 | +| SQID SBERT title cosine (no fine-tune) | 2024 | NDCG | US small test | 0.8292 | `2405.15190` Table 2 | + +- **Cost:** Cross-encoder or LLM ensembles over ~20–40 pairs per query. Not a first-stage ANN over a catalog. +- **Fit:** Gain table matches ours. Task, n, pool, and k do not. +- **Verdict:** **adopt** as the label protocol and as the **horizon-B ranking-in-pool target**. **reject** as a numeric bar for the 11-query / 200-doc slice. + +### KDD Cup 2022 as “SOTA on ESCI” + +- **Mechanism:** Same Task 1, private multilingual test. Top 10 private scores sit in **0.8998–0.9043** (AIcrowd). Jeronymo argues top-20 teams cluster near 0.90 nDCG@20 and that the dataset is too easy to discriminate retrievers (`2208.06264` abstract). +- **Cost:** Heavy fine-tune + ensemble. Wu et al. (`2208.00108`) rank 6th on Task 1 but do not publish a Task 1 nDCG in the paper body (leaderboard 6th is ETS-Lab 0.9014 private; Wu’s team name is not in the top-10 table — **not-checked** which public name maps to their 6th-place claim vs AIcrowd’s ETS-Lab). +- **Verdict:** **adopt** 0.9043 private NDCG as the published ceiling for *ranking-in-pool, multilingual Task 1*. Fashionable only if BrainAPI starts scoring labeled pools with a second-stage ranker. + +### Full-catalog ESCI retrieval (BLaIR) + +- **Mechanism:** Treat ESCI as product search over **1,367,729 items / 27,643 test queries**; report **NDCG@100** (`2403.03952` Table 3, Table 10). This is first-stage retrieval, not Task 1 reranking. +- **Reported (located evidence):** SFR-Embedding-Mistral **0.2560**; GritLM-7B 0.2537; e5-mistral-7b-instruct 0.2437; Qwen3-Embedding-8B 0.2328; text-embedding-3-large 0.2366; gemini-embedding-001 0.2233 (`2403.03952` Table 10). +- **Cost:** Embed the catalog; ANN. No cross-encoder over 40 candidates. +- **Fit:** Closer to “search a brain” than Task 1, but k=100, corpus ~1.4M, and nDCG@100 ≠ our nDCG@10. Recall@10 / MRR **not reported**. +- **Verdict:** **adapt** as the published bar if a later run is true first-stage over a large catalog. **reject** as a target for the 200-doc slice (our 0.758 lives on a different y-axis). + +### Papers that use ESCI but are not SOTA bars here + +- **ARR / SToICaL (`2601.05588`, 2026):** Custom in-context ranking, Gecko-mined lists, log-scale gains, n=310, nDCG ~95–97 and a nonstandard R@k (`|π[:k] ∩ π̂[:k]| / k`). **reject** — not ESCI Task 1 gains, not our recall. +- **GraphRAG / EDRM / LoCoMo:** Wrong task. Out of scope. +- **WANDS (ECIR 2022 Chen et al.):** Dataset paper identity **support-located** (2026-08-20): DOI [10.1007/978-3-030-99736-6_9](https://doi.org/10.1007/978-3-030-99736-6_9), OpenAlex `W4225565720`, Springer LNCS, `is_oa=false`, abstract index empty. arXiv title search did not return the chapter. Dataset size **located** on GitHub `wayfair/WANDS`: 480 queries / 42,994 products / 233,448 labels. Published BM25/hybrid **nDCG still not located**. Do not invent a paper nDCG. Medium notebooks are not primary. Local first-stage control is a different y-axis (≤80q / ≤2000 docs / linear Exact=1, Partial=0.5). See [23-search-wands-quality.md](23-search-wands-quality.md). +- **TREC 2023 Product Search (`2311.07861`):** Converts SQD toward end-to-end retrieval. **No harness in this repo.** Leftover. +- **BEIR (`2104.08663`):** No search harness here. Leftover. + +--- + +## Implementation plan + +No ranking-code change. Protocol only. Reuse `searchbenchesci20` with `--skip-ingest`. Never wipe brains. + +### Phase 1 — Slice-internal aims (this week) + +Treat passages-only **0.758 / 0.511 / 0.848** as the control. “Beating the slice” means: + +| Metric | Now | Structural ceiling (slice-internal) | What a real win looks like | What is not a win | +| --- | --- | --- | --- | --- | +| nDCG@10 | 0.758 | Not 1.0: several queries have mixed E/S gold; `esci-72` is 0/11 of the mean | +0.02–0.05 with Recall@20 not down; or `esci-72` leaving 0 (alone +~0.09 if that query went to 1.0) | Graph fusion that drops nDCG while MRR stays flat | +| Recall@10 | 0.511 | **0.667** mean of `min(10,\|gold\|)/\|gold\|`; **0.5** on the six `|gold|=20` queries | Move `esci-72` (0 vs ceiling 1.0), `esci-19` (0.333 vs 0.667), `esci-37` (0.667 vs 1.0) | Raising retrieve `k` then cutting @10 | +| MRR | 0.848 | ~0.94 if only `esci-72` is 0 and is fixed to rank-1 | Almost only the total-miss query | Any rank-2–10 cleanup | + +**Task 1 — Publish slice ceilings next to every report** + +- **Acceptance:** Report or canvas shows Recall@10 ceiling 0.667 and per-query `|gold|`. +- **Verification:** `python` over `search_esci_slice.jsonl` gold sizes (already computed 2026-08-18). +- **Scope:** S. Files: report helper or canvas only. + +**Checkpoint:** Humans agree the 0.758/0.511/0.848 triple is a **control**, not a SOTA claim. + +### Phase 2 — Comparable confirmatory protocol + +Pick one of two designs. Do not mix their numbers. + +**Design B-rank (comparable to published 0.85–0.90):** For each query, score **only the original labeled candidates** (≤40), same E/S/C/I gains, report nDCG@20 **and** full-list nDCG, US `small_version` test, n in the thousands if ingest allows, else the 74-query 2000-doc file **restricted to each query’s gold pool**. Cite Reddy EN CE **0.857** as the organizer-style bar and www **0.9043** as the 2022 ceiling. + +**Design B-retrieve (comparable to BLaIR, not to 0.90):** Search a shared corpus. Report nDCG@10 plus nDCG@100 if k allows. Cite SFR-Embedding-Mistral **0.256 nDCG@100** on 1.37M items only if the corpus is actually large. The 2000-doc file is still not that condition. + +**Task 2 — Ranking-in-pool eval mode (if B-rank is chosen)** + +- **Acceptance:** Harness can restrict hits to `gold_grades` keys plus labeled irrelevants for that `qid`; metrics include nDCG@20. +- **Verification:** One `--skip-ingest` run on a toy query with a 40-id pool; nDCG matches a manual Terrier-style recompute. +- **Scope:** M. Files: `benchmarks/search/evaluate.py`, `metrics.py`, `report.py`. +- **Depends:** Maintainer answer to open question 1. + +**Task 3 — 2000-doc condition as a separate row** + +- **Acceptance:** `search_esci.jsonl` run is labeled `dataset=search_esci.jsonl`, `n_queries=74`, never averaged with the 11-query slice. +- **Verification:** `./search.sh evaluate --dataset data/search_esci.jsonl --brain searchbenchesci --skip-ingest` (or a new `searchbench*` brain — no wipe of `searchbenchesci20`). +- **Scope:** S. Eval only. + +**Checkpoint:** A table with three rows that are never compared as one number: slice n=11, 2000-doc retrieve, ranking-in-pool (if implemented). + +### Measured n=11 ranking-in-pool smoke (2026-08-18) + +Not a published-number claim. Rows must not be averaged. + +| Row | nDCG@10 | nDCG@20 | full nDCG | n | Cite 0.857? | +| --- | --- | --- | --- | --- | --- | +| Slice first-stage passages | 0.758 | — | — | 11 | no | +| BrainAPI post-filter in-pool | 0.788 | 0.811 | 0.814 | 11 | no | +| MiniLM CE-on-pool (zero-shot MS MARCO) | 0.598 | 0.669 | 0.786 | 11 | **only this arm’s protocol**, still not the Reddy split | + +Harness: `--rank-pool` on evaluate; `rank-pool-ce` subcommand. Catalog persists `candidate_grades` including I=0. `searchbenchesci20` was not re-ingested. + +### 74-query brain `searchbenchesci74` (2026-08-18) + +New brain; dataset `search_esci_74.jsonl` (74 queries, 2043 pool products, mean 28.8 labeled candidates). Passages ingest onto this brain is in progress and is **not** required for CE-on-pool. + +Cite Reddy **0.857** only against CE-on-pool. n=74 is still not the 4,477-query public test. + +| Arm | nDCG@20 | full nDCG | n | vs 0.857 | +| --- | --- | --- | --- | --- | +| Zero-shot MS MARCO MiniLM-L-6 | 0.631 | 0.717 | 74 | below | +| Fine-tuned 1 epoch, 30k US train pairs, Exact→1 else 0, test qids held out | 0.669 | 0.745 | 74 | below | + +**Located evidence:** one-epoch Exact-vs-rest fine-tune lifted nDCG@20 +0.038 vs zero-shot and did not beat Reddy 0.857 / KDD 0.9043 (those use full Task 1 and, for the winner, ensembles). + +### 74-query CE-on-pool and first-stage (2026-08-18, this workstream) + +Do not average with n=11 or with Reddy n≈4477. Cite **0.857 only against CE-on-pool**. + +| Arm | n | nDCG@10 | nDCG@20 | full nDCG | notes | +| --- | --- | --- | --- | --- | --- | +| Live passages skip-ingest (control) | 74 | **0.495** | 0.549 | 0.549 | R@10 0.377, R@20 0.592, MRR 0.758, p50 62 ms; `search-esci-74-passages-control` | +| Local BM25 all-text | 74 | 0.424 | 0.472 | — | harness JSONL; not BrainAPI | +| Local BM25 title-boost | 74 | 0.458 | 0.508 | — | best BM25 variant; **below** 0.495 | +| Local SPLADE-cocondenser | 74 | 0.465 | 0.520 | — | `naver/splade-cocondenser-ensembledistil`; **below** 0.495 | +| CE-on-pool FT Exact→1 MiniLM-L-6 30k | 74 | — | 0.669 | 0.745 | prior row | +| CE-on-pool 4-class MiniLM-L-12, 40k US train, catalog fields, score 1·P(E)+0.1·P(S)+0.01·P(C) | 74 | 0.597 | **0.678** | **0.747** | `esci-minilm-l12-4class`; **> 0.669 progress**; **not** 0.857 | + +**Calls:** Reddy EN CE **0.857 was not beaten**. First-stage nDCG@10 did **not** beat 0.495 by ≥0.02 (live control matched 0.495; local BM25/SPLADE were lower). n=74 is still not the public test. + +### 74-query 4-class weighted CE (2026-08-18, isolated workstream) + +Ranking-in-pool only (I=0 stays in the list). Dataset `search_esci_74.jsonl`. Brain `searchbenchesci74` was **not** wiped or re-ingested. Did **not** repeat Exact→1 MiniLM-L-6. Cite **0.857 only against this protocol**; n=74 is not Reddy’s ~4,477-query EN public test. + +**Chosen path:** team www (`2208.02958`) 4-class softmax → score `1.0 P(E)+0.1 P(S)+0.01 P(C)+0.0 P(I)`, backbone `cross-encoder/ms-marco-MiniLM-L-12-v2` (Reddy footnote 6, different **loss** from sibling Exact→1). New files: `benchmarks/search/finetune_esci_4class.py`, `benchmarks/search/rank_pool_4class.py`. Ledger runs under `benchmarks.search` only. + +**Rejected (same lookup):** Reddy Exact→1 MiniLM (sibling already running that replica). Jeronymo mMonoT5-3.7B (`2208.06264`) 72 h TPU. www ensemble / DeBERTa-v3-large / RemBERT (0.90 is ensemble). SQID unfine-tuned title SBERT 0.8292 (`2405.15190` Table 2) already below 0.857. Wu (`2208.00108`) DeBERTaV3 + LightGBM group features. Pool-only ColBERT: extra infra, not a published 0.85–0.90 recipe we could finish in hours. `sentencepiece` is now in the parent venv (0.2.2); a **v3-base** 4-class slice was run 2026-08-19 (pool nDCG@20 **0.710**, still below 0.857; first-stage catalog R@10 **null**). v3-large / ensemble remain out of local MPS this pass. + +**Critique (why 0.857/0.90 may be unreachable with this small CE on n=74):** construct-valid ranking-in-pool + ESCI gains. External validity fails: n=74 vs ~4477 (no interval). Training used 80k/419,653 US small-train pairs, not full KDD train+aug+ensemble. Inverse-freq class weights (C weight 5.53) **hurt** nDCG: on Exact golds mean P(I)>P(E). Unweighted 4-class (www-like) restored E>S>C>I in expected gain. Second epoch +0.006 only. 0.90 remains ensemble/3.7B. + +| Arm | nDCG@20 | full nDCG | n | vs 0.669 / 0.857 | +| --- | --- | --- | --- | --- | +| Inverse-freq 4-class, 80k, 1 ep (`search-esci-74-ce-pool-4class`, `esci-minilm-l12-4class`) | 0.670 | 0.742 | 74 | ~tied / below | +| Unweighted 4-class, 80k, 1 ep (`search-esci-74-ce-pool-4class-nowt`) | 0.688 | 0.759 | 74 | **+0.019** / below | +| Unweighted 4-class, 80k, 2 ep (`search-esci-74-ce-pool-4class-e2`) | 0.695 | 0.765 | 74 | **+0.026** / below | +| Unweighted 4-class DeBERTa-v3-base, 80k, 2 ep (`search-esci-74-ce-pool-deberta-base`) | **0.710** | **0.777** | 74 | **+0.041** / below | + +**Call:** Reddy EN CE **0.857 was not beaten**. Progress vs our Exact→1 FT 0.669: yes (best MiniLM 0.695; DeBERTa-v3-base **0.710** on the same 74 pools). On-disk `esci-minilm-l12-4class` is this workstream’s 80k inverse-freq checkpoint, not the 40k row above. Do not average pool **0.710** with first-stage nDCG@10. + +### Deeper k then harness CE (2026-08-18, not 0.857) + +Separate y-axis from the CE-on-pool rows above. First-stage retrieve from the 2043-doc shared corpus, then (Phase 2) reorder **retrieved** `hit_ids` only. Graph off. Default API `k` / `RERANK_MAX_K=10` unchanged. Cite **0.857 only against CE-on-pool**; do not average with 0.695. + +**Phase 1 (located evidence).** Predeclared win: pool_coverage ≥ 0.57 **or** Recall@50 ≥ 0.65. + +| Arm | k | Recall@50 | pool_coverage | nDCG@10 | p50 ms | +| --- | --- | --- | --- | --- | --- | +| Passages control | 20 | — | 0.517 (`--rank-pool`) | **0.495** | 62 | +| `search-esci-74-passages-k50` | 50 | **0.834** | — | 0.500 | 60 | +| `search-esci-74-passages-k50-pool` | 50 | 0.832 | **0.778** | 0.567 | 60 | +| `search-esci-74-passages-k100` | 100 | 0.831 | — | 0.500 | 58 | +| `search-esci-74-passages-k100-pool` | 100 | 0.906 | **0.846** | 0.574 | 59 | + +**Phase 1 call:** win (Recall@50 0.834; coverage 0.778). nDCG@10 stayed flat, as predicted: extra golds are below rank 10. `missing_from_brain` stayed 0. k=100 does not raise Recall@50 vs k=50. + +**Phase 2 (located evidence).** Harness `rerank-retrieved` on `search-esci-74-passages-k50`. Win needed nDCG@10 ≥ 0.515 with Recall@50 not down. + +| Arm | nDCG@10 | Recall@50 | +| --- | --- | --- | +| k=50 first-stage | 0.500 | 0.834 | +| MiniLM-L-6 on retrieved hits | 0.448 | 0.834 | +| 4-class L-12 e2 on retrieved hits | 0.467 | 0.834 | + +**Phase 2 call:** null. Both CEs moved nDCG@10 **down**. Same hit set, so Recall@50 is unchanged. Do not treat this as a Reddy comparison. + +### Dual-k matching (2026-08-18, not 0.857) + +Goal: more gold in **both** Recall@10 and Recall@50. Predeclared win: ≥0.397 and ≥0.854. Sidecar / skip-ingest only. `searchbenchesci74` not wiped. + +**Task 1 located evidence.** k=50 golds: 424 in top-10, 728 in ranks 11–50, 118 missed (5 total-miss queries). Counts from `miss_strata.json` on `search-esci-74-passages-k50`. + +**Task 2 located evidence.** Rewrote `esci-113` and `esci-267` only; skipped `esci-72`. Live k=50 qrewrite Recall@10 **0.379** / Recall@50 **0.834** (unchanged). Those two qids stayed at 0. + +**Task 3 located evidence.** Local ANCE MiniLM (`search-esci-74-dense-ance-k50`): Recall@10 **0.302**, Recall@50 **0.750**, nDCG@10 **0.396**. Dual win **failed**. Not BrainAPI embeddings; not 0.857. + +**Call:** no first-stage matching arm beat passages on both cutoffs. Stop encoder fishing on this isolation set. Do not fuse the sidecar. Do not mix with CE-on-pool 0.695. + +### Retrieved-neg CE, ColBERT, BGE (2026-08-18, not 0.857) + +Separate y-axis from CE-on-pool **0.695** / Reddy **0.857**. Frozen `searchbenchesci74`. Graph off. `RERANK_MAX_K=10` unchanged. n=74. + +| Arm | protocol | nDCG@10 | Recall@10 | Recall@50 | vs gate | +| --- | --- | --- | --- | --- | --- | +| k=50 passages | hybrid RRF | **0.500** | 0.379 | **0.834** | control | +| retrieved-neg 4-class L-12 | CE on stored k=50 hits | 0.465 | 0.351 | 0.834 | nDCG@10 ≥0.515 **failed**; R@50 held | +| ColBERT MaxSim sidecar | local JSONL, not fused | 0.434 | 0.311 | 0.714 | dual-k **failed** | +| BGE-base (not MiniLM) | local JSONL, no `query:` | 0.441 | 0.322 | 0.799 | dual-k **failed** | + +ColBERT p50 encode+retrieve **10063 ms** (200 ms labeled, not a stop). BGE p50 **408 ms**. MiniLM ANCE was **0.302 / 0.750**, not this BGE row. + +**Call:** all three null vs their predeclared gates. Passages hybrid remains the first-stage control. Do not average with 0.695. + +### Exhaustive catalog then two-stage (2026-08-19, not 0.857) + +Separate y-axis from CE-on-pool **0.695** / Reddy **0.857**. Frozen `searchbenchesci74`. Graph off. Default `RERANK_MAX_K=10` unchanged. n=74. Cite **0.857 only against CE-on-pool**. + +| Arm | protocol | nDCG@10 | Recall@10 | Recall@50 | vs gate | +| --- | --- | --- | --- | --- | --- | +| k=50 passages | hybrid RRF | **0.500** | 0.379 | **0.834** | control | +| Exhaustive 4-class L-12 e2 | exhaustive-catalog, 2043 docs | 0.416 | 0.266 | 0.603 | nDCG@10 ≥0.515 **and** R@10 ≥0.397 **failed** | +| Union passages+BGE+ColBERT | harness-union RRF | 0.479 | 0.363 | 0.827 | dual-k **failed**; R@10 diluted below 0.379 | +| `mode=catalog` + plugin CE | live two-stage, rerank 50 | 0.467 | 0.341 | **0.834** | nDCG@10 ≥0.515 **failed**; R@50 held | +| Frozen-head cascade | harness; hybrid top-10 frozen | **0.500** | **0.379** | **0.889** | R@10 held **and** R@50 ≥0.854 **win** | + +U01: BGE top-50 held **40** gold ASINs absent from passages k=50 (19 qids); ColBERT **21** (13 qids). Unique golds were nonzero, so the union included ColBERT. Cascade injected **51** unique golds (set union) into ranks 11–50. Exhaustive p50 **94 s** is labeled, not a 200 ms claim. Catalog p50 retrieve **59 ms**; p50 client wall **877 ms** (not the ADR-007 default). + +**Call:** A/B quality null; C ships the opt-in hook; frozen-head cascade is the first dual-k **win** on this isolation set. Do not average with 0.695. Do not treat C’s 0.467 as a Reddy comparison. Cascade is harness-only until a live fill-tail path is an explicit product decision. + +### DeBERTa-v3-base 4-class catalog slice (2026-08-19, not 0.857) + +C06 was previously “out of compute this pass.” This pass ran a **single** `microsoft/deberta-v3-base` 4-class CE (not v3-large, not www ensemble). Same 80k/2ep/unweighted recipe as MiniLM nowt-e2. Frozen `searchbenchesci74` not wiped. Default `RERANK_MAX_K=10` unchanged. n=74, no p-values. + +**Ranking-in-pool (separate y-axis).** `search-esci-74-ce-pool-deberta-base` nDCG@20 **0.710** vs MiniLM e2 **0.695**. Still below Reddy **0.857**. n=74 ≠ ~4477. + +**First-stage (R@10 gate).** Harness `search-esci-74-passages-k50-ce-deberta` on stored k=50 hits: + +| Arm | protocol | nDCG@10 | Recall@10 | Recall@50 | vs gate | +| --- | --- | --- | --- | --- | --- | +| k=50 passages | hybrid RRF | **0.500** | **0.379** | **0.834** | control | +| MiniLM 4-class e2 on k=50 hits | CE on stored hits | 0.467 | 0.341 | **0.834** | nDCG@10 ≥0.515 **failed** | +| DeBERTa-v3-base 4-class on k=50 hits | CE on stored hits | 0.510 | 0.363 | **0.834** | R@10 ≥0.397 **and** nDCG@10 ≥0.515 **failed** | + +**Call:** first-stage **null**. Recall@10 **0.363** missed **0.397** (below passages). nDCG@10 **0.510** missed **0.515**. Recall@50 held. Live `mode=catalog` skipped (gates miss; MiniLM catalog already matched harness). Cascade remains the Recall@50 result. Do not average with pool **0.710** / MiniLM **0.695** / Reddy **0.857**. v3-large / ensemble still out. + +### Phase 3 — Optional adjacent harness + +WANDS first-stage control is a **separate y-axis** from ESCI n=74. Chen et al. identity located (DOI `10.1007/978-3-030-99736-6_9`); published nDCG **not located**. Do not set a Chen-paper nDCG aim. Do not mix with ESCI 0.500 / C6 0.544 / Reddy 0.857. BEIR / TREC Product Search: do not add. + +--- + +## Risks + +| Risk | Detection | +| --- | --- | +| Quoting 0.9043 as the target for the 11-query slice | Dataset + n_queries + “ranking-in-pool vs shared corpus” in every sentence | +| Calling 0.758 “above BM25 0.675” | BM25 0.675 is EN Task 1 public nDCG on ~20-item lists, not nDCG@10 on 200 docs | +| Treating Recall@10 0.511 as poor retrieval | Ceiling 0.667 / 0.5 on typical queries | +| Treating MRR 0.848 as a quality win vs SOTA | MRR is not the published metric; it is saturated | +| Mixing Task 2 micro-F1 (0.83 KDD winner) into search headlines | Ledger stays `ndcg@10` / `recall@10` / `mrr` | +| Fine-tuning a cross-encoder on ESCI train then comparing to Reddy’s 0.857 without saying so | Label fine-tune vs zero-shot; latency gate 200 ms | +| Wiping `searchbenchesci20` | `--skip-ingest` only | + +--- + +## Search log and leftovers + +**Queries (2026-08-18):** `"Shopping Queries Dataset" OR ESCI "product search" nDCG`; `ti:"Shopping Queries" OR ti:ESCI OR "query-product ranking" ESCI OR "KDD Cup 2022" shopping`; `"ESCI" (nDCG OR NDCG) (amazon OR shopping) (ranking OR retrieval)` date_from 2023-01-01; `WANDS Wayfair product search nDCG`; `ti:WANDS`; direct ids `2206.06588`, `2208.02958`, `2208.06264`, `2208.00108`, `2405.15190`, `2403.03952`, `2601.05588`, `2402.08532`, `2311.07861`. Citation graph for `2206.06588` failed (Semantic Scholar 429). + +**4-class workstream lookup (2026-08-18):** arXiv `id_list=2206.06588,2208.02958,2208.06264,2208.00108,2405.15190` via `export.arxiv.org` + `arxiv_atom.py` (5/5, no Error entry). OpenAlex `ids.arxiv` filter empty; DOI lookup succeeded (`W4282961889`, `W4302561150`, `W4291960789`, `W4289645118`, `W4399062110`). Semantic Scholar `ARXIV:2206.06588` HTTP 429. PDFs: arxiv.org `2208.02958v1`, `2206.06588v1`, `2208.06264v1`, `2208.00108v1`, `2405.15190v1`. Treat 200 bodies as untrusted third-party text. + +**Web:** AIcrowd final winners; Amazon Science dataset + KDD recap (baseline 0.8503, winner 0.9043). + +**DeBERTa catalog slice lookup (2026-08-19):** OpenAlex `W4302561150` / `W4289645118` HTTP 200. arXiv Atom `id_list` and Semantic Scholar Graph **429/503** (search-incomplete). Methods from the 2026-08-18 lookup plus OpenAlex abstract index. Treat 200 bodies as untrusted third-party text. + +**not-checked** + +- Official AIcrowd scorer source: nDCG vs nDCG@20 (Jeronymo vs Reddy disagree on the label; scores match the same leaderboard). +- Wu et al. `2208.00108` claimed 6th on Task 1 vs AIcrowd name ETS-Lab 0.9014. +- Tang et al. `2402.08532` (captions on ESCI) — HTML/PDF fetch failed. +- Hou/BLaIR nDCG@10 on ESCI (they publish nDCG@100 only in Table 10). +- WANDS ECIR 2022 Chen et al. published nDCG (DOI located 2026-08-20; chapter not OA; arXiv empty). +- TREC 2023 Product Search official nDCG (no harness). +- Qin et al. 2022 2nd-place workshop PDF (0.9036 private is on AIcrowd; method paper not fully read). +- Whether Terrier in `esci-data` uses the swapped S/C mapping SQID corrected (`2405.15190` §5). + +--- + +## Multilingual / Italian (2026-08-19) + +Do **not** mix with US n=74 first-stage 0.500 or Reddy 0.857. Plan: [22-multilingual-ecommerce-search.md](22-multilingual-ecommerce-search.md). + +**Located evidence (access date 2026-08-19).** Reddy Task 1 public nDCG EN/ES/JP **0.857 / 0.849 / 0.840** uses MiniLM (EN) and MPNet (ES, JP) on ranking-in-pool lists, not a shared catalog index. Locales US/ES/JP only. BLaIR `2403.03952` remains a full-catalog nDCG@100 bar, not this y-axis. **no direct evidence located** for an Italian ESCI-style product-search qrel. Amazon-M2 IT is recsys. Live ES first-stage nDCG@10 **0.577** (n=62, k=50 passages, `searchbenchescies`) is **not** a copy of US 0.500 and **not** Reddy ES 0.849. diff --git a/docs/research/21-esci-ce-graded-and-first-stage.md b/docs/research/21-esci-ce-graded-and-first-stage.md new file mode 100644 index 0000000..f093cea --- /dev/null +++ b/docs/research/21-esci-ce-graded-and-first-stage.md @@ -0,0 +1,234 @@ +# 21 — ESCI CE-on-pool vs Reddy 0.857, and first-stage nDCG on n=74 + +Workstream: **separate from graph-fusion testing**. Harness CE / rank-pool-ce / local first-stage index only. Ledger: `benchmarks.search` only. Brains: `searchbench*` only; no wipe of `searchbenchesci20` / `searchbenchesci74`. + +**Focal questions** + +1. Can a MiniLM-scale CE, trained and scored on labeled pools, beat **our FT 0.669 nDCG@20** and, on a comparable protocol only, Reddy EN CE **0.857**? +2. Can a first-stage intervention raise nDCG@10 on the shared 2043-doc `search_esci_74.jsonl` corpus by **≥0.02** vs **0.495**, without mixing y-axes with 0.857? + +n=74 is **not** Reddy’s ~4477-query US public test. Do not claim full-ESCI SOTA. + +Search date: **2026-08-18**. Origin of ideas: AI-assisted. Stage labels follow the scientific-brainstorming skill. + +--- + +## Scope + +- Purpose: pick 1–2 harness interventions; measure them. +- In: `finetune_esci_ce.py`, `rank_pool.py`, `plugins/search-rerank/`, new harness scripts, this note plus append-only claims in `19` / `20`. +- Out: `search.py`, `hybrid.py`, `fact_filter.py`, `graph_channels.py`, `replay_fusion.py`, `evaluate.py` instrumentation, fusion unit tests, production `RERANK_MAX_K`, default `channels=["passages"]`, default rerank. +- Constraints: `--brain` on parent `./search.sh`; `BRAINPAT_TOKEN` never printed; this checkout not `~/.brainapi/source`. + +--- + +## Independent ideas (before literature) + +Stage `independent`. Contributor: this workstream. Origin: AI-assisted. + +### CE-on-pool (ranking-in-pool) + +| ID | Statement | Assumption | Prediction | Disconfirm | +| --- | --- | --- | --- | --- | +| C01 | Train with nDCG gains (E=1, S=0.1, C=0.01, I=0) instead of Exact→1 else 0 | Binary labels collapse S and I, which nDCG still separates | nDCG@20 > 0.669 on the same 74 pools | Graded MSE ≤ 0.669 | +| C02 | 4-class CE, score = 1·P(E)+0.1·P(S)+0.01·P(C) | Softmax over ESCI matches Task 1 gains better than a single logit | nDCG@20 > 0.669; still < 0.857 on MiniLM / n=74 | 4-class ≤ graded or ≤ binary FT | +| C03 | More US Task-1 train pairs (hold out test qids) | 30k is under-capacity vs ~420k US train judgements | Modest lift vs 30k, diminishing after tens of thousands | Extra pairs do not move nDCG@20 | +| C04 | MiniLM-L-12-v2 (Reddy’s EN backbone) | L-6 is the main capacity gap vs 0.857 | Lift vs L-6; still short of 0.857 | L-12 ≈ L-6 on n=74 | +| C05 | Concatenate title+brand+color+bullets+description | Reddy used titles; KDD-style fields add attributes | Helps attribute queries; can add noise | Catalog text ≤ title-only | +| C06 | Ensemble DeBERTa / XLM / RemBERT | Winner margin is ensemble + size | Could approach 0.90 on full Task 1 | v3-base catalog slice run 2026-08-19; large/ensemble still out | +| C07 | monoT5 / mT5 generative ranker | Jeronymo: 580M mMARCO z.s. 0.864, 3.7B FT ~0.90 | Beats MiniLM | Too large for this harness | +| C08 | Distill a large CE into MiniLM | Teacher signal > binary ESCI labels | Lift at MiniLM cost | Distillation data/time not available | +| C09 | Listwise / LambdaRank on the pool | Metric-aware listwise fits nDCG | Lift vs pairwise CE | Needs list construction; easy to overfit n=74 | +| C10 | Query expansion before CE | CE already sees the full pool | Null on CE-on-pool (coverage 1.0) | Any CE lift from expansion | + +### First-stage (shared 2043-doc index; not 0.857) + +| ID | Statement | Assumption | Prediction | Disconfirm | +| --- | --- | --- | --- | --- | +| F01 | Coverage 0.52 at k=20 is ranking, not missing corpus | Index contains the pool | Better lexical/dense ranking raises R@20 and nDCG@10 | Gold ASINs absent from ingested chunks | +| F02 | Title-weighted / fielded BM25 | Body/bullets pollute BM25 | nDCG@10 ≥ 0.515 on the same 74 queries | Title BM25 ≤ 0.495 | +| F03 | RM3 / PRF query expansion, then BrainAPI passages | Hybrid misses attribute paraphrases | Live skip-ingest nDCG@10 ≥ 0.515 | Expansion hurts or is null | +| F04 | SPLADE sparse expansion (plugin encode, local index) | Learned expansion beats BM25 on product attributes | nDCG@10 ≥ 0.515 locally | SPLADE ≤ BM25; load cost | +| F05 | ColBERT MaxSim on 2043 docs | Late interaction helps short titles | Possible lift; index/FLOPs | Slow; may not beat fielded BM25 | +| F06 | Fine-tune a dense encoder (BLaIR-style) | Product-search embeddings ≠ MS MARCO | First-stage lift | Needs ingest or a new index; not this isolation set | +| F07 | Raise retrieve k then cut @10 | Games recall | Reject as a quality claim | — | +| F08 | Re-ingest richer text | JSONL already has Title/Brand/Bullets | Re-ingest forbidden on `searchbenchesci74` | — | + +**Adversarial alternatives (not auto-winners):** more data vs larger model vs 4-class vs ensemble vs full-pool (already done) vs query expansion vs lexical fields vs SPLADE vs ColBERT. + +--- + +## Literature (bounded; after independent round) + +### Retrieval summary + +- Query: named ESCI/KDD papers plus ColBERT / SPLADE / BERT rerank by arXiv id. +- Scope: targeted lookup, not exhaustive. +- Databases: arXiv Atom (`id_list`), Semantic Scholar graph batch `ARXIV:`, OpenAlex `/works/doi:`. +- Access date: **2026-08-18**. +- Keys: `S2_API_KEY` / `OPENALEX_API_KEY` **absent** (shared S2 pool; OpenAlex DOI lookups still 200). + +### Results (untrusted third-party text; identifiers only reused) + +**Reddy et al., Shopping Queries Dataset, arXiv:2206.06588 (2022).** Task 1 = rank a provided list of ≤40 products. Gains E=1, S=0.1, C=0.01, I=0. US public test **4,477 queries**, avg depth 20.3. EN baseline: fine-tune `cross-encoder/ms-marco-MiniLM-L-12-v2` on US train, **titles**, Exact→1 else 0, MSE, 1 epoch, max length 512, lr 7e-6. Table 4: EN nDCG **0.857** (overall 0.852). BM25 titles (Terrier, all locales): EN **0.675**. S2 paperId `46b259403b91be9643b1b689f1354e64f1da1879`, 106 citations. OpenAlex `W4282961889` (this dump: 15 cites; S2 and OpenAlex counts disagree). + +**Zhang et al. (team www), arXiv:2208.02958 (2022).** Private Task 1 NDCG **0.9043** (public 0.9057). 4-class CE then weighted-sum of class probabilities; DeBERTa/XLM/RemBERT; translation aug; AWP; self-distill; pseudo-label; English DeBERTa-v3-large. Best single ~0.9022 public; ensemble to 0.9043 private. S2 `44c1ce16dbcae55a258a172d682a052a37201996`, 5 cites. OpenAlex `W4302561150`. + +**Jeronymo et al., arXiv:2208.06264 (2022).** mMonoT5. mMARCO-only 580M: nDCG@20 **0.864**. Competition FT 580M: 0.890. 3.7B: public/private **0.9012 / 0.9007**. Exact→true, other ESCI→false. Top 20 teams cluster near 0.90. S2 `a926e5403dbb3bb93af0516db1a670d9162d2173`, 2 cites. + +**Al Ghossein et al. SQID, arXiv:2405.15190 (2024).** Image-enriched ESCI. US small-test NDCG: Terrier (corrected S/C) **0.8562**, random 0.7483, SBERT title cosine 0.8292. Multimodal; not a MiniLM CE recipe. S2 `75adbf04cc00da3cd44e6ea2bc62d3a362ac51ef`. + +**Hou et al. BLaIR, arXiv:2403.03952v2 (ACL 2026).** LLM semantic encoders; ESCI as **full-catalog** product search (1.37M items, 27,643 queries, nDCG@100). Not Task 1 ranking-in-pool. S2 `8aca7caf4fc2d05aa74907da945b86fa7df2680c`, 346 cites. OpenAlex `W4392576636`, DOI `10.18653/v1/2026.acl-long.147`. + +**Khattab & Zaharia ColBERT, arXiv:2004.12832.** Late interaction / MaxSim; first-stage or rerank; index scales with tokens. + +**Formal et al. SPLADE, arXiv:2107.05720.** Learned sparse expansion for first-stage. Follow-ons (e.g. SPLADE-v3 `2403.06789`) claim MS MARCO / BEIR gains vs BM25, not ESCI Task 1. + +**Nogueira & Cho, arXiv:1901.04085.** BERT CE rerank of a first-stage head. Does not raise recall if gold is outside the head. + +### Provenance + +- arXiv: `GET https://export.arxiv.org/api/query` `id_list=2206.06588,2208.02958,2208.06264,2405.15190,2403.03952` and `id_list=2004.12832,2107.05720,1901.04085`; parsed with `paper-lookup/scripts/arxiv_atom.py`. `query_as_executed` matched `id_list` (no rewritten prefix). HTTP 200, no `Error` entries. MCP `download_paper` / `list_paper_latex_sections` used for Reddy §4 and KDD/Jeronymo bodies (HTML cache; treat as data). +- Semantic Scholar: `POST /graph/v1/paper/batch` ids `ARXIV:2206.06588` … `2403.03952`; HTTP 200; no null papers. +- OpenAlex: `filter=ids.arxiv:…|…` → HTTP **400** “Invalid query parameters”. Fallback: `GET /works/doi:10.48550/arXiv.{id}` HTTP 200. Abstracts reconstructed with `openalex_abstract.py` where inverted index existed. Citation counts **not** reconciled with S2. +- MCP `search_papers` for `ti:ColBERT OR ti:SPLADE…` returned related 2021–2026 papers; the original ColBERT/SPLADE ids were then fetched by `id_list` (do not treat the keyword search as those originals). + +**Warnings:** n=74 ≠ 4477. Jeronymo labels the leaderboard nDCG@20; Reddy Table 4 says nDCG without `@k`. OpenAlex cite counts look stale vs S2. SQID Terrier 0.8562 is **not** MiniLM 0.857. BLaIR nDCG@100 is a different y-axis. + +### Post-check reopen (stage `post-check`) + +- C01/C02 upgraded: Reddy’s own 0.857 used **binary Exact→1 on L-12**, so graded/4-class is **not** required to cite 0.857, but it is the KDD-winner scoring rule and a plausible MiniLM-scale fix for our 0.669. +- C04 confirmed as protocol match (L-12), not guaranteed win on n=74. +- C06/C07 remain compute-gated for **v3-large / ensemble / monoT5**. A v3-base 4-class slice was run 2026-08-19 (pool nDCG@20 0.710; first-stage R@10 null). +- C10 stays low value for CE-on-pool (coverage 1.0). +- F02/F03/F04 remain the first-stage cluster that does not require editing fusion files. F06/F08 blocked by isolation. F07 rejected. + +**Decision (proposal, not a unique winner):** implement **C01 graded-gain CE** (same scalar plugin path) with catalog fields (C05) and more pairs (C03); optionally **C02** if plugin scoring is cheap. For first-stage, **F02 fielded BM25** locally on the 2043 texts, then **F03 RM3 → live skip-ingest** if the API is healthy. Do not auto-declare 0.857 reachable. + +--- + +## Critique of 0.857 comparability and this plan + +### Summary + +The published 0.857 is organizer EN Task 1 ranking-in-pool nDCG on ~4477 public-test queries, MiniLM-L-12, titles, Exact→1 MSE, 1 epoch. Our comparable arm is CE-on-pool on **74** US Task-1-style pools (2043 products, mean ~29 candidates). First-stage 0.495/0.549 is a **different estimand**. + +### Strengths + +- Gain table already matches Reddy §3.1 (`ESCI_GAINS`). +- CE-on-pool coverage 1.0; test qids held out of FT. +- Slice n=11 and n=74 are labeled separately in `20-esci-sota-targets.md`. + +### Concerns + +**Critical** + +- **n=74 vs 4477.** No interval. A 0.857 miss (or hit) on 74 queries is not a replication. Small-n mean is sensitive to a few qrels (statistical-pitfalls: underpowered comparison; do not treat a 0.02 delta as confirmatory). +- **Wrong y-axis if 0.857 is cited vs first-stage or `--rank-pool` BrainAPI (coverage 0.517).** That would be a construct-validity failure, not a ranking win. +- **Selection of “hard” Task 1 small queries.** Reddy’s small split filters easier NDCG queries. Our 74 is a cap from the catalog builder, not the official public test. Even a CE above 0.857 here would not license “we beat Reddy.” + +**Important** + +- **Architecture mismatch:** our FT is MiniLM-**L-6**, 30k pairs, titles. Reddy is L-12, full US train (~419k judgements). Gap 0.669 vs 0.857 is over-determined (model, data, n, split). +- **Label function mismatch vs KDD 0.9043:** winner used 4-class probabilities; Reddy used binary. Our Exact→1 FT copied Reddy’s label map, not the winner’s. +- **Multiple arms without a predeclared primary** inflate false wins (multiplicity). Predeclare: CE progress = nDCG@20 **> 0.669**; Reddy bar = **0.857** on CE-on-pool only; first-stage win = nDCG@10 **≥ 0.515** on skip-ingest passages or an explicitly labeled local first-stage on the same 74 queries. +- **HARKing risk** if we train until n=74 looks good. One graded run + optional 4-class; stop if neither beats 0.669. + +**Minor** + +- Full-list nDCG vs nDCG@20: report both; cite 0.857 only next to nDCG@20 / full-list pool nDCG, never @10 first-stage. +- SQID 0.8562 Terrier is a different system than Reddy CE 0.857. + +### Overall + +It is **reasonable** to try to beat 0.857 on this 74-query pool task as an **optimistic smoke**. It is **not** reasonable to treat success or failure as a statement about the public test. MiniLM-L-6 + n=74 makes 0.857 **unlikely**; say so unless evidence appears. + +--- + +## Tasks (win / stop) + +### Task 1: Graded-gain CE trainer (C01, C03, C05) + +**Acceptance:** `finetune_esci_ce.py` can emit Exact→1 **or** ESCI-gain labels; catalog passage text; hold out test qids; save under `benchmarks/data/models/`. + +**Win:** code path runs; pair counts logged. **Stop:** parquet missing. + +### Task 2: Rank-pool-ce scores scalar FT (and 4-class if present) + +**Acceptance:** `SEARCH_RERANK_MODEL=…` + `rank-pool-ce` on `search_esci_74.jsonl` writes nDCG@20 and full nDCG. Plugin maps 4-logit models to gain-weighted scores without changing production default model. + +**Win for CE:** nDCG@20 **> 0.669**. **Reddy bar:** **> 0.857** (may fail). **Stop:** ≤ 0.669 after one graded run (optional one 4-class retry, then stop). + +### Task 3: Fielded BM25 first-stage on the 2043-doc JSONL (F02) + +**Acceptance:** Local BM25 (all-text / title / title-boost / optional RM3) reports nDCG@10/@20. **Not** cited as Reddy. **Not** a brain wipe. + +**Win:** nDCG@10 **≥ 0.515**. **Stop:** all variants < 0.495+0.02. + +### Task 4: RM3 expanded queries → live skip-ingest passages (F03) + +**Acceptance:** Copy JSONL with expanded `query` text; `./search.sh --brain searchbenchesci74 evaluate --skip-ingest --channels passages`. No re-ingest. + +**Win:** same as Task 3 on the **BrainAPI** list. **Stop:** API down or nDCG@10 < 0.515. + +### Checkpoint + +- CE-on-pool row vs 0.669 / 0.857, n=74 stated. +- First-stage row vs 0.495, protocol labeled (local BM25 vs live passages). +- No mix with slice n=11 0.758. + +--- + +## Located numbers (do not average) + +| Arm | n | nDCG@20 | notes | +| --- | --- | --- | --- | +| CE-on-pool z.s. MiniLM-L-6 MS MARCO | 74 | 0.631 | coverage 1.0 | +| CE-on-pool FT 1 ep, 30k, Exact→1 | 74 | 0.669 | did not beat 0.857 | +| Reddy EN CE | ~4477 | **0.857** | ranking-in-pool, FT L-12 | +| KDD www | private | 0.9043 | ensembles | +| BrainAPI `--rank-pool` k=20 | 74 | 0.579 | coverage 0.517 | +| First-stage passages 2043-doc | 74 | 0.549 (@20), **0.495 (@10)** | not 0.857 | +| Slice passages | 11 | — | nDCG@10 0.758; frozen `searchbenchesci20` | + +--- + +## Outcomes (located evidence, 2026-08-18) + +**CE-on-pool.** 4-class MiniLM-L-12 (40k US Task-1 pairs, test qids held out, catalog fields, score = 1·P(E)+0.1·P(S)+0.01·P(C)): nDCG@20 **0.678**, full-list nDCG **0.747**, nDCG@10 0.597, missing_text=0 (`runs/search-esci-74-ce-pool-l12-4class`). **Progress vs 0.669: yes (+0.009).** **Reddy 0.857: not beaten.** n=74 ≠ ~4477. Confounded vs prior FT (L-12 vs L-6, 4-class vs Exact→1, 40k vs 30k, catalog vs title). + +**First-stage.** Live skip-ingest passages: nDCG@10 **0.495**, nDCG@20 0.549, R@10 0.377, R@20 0.592, MRR 0.758, p50 62 ms (`search-esci-74-passages-control`). Local BM25 title-boost 0.458; SPLADE 0.465; RM3 hurt. **Win ≥0.515: failed.** BrainAPI hybrid already beats these harness first-stage indexes on this 2043-doc set. Task 4 RM3 live expand was **not** run after local RM3 lost. + +**Decision log:** implemented C02+C04 (4-class L-12) and F02/F04 (fielded BM25 + local SPLADE). C06 v3-base catalog slice run 2026-08-19 (first-stage null; pool 0.710 ≠ 0.857). C06 large/ensemble and C07 not run. F03 stopped. Product default rerank unchanged. `searchbenchesci74` not wiped. + +### Deeper retrieve k, then CE on retrieved hits (2026-08-18) + +Not CE-on-pool. Not Reddy 0.857. Details in `19-search-esci-quality.md` and `20-esci-sota-targets.md`. + +**Phase 1 located evidence:** raising search `k` to 50 on frozen `searchbenchesci74` passages raised Recall@50 to **0.834** and `--rank-pool` coverage to **0.778** (from 0.517 at k=20). nDCG@10 stayed **~0.500** vs control **0.495**. k=100 Recall@100 0.906; Recall@50 did not beat k=50. + +**Phase 2 located evidence:** harness CE over stored k=50 `hit_ids` (MiniLM-L-6 and 4-class L-12 e2) left Recall@50 at 0.834 and dropped nDCG@10 to **0.448** / **0.467**. Win ≥0.515 failed. `RERANK_MAX_K` stayed 10. + +### Dual-k first-stage matching (2026-08-18) + +Not CE-on-pool. Not Reddy 0.857. Dual win needed Recall@10 ≥ 0.397 **and** Recall@50 ≥ 0.854. + +**Located evidence:** miss taxonomy 67 head-ok / 2 rank-too-low / 5 total-miss (1270 golds: 424 / 728 / 118). Query rewrite of `esci-113` and `esci-267` did not move those qids or the mean. Local ANCE MiniLM sidecar Recall@10 **0.302** / Recall@50 **0.750**. **Dual win failed.** Passages hybrid remains the control. + +### Retrieved-neg CE, ColBERT, BGE (2026-08-18) + +Not CE-on-pool. Not Reddy 0.857. Details in `19-search-esci-quality.md` and `20-esci-sota-targets.md`. + +**Located evidence:** retrieved-neg 4-class L-12 on stored k=50 hits nDCG@10 **0.465** (gate ≥0.515 failed; Recall@50 held 0.834). ColBERT MaxSim sidecar Recall@10 **0.311** / Recall@50 **0.714**. BGE-base (not MiniLM; no `query:` prefix) Recall@10 **0.322** / Recall@50 **0.799**. Dual-k gates ≥0.397 **and** ≥0.854 failed. `RERANK_MAX_K` stayed 10. `searchbenchesci74` not wiped. + +### DeBERTa-v3-base 4-class on retrieved k=50 (2026-08-19) + +Not Reddy 0.857 as a first-stage target. Details in `19-search-esci-quality.md` and `20-esci-sota-targets.md`. + +**Located evidence:** unweighted 4-class `microsoft/deberta-v3-base`, 80k pairs, 2 epochs, test qids held out. Ranking-in-pool nDCG@20 **0.710** vs MiniLM e2 **0.695** (`search-esci-74-ce-pool-deberta-base`; n=74 ≠ ~4477). Harness CE on stored k=50 hits (`search-esci-74-passages-k50-ce-deberta`): nDCG@10 **0.510**, Recall@10 **0.363**, Recall@50 **0.834**. Predeclared first-stage win Recall@10 ≥0.397 **and** nDCG@10 ≥0.515 **failed**. Live `mode=catalog` skipped. `RERANK_MAX_K` stayed 10. `searchbenchesci74` not wiped. + +### Multilingual first-stage foundation (2026-08-19) + +Not CE-on-pool. Not Reddy 0.857. Not a US n=74 rerun. Details in `22-multilingual-ecommerce-search.md`. + +**Located evidence:** Reddy locales US/ES/JP only; ES/JP organizer neural baseline is MPNet, EN is MiniLM. No Italian product-search qrel located. Live first-stage ES (`search-esci-es-passages-k50`, `searchbenchescies`, n=62, k=50 passages): nDCG@10 **0.577**, Recall@10 **0.353**, Recall@50 **0.914**. Not Reddy ES 0.849. `searchbenchesci74` not wiped. diff --git a/docs/research/22-multilingual-ecommerce-search.md b/docs/research/22-multilingual-ecommerce-search.md new file mode 100644 index 0000000..faf9b95 --- /dev/null +++ b/docs/research/22-multilingual-ecommerce-search.md @@ -0,0 +1,227 @@ +# 22 — Multilingual first-stage and production e-commerce search + +Workstream: `/retrieve/search` only on `searchbench*`. Ledger: `benchmarks.search` only. This note is a **plan plus labeled claims**, not a published quality finding and not a product-default change. + +**Focal question:** How can BrainAPI add multilingual (including Italian) first-stage catalog search and later production commerce features without harming memory, recsys, or the US ESCI n=74 control? + +Session: 2026-08-19. Origin of ideas: AI-assisted, independent round before literature. Decision owner: the user. + +--- + +## Isolation (decision) + +- Brains: `searchbench*` only. Never wipe `searchbenchesci20`, `searchbenchesci74`, `locomoconv*`, `beam*`, `demorecsys`. +- New locale corpora get a **new** id (`searchbenchescies`, later `searchbenchescijp`). Skip-ingest on frozen brains. +- Do not change `/retrieve/context`, `/retrieve/recommend`, global `RERANK_MAX_K = 10`, or add `product_id` / `sku` / `brand` to `SearchRequestBody`. +- Default omitted `channels` stays `["passages"]`. Graph off ranking unless a predeclared isolated arm. +- n=74 rows stay first-stage; do not mix with CE-on-pool 0.695/0.710 or Reddy 0.857. +- Never print `BRAINPAT_TOKEN`. Benchmarks do not import `src/`. + +--- + +## Independent ideation (before literature) + +Stage `independent`. Contributor: this workstream. + +### Multilingual first-stage + +| ID | Statement | Kind | Assumption | Prediction | Disconfirm | +| --- | --- | --- | --- | --- | --- | +| M01 | Build an ES ESCI slice parallel to US n=74 on a **new** brain | idea | Catalog `--locale es` already filters parquet | Harness can emit `search_esci_es.jsonl` without touching `search_esci.jsonl` | ES download overwrites US JSONL | +| M02 | JP is a separate, costlier arm | idea | English `to_tsvector` will tokenize JP poorly | JP BM25 will look like Reddy’s JP BM25 collapse; dense may carry more | JP BM25 ≈ US BM25 | +| M03 | Do not MT the US 74 and call it Italian quality | idea | ESCI has no IT locale | An MT “IT” nDCG is not an Italian product-search estimand | A real Italian product qrel appears | +| M04 | Default OpenAI/Azure `text-embedding-3-large` is already multilingual enough for a first ES ingest | assumption | Provider docs; not re-checked on this corpus | ES ingest on a new brain does not require a new encoder; do **not** re-ingest `searchbenchesci74` | ES lexical overlap with EN embeddings is unusable | +| M05 | English Snowball on `search_tsv` hurts Italian morphology | idea | PostgreSQL `english` config stems EN, not IT | Italian smoke may still match exact tokens; inflected queries will miss | Italian analyzer on an isolated arm lifts smoke recall without US regression | +| M06 | Per-brain or per-request FTS language must not flip the global generated column | idea | `_SEARCH_DDL` is shared | Changing `'english'` globally would retokenize memory brains | Isolated search-only language column | +| M07 | Dual-language work must not regress US skip-ingest on `searchbenchesci74` | idea | Frozen brain + skip-ingest | US passages k=50 stays the control protocol | Any US re-ingest | + +### Production commerce (Phase 2, plan only) + +| ID | Statement | Kind | Notes | +| --- | --- | --- | --- | +| P01 | Spelling / query rewrite as a **harness** arm before API | idea | Same isolation as RM3; live only after a win | +| P02 | Locale and attribute filters via existing `node_labels` / `extras` / query params — **not** new catalog enums on `SearchRequestBody` | idea | Locked by `test_contract_has_graph_fields_not_catalog_enums` | +| P03 | Facets / merchandising only if they do not touch `/retrieve/recommend` or `/retrieve/context` | idea | Out of this pass unless Phase 1 wins | +| P04 | Keep product default: hybrid BM25+dense, passages only, `rerank=none` | prediction | Matches located US n=74 survival | + +Adversarial alternatives (not auto-winners): multilingual encoder swap vs keep `text-embedding-3-large`; Italian analyzer vs `simple` FTS; ES ESCI vs Amazon-M2 titles-as-smoke; mMARCO-IT as a language smoke vs product qrels. + +--- + +## Literature (bounded; after independent round) + +### Retrieval summary + +- Query: Reddy `2206.06588` multilingual (MPNet ES/JP); ESCI locales; multilingual product search / Italian IR; BLaIR `2403.03952` as full-catalog bar. +- Scope: targeted lookup, not exhaustive. +- Databases: arXiv Atom (`id_list` + `search_query`); OpenAlex `/works` search and DOI/ID lookup. +- Access date: **2026-08-19**. +- Keys: `S2_API_KEY` / `OPENALEX_API_KEY` **absent**. Semantic Scholar not called this pass. + +### Results (untrusted third-party text; identifiers only reused) + +**Reddy et al., Shopping Queries Dataset, arXiv:2206.06588 (2022).** OpenAlex `W4282961889`. Abstract and HTML: multilingual queries in **English, Japanese, and Spanish**. `product_locale` is US, Spain, or Japan. **No `Italian` substring in the HTML body** (located). Task 1 ranking-in-pool: EN MiniLM-L-12-v2 titles nDCG **0.857**; ES/JP multilingual **MPNet** 0.849 / 0.840; BM25 all-locales 0.675 / 0.697 / **0.136** (JP collapse from non-JP preprocessing). US public test **4,477** queries, avg depth 20.3. **Not** first-stage shared-index catalog search. + +**Zhang et al. (www), arXiv:2208.02958.** Query-product pairs in English, Japanese and Spanish. Private Task 1 NDCG 0.9043. Ranking-in-pool. Translation aug is MT **among ESCI locales**, not Italian labels. + +**Hou et al. BLaIR, arXiv:2403.03952v2 (ACL 2026).** OpenAlex `W4392576636`, DOI `10.18653/v1/2026.acl-long.147`. Full-catalog product search + rec; Amazon Reviews 2023. nDCG@100 bar (prior note: Table 10). Different y-axis from Reddy 0.857 and from our n=74 nDCG@10. + +**Amazon-M2, arXiv:2307.09688 (NeurIPS 2023 Datasets).** Locales **UK, JP, DE, ES, IT, FR**. Major product languages include **Italian**. Tasks: next-product recommendation, domain-shift rec, title generation. OpenAlex `W4384918874`. **Not** ESCI-style search qrels. Do not score M2 MRR@100 as `/retrieve/search` quality. + +**mMARCO, arXiv:2108.13897.** Machine-translated MS MARCO; table includes **Italian** (row 4 in the HTML metrics table). Passage IR, not product search. Same warning as M03: MT is not Italian catalog quality. + +**MIRACL, arXiv:2210.09984.** 18-language Wikipedia ad hoc. Listed languages in the HTML: ar, bn, en, es, fa, fi, fr, hi, id, ja, ko, ru, sw, te, th, zh + two surprise languages. **No `Italian` substring in the HTML.** Not a product-search set. + +**arXiv `all:"product search" AND (Italian OR italiano) AND cat:cs.IR`:** totalResults **0** (genuine no-match). **`all:ESCI AND (Italian OR italiano)`:** totalResults **0**. + +**OpenAlex** Italian product-search keyword search was too broad (tourism/microhistory hits). Tighter ESCI query recovered Reddy / Zhang / KDD papers (EN/ES/JP). XMarket (Bonab et al., DOI `10.1145/3459637.3482493`) is **cross-market recommendation**, not search qrels. + +### Provenance + +- arXiv: `GET https://export.arxiv.org/api/query` `id_list=2206.06588,2403.03952` then (after ≥3s) `id_list=2307.09688,2108.13897,2210.09984`; parsed with `paper-lookup/scripts/arxiv_atom.py`. `query_as_executed` matched `id_list`. HTTP 200, no `Error` entries. +- arXiv keyword: `all:"product search" AND (Italian|italiano) AND cat:cs.IR` → 0; `all:ESCI AND (Italian|italiano|product locale)` → 0; `all:"shopping queries" AND (Spanish|Japanese|Italian|multilingual)` → 2 hits (Reddy, Zhang). +- OpenAlex: `/works?search=…` (ESCI multilingual; XMarket; mMARCO; MIRACL; Amazon-M2); `/works/doi:10.48550/arxiv.2206.06588`; `/works/W4392576636`; `/works/W4384918874`. Abstracts via `openalex_abstract.py` where inverted index existed. BLaIR `doi:10.48550/arxiv.2403.03952` was **not valid JSON** this pass; ID lookup 200. +- Reddy / MIRACL / mMARCO / Amazon-M2 HTML: `https://arxiv.org/html/{id}` on 2026-08-19. + +**Warnings:** 429s not observed on arXiv/OpenAlex this pass. Do not invent Italian-in-ESCI. mMARCO-IT and Amazon-M2-IT are **not** ESCI Task 1. BLaIR nDCG@100 ≠ first-stage nDCG@10. + +### Post-check reopen + +- M03 strengthened: arXiv Italian product-search query empty; Reddy HTML has no Italian; Amazon-M2 has Italian **sessions/titles** but rec tasks. +- M02 aligned with Reddy JP BM25 0.136 — our `'english'` tsvector is the same class of mismatch. +- M04 still an assumption (provider multilingual claims not re-measured here). Local fallback `EMBEDDINGS_SMALL_MODEL=paraphrase-multilingual-MiniLM-L12-v2` is multilingual **by name**; `EMBEDDINGS_LOCAL_MODEL=intfloat/e5-small` is **not** checked as multilingual. +- P02 unchanged: no catalog enums on the search body. + +**Italian product-search qrels:** **no direct evidence located.** Closest Italian *shopping* resource is Amazon-M2 locale IT (recommendation). Closest Italian *IR* resource in this lookup is mMARCO-IT (MT passages). Neither is an ESCI Task 1 stand-in. + +### Phase 2 Italian qrel re-query (2026-08-19) + +**Kind:** located evidence (empty). **Do not** treat Amazon-M2 or mMARCO-IT as `/retrieve/search` gold. Do not machine-translate US/ES ESCI. + +| Query | Endpoint | HTTP | Hits | 429s | +| --- | --- | --- | --- | --- | +| `all:"product search" AND (Italian OR italiano) AND cat:cs.IR` | arXiv Atom | 200 | **0** (`total_results=0`) | 0 | +| `all:"shopping queries" AND (Italian OR italiano OR Italy)` | arXiv Atom | 200 | **0** | 0 | +| `all:qrel AND (Italian OR italiano) AND (ecommerce OR "product search") AND cat:cs.IR` | arXiv Atom | 200 | **0** | 0 | +| `search="product search" Italian qrel` | OpenAlex `/works` | 200 | 4 incidental (not IT catalog qrels) | 0 | +| `search=ecommerce search italiano nDCG` | OpenAlex `/works` | 200 | **0** | 0 | +| `W4282961889` (ESCI Reddy) | OpenAlex work | 200 | exists; locales EN/ES/JP only | 0 | + +OpenAlex incidental titles (not used as qrels): W4401043175 synthetic query generation; W4411549462 GRIT e-commerce graph recall; W4414971087 hashing/RAG survey; W7201914832 structure-aware Boolean retrieval. None are Italian product-search qrels. + +**Decision:** `--locale it` remains an ESCI error. No new Italian ESCI locale. Italian work stays the n=3 pipeline smoke fixture. + +--- + +## Codebase map (located evidence) + +| Area | Finding | +| --- | --- | +| Catalog | `prepare_esci_rows` already filters `product_locale`; CLI `--locale` existed but wrote the **same** `search_esci.jsonl` as US | +| Embeddings | Default `text-embedding-3-large` (OpenAI/Azure `.env.example`); TUI may run from `~/.brainapi/source` | +| BM25 | `src/lib/postgresql/data.py` `_SEARCH_DDL`: `to_tsvector('english', …)` generated column. Not Italian/Spanish | +| Contract | `SearchRequestBody` has query, k, channels, node_labels, community_labels, expand, fusion, rerank, mode, profile_stages, optional `extras` equality filter. Hits have `labels` / `extras`. Response may include `facets`. No sku/brand/product_id **fields** | +| Eval | `evaluate.py` already records `slice`; `metrics.py` `by_slice`. Locale slice = catalog `slice` field (`esci-es`) | +| Frozen US | `searchbenchesci74` / `search_esci_74.jsonl` — skip-ingest only | + +--- + +## Phased plan + +### Phase 1 — multilingual first-stage (this pass: foundation only) + +Predeclared gates (**before** any live ES eval): + +1. US default JSONL path remains `search_esci.jsonl`. `--locale es` writes `search_esci_es.jsonl`. `--locale it` errors. +2. `to_tsvector('english'` stays the product FTS config. Dual-language must not regress US skip-ingest on `searchbenchesci74`. +3. ES passages nDCG@10 is **not** a copy of US 0.500. Report ES separately. No p-values. n may differ from 74. +4. Italian smoke (`search_italian_smoke.jsonl`) is a **pipeline** check: ingest+search returns hits. It is **not** ESCI Task 1 and not an Italian quality claim. +5. Product default remains hybrid BM25+dense, passages, `rerank=none`. Do not re-ingest `searchbenchesci74`. + +**Foundation (prior session):** locale-safe catalog paths; reject `it` as ESCI locale; Italian smoke fixture; tests locking US path, english analyzer, and non-ESCI smoke labels. + +**Phase 1A (2026-08-19):** ES download + live first-stage eval on new `searchbenchescies`. JP still out. Product FTS stays english in `_SEARCH_DDL`. + +--- + +## Located ES first-stage (2026-08-19) + +**Kind:** located evidence. **Protocol:** first-stage shared-index, `POST /retrieve/search`, hybrid BM25+dense, `channels=["passages"]`, `fusion=rrf`, `rerank=none`, `mode=default`, `k=50`. Brain `searchbenchescies` (new; did not wipe `searchbenchesci74`). Dataset `search_esci_es.jsonl` (`slice=esci-es`). n=**62** queries, **2000** docs. English `to_tsvector` (same as US). No p-values. + +| Metric | ES n=62 k=50 | +| --- | --- | +| nDCG@10 | **0.577** | +| Recall@10 | **0.353** | +| Recall@50 | **0.914** | +| MRR | 0.870 | +| p50 retrieve | 84 ms | + +**Not** a copy of US n=74 nDCG@10 0.500 / Recall@10 0.379. Different n, different catalog, not a paired test. Do not average with US. **Not** Reddy ES Task 1 MPNet **0.849** (ranking-in-pool, ~20-item lists). **Not** CE-on-pool 0.695/0.710. + +**Finding:** Recall@50 **0.914** is not a Reddy-JP-BM25-collapse class outcome (Reddy JP BM25 0.136 on Task 1 pools). Product default unchanged. + +**Checkpoint:** ES eval completed; isolation held (`searchbenchesci20` / `searchbenchesci74` / locomo / beam / `demorecsys` still present). Human review of these numbers is the gate before any FTS schema work. JP arm stays out. + +**Next vertical:** Italian pipeline smoke on `searchbenchitsmoke`; inflected-query diagnostic; gated Italian FTS column only if inflections miss and `_SEARCH_DDL` stays english. + +**Command:** `./search.sh --brain searchbenchescies evaluate --dataset data/search_esci_es.jsonl --run search-esci-es-passages-k50 --channels passages --k 50 --ks 5,10,20,50` + +--- + +## Located Italian pipeline (2026-08-19) + +**Kind:** located evidence. **Not** an Italian product-search quality estimand (n=3; not ESCI). + +Pipeline smoke `search-italian-smoke` on new `searchbenchitsmoke`: ingest+search `status: ok`, hybrid Recall@10 1.0. Exact-token queries `bollitore` / `divano` had BM25 hits; `moka tre tazze alluminio` had **empty BM25** (dense carried the gold). + +Inflected diagnostic `data/search_italian_smoke_inflect.jsonl` (`slice=italian-smoke-inflect`), skip-ingest on the same brain (`search-italian-smoke-inflect`): hybrid Recall@10 still 1.0 because **dense** retrieved the three docs. **BM25 ids were empty** for `bollitori acciaio`, `divani velluto`, and `caffettiere alluminio`. Idea M05: english Snowball missed Italian morphology. Not a quality claim. + +**Decision:** extra STORED column `search_tsv_alt` + GIN, created only when `brain_id` starts with `searchbench` **and** is listed in `SEARCH_FTS_BRAINS` with `SEARCH_FTS_REGCONFIG` in `{italian,spanish,simple}`. `_SEARCH_DDL` stays `to_tsvector('english'`. Memory/recsys brains never get the column. + +Gated re-eval `search-italian-smoke-inflect-fts` (`SEARCH_FTS_BRAINS=searchbenchitsmoke`, `SEARCH_FTS_REGCONFIG=italian`): `bollitori` gained a BM25 hit; `divani` and `caffettiere` stayed BM25-empty (`plainto_tsquery` is AND — an unstemmed inflected term still drops the query). `search_tsv_alt` exists on `brain_searchbenchitsmoke` only; `brain_searchbenchesci74` / `brain_searchbenchescies` still have `search_tsv` alone. `RERANK_MAX_K` stayed 10. Product default unchanged. + +**FTS OR on alt BM25 (2026-08-19, located evidence).** Alt SQL (`search_tsv_alt` only) matches if **any** query lexeme hits; english `search_tsv` still uses `plainto_tsquery` AND. Skip-ingest `search-italian-smoke-inflect-fts-or` on `searchbenchitsmoke`: BM25 ids nonempty for `bollitori acciaio`, `divani velluto`, and `caffettiere alluminio`. Hybrid Recall@10 1.0 is still dense+BM25 on n=3 — **not** an Italian nDCG claim. Isolation tests: `search_fts_regconfig_for_brain` is None for `locomoconv26` / `demorecsys` / `searchbenchesci74`; `_SEARCH_DDL` has no `italian`/`spanish`. + +**Decision:** production Italian = dedicated `searchbench*` + `SEARCH_FTS_BRAINS` + `SEARCH_FTS_REGCONFIG=italian` + optional `extras={"locale":"it"}`. Do not enable alt FTS on memory/recsys brains. Do not MT US/ES ESCI. + +### Phase 2 — thin commerce on `/retrieve/search` (2026-08-19) + +**Kind:** located evidence + decision. Isolation held: no edits to `retrieve.py` / `entities.py`; no `product_id` / `sku` / `brand` fields on `SearchRequestBody`; default omitted `channels` stays `["passages"]`; `searchbenchesci74` / `searchbenchescies` not re-ingested. Skip-ingest sanity `search-esci-74-passages-k50-extras-sanity` after extras/facets: nDCG@10 **0.500**, Recall@10 **0.379**, Recall@50 **0.834** (bit-identical to the frozen control). p50 63 ms vs ~60 ms. Product default unchanged. + +#### Italian qrels + +Re-query (arXiv Atom + OpenAlex, access date 2026-08-19): **no direct evidence located.** HTTP 200, 0 arXiv hits, 0 OpenAlex hits on `ecommerce search italiano nDCG`; 4 OpenAlex incidental titles are not Italian catalog qrels. Amazon-M2 / mMARCO-IT still not `/retrieve/search` gold. `--locale it` still errors. + +#### 2.1 Spelling harness (null → not live) + +Miss-strata on `search-esci-es-passages-k50` / `search_esci_es.jsonl`: n=62, **0 total-miss** (`stratum_counts={'head-ok': 62}`). Query-level head-ok is not Recall@10 0.353 (item-level). Harness `spell-normalize` wrote `data/search_esci_es_spell.jsonl` (23 queries changed: NFKC, junk punctuation, conservative accent fold). No live LLM. + +Skip-ingest eval `search-esci-es-spell-k50` on `searchbenchescies`, passages k=50: + +| Metric | ES baseline | Spell arm | Gate | +| --- | --- | --- | --- | +| nDCG@10 | 0.577 | **0.595** | win if ≥ 0.597 — **miss** (+0.018) | +| Recall@10 | 0.353 | **0.362** | hold ≥ 0.353 — pass | +| Recall@50 | 0.914 | **0.914** | hold ≥ 0.914 — pass | + +No previously total-miss qid to lift. **Decision:** spelling stays harness-only. Production `search.py` query text is unchanged. Do not mix with Reddy ES 0.849 or US n=74 0.500. + +#### 2.2–2.3 Extras filter + hit-list facets + +`SearchRequestBody.extras: dict[str, str] | None` is an equality filter after fusion, before cutting `k`. Catalog keys live inside that object (and on `SearchHit.extras`), not as body fields named `brand`. Search ingest now copies JSONL `brand` / `color` / `locale` via `meta_keys`. `SearchResponse.facets` counts extras values on the **current** hit list. No merchandising, no `/retrieve/recommend`. + +Live on `searchbenchitsmoke` (3-doc re-ingest allowed): `extras={"locale":"it"}` keeps the three IT docs (Recall@10 1.0); facets `{locale: {it: 3}, brand: {CasaLuce, AtelierNord, FornoBasso}, color: {argento, blu navy, alluminio}}`. `extras={"color":"nope"}` returns empty hits / empty facets. p50 retrieve ~17 ms on this 3-doc brain — **not** a 200 ms SLO claim. Contract test still bans catalog enum field names. + +#### Encoder smoke (M04 Italian-only) + +Harness `local-dense` `paraphrase-multilingual-MiniLM-L12-v2` on `data/search_italian_smoke.jsonl`, labeled brain `searchbenchitmmini`, run `search-italian-minilm-pipeline`. **Did not** reuse `searchbenchitsmoke` 3-large vectors; **did not** re-ingest 74 or ES. n=3, `status: ok`, all three golds retrieved at k=10 (same pipeline recall as `search-italian-smoke` 3-large). **Not** a claim that MiniLM beats 3-large. **Not** Reddy. **Not** ES n=62. Product embedder unchanged. + +### Out of scope + +Wiping brains; raising default `RERANK_MAX_K`; CE as default; graph fusion as default; recsys; LoCoMo/BEAM; claiming Reddy 0.857; MT-US-as-Italian; Amazon-M2 MRR as search quality; merchandising that needs recommend/context. + +--- + +## Located US control (do not redo) + +Passages k=50 n=74: nDCG@10 **0.500**, Recall@10 **0.379**, Recall@50 **0.834**, p50 ~60ms. MiniLM/DeBERTa CE on those hits hurt Recall@10. Live `mode=catalog` MiniLM 0.467/0.341 null. Cascade R@50 0.889 harness-only (live sidecar not indexed). `fusion=cc` alpha sweep best nDCG@10 **0.493** (null vs 0.520). Literal residual Recall@50 **0.655** (null). Head LTR 5-fold CV on stored k=50: nDCG@10 **0.515**, Recall@10 **0.384** (null vs 0.520 / 0.397); overlap-only 0.476 / 0.339. LTR + MiniLM `ce_gain`: nDCG@10 **0.524**, Recall@10 **0.383**, Recall@50 **0.834** (harness win vs 0.520 / 0.379 hold; not live). LTR + DeBERTa-v3-base `ce_gain` RankNet CV: nDCG@10 **0.542**, Recall@10 **0.387**, Recall@50 **0.834** (win vs MiniLM 0.524; horizon 0.70 missed; not live). Gated LightGBM on the same DeBERTa matrix: nDCG@10 **0.516** (null vs RankNet). LTR + DeBERTa applied from 170 train hybrid lists (`searchbenchesciltr2`): nDCG@10 **0.544**, Recall@10 **0.391**, Recall@50 **0.834** (win vs 0.542; horizon missed; not live). LightGBM apply on the same 170 lists: nDCG@10 **0.533** (null vs RankNet 0.544; R@10 0.389 / R@50 0.834 held; not live). RankNet + hybrid-list DeBERTa `ce_gain` (`esci-deberta-v3-base-4class-hybrid170`): nDCG@10 **0.530** (null vs 0.544; R@10 0.385 / R@50 0.834 held; not live). Default path that survived: hybrid BM25+dense, passages, rerank=none, Italian via gated FTS+OR. diff --git a/docs/research/23-search-wands-quality.md b/docs/research/23-search-wands-quality.md new file mode 100644 index 0000000..0f8fe0a --- /dev/null +++ b/docs/research/23-search-wands-quality.md @@ -0,0 +1,100 @@ +# 23 — WANDS first-stage search quality + +Workstream: `/retrieve/search` first-stage quality on a frozen Wayfair WANDS slice. Ledger: `benchmarks.search` only. This document records the **control protocol** and **located evidence**. It is not Chen et al. SOTA and not an ESCI number. + +Every claim is labeled **idea**, **assumption**, **prediction**, **located evidence**, or **decision**. + +--- + +## Focal question + +What is BrainAPI’s live hybrid first-stage quality on a frozen WANDS catalog slice (passages, RRF, `rerank=none`, k=50), and what can that number actually support? + +**Claim this run can support:** on this slice, with this product path, nDCG@10 / Recall@k / p50 were X. + +**Claim this run cannot support:** we match Chen / WANDS SOTA / ESCI 0.500 / Reddy 0.857 / C6 0.544. + +--- + +## Protocol (decision) + +Mirror the ESCI n=74 first-stage control on a **different** brain, catalog, and gain table. + +| Knob | Value | +| --- | --- | +| Dataset | Frozen [`benchmarks/data/search_wands.jsonl`](../../benchmarks/data/search_wands.jsonl) | +| Caps used to build it | `--max-queries 80 --max-docs 2000 --candidates-per-query 40` | +| Actual n | **66** queries / **2000** docs (`_select_catalog` dropped queries with no remaining gold once `max_docs` filled) | +| Slice label | `wands-80-2000` (actual n=66) | +| Brain | `searchbenchwands` (new; never wipe `searchbenchesci*`, `searchbenchesciltr2`, `locomoconv*`, `beam*`, `demorecsys`) | +| Ingest | `skip_enrichment` (default). No `--ingest-graph`. No `--enrich`. | +| Retrieve | `--channels passages --k 50 --ks 5,10,20,50`, fusion RRF, `rerank=none`. No `--rank-pool`. | +| Run | `search-wands-passages-k50` | +| After first eval | `--skip-ingest` only | +| Download freeze | `catalog_overwrite_blocked`: `--force` cannot clobber `search_wands.jsonl` once it exists | + +```bash +./search.sh --brain searchbenchwands evaluate \ + --dataset data/search_wands.jsonl \ + --run search-wands-passages-k50 \ + --channels passages --k 50 --ks 5,10,20,50 +# after first eval: +./search.sh --brain searchbenchwands evaluate \ + --dataset data/search_wands.jsonl \ + --run search-wands-passages-k50 \ + --channels passages --k 50 --ks 5,10,20,50 --skip-ingest +``` + +**Metrics (decision):** + +- **nDCG@10:** linear DCG `rel / log2(rank+1)` with Exact=1, Partial=0.5, Irrelevant=0. Unlabeled retrieved docs gain 0. Affine-equivalent to linear 2/1/0; **not** TREC `2^rel-1`. +- **Recall@10 / @50:** binary over **Exact + Partial** (`gold_doc_ids` = gain > 0). Not ESCI E+S+C. +- **p50 retrieve ms** excluding embed.query. +- n=66, no CIs. Do not mix with ESCI 0.500 / C6 0.544. + +**Subset caveat:** full WANDS is 480 queries / 42,994 products / 233,448 labels. This control searches the ingested 2000-doc brain, not the 40-candidate judged pool and not the 43k catalog. + +Do **not** edit `src/services/api/retrieve.py`, `entities.py`, or `search.py` for this measurement. + +--- + +## Literature (located evidence, 2026-08-20) + +| Item | Status | +| --- | --- | +| Chen, Liu, Liu, Sun, Baltrunas, Schroeder. *WANDS: Dataset for Product Search Relevance Assessment*. ECIR 2022. DOI [10.1007/978-3-030-99736-6_9](https://doi.org/10.1007/978-3-030-99736-6_9). OpenAlex `W4225565720`. | **support-located** (identity). `is_oa=false`, `abstract_inverted_index=null`. Springer LNCS, not OA. | +| arXiv title search `ti:WANDS AND (Wayfair OR "product search relevance")` | **no Chen paper**. Hits were unrelated IR papers. | +| Dataset size ([wayfair/WANDS](https://github.com/wayfair/WANDS/)) | **located evidence:** 480 queries, 42,994 products, 233,448 labels. Exact / Partial / Irrelevant. | +| Published BM25 / hybrid nDCG in Chen et al. | **not located**. Do not invent. Medium notebooks are not primary. | + +--- + +## First-stage control numbers + +**Located evidence (2026-08-20).** Live `search-wands-passages-k50` on `searchbenchwands`. `eval.json`: `skip_ingest=false`, `ingest.status=completed`, `n_docs_mapped=2000`, `ingest_graph=false`, channels `passages`, fusion `rrf`, `rerank=none`, k=50. Ingest wall ~14 min. Ledger: `benchmarks.search` run `search-wands-passages-k50`. + +| Metric | Value | +| --- | --- | +| n | 66 queries / 2000 docs | +| nDCG@10 | **0.823** | +| nDCG@20 | 0.812 | +| Recall@10 | **0.269** | +| Recall@20 | 0.465 | +| Recall@50 | **0.837** | +| MRR | **0.925** | +| p50 retrieve | **87 ms** | +| p95 retrieve | 97 ms | + +Gold (Exact+Partial) min/median/max = 1 / 40 / 40; mean 34.1. Structural Recall@10 ceiling = mean `min(10,|gold|)/|gold|` = **0.365**. Observed 0.269 is below that ceiling, not below ESCI 0.379 (different gold density and gain table). 60/66 queries have a relevant at rank 1. Two queries have nDCG@10 = 0: `Bath Rugs & Mats` (Recall@50 also 0) and `Vanities` (first relevant at rank 27; Recall@50 = 1.0). + +**Do not mix** with US ESCI n=74 nDCG@10 0.500 / C6 0.544, Reddy 0.857, or unpublished Medium WANDS notebooks. Chen et al. nDCG remains **not located**. + +After this run, freeze the index: `--skip-ingest` only. Do not re-ingest `searchbenchwands`. Do not wipe ESCI brains. + +--- + +## Isolation + +- Brains: `searchbenchwands` only for this catalog. Skip-ingest on `searchbenchesci74`. +- Product default stays hybrid RRF, `rerank=none`, omitted `channels=["passages"]`. +- Ranking / LTR / CE / graph on WANDS are out of scope until a new human decision. diff --git a/docs/research/24-search-catalog-graph.md b/docs/research/24-search-catalog-graph.md new file mode 100644 index 0000000..c2215ce --- /dev/null +++ b/docs/research/24-search-catalog-graph.md @@ -0,0 +1,175 @@ +# 24 — Opt-in catalog graph + hybrid search + +Workstream: structured catalog ingest + `channels=["communities"]` product retrieve. Ledger: `benchmarks.search` only, **architecture demo**. This is not a quality-default flip and not a claim against frozen WANDS passages nDCG@10 **0.823** or ESCI n=74 **0.500**. + +Every claim is labeled **idea**, **assumption**, **prediction**, **located evidence**, or **decision**. + +--- + +## Focal question + +Can BrainAPI ship an **opt-in** catalog path: labeled product/hub nodes with `search_text` embeddings + node BM25, hub intersection (CLASS ∩ ATTR), then hybrid-rank **product nodes** inside the communities channel — without changing the live omitted-`channels` default (`["passages"]`)? + +**Claim this run can support (located evidence, 2026-08-21):** on brain `searchbenchwandsgraph`, isolated communities nDCG@10 **0.409** / Recall@10 **0.148** / Recall@50 **0.287** / p50 retrieve **5609 ms** (ex-embed); paired passages nDCG@10 **0.823** / Recall@10 **0.269** / Recall@50 **0.837** / p50 retrieve **99 ms**. Architecture demo only. + +**Claim the 2026-08-21 join round can support (decision + prediction):** architecture-demo that (a) paired passages nDCG@10 on `searchbenchwandsgraph` stays ~0.823, (b) catalog passage hits expose `node_id` = `doc_id` = ENTITY uuid, (c) that uuid round-trips to `GET /retrieve/entities/neighbors`. + +**Claim this run cannot support:** we beat frozen `searchbenchwands` 0.823, ESCI 0.500, Reddy 0.857, or Chen et al. Communities beat passages. We should change the product default. Mixing chunk ids and node ids in default RRF. + +--- + +## Architecture (decision) + +One KB, three contracts (ADR-007). Search stays `POST /retrieve/search`. Recs stay `/retrieve/recommend`. Interactions may be ingested onto the same graph; they are not scored as HitRate and `user_id` is not a `SearchRequestBody` field. Optional search `target` (query-gated rerank of retrieved hits) is [ADR-008](../decisions/008-query-gated-search-personalization.md) / [25-personalized-search.md](25-personalized-search.md), not this workstream. + +```text +catalog JSONL + → skip_enrichment chunk ingest (passages) + → POST /ingest/structured (HAS triples; entity uuid = doc_id) + → CLASS / ATTR / TYPE hubs + ENTITY products (search_text on nodes) + → channels=["communities"]: match hubs → intersect if ≥2 hub kinds else union + → RRF(node BM25 ∪ node dense) on the product set (inside graph_channels) +``` + +- **Passages stay the live default (decision, 2026-08-21).** Omitted `channels` remains `["passages"]`. First-stage is query → passage. The catalog graph is not a competing candidate generator. `rerank=none`. `search.py` fusion does not mix chunk ids and node ids on the default path. +- **`node_id` is a sidecar, not fusion (decision).** After ranking, catalog passage hits attach `SearchHit.node_id` (and unique `SearchResponse.node_ids` in hit order) by parsing `DOCID ` from fetched chunk text. `entity_uuid(doc_id)` is `str(doc_id)`, so the join is deterministic and does not extra-round-trip the graph. Graph-channel item hits set `node_id = id`. Missing marker → `node_id` is null. The field is **`node_id`**, not `product_id` / `sku` / `brand` / `user_id`. +- **Clients use existing graph reads.** `node_id` is the product ENTITY uuid for `/retrieve/entities/neighbors`, `/retrieve/entity/synergies`, `/retrieve/recommend`, entity info. Recsys and memory already consume the graph. This round does not change `/retrieve/context` (ADR-006). +- **Product ENTITY `search_text` is the catalog blob (decision).** Prefer `doc["text"]` when present, else title + class + features + description (plus hierarchy/brand/color when composed). Hub `search_text` stays short name/key text. +- **Static catalog attributes are a direct edge (decision, 2026-08-21).** Mapper `has_triple` writes `ENTITY -HAS-> CLASS|TYPE|ATTR` with no EVENT mid-node. Structured ingest collapses the same shape if an old 5-tuple `HAS` wrapper arrives with no `happened_at`. Keep EVENT for real occurrences (`USER -MADE-> View/Purchase -TARGETED-> PRODUCT`, with `happened_at`). Polarity is not a reason to keep HAS wrappers. **Located evidence:** `searchbenchwandsgraph` was persisted on the 2-edge wrapper (≈32k HAS events); that brain is not rewritten this change. +- **Catalog product path is explicit:** `channels=["communities"]` (optionally `entities`) remains an opt-in architecture demo. Do not re-eval communities nDCG as a quality gate this round. +- **Hubs, not Louvain.** Communities already means TYPE/CLASS/TOPIC/ATTR hubs. +- **Continuous attributes are not hubs.** Price / measures / rating stay node properties when present. Style / color / material / class become ATTR/CLASS hubs. +- **Eval isolation:** brain `searchbenchwandsgraph` only. Never wipe `searchbenchwands`, `searchbenchesci74`, `searchbenchescies`, `searchbenchesciltr2`, `locomoconv*`, `beam*`, `demorecsys`. Frozen JSONL `benchmarks/data/search_wands.jsonl` is reuse-only; do not wipe frozen structured brains. ENTITY text/embed refresh, if run, refuses `FROZEN_STRUCTURED_BRAINS` and does not resubmit Celery architect. + +Mapper lives in product code [`src/core/search/catalog_graph.py`](../../src/core/search/catalog_graph.py). The harness re-exports it from [`benchmarks/search/mapping.py`](../../benchmarks/search/mapping.py) (repo root on `sys.path`). `--ingest-graph` / `--interactions` targeting frozen structured brains raises `SystemExit`. + +--- + +## Literature (located evidence, 2026-08-20) + +**Support-located for query–document first-stage (not hub-walk):** + +- Nigam et al., *Semantic Product Search*, arXiv `1907.00937` (KDD 2019) — dense semantic matching complements lexical; query → document, not query → hub → walk. +- Choi et al., *Semantic Product Search for Matching Structured Product Catalogs*, arXiv `2008.08180` — fielded catalog text plus lexical features. +- Balog / Garigliotti entity-oriented search, arXiv `1802.08010` — query → typed entities → fulfill (hubs then products) is a graph-read pattern, not the live search default. + +**Challenge-located / do not overclaim (PKG as the only candidate generator):** + +- Xu et al., *Product Knowledge Graph Embedding for E-commerce*, arXiv `1911.12481`, DOI `10.1145/3336191.3371778` — product KG used in ranking **and** recs; does not validate symbolic hub intersection as first-stage IR. +- Ai et al. DREM, arXiv `1909.07212` — user–item KG embeddings for search; learned relations, not HAS expansion. +- Zhang et al. PKGM, arXiv `2105.00388` — PKG for recs/classification, not first-stage nDCG. +- No located paper that community-detection retrieval beats BM25+dense on WANDS/ESCI. Absence from this bound is not “never studied.” + +Prior BrainAPI **located evidence:** peer RRF of graph lists with passages **hurt** an ESCI slice (nDCG@10 0.682 vs 0.758 passages). Do not treat extra graph RRF peers as a quality win. + +--- + +## Protocol (decision) + +| Knob | Value | +| --- | --- | +| Dataset | Frozen [`benchmarks/data/search_wands.jsonl`](../../benchmarks/data/search_wands.jsonl) (reuse only) | +| Actual n | **66** queries / **2000** docs | +| Brain | `searchbenchwandsgraph` (**new**; never `searchbenchwands`) | +| Ingest | `skip_enrichment` chunks, then `--ingest-graph`. After first eval, `--skip-ingest` | +| Communities | `--channels communities --k 50 --ks 5,10,20,50`, fusion RRF, `rerank=none` | +| Paired passages | same brain, `--channels passages`, `--skip-ingest` | +| Runs | `search-wandsgraph-communities-k50`, `search-wandsgraph-passages-k50` | +| Ledger claim | `architecture-demo` | + +```bash +./search.sh --brain searchbenchwandsgraph evaluate \ + --dataset data/search_wands.jsonl \ + --run search-wandsgraph-communities-k50 \ + --channels communities --k 50 --ks 5,10,20,50 \ + --ingest-graph --skip-ingest --timeout 10800 +./search.sh --brain searchbenchwandsgraph evaluate \ + --dataset data/search_wands.jsonl \ + --run search-wandsgraph-passages-k50 \ + --channels passages --k 50 --ks 5,10,20,50 \ + --skip-ingest +./search.sh --brain searchbenchwandsgraph smoke \ + --dataset data/search_wands.jsonl \ + --interactions data/search_wands_interactions_smoke.jsonl \ + --channels events --skip-ingest --timeout 1800 +``` + +**Go/no-go for a later default (not executed this round):** communities nDCG@10 ≥ paired passages **on this brain**. **Located evidence:** 0.409 < 0.823 — gate fails. Still must not regress frozen ESCI 0.500 / WANDS 0.823 protocols. Do not quote either row as beating those frozen numbers. + +--- + +## Mapper / graph write (located evidence, 2026-08-20) + +Offline count on the frozen JSONL via `docs_to_triples` (not a quality claim): + +| Count | Value | +| --- | --- | +| HAS triples | 32208 | +| Product ENTITY nodes | 2000 | +| Hub nodes | 3641 | +| CLASS hubs (unique names) | 157 | +| ATTR hubs (unique names) | 3484 | +| TYPE hubs | 0 (frozen JSONL has no `hierarchy` / `category_hierarchy` field) | +| Docs with `class` | 1863 | +| Docs with `description` field | **0** | +| ATTR `modern` present | yes (`dsprimaryproductstyle : modern` and similar) | + +Those 32208 HAS triples were written as **two edges + EVENT mid-node** on `searchbenchwandsgraph`. **Decision (2026-08-21):** new mapper/ingest writes them as one `HAS` edge. Do not treat the live 37383/64416 counts as the post-change shape. Do not wipe that brain to apply this. + +**Located limitation (ingest already on disk, 2026-08-21):** the first `searchbenchwandsgraph` persist wrote title-only ENTITY `search_text` (mapper used title + missing `description` field). Frozen WANDS JSONL rows expose `title` / `class` / `features` and a full `text` blob, but not a separate `description` / `hierarchy` field. **Decision:** new mapper writes prefer `doc["text"]` (the same catalog blob passages embed). Already-persisted nodes do not rewrite until an ENTITY-only `search_text` + `nodes` vector refresh (2000 products, not 32k HAS events, not Celery architect). Overwrite of `search_wands.jsonl` remains blocked. Node BM25 after that refresh is **architecture-demo**, not a quality claim vs passages 0.823. + +**Assumption:** many furniture queries are class-like; CLASS∩ATTR intersection may rarely fire on this slice. Fixture tests guarantee the operator; WANDS eval is still valid on union+hybrid. Feature keys are capped (`FEATURE_KEY_CAP=16`) and measure/price tokens are skipped; unique ATTR values across 2000 products still numbered in the thousands. + +Structured ingest embeds `properties.search_text` into store `"nodes"` (cache key `uuid:{uuid}`). Nodes without `search_text` still embed **name only** (memory path). Node BM25 (`kg_nodes.search_tsv`, english) is gated on `SEARCH_ENABLED` like chunk DDL. Italian node FTS is not added this round. + +--- + +## Isolated architecture eval + +**Located evidence (2026-08-21).** Brain `searchbenchwandsgraph`. `rerank=none`, k=50, n=66. Ledger `claim: architecture-demo`. Do **not** quote vs frozen WANDS passages 0.823 or ESCI 0.500. + +Live graph after Celery persist (exact counts): **37383** `kg_nodes`, **64416** `kg_relationships` (32208 HAS triples × 2 EVENT-mid edges). `vectors_relationships` leftover HAS from a killed first run is not a success signal. + +Ingest harness timed out twice (`1800s` on batch 1; `10800s` on the remaining-triple parent while Celery was still writing). Persist finished in the worker; communities metrics below are **skip-ingest** search on that completed graph. Do not treat harness timeout as a failed graph write. + +| Run | Channels | nDCG@10 | Recall@10 | Recall@50 | p50 retrieve ms (ex-embed) | +| --- | --- | --- | --- | --- | --- | +| `search-wandsgraph-communities-k50` | communities | 0.409 | 0.148 | 0.287 | 5609 | +| `search-wandsgraph-passages-k50` | passages | 0.823 | 0.269 | 0.837 | 99 | + +ADR-007 architecture SLO: p50 < 200 ms excluding embed. Passages **99 ms** meets it. Communities **5609 ms** misses it (p50 embed 205 ms recorded separately). SLO check, not a quality win. + +Interactions smoke (`channels=["events"]`, `--skip-ingest`, synthetic view/cart/purchase on WANDS `doc_id`s `0` / `10` / `13`) **passed** as pipeline-only (`retrieve/search(wands-0) -> 0 hits`). Not mixed into the communities nDCG ledger row. Not recsys HitRate. No `user_id` on the search body. + +--- + +## Passages first-stage + node_id (located evidence, 2026-08-21) + +Skip-ingest only on `searchbenchwandsgraph`. No new architect persist. Ledger `claim: architecture-demo`. Do **not** mix with frozen ESCI 0.500. + +- Smoke (`channels=["passages"]`, 2 queries): catalog hits expose `node_id` = `doc_id` = ENTITY uuid (example `7468`); `GET /retrieve/entities/neighbors?uuid=7468` returned **200**. Neighbor `count` may be 0 depending on the neighbors filter; 200 is the round-trip gate. +- Skip-ingest k=50 eval `search-wandsgraph-passages-nodeid-k50`: nDCG@10 **0.823**, Recall@50 **0.837**, p50 retrieve **84 ms** (ex-embed). Same nDCG as paired passages `search-wandsgraph-passages-k50`. All 50 hits on `wands-0` had `node_id` matching DOCID. Join is string parse only. + +Graph shape at this check: **37393** `kg_nodes` / **64429** `kg_relationships` (~1996 ENTITY / ~31756 EVENT). Close to the persist snapshot; not a rewrite. + +ENTITY-only `search_text` + `nodes` vector refresh (`backfill-entity-text` on `searchbenchwandsgraph`, 2026-08-21): **1996** ENTITY nodes updated, **4** missing, **0** skipped. EVENT count stayed **31756**; total nodes stayed **37393**. Postgres node BM25 for `rubberwood` / `modern` hits product ENTITY `0` (`solid wood platform bed`). `channels=["entities"]` query `rubberwood` returned 200 with `node_id == id`. Refuses `FROZEN_STRUCTURED_BRAINS` (`searchbenchwands` exits 1). **Architecture-demo only** — not a quality claim vs passages 0.823. + +--- + +## Passages first-stage + node_id (decision, 2026-08-21) + +Recall@50 0.287 vs 0.837 was **candidate generation**, not ranking. Hub-walk cannot recover gold that sits outside the matched CLASS. Fanout/intersection/HAS-ANN remain valid **graph-channel** hygiene, not the search default. Keep `channels=["communities"]` as an opt-in demo; do not treat 0.409 as a quality gate. + +**Out of scope this round:** raising communities nDCG@10 from 0.409; flipping the live default; mixed-ID RRF; `product_id` on `SearchRequestBody` / `SearchHit`; wiping frozen brains; editing `retrieve.py` / `entities.py`; a new 20k-triple architect persist. + +--- + +## Explicitly later + +- Fusion of passages + communities in `search.py` (needs this `node_id` join, then rank **products**, not mixed chunk/node RRF). +- Communities fanout/union/HAS-ANN hygiene as opt-in channel quality. +- `/retrieve/context` using catalog node text. +- Quality-default flip. +- Range filters for price/measures; SKU/configurable-option child nodes; mood taxonomies beyond feature strings. +- Learned PKG embeddings (Xu/DREM/PKGM) or GNN recs on `/retrieve/search`. +- Query-gated personalized search is a **separate** workstream, already shipped: [25-personalized-search.md](25-personalized-search.md), [ADR-008](../decisions/008-query-gated-search-personalization.md). Not a first-stage change and not a quality-default flip. diff --git a/docs/research/25-personalized-search.md b/docs/research/25-personalized-search.md new file mode 100644 index 0000000..b41be86 --- /dev/null +++ b/docs/research/25-personalized-search.md @@ -0,0 +1,98 @@ +# 25 — Query-gated personalized search + +Workstream: optional `target` on `/retrieve/search` reranks **already retrieved** hits. Ledger, if used: `benchmarks.search` only, **architecture demo**. This is not a quality-default flip, not recsys HitRate, and not a claim against frozen WANDS passages nDCG@10 **0.823** or ESCI n=74 **0.500**. + +Every claim is labeled **idea**, **assumption**, **prediction**, **located evidence**, or **decision**. Binding contract: [ADR-008](../decisions/008-query-gated-search-personalization.md). Catalog graph / `node_id` join: [24-search-catalog-graph.md](24-search-catalog-graph.md). Recs isolation: [16-recsys-eval-protocol.md](16-recsys-eval-protocol.md). + +--- + +## Focal question + +Can BrainAPI personalize product search **without** (a) changing first-stage candidate generation, (b) sharing `/retrieve/recommend` request semantics, or (c) turning inferred taste into a hard `extras` filter? + +**Claim this ship can support (decision, 2026-08-21):** omit `target` → anonymous ranking unchanged. Set `target` → query-gated blend of retrieve score vs user ATTR prefs over hits that already have `node_id`. Unknown user and all-zero prefs preserve order. `extras` AND stays user-asserted. + +**Claim this ship cannot support:** we beat frozen WANDS 0.823 or ESCI 0.500. Recsys HitRate on `demorecsys` is search nDCG. HEM/TEM-class query–user–item training. Flipping live default `channels=["passages"]`. Using `user_id` on `SearchRequestBody`. + +--- + +## Architecture (decision) + +```text +query + → first-stage (default: passages BM25∪dense) + → extras AND (user-asserted only) + → node_id attach (DOCID parse / graph hit id) + → if target set: score given node_ids vs user prefs, blend with λ(query) + → cut to k +``` + +- **Identity is `target`, not `user_id` (decision).** Same field name as `/retrieve/recommend`. GET `/retrieve/search?target=` matches the body. Schema tests ban `user_id` / `product_id` / `sku` / `brand` on the search body. +- **Rerank, not candidate generation (decision).** Do not call `EntityRecommendRetriever.recommend()`. Communities as first-stage already failed on `searchbenchwandsgraph` (Recall@50 **0.287** vs passages **0.837**; [24](24-search-catalog-graph.md)). Personalize scores only the retrieved `node_id`s. +- **Query λ (decision).** Digits or SKU-like tokens → 0 (navigational / specific). One content token ≈ 0.85; two ≈ 0.5; three ≈ 0.25; four+ ≈ 0.1. Blend is min-max retrieve vs pref; λ=0 or all-zero prefs preserve retrieve order and never drop ids. +- **Pref vector (decision).** `user_pref_weights` sums write-time `USER -PREFERS-> ATTR` (long-term) **and** recency-decayed `USER → EVENT → ITEM → ATTR` (short-term, 14-day half-life). Search does **not** skip events when PREFERS exist — unlike recommend’s `_attribute_pref_targets`, which still does. Treat ENTITY and PRODUCT as items. +- **Write path (decision).** Dated view / cart / purchase / favorite stay EVENT wrappers. Selected `options` / `attributes` emit `USER -PREFERS-> ATTR` with the **same hub uuid** as catalog HAS (`hub:attr:70s`). Options are not merged into the product `catalog_doc` (would stamp the SKU). Catalog `brand` / `color` / `class` on the row still PRODUCT-HAS. Architect persists any no-event direct triple (`HAS`, `PREFERS`, later `AVOIDS`), not only static HAS. +- **I4 weights (decision).** Favorite / wishlist **0.7**; unknown behaviors **0.2** (was 1.0). Shared by `src/core/search/recommend.py` and `plugins/features-rec/models/mapping.py`. +- **I7 rejected (decision).** Inferred style is blend-only. Do not write `extras={style: 70s}` from history. User-asserted `extras` remains a hard AND filter, applied **before** personalize. +- **SLO (decision).** Omitted-`target` path is the ADR-007 200 ms default. Personalized path is opt-in; do not cite its latency as that SLO. +- **Hits without `node_id` stay at pref 0.** Catalog mode personalizes `k_ret` then cuts to request `k`. + +Code: [`src/core/search/personalize.py`](../../src/core/search/personalize.py), wired in [`src/services/api/controllers/search.py`](../../src/services/api/controllers/search.py) after extras / `node_id`. Mapper: [`src/core/search/catalog_graph.py`](../../src/core/search/catalog_graph.py) `prefers_triple` / `interaction_to_triples`. + +--- + +## Literature (located evidence, 2026-08-21) + +**Support-located for query-dependent personalization (not always-on):** + +- Ai, Hill, Vishwanathan, Croft, *A Zero Attention Model for Personalized Product Search*, arXiv `1908.11322` (CIKM 2019) — commercial logs: personalization **depends on query characteristics**; a zero vector lets the model pay **no** attention to history. Maps to λ=0 on specific / digit queries, not to training ZAM. +- Bi, Ai, Croft, *A Transformer-based Embedding Model for Personalized Product Search*, arXiv `2005.08936` (SIGIR 2020) — TEM: “personalization does not always improve product search quality”; undifferentiated personalization is the failure mode. Maps to query-gated blend. **Does not** authorize training TEM here. + +**Challenge-located / deferred (learned query–user–item first-stage):** + +- Ai, Zhang, Bi, Chen, Croft, *Learning a Hierarchical Embedding Model for Personalized Product Search*, SIGIR 2017, DOI `10.1145/3077136.3080813` — HEM jointly embeds query, user, item from purchase language. Needs query×user logs. **I6 deferred.** +- Ai et al. DREM, arXiv `1909.07212` — learned user–item KG for search; not ATTR-hub overlap on a retrieved list. Already bounded in [24](24-search-catalog-graph.md). +- Nigam et al., *Semantic Product Search*, arXiv `1907.00937` — first-stage remains query → document. Personalize does not replace it. + +**Wrong metric if cited as search quality:** + +- Recsys next-item HitRate on `demorecsys` via `/retrieve/recommend` ([16](16-recsys-eval-protocol.md)) is a different API, task, brain, and label. WANDS / ESCI have no users. + +--- + +## Protocol (decision) + +| Knob | Value | +| --- | --- | +| Identity | Optional `target`; omit = control | +| First-stage | Unchanged; default omitted `channels=["passages"]` | +| Filter | User-asserted `extras` AND, then personalize | +| Unit tests | `tests/test_search_personalize.py` (λ, blend, PREFERS+recency, extras AND, schema ban) | +| Optional live smoke | skip-ingest on `searchbenchwandsgraph` only; fixture `benchmarks/data/search_personalize_smoke.jsonl` (`wands-u70`, favorite + `style=70s` options). Mapping unit-tested; live ingest is maintainer-gated | +| Ledger claim | If ledgered: `architecture-demo` only | + +```bash +# Mapping-only fixture (gitignored under benchmarks/data/*). Compare omitted +# target vs target=wands-u70 on a generic query (lamp) vs a specific query +# (oak dining table 180cm). Skip-ingest on searchbenchwandsgraph after +# maintainer OK; do not wipe or re-architect. +# Fixture: benchmarks/data/search_personalize_smoke.jsonl +``` + +**Go/no-go for a later default (not executed):** personalized nDCG on a query×user labeled set beats omitted-`target` without regressing specific queries. WANDS/ESCI cannot run that gate. Do not quote vs frozen 0.823 / 0.500. + +--- + +## Isolation + +Never wipe `searchbenchwands`, `searchbenchesci74`, `searchbenchescies`, `searchbenchesciltr2`, `locomoconv*`, `beam*`, `demorecsys`. Optional smoke only on `searchbenchwandsgraph`. Do not score recsys HitRate as search nDCG. Do not mix architecture-demo numbers with frozen WANDS / ESCI quality rows. + +--- + +## Explicitly later + +- **I6 deferred:** HEM / TEM / ZAM training. Needs real query+user logs, not WANDS. Not in this ship. +- Learned query-conditioned attention (ZAM/TEM) instead of the token-count λ heuristic. +- Recommend `_attribute_pref_targets` combining PREFERS + EVENT (search already does; recs still skip EVENT when PREFERS exist). +- `AVOIDS` polarity as a direct edge. +- Using personalized search latency as the ADR-007 200 ms product claim. diff --git a/entrypoint.sh b/entrypoint.sh index ea25190..cea3e35 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,6 +1,19 @@ #!/bin/bash set -e +plugin_failure_policy="${PLUGIN_FAILURE_POLICY:-}" +if [ -z "$plugin_failure_policy" ]; then + if [ "${ENV:-production}" = "development" ]; then + plugin_failure_policy="warn" + else + plugin_failure_policy="fail" + fi +fi +if [ "$plugin_failure_policy" != "fail" ] && [ "$plugin_failure_policy" != "warn" ]; then + echo "[brainapi] PLUGIN_FAILURE_POLICY must be 'fail' or 'warn'" >&2 + exit 64 +fi + if [ -d /app/.cache ]; then chown -R appuser:appuser /app/.cache 2>/dev/null || true fi @@ -16,8 +29,16 @@ if [ -n "$BRAINAPI_PLUGINS" ]; then version="${plugin_spec##*:}" [ "$name" = "$version" ] && version="latest" echo "[brainapi] Installing plugin: $name v$version" - gosu appuser /app/.venv/bin/python -m src.core.plugins.cli plugins install "$name" --version "$version" || echo "[brainapi] WARNING: Failed to install plugin '$name'" + if ! setpriv --reuid=appuser --regid=appuser --init-groups -- \ + /app/.venv/bin/python -m src.core.plugins.cli plugins install "$name" --version "$version"; then + if [ "$plugin_failure_policy" = "fail" ]; then + echo "[brainapi] Failed to install required plugin '$name'" >&2 + exit 1 + fi + echo "[brainapi] WARNING: Failed to install plugin '$name'" >&2 + fi done fi -exec gosu appuser /app/.venv/bin/python "$@" +exec setpriv --reuid=appuser --regid=appuser --init-groups -- \ + /app/.venv/bin/python "$@" diff --git a/mcp-stdio-http-bridge/Cargo.lock b/mcp-stdio-http-bridge/Cargo.lock index 49d36ae..31f1e1a 100644 --- a/mcp-stdio-http-bridge/Cargo.lock +++ b/mcp-stdio-http-bridge/Cargo.lock @@ -1,21 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 - -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +version = 4 [[package]] name = "aho-corasick" @@ -32,21 +17,6 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link", -] - [[package]] name = "base64" version = "0.21.7" @@ -276,12 +246,6 @@ dependencies = [ "wasip3", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "h2" version = "0.3.27" @@ -322,12 +286,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - [[package]] name = "http" version = "0.2.12" @@ -379,7 +337,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -621,24 +579,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - [[package]] name = "mio" -version = "0.8.11" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -668,25 +617,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "num_cpus" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.21.3" @@ -920,12 +850,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "rustc-demangle" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" - [[package]] name = "rustix" version = "1.1.3" @@ -1097,6 +1021,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1180,28 +1114,26 @@ dependencies = [ [[package]] name = "tokio" -version = "1.35.1" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ - "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.5", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", diff --git a/poetry.lock b/poetry.lock index ff531ce..63f2a0c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,14 +2,14 @@ [[package]] name = "accelerate" -version = "1.12.0" +version = "1.14.0" description = "Accelerate" optional = false python-versions = ">=3.10.0" groups = ["main", "docling-ocr"] files = [ - {file = "accelerate-1.12.0-py3-none-any.whl", hash = "sha256:3e2091cd341423207e2f084a6654b1efcd250dc326f2a37d6dde446e07cabb11"}, - {file = "accelerate-1.12.0.tar.gz", hash = "sha256:70988c352feb481887077d2ab845125024b2a137a5090d6d7a32b57d03a45df6"}, + {file = "accelerate-1.14.0-py3-none-any.whl", hash = "sha256:e94390c2863b873be18f623f9df48a0d8fe5eff13ea7f1a00092b0a7904888c6"}, + {file = "accelerate-1.14.0.tar.gz", hash = "sha256:41b9c4377a54e0b460a959b0defa1b736e4ca0a2373252d9a539964c2afe3c8d"}, ] [package.dependencies] @@ -30,7 +30,7 @@ sagemaker = ["sagemaker"] test-dev = ["bitsandbytes", "datasets", "diffusers", "evaluate", "scikit-learn", "scipy", "timm", "torchdata (>=0.8.0)", "torchpippy (>=0.2.0)", "tqdm", "transformers"] test-fp8 = ["torchao"] test-prod = ["parameterized", "pytest (>=7.2.0)", "pytest-order", "pytest-subtests", "pytest-xdist"] -test-trackers = ["comet-ml", "dvclive", "matplotlib", "swanlab[dashboard]", "tensorboard", "trackio", "wandb"] +test-trackers = ["dvclive", "matplotlib", "swanlab[dashboard]", "tensorboard", "trackio", "wandb"] testing = ["bitsandbytes", "datasets", "diffusers", "evaluate", "parameterized", "pytest (>=7.2.0)", "pytest-order", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "timm", "torchdata (>=0.8.0)", "torchpippy (>=0.2.0)", "tqdm", "transformers"] [[package]] @@ -48,6 +48,18 @@ files = [ [package.dependencies] vine = ">=5.0.0,<6.0.0" +[[package]] +name = "annotated-doc" +version = "0.0.5" +description = "Document parameters, class attributes, return types, and variables inline, with Annotated." +optional = false +python-versions = ">=3.9" +groups = ["main", "docling-ocr", "local-ml"] +files = [ + {file = "annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101"}, + {file = "annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb"}, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -103,7 +115,7 @@ version = "4.11.0" description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc"}, {file = "anyio-4.11.0.tar.gz", hash = "sha256:82a8d0b81e318cc5ce71a5f1f8b5c4e63619620b63141ef8c995fa0db95a57c4"}, @@ -393,18 +405,6 @@ numpy = "*" [package.extras] doc = ["gitpython", "numpydoc", "sphinx"] -[[package]] -name = "cachetools" -version = "6.2.1" -description = "Extensible memoizing collections and decorators" -optional = false -python-versions = ">=3.9" -groups = ["main"] -files = [ - {file = "cachetools-6.2.1-py3-none-any.whl", hash = "sha256:09868944b6dde876dfd44e1d47e18484541eaf12f26f29b7af91b26cc892d701"}, - {file = "cachetools-6.2.1.tar.gz", hash = "sha256:3f391e4bd8f8bf0931169baf7456cc822705f4e2a31f840d218f445b9a854201"}, -] - [[package]] name = "catalogue" version = "2.0.10" @@ -480,7 +480,7 @@ version = "2025.10.5" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "certifi-2025.10.5-py3-none-any.whl", hash = "sha256:0f212c2744a9bb6de0c56639a6f68afe01ecd92d91f14ae897c4fe7bbeeef0de"}, {file = "certifi-2025.10.5.tar.gz", hash = "sha256:47c09d31ccf2acf0be3f701ea53595ee7e0b8fa08801c6624be771df09ae7b43"}, @@ -709,14 +709,14 @@ files = [ [[package]] name = "click" -version = "8.3.0" +version = "8.4.2" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" groups = ["main", "docling-ocr"] files = [ - {file = "click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc"}, - {file = "click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4"}, + {file = "click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76"}, + {file = "click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6"}, ] [package.dependencies] @@ -798,12 +798,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main", "docling-ocr"] +groups = ["main", "dev", "docling-ocr", "local-ml"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", docling-ocr = "platform_system == \"Windows\" or sys_platform == \"win32\""} +markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", dev = "sys_platform == \"win32\"", docling-ocr = "platform_system == \"Windows\" or sys_platform == \"win32\"", local-ml = "platform_system == \"Windows\""} [[package]] name = "colorlog" @@ -856,75 +856,65 @@ pybind11 = ">=2.10.0" [[package]] name = "cryptography" -version = "46.0.4" +version = "50.0.0" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false -python-versions = "!=3.9.0,!=3.9.1,>=3.8" -groups = ["main"] -files = [ - {file = "cryptography-46.0.4-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:281526e865ed4166009e235afadf3a4c4cba6056f99336a99efba65336fd5485"}, - {file = "cryptography-46.0.4-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5f14fba5bf6f4390d7ff8f086c566454bff0411f6d8aa7af79c88b6f9267aecc"}, - {file = "cryptography-46.0.4-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:47bcd19517e6389132f76e2d5303ded6cf3f78903da2158a671be8de024f4cd0"}, - {file = "cryptography-46.0.4-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:01df4f50f314fbe7009f54046e908d1754f19d0c6d3070df1e6268c5a4af09fa"}, - {file = "cryptography-46.0.4-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5aa3e463596b0087b3da0dbe2b2487e9fc261d25da85754e30e3b40637d61f81"}, - {file = "cryptography-46.0.4-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:0a9ad24359fee86f131836a9ac3bffc9329e956624a2d379b613f8f8abaf5255"}, - {file = "cryptography-46.0.4-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:dc1272e25ef673efe72f2096e92ae39dea1a1a450dd44918b15351f72c5a168e"}, - {file = "cryptography-46.0.4-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:de0f5f4ec8711ebc555f54735d4c673fc34b65c44283895f1a08c2b49d2fd99c"}, - {file = "cryptography-46.0.4-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:eeeb2e33d8dbcccc34d64651f00a98cb41b2dc69cef866771a5717e6734dfa32"}, - {file = "cryptography-46.0.4-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:3d425eacbc9aceafd2cb429e42f4e5d5633c6f873f5e567077043ef1b9bbf616"}, - {file = "cryptography-46.0.4-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:91627ebf691d1ea3976a031b61fb7bac1ccd745afa03602275dda443e11c8de0"}, - {file = "cryptography-46.0.4-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2d08bc22efd73e8854b0b7caff402d735b354862f1145d7be3b9c0f740fef6a0"}, - {file = "cryptography-46.0.4-cp311-abi3-win32.whl", hash = "sha256:82a62483daf20b8134f6e92898da70d04d0ef9a75829d732ea1018678185f4f5"}, - {file = "cryptography-46.0.4-cp311-abi3-win_amd64.whl", hash = "sha256:6225d3ebe26a55dbc8ead5ad1265c0403552a63336499564675b29eb3184c09b"}, - {file = "cryptography-46.0.4-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:485e2b65d25ec0d901bca7bcae0f53b00133bf3173916d8e421f6fddde103908"}, - {file = "cryptography-46.0.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:078e5f06bd2fa5aea5a324f2a09f914b1484f1d0c2a4d6a8a28c74e72f65f2da"}, - {file = "cryptography-46.0.4-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:dce1e4f068f03008da7fa51cc7abc6ddc5e5de3e3d1550334eaf8393982a5829"}, - {file = "cryptography-46.0.4-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:2067461c80271f422ee7bdbe79b9b4be54a5162e90345f86a23445a0cf3fd8a2"}, - {file = "cryptography-46.0.4-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:c92010b58a51196a5f41c3795190203ac52edfd5dc3ff99149b4659eba9d2085"}, - {file = "cryptography-46.0.4-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:829c2b12bbc5428ab02d6b7f7e9bbfd53e33efd6672d21341f2177470171ad8b"}, - {file = "cryptography-46.0.4-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:62217ba44bf81b30abaeda1488686a04a702a261e26f87db51ff61d9d3510abd"}, - {file = "cryptography-46.0.4-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:9c2da296c8d3415b93e6053f5a728649a87a48ce084a9aaf51d6e46c87c7f2d2"}, - {file = "cryptography-46.0.4-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:9b34d8ba84454641a6bf4d6762d15847ecbd85c1316c0a7984e6e4e9f748ec2e"}, - {file = "cryptography-46.0.4-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:df4a817fa7138dd0c96c8c8c20f04b8aaa1fac3bbf610913dcad8ea82e1bfd3f"}, - {file = "cryptography-46.0.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b1de0ebf7587f28f9190b9cb526e901bf448c9e6a99655d2b07fff60e8212a82"}, - {file = "cryptography-46.0.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9b4d17bc7bd7cdd98e3af40b441feaea4c68225e2eb2341026c84511ad246c0c"}, - {file = "cryptography-46.0.4-cp314-cp314t-win32.whl", hash = "sha256:c411f16275b0dea722d76544a61d6421e2cc829ad76eec79280dbdc9ddf50061"}, - {file = "cryptography-46.0.4-cp314-cp314t-win_amd64.whl", hash = "sha256:728fedc529efc1439eb6107b677f7f7558adab4553ef8669f0d02d42d7b959a7"}, - {file = "cryptography-46.0.4-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a9556ba711f7c23f77b151d5798f3ac44a13455cc68db7697a1096e6d0563cab"}, - {file = "cryptography-46.0.4-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8bf75b0259e87fa70bddc0b8b4078b76e7fd512fd9afae6c1193bcf440a4dbef"}, - {file = "cryptography-46.0.4-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3c268a3490df22270955966ba236d6bc4a8f9b6e4ffddb78aac535f1a5ea471d"}, - {file = "cryptography-46.0.4-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:812815182f6a0c1d49a37893a303b44eaac827d7f0d582cecfc81b6427f22973"}, - {file = "cryptography-46.0.4-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a90e43e3ef65e6dcf969dfe3bb40cbf5aef0d523dff95bfa24256be172a845f4"}, - {file = "cryptography-46.0.4-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a05177ff6296644ef2876fce50518dffb5bcdf903c85250974fc8bc85d54c0af"}, - {file = "cryptography-46.0.4-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:daa392191f626d50f1b136c9b4cf08af69ca8279d110ea24f5c2700054d2e263"}, - {file = "cryptography-46.0.4-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e07ea39c5b048e085f15923511d8121e4a9dc45cee4e3b970ca4f0d338f23095"}, - {file = "cryptography-46.0.4-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:d5a45ddc256f492ce42a4e35879c5e5528c09cd9ad12420828c972951d8e016b"}, - {file = "cryptography-46.0.4-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:6bb5157bf6a350e5b28aee23beb2d84ae6f5be390b2f8ee7ea179cda077e1019"}, - {file = "cryptography-46.0.4-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dd5aba870a2c40f87a3af043e0dee7d9eb02d4aff88a797b48f2b43eff8c3ab4"}, - {file = "cryptography-46.0.4-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:93d8291da8d71024379ab2cb0b5c57915300155ad42e07f76bea6ad838d7e59b"}, - {file = "cryptography-46.0.4-cp38-abi3-win32.whl", hash = "sha256:0563655cb3c6d05fb2afe693340bc050c30f9f34e15763361cf08e94749401fc"}, - {file = "cryptography-46.0.4-cp38-abi3-win_amd64.whl", hash = "sha256:fa0900b9ef9c49728887d1576fd8d9e7e3ea872fa9b25ef9b64888adc434e976"}, - {file = "cryptography-46.0.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:766330cce7416c92b5e90c3bb71b1b79521760cdcfc3a6a1a182d4c9fab23d2b"}, - {file = "cryptography-46.0.4-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c236a44acfb610e70f6b3e1c3ca20ff24459659231ef2f8c48e879e2d32b73da"}, - {file = "cryptography-46.0.4-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8a15fb869670efa8f83cbffbc8753c1abf236883225aed74cd179b720ac9ec80"}, - {file = "cryptography-46.0.4-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:fdc3daab53b212472f1524d070735b2f0c214239df131903bae1d598016fa822"}, - {file = "cryptography-46.0.4-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:44cc0675b27cadb71bdbb96099cca1fa051cd11d2ade09e5cd3a2edb929ed947"}, - {file = "cryptography-46.0.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be8c01a7d5a55f9a47d1888162b76c8f49d62b234d88f0ff91a9fbebe32ffbc3"}, - {file = "cryptography-46.0.4.tar.gz", hash = "sha256:bfd019f60f8abc2ed1b9be4ddc21cfef059c841d86d710bb69909a688cbb8f59"}, -] - -[package.dependencies] -cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} +python-versions = "!=3.9.0,!=3.9.1,>=3.9" +groups = ["main"] +files = [ + {file = "cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03"}, + {file = "cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645"}, + {file = "cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7"}, + {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3"}, + {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f"}, + {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae"}, + {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a"}, + {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987"}, + {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169"}, + {file = "cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f"}, + {file = "cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105"}, + {file = "cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef"}, + {file = "cryptography-50.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30"}, + {file = "cryptography-50.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:49e7d93abdbd2990caced757e5fade25302f719c3c8fb6e6fff2dde98999fc41"}, + {file = "cryptography-50.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc"}, + {file = "cryptography-50.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f"}, + {file = "cryptography-50.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3"}, + {file = "cryptography-50.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:df2a58a472f332225671c35b0a830208b86d004f82baa8530fa3782c85646533"}, + {file = "cryptography-50.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037"}, + {file = "cryptography-50.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f"}, + {file = "cryptography-50.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11"}, + {file = "cryptography-50.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:9aa87839c383bdbab6ef865787a1fb877af8dd03464c4400322726feaaadfc6d"}, + {file = "cryptography-50.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c"}, + {file = "cryptography-50.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c"}, + {file = "cryptography-50.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95"}, + {file = "cryptography-50.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:12b9c6996425c76ea6c457ace4f3073e715b8c545add07cd1a8f3a4f90691269"}, + {file = "cryptography-50.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07"}, + {file = "cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3"}, + {file = "cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f"}, + {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5"}, + {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f"}, + {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025"}, + {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a"}, + {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b"}, + {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708"}, + {file = "cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47"}, + {file = "cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9"}, + {file = "cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7"}, + {file = "cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba"}, + {file = "cryptography-50.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5e34edd123674534acd70147f0ca331eaa2c74e6325fb2028c886aa26ba0b68c"}, + {file = "cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8eb5e1172eb569ea8a872796576e6a67c276351728b6455d5beb01242b027c6a"}, + {file = "cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:910d11e1a385c654bf738bf3e6b8e6ed5de0f5610fcae2be9e5b398d8081d20e"}, + {file = "cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:62598a8a57f815db4c6259a4e97d857dab56697e7de8e8ab02352ab74da1995d"}, + {file = "cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:07479a1cb08219ab719147e742e76090c9c773321959bb94946fffdd397a6437"}, + {file = "cryptography-50.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c99c003e088647b8a5b7c145d6f78c335f6348332b62e142d411c4b63d1460b9"}, + {file = "cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9"}, +] + +[package.dependencies] +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] -docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox[uv] (>=2024.4.15)"] -pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.14)", "ruff (>=0.11.11)"] -sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==46.0.4)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] -test-randomorder = ["pytest-randomly"] [[package]] name = "cymem" @@ -1110,105 +1100,105 @@ idna = ["idna (>=3.10)"] trio = ["trio (>=0.30)"] wmi = ["wmi (>=1.5.1) ; platform_system == \"Windows\""] +[[package]] +name = "doclang" +version = "0.7.3" +description = "DocLang reference toolkit" +optional = false +python-versions = ">=3.10" +groups = ["main", "docling-ocr"] +files = [ + {file = "doclang-0.7.3-py3-none-any.whl", hash = "sha256:9440c4ca9f7e061a7b8d33bdf15b1029be69a4c13cd8952dd6ce541884e4c685"}, + {file = "doclang-0.7.3.tar.gz", hash = "sha256:ca50615357e46ebf9597bb9065b9112367103ec24bd539f8ae12649224cf50b0"}, +] + +[package.dependencies] +lxml = ">=4.8.0" +typer = ">=0.15.1" + +[package.extras] +schematron-saxon = ["saxonche (>=12.9.0)"] + [[package]] name = "docling" -version = "2.74.0" +version = "2.121.0" description = "SDK and CLI for parsing PDF, DOCX, HTML, and more, to a unified document representation for powering downstream workflows such as gen AI applications." optional = false python-versions = "<4.0,>=3.10" groups = ["main", "docling-ocr"] files = [ - {file = "docling-2.74.0-py3-none-any.whl", hash = "sha256:931e9a522cd295cf523eebc3b52e507c7cf8709e0e7e9a7a84192051f3d62848"}, - {file = "docling-2.74.0.tar.gz", hash = "sha256:107e7e0854c1ecb56c6381a07f87d587c3606386ca56f0d22917cb49e1349468"}, + {file = "docling-2.121.0-py3-none-any.whl", hash = "sha256:d1577ac7660ac6cb3ac7e213f1541f8bed1990ff8da4274d94b5bfc91e0fec55"}, + {file = "docling-2.121.0.tar.gz", hash = "sha256:c8970cbd3907892db218c7f342ff62db5b393ca3cf568db50c8f2384e248a22d"}, ] [package.dependencies] -accelerate = ">=1.0.0,<2" -beautifulsoup4 = ">=4.12.3,<5.0.0" -certifi = ">=2024.7.4" -defusedxml = ">=0.7.1,<0.8.0" -docling-core = {version = ">=2.62.0,<3.0.0", extras = ["chunking"]} -docling-ibm-models = ">=3.9.1,<4" -docling-parse = ">=5.3.2,<6.0.0" -filetype = ">=1.2.0,<2.0.0" -huggingface_hub = ">=0.23,<1" -lxml = ">=4.0.0,<7.0.0" -marko = ">=2.1.2,<3.0.0" -ocrmac = {version = ">=1.0.0,<2.0.0", markers = "sys_platform == \"darwin\""} -openpyxl = ">=3.1.5,<4.0.0" -pandas = ">=2.1.4,<3.0.0" -pillow = ">=10.0.0,<13.0.0" -pluggy = ">=1.0.0,<2.0.0" -polyfactory = ">=2.22.2" -pydantic = ">=2.0.0,<3.0.0" -pydantic-settings = ">=2.3.0,<3.0.0" -pylatexenc = ">=2.10,<3.0" -pypdfium2 = ">=4.30.0,<4.30.1 || >4.30.1,<6.0.0" -python-docx = ">=1.1.2,<2.0.0" -python-pptx = ">=1.0.2,<2.0.0" -rapidocr = ">=3.3,<4.0.0" -requests = ">=2.32.2,<3.0.0" -rtree = ">=1.3.0,<2.0.0" -scipy = ">=1.6.0,<2.0.0" -tqdm = ">=4.65.0,<5.0.0" -typer = ">=0.12.5,<0.22.0" +docling-slim = {version = "2.121.0", extras = ["standard"]} [package.extras] -asr = ["mlx-whisper (>=0.4.3) ; python_version >= \"3.10\" and sys_platform == \"darwin\" and platform_machine == \"arm64\"", "numba (>=0.63.0)", "openai-whisper (>=20250625)"] -easyocr = ["easyocr (>=1.7,<2.0)"] -ocrmac = ["ocrmac (>=1.0.0,<2.0.0) ; sys_platform == \"darwin\""] -onnxruntime = ["onnxruntime (<1.24) ; python_version < \"3.14\" and sys_platform == \"darwin\"", "onnxruntime-gpu (<1.24) ; python_version < \"3.14\" and (sys_platform == \"linux\" or sys_platform == \"win32\")"] -rapidocr = ["onnxruntime (>=1.7.0,<2.0.0) ; python_version < \"3.14\"", "rapidocr (>=3.3,<4.0.0)"] -tesserocr = ["tesserocr (>=2.7.1,<3.0.0)"] -vlm = ["accelerate (>=1.2.1,<2.0.0)", "mlx-vlm (>=0.3.0,<1.0.0) ; python_version >= \"3.10\" and sys_platform == \"darwin\" and platform_machine == \"arm64\"", "qwen-vl-utils (>=0.0.11)", "transformers (>=4.46.0,<5.0.0)"] +asr = ["docling-slim[format-audio] (==2.121.0)"] +easyocr = ["docling-slim[feat-ocr-easyocr] (==2.121.0)"] +htmlrender = ["docling-slim[format-html-render] (==2.121.0)"] +ocrmac = ["docling-slim[feat-ocr-mac] (==2.121.0)"] +onnxruntime = ["docling-slim[models-onnxruntime] (==2.121.0)"] +rapidocr = ["docling-slim[feat-ocr-rapidocr-onnx] (==2.121.0)"] +remote-serving = ["docling-slim[models-remote] (==2.121.0)"] +tesserocr = ["docling-slim[feat-ocr-tesserocr] (==2.121.0)"] +vlm = ["docling-slim[models-vlm-inline] (==2.121.0)"] +xbrl = ["docling-slim[format-xml-xbrl] (==2.121.0)"] [[package]] name = "docling-core" -version = "2.65.1" +version = "2.92.0" description = "A python library to define and validate data types in Docling." optional = false python-versions = "<4.0,>=3.10" groups = ["main", "docling-ocr"] files = [ - {file = "docling_core-2.65.1-py3-none-any.whl", hash = "sha256:fcdb30254bc5046b52b8bec5919de3b6cdbeed915399cede5351ff328cdd020d"}, - {file = "docling_core-2.65.1.tar.gz", hash = "sha256:3a143adb9cc613c503380eff92f5895078fc5a00fc7264f327d0d85ff60176cd"}, + {file = "docling_core-2.92.0-py3-none-any.whl", hash = "sha256:726d89c23197e53be2f7192bb4c00108ff545830cf9ab7b981fb014fa92b5c02"}, + {file = "docling_core-2.92.0.tar.gz", hash = "sha256:33fd25e38c199336447a21925374400aca13a6b9316a032f111972c2dc0f085c"}, ] [package.dependencies] defusedxml = ">=0.7.1,<0.8.0" +doclang = ">=0.7,<0.8" jsonref = ">=1.1.0,<2.0.0" jsonschema = ">=4.16.0,<5.0.0" latex2mathml = ">=3.77.0,<4.0.0" -pandas = ">=2.1.4,<3.0.0" +pandas = ">=2.1.4,<4.0.0" pillow = ">=10.0.0,<13.0.0" pydantic = ">=2.6.0,<2.10.0 || >2.10.0,<2.10.1 || >2.10.1,<2.10.2 || >2.10.2,<3.0.0" +pydantic-settings = ">=2.14.0" pyyaml = ">=5.1,<7.0.0" -semchunk = {version = ">=2.2.0,<3.0.0", optional = true, markers = "extra == \"chunking\""} -tabulate = ">=0.9.0,<0.10.0" -transformers = {version = ">=4.34.0,<5.0.0", optional = true, markers = "extra == \"chunking\""} +semchunk = {version = ">=2.2.0,<4.0.0", optional = true, markers = "extra == \"chunking\""} +tabulate = ">=0.9.0,<0.11.0" +transformers = [ + {version = ">=4.34.0,<5.0.dev0 || >=5.4.dev0,<5.9.0", optional = true, markers = "sys_platform == \"darwin\" and extra == \"chunking\""}, + {version = ">=4.34.0,<5.0.dev0 || >=5.4.dev0,<6.0.0", optional = true, markers = "sys_platform != \"darwin\" and extra == \"chunking\""}, +] tree-sitter = {version = ">=0.25.0,<0.27.0", optional = true, markers = "extra == \"chunking\""} tree-sitter-c = {version = ">=0.23.4", optional = true, markers = "extra == \"chunking\""} tree-sitter-javascript = {version = ">=0.23.1", optional = true, markers = "extra == \"chunking\""} tree-sitter-python = {version = ">=0.23.6", optional = true, markers = "extra == \"chunking\""} tree-sitter-typescript = {version = ">=0.23.2", optional = true, markers = "extra == \"chunking\""} -typer = ">=0.12.5,<0.22.0" +typer = ">=0.12.5,<0.27.0" typing-extensions = ">=4.12.2,<5.0.0" [package.extras] -chunking = ["semchunk (>=2.2.0,<3.0.0)", "transformers (>=4.34.0,<5.0.0)", "tree-sitter (>=0.25.0,<0.27.0)", "tree-sitter-c (>=0.23.4)", "tree-sitter-javascript (>=0.23.1)", "tree-sitter-python (>=0.23.6)", "tree-sitter-typescript (>=0.23.2)"] -chunking-openai = ["semchunk (>=2.2.0,<3.0.0)", "tiktoken (>=0.9.0,<0.13.0)", "tree-sitter (>=0.25.0,<0.27.0)", "tree-sitter-c (>=0.23.4)", "tree-sitter-javascript (>=0.23.1)", "tree-sitter-python (>=0.23.6)", "tree-sitter-typescript (>=0.23.2)"] -examples = ["datasets (>=4.0.0)", "matplotlib (>=3.7.0)", "openpyxl (>=3.1.5)", "pandas (>=2.3.3)"] +chunking = ["semchunk (>=2.2.0,<4.0.0)", "transformers (>=4.34.0,<5.0.dev0 || >=5.4.dev0,<5.9.0) ; sys_platform == \"darwin\"", "transformers (>=4.34.0,<5.0.dev0 || >=5.4.dev0,<6.0.0) ; sys_platform != \"darwin\"", "tree-sitter (>=0.25.0,<0.27.0)", "tree-sitter-c (>=0.23.4)", "tree-sitter-javascript (>=0.23.1)", "tree-sitter-python (>=0.23.6)", "tree-sitter-typescript (>=0.23.2)"] +chunking-openai = ["semchunk (>=2.2.0,<4.0.0)", "tiktoken (>=0.9.0,<0.13.0)", "tree-sitter (>=0.25.0,<0.27.0)", "tree-sitter-c (>=0.23.4)", "tree-sitter-javascript (>=0.23.1)", "tree-sitter-python (>=0.23.6)", "tree-sitter-typescript (>=0.23.2)"] +dclq = ["click (>=8.0.0)", "doclang (>=0.7,<0.8)", "lxml (>=6.0.2)", "typer (>=0.15.1,<0.25.0)"] +examples = ["datasets (>=4.0.0)", "matplotlib (>=3.7.0)", "openpyxl (>=3.1.5)"] [[package]] name = "docling-ibm-models" -version = "3.11.0" +version = "3.14.0" description = "This package contains the AI models used by the Docling PDF conversion package" optional = false python-versions = "<4.0,>=3.10" groups = ["main", "docling-ocr"] files = [ - {file = "docling_ibm_models-3.11.0-py3-none-any.whl", hash = "sha256:68f7961069d643bfdab21b1c9ef24a979db293496f4c2283d95b1025a9ac5347"}, - {file = "docling_ibm_models-3.11.0.tar.gz", hash = "sha256:454401563a8e79cb33b718bc559d9bacca8a0183583e48f8e616c9184c1f5eb1"}, + {file = "docling_ibm_models-3.14.0-py3-none-any.whl", hash = "sha256:795d39cd0f7b1e14a702e681b0ef0f9bd31deaedddb4e2686ad577296ecb8fc9"}, + {file = "docling_ibm_models-3.14.0.tar.gz", hash = "sha256:def964e3d524f66c7321ef9d48d4021278f14319f01d3f78058cd2324f641e22"}, ] [package.dependencies] @@ -1224,7 +1214,10 @@ safetensors = {version = ">=0.4.3,<1", extras = ["torch"]} torch = ">=2.2.2,<3.0.0" torchvision = ">=0,<1" tqdm = ">=4.64.0,<5.0.0" -transformers = ">=4.42.0,<5.0.0" +transformers = [ + {version = ">=4.42.0,<5.0.dev0 || >=5.4.dev0,<5.9.0", markers = "sys_platform == \"darwin\""}, + {version = ">=4.42.0,<5.0.dev0 || >=5.4.dev0,<6.0.0", markers = "sys_platform != \"darwin\""}, +] [package.extras] opencv-python = ["opencv-python (>=4.6.0.66,<5.0.0.0)"] @@ -1232,44 +1225,136 @@ opencv-python-headless = ["opencv-python-headless (>=4.6.0.66,<5.0.0.0)"] [[package]] name = "docling-parse" -version = "5.3.3" +version = "7.15.0" description = "Simple package to extract text with coordinates from programmatic PDFs" optional = false python-versions = ">=3.10" groups = ["main", "docling-ocr"] files = [ - {file = "docling_parse-5.3.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:aa880d0594fa5edf57522b7704950f9e2bc484de9a046d94839e9cc2cfa49941"}, - {file = "docling_parse-5.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c22ab0d86c042f8745be4bcb2edd2afd263ab978c73a553a754ae750d581f74"}, - {file = "docling_parse-5.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c19fdfb13b8015a8886694f5894874f579e134cc456ed49b86757c413a84aa78"}, - {file = "docling_parse-5.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:97acbcd4bd2d89ef4c28209ac138833bd4df432de3a533dda1fdfdc04e8db50b"}, - {file = "docling_parse-5.3.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:cb700693be4b7b1172a7201eb60ed4f8b2e47feb5366e1c25111fa6e6fa04f01"}, - {file = "docling_parse-5.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e420970361103a0c09b8edb220e9d30652d7a2a1a3021d35d5211cbc639fd2ea"}, - {file = "docling_parse-5.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e98cd302f8dd31159f7063befc51460e8ef0ad3c2c57c2c1ea711f18000335e"}, - {file = "docling_parse-5.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:ae22fb4857ce244948d1de67dc9c2863cfe251d153de70751d133ccadf0b7670"}, - {file = "docling_parse-5.3.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:087985ae6ae9cbe7690e60592b1e41a8a282ecb3509957833bd6931cd745fca5"}, - {file = "docling_parse-5.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4912ee081d20238e2de78bb0c76b23c6391f48f883ac70c9fcdd4ac629688434"}, - {file = "docling_parse-5.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37df23ab92620891278d3ee4576431652bb4e980fbcca69bc6814d32182d0a62"}, - {file = "docling_parse-5.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:4d37095f4bf471e86422828ce1615099d02f9fb352e5e2dfc766bac9a8005427"}, - {file = "docling_parse-5.3.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:3deaa7070b259ad3fa24b6d03d5283d65744f9d8573cd44f9aacee91ed1cd5c1"}, - {file = "docling_parse-5.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:872d9fd051caed6e866b0f7200c0aea3fb436eab6403cbefc1057c9a55ca4649"}, - {file = "docling_parse-5.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c606cd8b6815accb7e3e0c27ae06eda55677bb582d5be7ae6f814aecedd28ab"}, - {file = "docling_parse-5.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:54f7f04cf886cbce490723d2f05b936bf85a39fb7446f2de02bf03c1b2fb8ba8"}, - {file = "docling_parse-5.3.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:fa34828e47d94f304fb12c21e37ee1dd5f94b31d118adb8d0e69043c3353c5d0"}, - {file = "docling_parse-5.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bff9dd7eefe376f4558868a6393950196a3baa81c60ffc1eb87e1be8781aaf76"}, - {file = "docling_parse-5.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f3a997a9441a60bf6f1d5526a96e426a8414acb219c26fff0bf2932e863f418"}, - {file = "docling_parse-5.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:0d940a0048e0469ceba757966921806266a0ab5b54b422b62768430bfd25247b"}, - {file = "docling_parse-5.3.3.tar.gz", hash = "sha256:d3bc34bc236b205c466334870a4284c75ac5dc973294107f2523548fac2b3408"}, -] - -[package.dependencies] -docling-core = ">=2.65.1" + {file = "docling_parse-7.15.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:bfe13c2c0ed982b24ceefb2ace083a22aa632ba4e2b9427d274f992c41019df1"}, + {file = "docling_parse-7.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67f0ed5053426bd4f82b407ec48fbe501c706286fa7d6fd92ff7510aef2e24e6"}, + {file = "docling_parse-7.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7299cb9614ff0776618b155e2b55a92f2f27cf9828adfbcf90e5f1127b19ec5b"}, + {file = "docling_parse-7.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:2905b77daff3849c559cf6f6f39ed7fbd451b79e20948ed325b721014e0dccb9"}, + {file = "docling_parse-7.15.0-cp310-cp310-win_arm64.whl", hash = "sha256:7a70f37eba73e676e13a3b71f7bbbdafc291b94f8bb4426294d27b52bd6559fd"}, + {file = "docling_parse-7.15.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:b6453b243d998a73a2fbd1245d202dd315fb1f915e1ccfafdce98d391cade8ae"}, + {file = "docling_parse-7.15.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a00c19112264b4e6b0d7d5baa9ccd74997b6252924e512f1c88cf30de5bea1f5"}, + {file = "docling_parse-7.15.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4cb6494f8ca158c774cd7b68d96b11054ec8a92f64fb3e8cc355a8cc858997fc"}, + {file = "docling_parse-7.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:bbb301b0e7a283745fa54fa20c4279a95a0500a2228362cff5a4a6f82e740fd5"}, + {file = "docling_parse-7.15.0-cp311-cp311-win_arm64.whl", hash = "sha256:b59940048cf0216e09a9de9c5c1bd86d061f89e7fe0ca49f6388061626d79043"}, + {file = "docling_parse-7.15.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a425599d40aa8b72a443c5fc9e22c120163d398a341af988c953e0e70f1ad0f3"}, + {file = "docling_parse-7.15.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4ea8d79e870319a583a68b3e88e95d048dfa496d8b6e5928f5333fe26cc597e3"}, + {file = "docling_parse-7.15.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca982b7df91f05b0bdcb28f437b2637623baccd205b941297f43b9e93aeb1f14"}, + {file = "docling_parse-7.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:6143e824b00dbaf14bd3692792bd0b41348f582fa7ed221ff462079935c127f6"}, + {file = "docling_parse-7.15.0-cp312-cp312-win_arm64.whl", hash = "sha256:f528916ec21c1bec5916db2e9c6ecac4c472fedd6aae4a9cf308f59cf8fcbd7b"}, + {file = "docling_parse-7.15.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:01d853d3b7f9c231351f9d15c1cb0746ab56c3d8329a850a1b706ef279af09ed"}, + {file = "docling_parse-7.15.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa573d66c428ceed2fb3ae968823af79eb529abffaf95d60a7ed74cde9f9cc5b"}, + {file = "docling_parse-7.15.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3523b631ca10180fe414c2d9cd11579171289936353fa5af3dd29caba541458d"}, + {file = "docling_parse-7.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:6ff600dfeb2c2d9e0e66a00efeb15d59c8d543b38c6f1bcfcb1f86156216e086"}, + {file = "docling_parse-7.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:e4bc3bf49bfd56447f35d1fe2e86897893da3b6c154f0c44643ff311e12ff8b5"}, + {file = "docling_parse-7.15.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:59b0dfa1d1ac97bc2d9d7ec457299e97afcf408bf4126e0cdc790818b8781510"}, + {file = "docling_parse-7.15.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88dd0b133febd1ea21cffd43f3aafeb4161b33b74f0c59ae7c7420438d1abd4a"}, + {file = "docling_parse-7.15.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5fc8ddd9cb3e864f4894d89271fc239277bdf5efc6c8deeecf1560cb163b3c4c"}, + {file = "docling_parse-7.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:9c8d5aa23a90b5d082ed21b22a3384e13ceac4b45ce3d52230f5e05893f5cc6c"}, + {file = "docling_parse-7.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:b844748fdc27a48737b6d2f0ee0a07cb25cbd340ad8535ac2ae9048f9e214b18"}, + {file = "docling_parse-7.15.0.tar.gz", hash = "sha256:fcb1b4e9a0e447f29e8248e46eea502ab4c7a75fe733a2ed911635a9b74b5092"}, +] + +[package.dependencies] +docling-core = ">=2.85.0,<3.0.0" pillow = ">=10.0.0,<13.0.0" pydantic = ">=2.0.0" pywin32 = {version = ">=305", markers = "sys_platform == \"win32\""} -tabulate = ">=0.9.0,<1.0.0" + +[[package]] +name = "docling-slim" +version = "2.121.0" +description = "Modular version of the Docling package: SDK and CLI for parsing PDF, DOCX, HTML, and more, to a unified document representation for powering downstream workflows such as gen AI applications." +optional = false +python-versions = "<4.0,>=3.10" +groups = ["main", "docling-ocr"] +files = [ + {file = "docling_slim-2.121.0-py3-none-any.whl", hash = "sha256:c97fc9cd889b61a66b9cf5d2d87c6b6935f29fe7aacfd8fd39802be7425e4996"}, + {file = "docling_slim-2.121.0.tar.gz", hash = "sha256:f3069ded34f62f4e6a4d2ffda78cba0baad2ced4944a4b2d70cb646670ef327b"}, +] + +[package.dependencies] +accelerate = {version = ">=1.0.0,<2", optional = true, markers = "extra == \"standard\""} +beautifulsoup4 = {version = ">=4.12.3,<5.0.0", optional = true, markers = "extra == \"standard\""} +certifi = ">=2024.7.4" +defusedxml = {version = ">=0.7.1,<0.8.0", optional = true, markers = "extra == \"standard\""} +docling-core = [ + {version = ">=2.91.0,<3.0.0"}, + {version = ">=2.73.0,<3.0.0", extras = ["chunking"], optional = true, markers = "extra == \"standard\""}, +] +docling-ibm-models = {version = ">=3.13.0,<4", optional = true, markers = "extra == \"standard\""} +docling-parse = {version = ">=7.12.0,<8.0.0", optional = true, markers = "extra == \"standard\""} +filetype = ">=1.2.0,<2.0.0" +httpx = {version = ">=0.28,<1.0.0", optional = true, markers = "extra == \"standard\""} +huggingface-hub = {version = ">=0.23,<2", optional = true, markers = "extra == \"standard\""} +mail-parser = {version = ">=4.1.4,<5.0.0", optional = true, markers = "extra == \"standard\""} +marko = {version = ">=2.1.2,<3.0.0", optional = true, markers = "extra == \"standard\""} +numpy = {version = ">=1.24.0,<3.0.0", optional = true, markers = "extra == \"standard\""} +openpyxl = {version = ">=3.1.5,<4.0.0", optional = true, markers = "extra == \"standard\""} +pillow = {version = ">=10.0.0,<13.0.0", optional = true, markers = "extra == \"standard\""} +pluggy = ">=1.0.0,<2.0.0" +polyfactory = {version = ">=2.22.2", optional = true, markers = "extra == \"standard\""} +pydantic = ">=2.0.0,<3.0.0" +pydantic-settings = ">=2.3.0,<3.0.0" +pylatexenc = {version = ">=2.10,<3.0", optional = true, markers = "extra == \"standard\""} +pypdfium2 = {version = ">=4.30.0,<4.30.1 || >4.30.1,<6.0.0", optional = true, markers = "extra == \"standard\""} +python-docx = {version = ">=1.2.0,<2.0.0", optional = true, markers = "extra == \"standard\""} +python-dotenv = {version = ">=1.0,<2.0", optional = true, markers = "extra == \"standard\""} +python-oxmsg = {version = ">=0.0.2,<0.1.0", optional = true, markers = "extra == \"standard\""} +python-pptx = {version = ">=1.0.2,<2.0.0", optional = true, markers = "extra == \"standard\""} +rapidocr = {version = ">=3.9.1,<4.0.0", optional = true, markers = "extra == \"standard\""} +requests = ">=2.32.2,<3.0.0" +rich = {version = ">=13.0.0", optional = true, markers = "extra == \"standard\""} +rtree = {version = ">=1.3.0,<2.0.0", optional = true, markers = "extra == \"standard\""} +scipy = {version = ">=1.6.0,<2.0.0", optional = true, markers = "extra == \"standard\""} +torch = {version = ">=2.2.2,<3.0.0", optional = true, markers = "extra == \"standard\""} +torchvision = {version = ">=0,<1", optional = true, markers = "extra == \"standard\""} +tqdm = ">=4.65.0,<5.0.0" +typer = {version = ">=0.12.5,<0.27.0", optional = true, markers = "extra == \"standard\""} +websockets = {version = ">=14.0,<17.0", optional = true, markers = "extra == \"standard\""} [package.extras] -perf-tools = ["pdfplumber (>=0.11.7)", "pymupdf (>=1.26.4)", "pypdfium2 (>=4.30.0)"] +all = ["accelerate (>=1.0.0,<2)", "accelerate (>=1.2.1,<2.0.0)", "arelle-release (>=2.38.17,<3.0.0)", "beautifulsoup4 (>=4.12.3,<5.0.0)", "defusedxml (>=0.7.1,<0.8.0)", "docling-core[chunking] (>=2.73.0,<3.0.0)", "docling-ibm-models (>=3.13.0,<4)", "docling-parse (>=7.12.0,<8.0.0)", "easyocr (>=1.7,<2.0)", "httpx (>=0.28,<1.0.0)", "huggingface-hub (>=0.23,<2)", "lxml (>=4.0.0,<7.0.0)", "mail-parser (>=4.1.4,<5.0.0)", "marko (>=2.1.2,<3.0.0)", "mlx-vlm (>=0.4.3,<1.0.0) ; python_version >= \"3.10\" and sys_platform == \"darwin\" and platform_machine == \"arm64\"", "mlx-whisper (>=0.4.3) ; python_version >= \"3.10\" and sys_platform == \"darwin\" and platform_machine == \"arm64\"", "numba (>=0.63.0)", "numpy (>=1.24.0,<3.0.0)", "ocrmac (>=1.0.0,<2.0.0) ; sys_platform == \"darwin\"", "onnxruntime (<1.24) ; python_version < \"3.14\" and sys_platform == \"darwin\"", "onnxruntime-gpu (<1.24) ; python_version < \"3.14\" and (sys_platform == \"linux\" or sys_platform == \"win32\")", "openai-whisper (>=20250625)", "openpyxl (>=3.1.5,<4.0.0)", "pandas (>=2.1.4,<4.0.0)", "peft (>=0.18.1)", "pillow (>=10.0.0,<13.0.0)", "playwright (>=1.58.0)", "polyfactory (>=2.22.2)", "pylatexenc (>=2.10,<3.0)", "pypdfium2 (>=4.30.0,!=4.30.1,<6.0.0)", "python-docx (>=1.2.0,<2.0.0)", "python-dotenv (>=1.0,<2.0)", "python-oxmsg (>=0.0.2,<0.1.0)", "python-pptx (>=1.0.2,<2.0.0)", "qwen-vl-utils (>=0.0.11)", "rapidocr (>=3.9.1,<4.0.0)", "rich (>=13.0.0)", "rtree (>=1.3.0,<2.0.0)", "scikit-image (>=0.19)", "scipy (>=1.6.0,<2.0.0)", "tesserocr (>=2.7.1,<3.0.0)", "torch (>=2.2.2,<3.0.0)", "torchvision (>=0,<1)", "transformers (>=4.42.0,<5.0.dev0 || >=5.4.dev0,<5.9.0) ; sys_platform == \"darwin\"", "transformers (>=4.42.0,<5.0.dev0 || >=5.4.dev0,<6.0.0) ; sys_platform != \"darwin\"", "tritonclient[grpc] (>=2.65.0,<3.0.0)", "typer (>=0.12.5,<0.27.0)", "websockets (>=14.0,<17.0)", "whisper-s2t-reborn (>=1.7.1,<2.0.0) ; sys_platform != \"darwin\" or platform_machine != \"arm64\""] +cli = ["python-dotenv (>=1.0,<2.0)", "rich (>=13.0.0)", "typer (>=0.12.5,<0.27.0)"] +convert-core = ["numpy (>=1.24.0,<3.0.0)", "pillow (>=10.0.0,<13.0.0)", "rtree (>=1.3.0,<2.0.0)", "scipy (>=1.6.0,<2.0.0)"] +extract-core = ["numpy (>=1.24.0,<3.0.0)", "pillow (>=10.0.0,<13.0.0)", "polyfactory (>=2.22.2)", "rtree (>=1.3.0,<2.0.0)", "scipy (>=1.6.0,<2.0.0)"] +feat-chunking = ["docling-core[chunking] (>=2.73.0,<3.0.0)"] +feat-ocr-easyocr = ["easyocr (>=1.7,<2.0)", "scikit-image (>=0.19)"] +feat-ocr-mac = ["ocrmac (>=1.0.0,<2.0.0) ; sys_platform == \"darwin\""] +feat-ocr-nemotron = ["nemotron-ocr (>=2.0.0) ; python_version == \"3.12\" and sys_platform == \"linux\" and platform_machine == \"x86_64\""] +feat-ocr-rapidocr = ["rapidocr (>=3.9.1,<4.0.0)"] +feat-ocr-rapidocr-onnx = ["onnxruntime (>=1.7.0,<2.0.0) ; python_version < \"3.14\"", "rapidocr (>=3.9.1,<4.0.0)"] +feat-ocr-tesserocr = ["pandas (>=2.1.4,<4.0.0)", "tesserocr (>=2.7.1,<3.0.0)"] +format-audio = ["mlx-whisper (>=0.4.3) ; python_version >= \"3.10\" and sys_platform == \"darwin\" and platform_machine == \"arm64\"", "numba (>=0.63.0)", "openai-whisper (>=20250625)", "whisper-s2t-reborn (>=1.7.1,<2.0.0) ; sys_platform != \"darwin\" or platform_machine != \"arm64\""] +format-docx = ["python-docx (>=1.2.0,<2.0.0)"] +format-email = ["beautifulsoup4 (>=4.12.3,<5.0.0)", "mail-parser (>=4.1.4,<5.0.0)", "python-oxmsg (>=0.0.2,<0.1.0)"] +format-html = ["beautifulsoup4 (>=4.12.3,<5.0.0)"] +format-html-render = ["playwright (>=1.58.0)"] +format-iwork = ["defusedxml (>=0.7.1,<0.8.0)"] +format-latex = ["pylatexenc (>=2.10,<3.0)"] +format-markdown = ["marko (>=2.1.2,<3.0.0)"] +format-office = ["openpyxl (>=3.1.5,<4.0.0)", "python-docx (>=1.2.0,<2.0.0)", "python-pptx (>=1.0.2,<2.0.0)"] +format-opendocument = ["odfdo (>=3.22.0,<4.0.0)"] +format-pdf = ["docling-parse (>=7.12.0,<8.0.0)", "pypdfium2 (>=4.30.0,!=4.30.1,<6.0.0)"] +format-pdf-docling = ["docling-parse (>=7.12.0,<8.0.0)", "pypdfium2 (>=4.30.0,!=4.30.1,<6.0.0)"] +format-pdf-pypdfium2 = ["pypdfium2 (>=4.30.0,!=4.30.1,<6.0.0)"] +format-pptx = ["python-pptx (>=1.0.2,<2.0.0)"] +format-video = ["librosa (>=0.10.0,<1.0.0)", "mlx-whisper (>=0.4.3) ; python_version >= \"3.10\" and sys_platform == \"darwin\" and platform_machine == \"arm64\"", "numba (>=0.63.0)", "openai-whisper (>=20250625)", "resemblyzer (>=0.1.4,<0.2.0)", "scikit-learn (>=1.3.0,<2.0.0)", "soundfile (>=0.12.1,<1.0.0)", "whisper-s2t-reborn (>=1.7.1,<2.0.0) ; sys_platform != \"darwin\" or platform_machine != \"arm64\""] +format-web = ["beautifulsoup4 (>=4.12.3,<5.0.0)", "marko (>=2.1.2,<3.0.0)"] +format-xlsx = ["openpyxl (>=3.1.5,<4.0.0)"] +format-xml-jats = ["beautifulsoup4 (>=4.12.3,<5.0.0)", "lxml (>=4.0.0,<7.0.0)"] +format-xml-uspto = ["beautifulsoup4 (>=4.12.3,<5.0.0)", "defusedxml (>=0.7.1,<0.8.0)"] +format-xml-xbrl = ["arelle-release (>=2.38.17,<3.0.0)"] +models-local = ["accelerate (>=1.0.0,<2)", "defusedxml (>=0.7.1,<0.8.0)", "docling-ibm-models (>=3.13.0,<4)", "huggingface-hub (>=0.23,<2)", "torch (>=2.2.2,<3.0.0)", "torchvision (>=0,<1)"] +models-onnxruntime = ["onnxruntime (<1.24) ; python_version < \"3.14\" and sys_platform == \"darwin\"", "onnxruntime-gpu (<1.24) ; python_version < \"3.14\" and (sys_platform == \"linux\" or sys_platform == \"win32\")"] +models-remote = ["tritonclient[grpc] (>=2.65.0,<3.0.0)"] +models-vlm-inline = ["accelerate (>=1.2.1,<2.0.0)", "mlx-vlm (>=0.4.3,<1.0.0) ; python_version >= \"3.10\" and sys_platform == \"darwin\" and platform_machine == \"arm64\"", "peft (>=0.18.1)", "qwen-vl-utils (>=0.0.11)", "transformers (>=4.42.0,<5.0.dev0 || >=5.4.dev0,<5.9.0) ; sys_platform == \"darwin\"", "transformers (>=4.42.0,<5.0.dev0 || >=5.4.dev0,<6.0.0) ; sys_platform != \"darwin\""] +service-client = ["httpx (>=0.28,<1.0.0)", "python-dotenv (>=1.0,<2.0)", "rich (>=13.0.0)", "typer (>=0.12.5,<0.27.0)", "websockets (>=14.0,<17.0)"] +standard = ["accelerate (>=1.0.0,<2)", "beautifulsoup4 (>=4.12.3,<5.0.0)", "defusedxml (>=0.7.1,<0.8.0)", "docling-core[chunking] (>=2.73.0,<3.0.0)", "docling-ibm-models (>=3.13.0,<4)", "docling-parse (>=7.12.0,<8.0.0)", "httpx (>=0.28,<1.0.0)", "huggingface-hub (>=0.23,<2)", "mail-parser (>=4.1.4,<5.0.0)", "marko (>=2.1.2,<3.0.0)", "numpy (>=1.24.0,<3.0.0)", "openpyxl (>=3.1.5,<4.0.0)", "pillow (>=10.0.0,<13.0.0)", "polyfactory (>=2.22.2)", "pylatexenc (>=2.10,<3.0)", "pypdfium2 (>=4.30.0,!=4.30.1,<6.0.0)", "python-docx (>=1.2.0,<2.0.0)", "python-dotenv (>=1.0,<2.0)", "python-oxmsg (>=0.0.2,<0.1.0)", "python-pptx (>=1.0.2,<2.0.0)", "rapidocr (>=3.9.1,<4.0.0)", "rich (>=13.0.0)", "rtree (>=1.3.0,<2.0.0)", "scipy (>=1.6.0,<2.0.0)", "torch (>=2.2.2,<3.0.0)", "torchvision (>=0,<1)", "typer (>=0.12.5,<0.27.0)", "websockets (>=14.0,<17.0)"] [[package]] name = "docstring-parser" @@ -1320,25 +1405,27 @@ tzdata = ["tzdata"] [[package]] name = "fastapi" -version = "0.119.0" +version = "0.141.1" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "fastapi-0.119.0-py3-none-any.whl", hash = "sha256:90a2e49ed19515320abb864df570dd766be0662c5d577688f1600170f7f73cf2"}, - {file = "fastapi-0.119.0.tar.gz", hash = "sha256:451082403a2c1f0b99c6bd57c09110ed5463856804c8078d38e5a1f1035dbbb7"}, + {file = "fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3"}, + {file = "fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1"}, ] [package.dependencies] -pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<2.0.0 || >2.0.0,<2.0.1 || >2.0.1,<2.1.0 || >2.1.0,<3.0.0" -starlette = ">=0.40.0,<0.49.0" +annotated-doc = ">=0.0.2" +pydantic = ">=2.9.0" +starlette = ">=0.46.0" typing-extensions = ">=4.8.0" +typing-inspection = ">=0.4.2" [package.extras] -all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "orjson (>=3.2.1)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "ujson (>=4.0.1,!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0)", "uvicorn[standard] (>=0.12.0)"] -standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] -standard-no-fastapi-cloud-cli = ["email-validator (>=2.0.0)", "fastapi-cli[standard-no-fastapi-cloud-cli] (>=0.0.8)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] +all = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.32)", "httpx (>=0.23.0,<1.0.0)", "itsdangerous (>=1.1.0)", "jinja2 (>=3.1.5)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "pyyaml (>=5.3.1)", "uvicorn[standard] (>=0.12.0)"] +standard = ["email-validator (>=2.0.0)", "fastapi-cli[standard] (>=0.0.32)", "fastar (>=0.9.0)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] +standard-no-fastapi-cloud-cli = ["email-validator (>=2.0.0)", "fastapi-cli[standard-no-fastapi-cloud-cli] (>=0.0.32)", "httpx (>=0.23.0,<1.0.0)", "jinja2 (>=3.1.5)", "pydantic-extra-types (>=2.0.0)", "pydantic-settings (>=2.0.0)", "python-multipart (>=0.0.18)", "uvicorn[standard] (>=0.12.0)"] [[package]] name = "filelock" @@ -1346,7 +1433,7 @@ version = "3.20.1" description = "A platform independent file lock." optional = false python-versions = ">=3.10" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "filelock-3.20.1-py3-none-any.whl", hash = "sha256:15d9e9a67306188a44baa72f569d2bfd803076269365fdea0934385da4dc361a"}, {file = "filelock-3.20.1.tar.gz", hash = "sha256:b8360948b351b80f420878d8516519a2204b07aefcdcfd24912a5d33127f188c"}, @@ -1370,7 +1457,7 @@ version = "2025.12.0" description = "File-system specification" optional = false python-versions = ">=3.10" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "fsspec-2025.12.0-py3-none-any.whl", hash = "sha256:8bf1fe301b7d8acfa6e8571e3b1c3d158f909666642431cc78a1b7b4dbc5ec5b"}, {file = "fsspec-2025.12.0.tar.gz", hash = "sha256:c505de011584597b1060ff778bb664c1bc022e87921b0e4f10cc9c44f9635973"}, @@ -1442,86 +1529,93 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] [[package]] name = "google-auth" -version = "2.41.1" +version = "2.56.3" description = "Google Authentication Library" optional = false -python-versions = ">=3.7" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "google_auth-2.41.1-py2.py3-none-any.whl", hash = "sha256:754843be95575b9a19c604a848a41be03f7f2afd8c019f716dc1f51ee41c639d"}, - {file = "google_auth-2.41.1.tar.gz", hash = "sha256:b76b7b1f9e61f0cb7e88870d14f6a94aeef248959ef6992670efee37709cbfd2"}, + {file = "google_auth-2.56.3-py3-none-any.whl", hash = "sha256:8ec438808f813ad034535000261eed1067475d229d05bbf4216e78c3f2362e53"}, + {file = "google_auth-2.56.3.tar.gz", hash = "sha256:40e229fc901f0a305b553050e5fce562d509bee0435be053abfa91582b51b90c"}, ] [package.dependencies] -cachetools = ">=2.0.0,<7.0" +cryptography = [ + {version = ">=38.0.3", markers = "python_version < \"3.14\""}, + {version = ">=41.0.5", markers = "python_version >= \"3.14\""}, +] pyasn1-modules = ">=0.2.1" -rsa = ">=3.1.4,<5" +requests = {version = ">=2.30.0,<3.0.0", optional = true, markers = "extra == \"requests\""} [package.extras] -aiohttp = ["aiohttp (>=3.6.2,<4.0.0)", "requests (>=2.20.0,<3.0.0)"] -enterprise-cert = ["cryptography", "pyopenssl"] -pyjwt = ["cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "pyjwt (>=2.0)"] -pyopenssl = ["cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +aiohttp = ["aiohttp (>=3.8.0,<4.0.0) ; python_version < \"3.14\"", "aiohttp (>=3.9.0,<4.0.0) ; python_version >= \"3.14\"", "requests (>=2.30.0,<3.0.0)"] +cryptography = ["cryptography (>=38.0.3) ; python_version < \"3.14\"", "cryptography (>=41.0.5) ; python_version >= \"3.14\""] +enterprise-cert = ["cryptography (>=38.0.3) ; python_version < \"3.14\"", "cryptography (>=41.0.5) ; python_version >= \"3.14\""] +grpc = ["grpcio (>=1.59.0,<2.0.0) ; python_version < \"3.14\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\""] +pyjwt = ["pyjwt (>=2.0)"] +pyopenssl = ["cryptography (>=38.0.3) ; python_version < \"3.14\"", "cryptography (>=41.0.5) ; python_version >= \"3.14\""] reauth = ["pyu2f (>=0.1.5)"] -requests = ["requests (>=2.20.0,<3.0.0)"] -testing = ["aiohttp (<3.10.0)", "aiohttp (>=3.6.2,<4.0.0)", "aioresponses", "cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (<39.0.0) ; python_version < \"3.8\"", "cryptography (>=38.0.3)", "cryptography (>=38.0.3)", "flask", "freezegun", "grpcio", "mock", "oauth2client", "packaging", "pyjwt (>=2.0)", "pyopenssl (<24.3.0)", "pyopenssl (>=20.0.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.20.0,<3.0.0)", "responses", "urllib3"] -urllib3 = ["packaging", "urllib3"] +requests = ["requests (>=2.30.0,<3.0.0)"] +rsa = ["rsa (>=4.0.0,<5)"] +testing = ["aiohttp (>=3.8.0,<4.0.0) ; python_version < \"3.14\"", "aiohttp (>=3.9.0,<4.0.0) ; python_version >= \"3.14\"", "aioresponses", "flask", "freezegun", "grpcio (>=1.59.0,<2.0.0) ; python_version < \"3.14\"", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "packaging (>=20.0)", "pyjwt (>=2.0)", "pytest", "pytest-asyncio", "pytest-cov", "pytest-localserver", "pyu2f (>=0.1.5)", "requests (>=2.30.0,<3.0.0)", "responses", "urllib3 (>=1.26.15,<3.0.0)"] +urllib3 = ["packaging (>=20.0)", "urllib3 (>=1.26.15,<3.0.0)"] [[package]] name = "google-cloud-aiplatform" -version = "1.122.0" +version = "1.165.1" description = "Vertex AI API client library" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "google_cloud_aiplatform-1.122.0-py2.py3-none-any.whl", hash = "sha256:389bc24c5f710b7c58df2b95f598ef7c6e90c116608484a171f4da03bf6ea249"}, - {file = "google_cloud_aiplatform-1.122.0.tar.gz", hash = "sha256:949361abdf4ba60911661ac3acb5a139e9b97b603d83aac1d4932dcdaba0a748"}, + {file = "google_cloud_aiplatform-1.165.1-py2.py3-none-any.whl", hash = "sha256:93874bd7993d1d901291595df693a8b9d4d7c89b93d4f41f8664d73f088dec89"}, + {file = "google_cloud_aiplatform-1.165.1.tar.gz", hash = "sha256:bd62ba7590255cacd66f9d0439eb731060af460d35bb48dbbeeefce9dfc0a359"}, ] [package.dependencies] +certifi = ">=2023.7.22" docstring_parser = "<1" google-api-core = {version = ">=1.34.1,<2.0.dev0 || >=2.8.dev0,<3.0.0", extras = ["grpc"]} -google-auth = ">=2.14.1,<3.0.0" +google-auth = ">=2.47.0,<3.0.0" google-cloud-bigquery = ">=1.15.0,<3.20.0 || >3.20.0,<4.0.0" google-cloud-resource-manager = ">=1.3.3,<3.0.0" google-cloud-storage = [ - {version = ">=1.32.0,<3.0.0", markers = "python_version < \"3.13\""}, - {version = ">=2.10.0,<3.0.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.32.0,<4.0.0", markers = "python_version < \"3.13\""}, + {version = ">=3.10.0,<4.0.0", markers = "python_version >= \"3.13\""}, ] -google-genai = ">=1.37.0,<2.0.0" +google-genai = {version = ">=1.66.0,<3.0.0", markers = "python_version >= \"3.10\""} packaging = ">=14.3" proto-plus = ">=1.22.3,<2.0.0" protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" pydantic = "<3" -shapely = "<3.0.0" typing_extensions = "*" [package.extras] -adk = ["google-adk (>=1.0.0,<2.0.0)", "opentelemetry-instrumentation-google-genai (>=0.3b0,<1.0.0)"] -ag2 = ["ag2[gemini]", "openinference-instrumentation-autogen (>=0.1.6,<0.2)"] -ag2-testing = ["absl-py", "ag2[gemini]", "cloudpickle (>=3.0,<4.0)", "google-cloud-trace (<2)", "openinference-instrumentation-autogen (>=0.1.6,<0.2)", "opentelemetry-exporter-gcp-trace (<2)", "opentelemetry-sdk (<2)", "pydantic (>=2.11.1,<3)", "pytest-xdist", "typing_extensions"] -agent-engines = ["cloudpickle (>=3.0,<4.0)", "google-cloud-logging (<4)", "google-cloud-trace (<2)", "opentelemetry-exporter-gcp-trace (<2)", "opentelemetry-sdk (<2)", "packaging (>=24.0)", "pydantic (>=2.11.1,<3)", "typing_extensions"] -autologging = ["mlflow (>=1.27.0,<=2.16.0)"] +adk = ["aiohttp", "google-adk (>=1.5.0,<3.0.0)"] +adk-testing = ["absl-py", "aiohttp", "cloudpickle (>=3.0,<4.0)", "google-adk (>=1.5.0,<3.0.0)", "opentelemetry-exporter-gcp-logging (>=1.11.0a0,<2.0.0)", "opentelemetry-exporter-otlp-proto-http (<2)", "opentelemetry-instrumentation-google-genai (>=0.3b0,<1.0.0)", "opentelemetry-resourcedetector-gcp (>=1.5.0.dev0)", "opentelemetry-sdk (<2)", "pydantic (>=2.11.1,<3)", "pytest-xdist", "typing_extensions"] +ag2 = ["autogen[gemini] (>=0.14.1,<1)", "openinference-instrumentation-autogen (>=0.1.6,<0.2)"] +ag2-testing = ["absl-py", "aiohttp", "autogen[gemini] (>=0.14.1,<1)", "cloudpickle (>=3.0,<4.0)", "openinference-instrumentation-autogen (>=0.1.6,<0.2)", "opentelemetry-exporter-gcp-logging (>=1.11.0a0,<2.0.0)", "opentelemetry-exporter-otlp-proto-http (<2)", "opentelemetry-instrumentation-google-genai (>=0.3b0,<1.0.0)", "opentelemetry-resourcedetector-gcp (>=1.5.0.dev0)", "opentelemetry-sdk (<2)", "pydantic (>=2.11.1,<3)", "pytest-xdist", "typing_extensions"] +agent-engines = ["aiohttp", "cloudpickle (>=3.0,<4.0)", "google-cloud-logging (<4)", "opentelemetry-exporter-gcp-logging (>=1.11.0a0,<2.0.0)", "opentelemetry-exporter-otlp-proto-http (<2)", "opentelemetry-sdk (<2)", "packaging (>=24.0)", "pydantic (>=2.11.1,<3)", "typing_extensions"] +autologging = ["mlflow (>=1.27.0) ; python_version >= \"3.13\"", "mlflow (>=1.27.0,<=2.16.0) ; python_version < \"3.13\""] cloud-profiler = ["tensorboard-plugin-profile (>=2.4.0,<2.18.0)", "werkzeug (>=2.0.0,<4.0.0)"] -datasets = ["pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0.0) ; python_version < \"3.11\""] +datasets = ["pyarrow (>=10.0.1) ; python_version == \"3.10\"", "pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0.0) ; python_version < \"3.10\""] endpoint = ["requests (>=2.28.1)", "requests-toolbelt (<=1.0.0)"] -evaluation = ["jsonschema", "litellm (>=1.72.4,<=1.76.3)", "pandas (>=1.0.0)", "pyyaml", "ruamel.yaml", "scikit-learn (<1.6.0) ; python_version <= \"3.10\"", "scikit-learn ; python_version > \"3.10\"", "tqdm (>=4.23.0)"] -full = ["docker (>=5.0.3)", "explainable-ai-sdk (>=1.0.0) ; python_version < \"3.13\"", "fastapi (>=0.71.0,<=0.114.0)", "google-cloud-bigquery", "google-cloud-bigquery-storage", "google-vizier (>=0.1.6)", "httpx (>=0.23.0,<=0.28.1)", "immutabledict", "jsonschema", "lit-nlp (==0.4.0)", "litellm (>=1.72.4,<=1.76.3)", "mlflow (>=1.27.0,<=2.16.0)", "numpy (>=1.15.0)", "pandas (>=1.0.0)", "pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0.0) ; python_version < \"3.11\"", "pyarrow (>=6.0.1)", "pyyaml", "pyyaml (>=5.3.1,<7)", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<2.10.dev0 || ==2.33.* || >=2.42.dev0,<=2.42.0) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.47.1) ; python_version == \"3.11\"", "requests (>=2.28.1)", "requests-toolbelt (<=1.0.0)", "ruamel.yaml", "scikit-learn (<1.6.0) ; python_version <= \"3.10\"", "scikit-learn ; python_version > \"3.10\"", "starlette (>=0.17.1)", "tensorboard-plugin-profile (>=2.4.0,<2.18.0)", "tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\"", "tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\"", "tqdm (>=4.23.0)", "urllib3 (>=1.21.1,<1.27)", "uvicorn[standard] (>=0.16.0)", "werkzeug (>=2.0.0,<4.0.0)"] -langchain = ["langchain (>=0.3,<0.4)", "langchain-core (>=0.3,<0.4)", "langchain-google-vertexai (>=2.0.22,<3)", "langgraph (>=0.2.45,<0.4)", "openinference-instrumentation-langchain (>=0.1.19,<0.2)"] -langchain-testing = ["absl-py", "cloudpickle (>=3.0,<4.0)", "google-cloud-trace (<2)", "langchain (>=0.3,<0.4)", "langchain-core (>=0.3,<0.4)", "langchain-google-vertexai (>=2.0.22,<3)", "langgraph (>=0.2.45,<0.4)", "openinference-instrumentation-langchain (>=0.1.19,<0.2)", "opentelemetry-exporter-gcp-trace (<2)", "opentelemetry-sdk (<2)", "pydantic (>=2.11.1,<3)", "pytest-xdist", "typing_extensions"] -lit = ["explainable-ai-sdk (>=1.0.0) ; python_version < \"3.13\"", "lit-nlp (==0.4.0)", "pandas (>=1.0.0)", "tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\""] +evaluation = ["jsonschema", "litellm (>=1.83.7,<1.86.0) ; python_version < \"3.14\"", "litellm (>=1.93.0,<1.97.0) ; python_version >= \"3.14\"", "pandas (>=1.0.0)", "pyyaml", "ruamel.yaml", "scikit-learn (<1.6.0) ; python_version <= \"3.10\"", "scikit-learn ; python_version > \"3.10\"", "tqdm (>=4.23.0)"] +full = ["docker (>=5.0.3)", "explainable-ai-sdk (>=1.0.0) ; python_version < \"3.13\"", "fastapi (>=0.71.0,<=0.124.4)", "google-cloud-bigquery", "google-cloud-bigquery-storage", "google-vizier (>=0.1.6)", "httpx (>=0.23.0,<=0.28.1)", "immutabledict", "jsonschema", "lit-nlp (==0.4.0) ; python_version < \"3.13\"", "litellm (>=1.83.7,<1.86.0) ; python_version < \"3.14\"", "litellm (>=1.93.0,<1.97.0) ; python_version >= \"3.14\"", "mlflow (>=1.27.0) ; python_version >= \"3.13\"", "mlflow (>=1.27.0,<=2.16.0) ; python_version < \"3.13\"", "numpy (>=1.15.0)", "pandas (>=1.0.0)", "pyarrow (>=10.0.1) ; python_version == \"3.10\"", "pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0.0) ; python_version < \"3.10\"", "pyarrow (>=6.0.1)", "pyyaml", "pyyaml (>=5.3.1,<7)", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<2.10.dev0 || ==2.33.* || >=2.42.dev0,<=2.42.0) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.47.1) ; python_version == \"3.11\"", "requests (>=2.28.1)", "requests-toolbelt (<=1.0.0)", "ruamel.yaml", "scikit-learn (<1.6.0) ; python_version <= \"3.10\"", "scikit-learn ; python_version > \"3.10\"", "starlette (>=0.17.1)", "tensorboard-plugin-profile (>=2.4.0,<2.18.0)", "tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\"", "tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\"", "tqdm (>=4.23.0)", "urllib3 (>=1.21.1,<1.27)", "uvicorn[standard] (>=0.16.0)", "werkzeug (>=2.0.0,<4.0.0)"] +langchain = ["langchain (>=1.0.0,<2.0.0)", "langchain-classic", "langchain-core (>=1.0.0,<2.0.0)", "langchain-google-genai (>=4.0.0,<4.2.3)", "langgraph (>=1.0.0,<2.0.0)", "openinference-instrumentation-langchain (>=0.1.19,<0.2)"] +langchain-testing = ["absl-py", "aiohttp", "cloudpickle (>=3.0,<4.0)", "langchain (>=1.0.0,<2.0.0)", "langchain-classic", "langchain-core (>=1.0.0,<2.0.0)", "langchain-google-genai (>=4.0.0,<4.2.3)", "langgraph (>=1.0.0,<2.0.0)", "openinference-instrumentation-langchain (>=0.1.19,<0.2)", "opentelemetry-exporter-gcp-logging (>=1.11.0a0,<2.0.0)", "opentelemetry-exporter-otlp-proto-http (<2)", "opentelemetry-instrumentation-google-genai (>=0.3b0,<1.0.0)", "opentelemetry-resourcedetector-gcp (>=1.5.0.dev0)", "opentelemetry-sdk (<2)", "pydantic (>=2.11.1,<3)", "pytest-xdist", "typing_extensions"] +lit = ["explainable-ai-sdk (>=1.0.0) ; python_version < \"3.13\"", "lit-nlp (==0.4.0) ; python_version < \"3.13\"", "pandas (>=1.0.0)", "tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\""] llama-index = ["llama-index", "llama-index-llms-google-genai", "openinference-instrumentation-llama-index (>=3.0,<4.0)"] -llama-index-testing = ["absl-py", "cloudpickle (>=3.0,<4.0)", "google-cloud-trace (<2)", "llama-index", "llama-index-llms-google-genai", "openinference-instrumentation-llama-index (>=3.0,<4.0)", "opentelemetry-exporter-gcp-trace (<2)", "opentelemetry-sdk (<2)", "pydantic (>=2.11.1,<3)", "pytest-xdist", "typing_extensions"] +llama-index-testing = ["absl-py", "aiohttp", "cloudpickle (>=3.0,<4.0)", "llama-index", "llama-index-llms-google-genai", "openinference-instrumentation-llama-index (>=3.0,<4.0)", "opentelemetry-exporter-gcp-logging (>=1.11.0a0,<2.0.0)", "opentelemetry-exporter-otlp-proto-http (<2)", "opentelemetry-instrumentation-google-genai (>=0.3b0,<1.0.0)", "opentelemetry-resourcedetector-gcp (>=1.5.0.dev0)", "opentelemetry-sdk (<2)", "pydantic (>=2.11.1,<3)", "pytest-xdist", "typing_extensions"] metadata = ["numpy (>=1.15.0)", "pandas (>=1.0.0)"] pipelines = ["pyyaml (>=5.3.1,<7)"] -prediction = ["docker (>=5.0.3)", "fastapi (>=0.71.0,<=0.114.0)", "httpx (>=0.23.0,<=0.28.1)", "starlette (>=0.17.1)", "uvicorn[standard] (>=0.16.0)"] +prediction = ["docker (>=5.0.3)", "fastapi (>=0.71.0,<=0.124.4)", "httpx (>=0.23.0,<=0.28.1)", "starlette (>=0.17.1)", "uvicorn[standard] (>=0.16.0)"] private-endpoints = ["requests (>=2.28.1)", "urllib3 (>=1.21.1,<1.27)"] ray = ["google-cloud-bigquery", "google-cloud-bigquery-storage", "immutabledict", "pandas (>=1.0.0)", "pyarrow (>=6.0.1)", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<2.10.dev0 || ==2.33.* || >=2.42.dev0,<=2.42.0) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.47.1) ; python_version == \"3.11\""] ray-testing = ["google-cloud-bigquery", "google-cloud-bigquery-storage", "immutabledict", "pandas (>=1.0.0)", "pyarrow (>=6.0.1)", "pytest-xdist", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<2.10.dev0 || ==2.33.* || >=2.42.dev0,<=2.42.0) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.47.1) ; python_version == \"3.11\"", "ray[train]", "scikit-learn (<1.6.0)", "tensorflow ; python_version < \"3.13\"", "torch (>=2.0.0,<2.1.0)", "xgboost", "xgboost_ray"] -reasoningengine = ["cloudpickle (>=3.0,<4.0)", "google-cloud-trace (<2)", "opentelemetry-exporter-gcp-trace (<2)", "opentelemetry-sdk (<2)", "pydantic (>=2.11.1,<3)", "typing_extensions"] +reasoningengine = ["aiohttp", "cloudpickle (>=3.0,<4.0)", "opentelemetry-exporter-gcp-logging (>=1.11.0a0,<2.0.0)", "opentelemetry-exporter-otlp-proto-http (<2)", "opentelemetry-instrumentation-google-genai (>=0.3b0,<1.0.0)", "opentelemetry-resourcedetector-gcp (>=1.5.0.dev0)", "opentelemetry-sdk (<2)", "pydantic (>=2.11.1,<3)", "typing_extensions"] tensorboard = ["tensorboard-plugin-profile (>=2.4.0,<2.18.0)", "werkzeug (>=2.0.0,<4.0.0)"] -testing = ["aiohttp", "bigframes ; python_version >= \"3.10\"", "docker (>=5.0.3)", "explainable-ai-sdk (>=1.0.0) ; python_version < \"3.13\"", "fastapi (>=0.71.0,<=0.114.0)", "google-api-core (>=2.11,<3.0.0)", "google-cloud-bigquery", "google-cloud-bigquery-storage", "google-vizier (>=0.1.6)", "google-vizier (>=0.1.6)", "grpcio-testing", "grpcio-tools (>=1.63.0) ; python_version >= \"3.13\"", "httpx (>=0.23.0,<=0.28.1)", "immutabledict", "immutabledict", "ipython", "jsonschema", "kfp (>=2.6.0,<3.0.0) ; python_version < \"3.13\"", "lit-nlp (==0.4.0)", "litellm (>=1.72.4,<=1.76.3)", "mlflow (>=1.27.0,<=2.16.0)", "mock", "nltk", "numpy (>=1.15.0)", "pandas (>=1.0.0)", "protobuf (<=5.29.4)", "pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0.0) ; python_version < \"3.11\"", "pyarrow (>=6.0.1)", "pytest-asyncio", "pytest-cov", "pytest-xdist", "pyyaml", "pyyaml (>=5.3.1,<7)", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<2.10.dev0 || ==2.33.* || >=2.42.dev0,<=2.42.0) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.47.1) ; python_version == \"3.11\"", "requests (>=2.28.1)", "requests-toolbelt (<=1.0.0)", "requests-toolbelt (<=1.0.0)", "ruamel.yaml", "scikit-learn (<1.6.0) ; python_version <= \"3.10\"", "scikit-learn (<1.6.0) ; python_version <= \"3.10\"", "scikit-learn ; python_version > \"3.10\"", "scikit-learn ; python_version > \"3.10\"", "sentencepiece (>=0.2.0)", "starlette (>=0.17.1)", "tensorboard-plugin-profile (>=2.4.0,<2.18.0)", "tensorboard-plugin-profile (>=2.4.0,<2.18.0)", "tensorflow (==2.14.1) ; python_version <= \"3.11\"", "tensorflow (==2.19.0) ; python_version > \"3.11\" and python_version < \"3.13\"", "tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\"", "tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\"", "torch (>=2.0.0,<2.1.0) ; python_version <= \"3.11\"", "torch (>=2.2.0) ; python_version > \"3.11\" and python_version < \"3.13\"", "tqdm (>=4.23.0)", "urllib3 (>=1.21.1,<1.27)", "uvicorn[standard] (>=0.16.0)", "werkzeug (>=2.0.0,<4.0.0)", "werkzeug (>=2.0.0,<4.0.0)", "xgboost"] +testing = ["Pillow", "aiohttp", "bigframes ; python_version >= \"3.10\" and python_version < \"3.14\"", "docker (>=5.0.3)", "explainable-ai-sdk (>=1.0.0) ; python_version < \"3.13\"", "fastapi (>=0.71.0,<=0.124.4)", "google-api-core (>=2.11,<3.0.0)", "google-cloud-bigquery", "google-cloud-bigquery-storage", "google-vizier (>=0.1.6)", "google-vizier (>=0.1.6)", "grpcio-testing", "grpcio-tools (>=1.63.0) ; python_version >= \"3.13\"", "httpx (>=0.23.0,<=0.28.1)", "immutabledict", "immutabledict", "ipython", "jsonschema", "kfp (>=2.6.0,<3.0.0) ; python_version < \"3.13\"", "lit-nlp (==0.4.0) ; python_version < \"3.13\"", "litellm (>=1.83.7,<1.86.0) ; python_version < \"3.14\"", "litellm (>=1.93.0,<1.97.0) ; python_version >= \"3.14\"", "mlflow (>=1.27.0) ; python_version >= \"3.13\"", "mlflow (>=1.27.0,<=2.16.0) ; python_version < \"3.13\"", "mock", "nltk", "numpy (>=1.15.0)", "pandas (>=1.0.0)", "protobuf (<=5.29.4) ; python_version < \"3.14\"", "protobuf (>=5.29.4) ; python_version >= \"3.14\"", "pyarrow (>=10.0.1) ; python_version == \"3.10\"", "pyarrow (>=10.0.1) ; python_version == \"3.11\"", "pyarrow (>=14.0.0) ; python_version >= \"3.12\"", "pyarrow (>=3.0.0,<8.0.0) ; python_version < \"3.10\"", "pyarrow (>=6.0.1)", "pytest-asyncio", "pytest-cov", "pytest-xdist", "pyyaml", "pyyaml (>=5.3.1,<7)", "ray[default] (>=2.4,<2.5.dev0 || >2.9.0,!=2.9.1,!=2.9.2,<2.10.dev0 || ==2.33.* || >=2.42.dev0,<=2.42.0) ; python_version < \"3.11\"", "ray[default] (>=2.5,<=2.47.1) ; python_version == \"3.11\"", "requests (>=2.28.1)", "requests-toolbelt (<=1.0.0)", "requests-toolbelt (<=1.0.0)", "ruamel.yaml", "scikit-learn (<1.6.0) ; python_version <= \"3.10\"", "scikit-learn (<1.6.0) ; python_version <= \"3.10\"", "scikit-learn ; python_version > \"3.10\"", "scikit-learn ; python_version > \"3.10\"", "sentencepiece (>=0.2.0)", "starlette (>=0.17.1)", "tensorboard-plugin-profile (>=2.4.0,<2.18.0)", "tensorboard-plugin-profile (>=2.4.0,<2.18.0)", "tensorflow (==2.14.1) ; python_version <= \"3.11\"", "tensorflow (==2.19.0) ; python_version > \"3.11\" and python_version < \"3.13\"", "tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\"", "tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\"", "torch (>=2.0.0,<2.1.0) ; python_version <= \"3.11\"", "torch (>=2.2.0) ; python_version > \"3.11\" and python_version < \"3.13\"", "tqdm (>=4.23.0)", "urllib3 (>=1.21.1,<1.27)", "uvicorn[standard] (>=0.16.0)", "werkzeug (>=2.0.0,<4.0.0)", "werkzeug (>=2.0.0,<4.0.0)", "xgboost"] tokenization = ["sentencepiece (>=0.2.0)"] vizier = ["google-vizier (>=0.1.6)"] xai = ["tensorflow (>=2.3.0,<3.0.0) ; python_version < \"3.13\""] @@ -1606,27 +1700,51 @@ protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4 [[package]] name = "google-cloud-storage" -version = "2.19.0" +version = "3.13.1" description = "Google Cloud Storage API client library" optional = false -python-versions = ">=3.7" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "google_cloud_storage-2.19.0-py2.py3-none-any.whl", hash = "sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba"}, - {file = "google_cloud_storage-2.19.0.tar.gz", hash = "sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2"}, + {file = "google_cloud_storage-3.13.1-py3-none-any.whl", hash = "sha256:98208de6c21e85cecd3eb44551894efff33d98365500e178867d4305854a770a"}, + {file = "google_cloud_storage-3.13.1.tar.gz", hash = "sha256:a80bf8cac2794808aa61c50c5f769ecbbe2d10331bacd0d69d30e59b14b346b2"}, ] [package.dependencies] -google-api-core = ">=2.15.0,<3.0.0dev" -google-auth = ">=2.26.1,<3.0dev" -google-cloud-core = ">=2.3.0,<3.0dev" -google-crc32c = ">=1.0,<2.0dev" -google-resumable-media = ">=2.7.2" -requests = ">=2.18.0,<3.0.0dev" +google-api-core = ">=2.27.0,<3.0.0" +google-auth = ">=2.26.1,<3.0.0" +google-cloud-core = ">=2.4.2,<3.0.0" +google-crc32c = ">=1.6.0,<2.0.0" +google-resumable-media = ">=2.7.2,<3.0.0" +requests = ">=2.22.0,<3.0.0" [package.extras] -protobuf = ["protobuf (<6.0.0dev)"] -tracing = ["opentelemetry-api (>=1.1.0)"] +grpc = ["google-api-core[grpc] (>=2.27.0,<3.0.0)", "grpc-google-iam-v1 (>=0.14.2,<1.0.0)", "grpcio (>=1.59.0,<2.0.0)", "grpcio (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "grpcio-status (>=1.59.0,<2.0.0)", "grpcio-status (>=1.75.1,<2.0.0) ; python_version >= \"3.14\"", "proto-plus (>=1.26.1,<2.0.0)", "protobuf (>=6.33.5,<8.0.0)"] +protobuf = ["protobuf (>=6.33.5,<8.0.0)"] +testing = ["PyYAML", "black", "brotli", "coverage", "flake8", "google-cloud-iam", "google-cloud-kms", "google-cloud-pubsub", "google-cloud-testutils", "google-cloud-testutils", "mock", "numpy", "opentelemetry-sdk", "psutil", "py-cpuinfo", "pyopenssl", "pytest", "pytest-asyncio", "pytest-benchmark", "pytest-cov", "pytest-rerunfailures", "pytest-xdist"] +tracing = ["opentelemetry-api (>=1.1.0,<2.0.0)"] + +[[package]] +name = "google-cloud-vectorsearch" +version = "0.11.1" +description = "Google Cloud Vectorsearch API client library" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "google_cloud_vectorsearch-0.11.1-py3-none-any.whl", hash = "sha256:005e3829e0303853ffa114483283b6bc747537b155fc90aca36e815d1c649ec3"}, + {file = "google_cloud_vectorsearch-0.11.1.tar.gz", hash = "sha256:fb4723a6668fc633a1e780dec85f436a417228f346b25477afbcf288c311c393"}, +] + +[package.dependencies] +google-api-core = {version = ">=2.24.2,<3.0.0", extras = ["grpc"]} +google-auth = ">=2.14.1,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +grpcio = [ + {version = ">=1.59.0,<2.0.0"}, + {version = ">=1.75.1,<2.0.0", markers = "python_version >= \"3.14\""}, +] +proto-plus = ">=1.26.1,<2.0.0" +protobuf = ">=6.33.5,<8.0.0" [[package]] name = "google-crc32c" @@ -1677,29 +1795,32 @@ testing = ["pytest"] [[package]] name = "google-genai" -version = "1.46.0" +version = "1.75.0" description = "GenAI Python SDK" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "google_genai-1.46.0-py3-none-any.whl", hash = "sha256:879c4a260d630db0dcedb5cc84a9d7b47acd29e43e9dc63541b511b757ea7296"}, - {file = "google_genai-1.46.0.tar.gz", hash = "sha256:6824c31149fe3b1c7285b25f79b924c5f89fd52466f62e30f76954f8104fe3a7"}, + {file = "google_genai-1.75.0-py3-none-any.whl", hash = "sha256:8dc4c096e7d6288c3087f6893f582fe52468932464781edb8193bd92b9fefb2c"}, + {file = "google_genai-1.75.0.tar.gz", hash = "sha256:56bac3991b311c93f980c0a2abcd287b672146905df1fbd71c92ed633d5a07cf"}, ] [package.dependencies] anyio = ">=4.8.0,<5.0.0" -google-auth = ">=2.14.1,<3.0.0" +distro = ">=1.7.0,<2" +google-auth = {version = ">=2.48.1,<3.0.0", extras = ["requests"]} httpx = ">=0.28.1,<1.0.0" -pydantic = ">=2.0.0,<3.0.0" +pydantic = ">=2.9.0,<3.0.0" requests = ">=2.28.1,<3.0.0" +sniffio = "*" tenacity = ">=8.2.3,<9.2.0" -typing-extensions = ">=4.11.0,<5.0.0" -websockets = ">=13.0.0,<15.1.0" +typing-extensions = ">=4.14.0,<5.0.0" +websockets = ">=13.0.0,<17.0" [package.extras] -aiohttp = ["aiohttp (<4.0.0)"] +aiohttp = ["aiohttp (>=3.10.11,<4.0.0)"] local-tokenizer = ["protobuf", "sentencepiece (>=0.2.0)"] +pyopenssl = ["pyopenssl"] [[package]] name = "google-resumable-media" @@ -1856,7 +1977,7 @@ version = "0.16.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, @@ -1864,35 +1985,30 @@ files = [ [[package]] name = "hf-xet" -version = "1.2.0" +version = "1.6.0" description = "Fast transfer of large files with the Hugging Face Hub." optional = false python-versions = ">=3.8" -groups = ["main", "docling-ocr"] -markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" -files = [ - {file = "hf_xet-1.2.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:ceeefcd1b7aed4956ae8499e2199607765fbd1c60510752003b6cc0b8413b649"}, - {file = "hf_xet-1.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b70218dd548e9840224df5638fdc94bd033552963cfa97f9170829381179c813"}, - {file = "hf_xet-1.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d40b18769bb9a8bc82a9ede575ce1a44c75eb80e7375a01d76259089529b5dc"}, - {file = "hf_xet-1.2.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd3a6027d59cfb60177c12d6424e31f4b5ff13d8e3a1247b3a584bf8977e6df5"}, - {file = "hf_xet-1.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6de1fc44f58f6dd937956c8d304d8c2dea264c80680bcfa61ca4a15e7b76780f"}, - {file = "hf_xet-1.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f182f264ed2acd566c514e45da9f2119110e48a87a327ca271027904c70c5832"}, - {file = "hf_xet-1.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:293a7a3787e5c95d7be1857358a9130694a9c6021de3f27fa233f37267174382"}, - {file = "hf_xet-1.2.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:10bfab528b968c70e062607f663e21e34e2bba349e8038db546646875495179e"}, - {file = "hf_xet-1.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2a212e842647b02eb6a911187dc878e79c4aa0aa397e88dd3b26761676e8c1f8"}, - {file = "hf_xet-1.2.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30e06daccb3a7d4c065f34fc26c14c74f4653069bb2b194e7f18f17cbe9939c0"}, - {file = "hf_xet-1.2.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:29c8fc913a529ec0a91867ce3d119ac1aac966e098cf49501800c870328cc090"}, - {file = "hf_xet-1.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e159cbfcfbb29f920db2c09ed8b660eb894640d284f102ada929b6e3dc410a"}, - {file = "hf_xet-1.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c91d5ae931510107f148874e9e2de8a16052b6f1b3ca3c1b12f15ccb491390f"}, - {file = "hf_xet-1.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:210d577732b519ac6ede149d2f2f34049d44e8622bf14eb3d63bbcd2d4b332dc"}, - {file = "hf_xet-1.2.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:46740d4ac024a7ca9b22bebf77460ff43332868b661186a8e46c227fdae01848"}, - {file = "hf_xet-1.2.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:27df617a076420d8845bea087f59303da8be17ed7ec0cd7ee3b9b9f579dff0e4"}, - {file = "hf_xet-1.2.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3651fd5bfe0281951b988c0facbe726aa5e347b103a675f49a3fa8144c7968fd"}, - {file = "hf_xet-1.2.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d06fa97c8562fb3ee7a378dd9b51e343bc5bc8190254202c9771029152f5e08c"}, - {file = "hf_xet-1.2.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4c1428c9ae73ec0939410ec73023c4f842927f39db09b063b9482dac5a3bb737"}, - {file = "hf_xet-1.2.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a55558084c16b09b5ed32ab9ed38421e2d87cf3f1f89815764d1177081b99865"}, - {file = "hf_xet-1.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:e6584a52253f72c9f52f9e549d5895ca7a471608495c4ecaa6cc73dba2b24d69"}, - {file = "hf_xet-1.2.0.tar.gz", hash = "sha256:a8c27070ca547293b6890c4bf389f713f80e8c478631432962bb7f4bc0bd7d7f"}, +groups = ["main", "docling-ocr", "local-ml"] +markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" +files = [ + {file = "hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d"}, + {file = "hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675"}, + {file = "hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b"}, + {file = "hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522"}, + {file = "hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e"}, + {file = "hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9"}, + {file = "hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338"}, + {file = "hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765"}, + {file = "hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d"}, + {file = "hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a"}, + {file = "hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f"}, + {file = "hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7"}, + {file = "hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb"}, + {file = "hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c"}, + {file = "hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b"}, + {file = "hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3"}, + {file = "hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef"}, ] [package.extras] @@ -1904,7 +2020,7 @@ version = "1.0.9" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, @@ -1926,7 +2042,7 @@ version = "0.28.1" description = "The next generation HTTP client." optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, @@ -1959,57 +2075,54 @@ files = [ [[package]] name = "huggingface-hub" -version = "0.36.0" +version = "1.16.1" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false -python-versions = ">=3.8.0" -groups = ["main", "docling-ocr"] +python-versions = ">=3.10.0" +groups = ["main", "docling-ocr", "local-ml"] files = [ - {file = "huggingface_hub-0.36.0-py3-none-any.whl", hash = "sha256:7bcc9ad17d5b3f07b57c78e79d527102d08313caa278a641993acddcb894548d"}, - {file = "huggingface_hub-0.36.0.tar.gz", hash = "sha256:47b3f0e2539c39bf5cde015d63b72ec49baff67b6931c3d97f3f84532e2b8d25"}, + {file = "huggingface_hub-1.16.1-py3-none-any.whl", hash = "sha256:64340de934b9ce37857ef85a82de72f5629e8a270f9119eabb12bf495eb53c22"}, + {file = "huggingface_hub-1.16.1.tar.gz", hash = "sha256:7f1dc4c5ec21aed69be630ad0c3378616be16f3de1a47b141c0e812965d9c832"}, ] [package.dependencies] -filelock = "*" +filelock = ">=3.10.0" fsspec = ">=2023.5.0" -hf-xet = {version = ">=1.1.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} +hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} +httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" -requests = "*" tqdm = ">=4.42.1" -typing-extensions = ">=3.7.4.3" +typer = ">=0.20.0" +typing-extensions = ">=4.1.0" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0) ; python_version >= \"3.9\"", "mypy (>=1.14.1,<1.15.0) ; python_version == \"3.8\"", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0) ; python_version >= \"3.9\"", "mypy (>=1.14.1,<1.15.0) ; python_version == \"3.8\"", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -hf-transfer = ["hf-transfer (>=0.1.4)"] -hf-xet = ["hf-xet (>=1.1.2,<2.0.0)"] -inference = ["aiohttp"] -mcp = ["aiohttp", "mcp (>=1.8.0)", "typer"] +gradio = ["gradio (>=5.0.0)", "requests"] +hf-xet = ["hf-xet (>=1.4.3,<2.0.0)"] +mcp = ["mcp (>=1.8.0)"] oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] -quality = ["libcst (>=1.4.0)", "mypy (==1.15.0) ; python_version >= \"3.9\"", "mypy (>=1.14.1,<1.15.0) ; python_version == \"3.8\"", "ruff (>=0.9.0)", "ty"] -tensorflow = ["graphviz", "pydot", "tensorflow"] -tensorflow-testing = ["keras (<3.0)", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1.3.2)", "fastapi", "gradio (>=4.0.0)", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] +testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] +typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] [[package]] name = "idna" -version = "3.11" +version = "3.19" description = "Internationalized Domain Names in Applications (IDNA)" optional = false -python-versions = ">=3.8" -groups = ["main", "docling-ocr"] +python-versions = ">=3.9" +groups = ["main", "docling-ocr", "local-ml"] files = [ - {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, - {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, + {file = "idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4"}, + {file = "idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15"}, ] [package.extras] -all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +all = ["coverage (>=7.10.0)", "hypothesis (>=6.141.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.16.0)", "ty (>=0.0.37)"] [[package]] name = "iniconfig" @@ -2017,7 +2130,7 @@ version = "2.3.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, @@ -2045,7 +2158,7 @@ version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, @@ -2187,7 +2300,7 @@ version = "1.5.3" description = "Lightweight pipelining with Python functions" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "local-ml"] files = [ {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, @@ -2322,24 +2435,26 @@ zookeeper = ["kazoo (>=2.8.0)"] [[package]] name = "langchain" -version = "1.0.0" +version = "1.3.16" description = "Building applications with LLMs through composability" optional = false python-versions = "<4.0.0,>=3.10.0" groups = ["main"] files = [ - {file = "langchain-1.0.0-py3-none-any.whl", hash = "sha256:8c95e41250fc86d09a978fbdf999f86c18d50a28a2addc5da88546af00a1ad15"}, - {file = "langchain-1.0.0.tar.gz", hash = "sha256:56bf90d935ac1dda864519372d195ca58757b755dd4c44b87840b67d069085b7"}, + {file = "langchain-1.3.16-py3-none-any.whl", hash = "sha256:67dcb28d69f96018f3062da2075f99b25ea048af7f5baae570afe1d91e033743"}, + {file = "langchain-1.3.16.tar.gz", hash = "sha256:781fd3bca96813e9d0d74c58a323d0ac05a0c0d5cb2c56ac34185f4f3ec0c4bb"}, ] [package.dependencies] -langchain-core = ">=1.0.0,<2.0.0" -langgraph = ">=1.0.0,<1.1.0" +langchain-core = ">=1.6.0,<2.0.0" +langgraph = ">=1.2.11,<1.3.0" pydantic = ">=2.7.4,<3.0.0" [package.extras] anthropic = ["langchain-anthropic"] aws = ["langchain-aws"] +azure-ai = ["langchain-azure-ai"] +baseten = ["langchain-baseten (>=0.2.0)"] community = ["langchain-community"] deepseek = ["langchain-deepseek"] fireworks = ["langchain-fireworks"] @@ -2347,6 +2462,7 @@ google-genai = ["langchain-google-genai"] google-vertexai = ["langchain-google-vertexai"] groq = ["langchain-groq"] huggingface = ["langchain-huggingface"] +meta = ["langchain-meta"] mistralai = ["langchain-mistralai"] ollama = ["langchain-ollama"] openai = ["langchain-openai"] @@ -2380,19 +2496,20 @@ valkey = ["valkey-glide-sync (>=2.0.0)"] [[package]] name = "langchain-core" -version = "1.4.0" +version = "1.6.0" description = "Building applications with LLMs through composability" optional = false python-versions = "<4.0.0,>=3.10.0" groups = ["main"] files = [ - {file = "langchain_core-1.4.0-py3-none-any.whl", hash = "sha256:23cbbdb46e38ddd1dd5247e6167e96013eae74bea4c5949c550809970a9e565c"}, - {file = "langchain_core-1.4.0.tar.gz", hash = "sha256:1dc341eed802ed9c117c0df3923c991e5e9e226571e5725c194eeb5bd93d1a7f"}, + {file = "langchain_core-1.6.0-py3-none-any.whl", hash = "sha256:d8bb924cd413955d9d3192ccced140407427c3ff51e50ffb941222648da92cb2"}, + {file = "langchain_core-1.6.0.tar.gz", hash = "sha256:dc72e36678ed26683ec0ad8829b44011fba461d10f9b31dbd9110215e5ec2333"}, ] [package.dependencies] +httpx = ">=0.23.0,<1.0.0" jsonpatch = ">=1.33.0,<2.0.0" -langchain-protocol = ">=0.0.14" +langchain-protocol = ">=0.0.17" langsmith = ">=0.3.45,<1.0.0" packaging = ">=23.2.0" pydantic = ">=2.7.4,<3.0.0" @@ -2403,59 +2520,62 @@ uuid-utils = ">=0.12.0,<1.0" [[package]] name = "langchain-google-vertexai" -version = "3.0.1" +version = "3.2.4" description = "An integration package connecting Google VertexAI and LangChain" optional = false python-versions = "<4.0.0,>=3.10.0" groups = ["main"] files = [ - {file = "langchain_google_vertexai-3.0.1-py3-none-any.whl", hash = "sha256:d6a94857a2413c1a81fe8d20f2a40e9ce999120fee93033a00ce50351e24855a"}, - {file = "langchain_google_vertexai-3.0.1.tar.gz", hash = "sha256:f18249bbdd8a3bcff1f38d40ca7ae2132edcb8d7af51896581dd24c835940e69"}, + {file = "langchain_google_vertexai-3.2.4-py3-none-any.whl", hash = "sha256:65b5615e596fdabc2e149f0160fded88bebef2bbc1ea70095ff81714f7570183"}, + {file = "langchain_google_vertexai-3.2.4.tar.gz", hash = "sha256:f0c16b4b639037f60357dbf6d4aa3b0f301f23be6b33b0688c75902235cb9434"}, ] [package.dependencies] bottleneck = ">=1.4.0,<2.0.0" google-cloud-aiplatform = ">=1.97.0,<2.0.0" -google-cloud-storage = ">=2.18.0,<3.0.0" +google-cloud-storage = ">=2.18.0,<4.0.0" +google-cloud-vectorsearch = ">=0.2.0" httpx = ">=0.28.0,<1.0.0" httpx-sse = ">=0.4.0,<1.0.0" -langchain-core = ">=1.0.0,<2.0.0" +langchain-core = ">=1.3.2,<2.0.0" numexpr = ">=2.8.6,<3.0.0" -pyarrow = ">=19.0.1,<22.0.0" +pyarrow = ">=19.0.1,<24.0.0" pydantic = ">=2.9.0,<3.0.0" validators = ">=0.22.0,<1.0.0" [package.extras] anthropic = ["anthropic (>=0.35.0,<1.0.0)"] mistral = ["langchain-mistralai (>=0.2.0,<2.0.0)"] +vectorsearch-v2 = ["google-cloud-vectorsearch"] [[package]] name = "langchain-openai" -version = "1.0.0" +version = "1.6.0" description = "An integration package connecting OpenAI and LangChain" optional = false python-versions = "<4.0.0,>=3.10.0" groups = ["main"] files = [ - {file = "langchain_openai-1.0.0-py3-none-any.whl", hash = "sha256:e42b856ae53d2f6f3afd2ecb875056025d8c0b8f313a89fc66eb682c6b958111"}, - {file = "langchain_openai-1.0.0.tar.gz", hash = "sha256:14b4f8e77e2ace62e390fe4824f08718b227ed86b476983ce4f0063fa258e966"}, + {file = "langchain_openai-1.6.0-py3-none-any.whl", hash = "sha256:648112bbd135aa51d60d4aa2fd4ad353628a8d92a2f8e779244a0d9869a16f18"}, + {file = "langchain_openai-1.6.0.tar.gz", hash = "sha256:7c545cc1f2bee048a7e219f375b78cd820b32c876ccf825320e25e1e94132577"}, ] [package.dependencies] -langchain-core = ">=1.0.0,<2.0.0" -openai = ">=1.109.1,<3.0.0" +certifi = ">=2024.6.2" +langchain-core = ">=1.6.0,<2.0.0" +openai = ">=2.45.0,<4.0.0" tiktoken = ">=0.7.0,<1.0.0" [[package]] name = "langchain-protocol" -version = "0.0.16" +version = "0.0.18" description = "Python bindings for the LangChain agent streaming protocol" optional = false python-versions = "<4.0.0,>=3.10.0" groups = ["main"] files = [ - {file = "langchain_protocol-0.0.16-py3-none-any.whl", hash = "sha256:3658c142c5d0fb3a023a4be442ce4c15c6d626aab6135eb79a76dc64ad19c3c3"}, - {file = "langchain_protocol-0.0.16.tar.gz", hash = "sha256:806c7cdd951b1c4f692fa40fce60821ff0f221d4360e27673ddf2c2b99c2b7ff"}, + {file = "langchain_protocol-0.0.18-py3-none-any.whl", hash = "sha256:70b53a86fbf9cedc863555effe44da192ab02d556ddbf2cf95b8873adcf41b5a"}, + {file = "langchain_protocol-0.0.18.tar.gz", hash = "sha256:ec3e11782f1ed0c9db38e5a9ed01b0e7a0d3fba406faa8aef6594b73c56a63e6"}, ] [package.dependencies] @@ -2495,71 +2615,74 @@ six = "*" [[package]] name = "langgraph" -version = "1.0.0" +version = "1.2.11" description = "Building stateful, multi-actor applications with LLMs" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "langgraph-1.0.0-py3-none-any.whl", hash = "sha256:4d478781832a1bc67e06c3eb571412ec47d7c57a5467d1f3775adf0e9dd4042c"}, - {file = "langgraph-1.0.0.tar.gz", hash = "sha256:5f83ed0e9bbcc37635bc49cbc9b3d9306605fa07504f955b7a871ed715f9964c"}, + {file = "langgraph-1.2.11-py3-none-any.whl", hash = "sha256:8bab70de7b2d00b5300fb289bcf38d8b241400f3184c1e95e8ce706fb0e8686b"}, + {file = "langgraph-1.2.11.tar.gz", hash = "sha256:9ecfe11e50d338b34b15cf4d8a442642de103e8ae6971320efba84e4542eb363"}, ] [package.dependencies] -langchain-core = ">=0.1" -langgraph-checkpoint = ">=2.1.0,<3.0.0" -langgraph-prebuilt = ">=1.0.0,<1.1.0" -langgraph-sdk = ">=0.2.2,<0.3.0" +langchain-core = ">=1.4.7,<2" +langgraph-checkpoint = ">=4.1.0,<5.0.0" +langgraph-prebuilt = ">=1.1.0,<1.2.0" +langgraph-sdk = ">=0.4.2,<0.5.0" pydantic = ">=2.7.4" xxhash = ">=3.5.0" [[package]] name = "langgraph-checkpoint" -version = "2.1.2" +version = "4.2.0" description = "Library with base interfaces for LangGraph checkpoint savers." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "langgraph_checkpoint-2.1.2-py3-none-any.whl", hash = "sha256:911ebffb069fd01775d4b5184c04aaafc2962fcdf50cf49d524cd4367c4d0c60"}, - {file = "langgraph_checkpoint-2.1.2.tar.gz", hash = "sha256:112e9d067a6eff8937caf198421b1ffba8d9207193f14ac6f89930c1260c06f9"}, + {file = "langgraph_checkpoint-4.2.0-py3-none-any.whl", hash = "sha256:0547fd228935a0b758865de3a3d6d7a2537c308895d0f9ab092ce9151b5da942"}, + {file = "langgraph_checkpoint-4.2.0.tar.gz", hash = "sha256:51a593b6bee684b0818e5d6e58e28ab340c6db7794575056ce7bd1b746a84ed7"}, ] [package.dependencies] langchain-core = ">=0.2.38" -ormsgpack = ">=1.10.0" +ormsgpack = ">=1.12.0" [[package]] name = "langgraph-prebuilt" -version = "1.0.0" +version = "1.1.0" description = "Library with high-level APIs for creating and executing LangGraph agents and tools." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "langgraph_prebuilt-1.0.0-py3-none-any.whl", hash = "sha256:ceaae4c5cee8c1f9b6468f76c114cafebb748aed0c93483b7c450e5a89de9c61"}, - {file = "langgraph_prebuilt-1.0.0.tar.gz", hash = "sha256:eb75dad9aca0137451ca0395aa8541a665b3f60979480b0431d626fd195dcda2"}, + {file = "langgraph_prebuilt-1.1.0-py3-none-any.whl", hash = "sha256:51e311747d755b751d5c6b39b0c1446124d3a7643d2515017e6714b323508fc9"}, + {file = "langgraph_prebuilt-1.1.0.tar.gz", hash = "sha256:3c579cf6eed2d17f9c157c2d0fcaddcd8688524e7022d3b22b37a3bf4589d528"}, ] [package.dependencies] -langchain-core = ">=0.3.67" -langgraph-checkpoint = ">=2.1.0,<3.0.0" +langchain-core = ">=1.3.1" +langgraph-checkpoint = ">=2.1.0,<5.0.0" [[package]] name = "langgraph-sdk" -version = "0.2.9" +version = "0.4.3" description = "SDK for interacting with LangGraph API" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "langgraph_sdk-0.2.9-py3-none-any.whl", hash = "sha256:fbf302edadbf0fb343596f91c597794e936ef68eebc0d3e1d358b6f9f72a1429"}, - {file = "langgraph_sdk-0.2.9.tar.gz", hash = "sha256:b3bd04c6be4fa382996cd2be8fbc1e7cc94857d2bc6b6f4599a7f2a245975303"}, + {file = "langgraph_sdk-0.4.3-py3-none-any.whl", hash = "sha256:1b7920b39b6dc439843d122a06f04a0cb8b65c02fd086ddf68e18596a230a0e7"}, + {file = "langgraph_sdk-0.4.3.tar.gz", hash = "sha256:f101cc043ddd7400ceaef66d934c42fa2f0e7d4ea5029754d286260d39038251"}, ] [package.dependencies] httpx = ">=0.25.2" -orjson = ">=3.10.1" +langchain-core = ">=1.4.0,<2" +langchain-protocol = ">=0.0.15" +orjson = ">=3.11.5" +websockets = ">=14,<17" [[package]] name = "langid" @@ -2577,31 +2700,45 @@ numpy = "*" [[package]] name = "langsmith" -version = "0.4.37" -description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +version = "0.11.1" +description = "Client library to connect to the LangSmith Observability and Evaluation Platform." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "langsmith-0.4.37-py3-none-any.whl", hash = "sha256:e34a94ce7277646299e4703a0f6e2d2c43647a28e8b800bb7ef82fd87a0ec766"}, - {file = "langsmith-0.4.37.tar.gz", hash = "sha256:d9a0eb6dd93f89843ac982c9f92be93cf2bcabbe19957f362c547766c7366c71"}, + {file = "langsmith-0.11.1-py3-none-any.whl", hash = "sha256:cfc3437a9cf27440cd0095c24df945edbceb6df10b579bc8b3980b4ad367835f"}, + {file = "langsmith-0.11.1.tar.gz", hash = "sha256:47998977366acb3ba3093881fd465cbf11a5f8c2f4e87e40a17dda203f6dedf4"}, ] [package.dependencies] +anyio = ">=3.5.0" +distro = ">=1.7.0" httpx = ">=0.23.0,<1" orjson = {version = ">=3.9.14", markers = "platform_python_implementation != \"PyPy\""} packaging = ">=23.2" -pydantic = ">=1,<3" +pydantic = ">=2,<3" requests = ">=2.0.0" requests-toolbelt = ">=1.0.0" +sniffio = ">=1.1" +typing-extensions = ">=4.0.0" +uuid-utils = ">=0.12.0,<1.0" +websockets = ">=15.0" +xxhash = ">=3.0.0" zstandard = ">=0.23.0" [package.extras] claude-agent-sdk = ["claude-agent-sdk (>=0.1.0) ; python_version >= \"3.10\""] +gemini-live = ["google-genai (>=1.75)"] +google-adk = ["google-adk (>=1.0.0)", "wrapt (>=1.16.0)"] +google-adk-live = ["google-adk (>=2.3.0)"] langsmith-pyo3 = ["langsmith-pyo3 (>=0.1.0rc2)"] +livekit = ["cachetools (>=5.0.0)", "livekit-agents (>=1.0)", "opentelemetry-api (>=1.30.0)", "opentelemetry-exporter-otlp-proto-http (>=1.30.0)", "opentelemetry-sdk (>=1.30.0)"] openai-agents = ["openai-agents (>=0.0.3)"] +openai-realtime = ["openai (>=1.50)", "openai-agents (>=0.0.3)"] otel = ["opentelemetry-api (>=1.30.0)", "opentelemetry-exporter-otlp-proto-http (>=1.30.0)", "opentelemetry-sdk (>=1.30.0)"] +pipecat = ["cachetools (>=5.0.0)", "opentelemetry-api (>=1.30.0)", "opentelemetry-exporter-otlp-proto-http (>=1.30.0)", "opentelemetry-sdk (>=1.30.0)", "pipecat-ai (>=1.0) ; python_version >= \"3.11\""] pytest = ["pytest (>=7.0.0)", "rich (>=13.9.4)", "vcrpy (>=7.0.0)"] +strands-agents = ["opentelemetry-api (>=1.30.0)", "opentelemetry-exporter-otlp-proto-http (>=1.30.0)", "opentelemetry-sdk (>=1.30.0)", "strands-agents (>=0.1.0)", "strands-agents-tools (>=0.2.0)"] vcr = ["vcrpy (>=7.0.0)"] [[package]] @@ -2723,6 +2860,35 @@ files = [ [package.dependencies] rapidfuzz = ">=3.9.0,<4.0.0" +[[package]] +name = "lightgbm" +version = "4.7.0" +description = "LightGBM Python-package" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "lightgbm-4.7.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:dfc1cfe8e760387be1e7ba7a214688be21fdff96e4ed9749188f83e1877c2477"}, + {file = "lightgbm-4.7.0-py3-none-macosx_12_0_arm64.whl", hash = "sha256:129535462686f274df179133643118c5c5c5667167fe6c3a28d955f0b3c8e868"}, + {file = "lightgbm-4.7.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d4529acec5c6fefe4768302a529707d0ead90f6a6f42df694b856212e09695b8"}, + {file = "lightgbm-4.7.0-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d23e922acd891e77212e4d0fbcee9ba973c96dee479491341d05ba595357ebb7"}, + {file = "lightgbm-4.7.0-py3-none-win_amd64.whl", hash = "sha256:f42d1e5b32b6f170e606d7c689c6165671da98d7bf37f1addec2623efc8740c9"}, + {file = "lightgbm-4.7.0.tar.gz", hash = "sha256:f8e20f682c9aabd000bcf4a7ed8aa6f473c1adfecccae34ec24e823d156f4af0"}, +] + +[package.dependencies] +narwhals = ">=1.15" +numpy = ">=1.21.3" +scipy = "*" + +[package.extras] +arrow = ["narwhals[pyarrow]", "pyarrow (>=16.0.0)"] +dask = ["dask[array,dataframe,distributed] (>=2.0.0)", "narwhals[pandas]", "pandas (>=1.3.4)"] +pandas = ["narwhals[pandas]", "pandas (>=1.3.4)"] +plotting = ["graphviz", "matplotlib"] +polars = ["narwhals[polars]", "polars (>=1.0.0)"] +scikit-learn = ["scikit-learn (>=1.0.2)"] + [[package]] name = "lxml" version = "6.0.2" @@ -2879,13 +3045,28 @@ html-clean = ["lxml_html_clean"] html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] +[[package]] +name = "mail-parser" +version = "4.6.4" +description = "A tool that parses emails by enhancing the Python standard library, extracting all details into a comprehensive object." +optional = false +python-versions = ">=3.9" +groups = ["main", "docling-ocr"] +files = [ + {file = "mail_parser-4.6.4-py3-none-any.whl", hash = "sha256:bc6e437b3afe38091893e7b6ea49c7f2188ad7616abdd4fa3059e85f1b69efce"}, + {file = "mail_parser-4.6.4.tar.gz", hash = "sha256:1929109b7934ee061c1533a897b18c57fd0adb3548b79b5e28d901819bade872"}, +] + +[package.extras] +outlook = ["extract-msg (>=0.54)"] + [[package]] name = "markdown-it-py" version = "4.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.10" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147"}, {file = "markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3"}, @@ -2926,7 +3107,7 @@ version = "3.0.3" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, @@ -3033,28 +3214,37 @@ files = [ [[package]] name = "mcp" -version = "1.26.0" +version = "1.29.0" description = "Model Context Protocol SDK" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "mcp-1.26.0-py3-none-any.whl", hash = "sha256:904a21c33c25aa98ddbeb47273033c435e595bbacfdb177f4bd87f6dceebe1ca"}, - {file = "mcp-1.26.0.tar.gz", hash = "sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66"}, + {file = "mcp-1.29.0-py3-none-any.whl", hash = "sha256:f5a075bb611f23d6f4d080c6a1699fa62772eebc562ba9e66b306ddde1c755f7"}, + {file = "mcp-1.29.0.tar.gz", hash = "sha256:52d01f334de1868cc3bb2d6604931126a67631f99a6c5d3b82ba47290315ec36"}, ] [package.dependencies] anyio = ">=4.5" -httpx = ">=0.27.1" +httpx = ">=0.27.1,<1.0.0" httpx-sse = ">=0.4" jsonschema = ">=4.20.0" -pydantic = ">=2.11.0,<3.0.0" +pydantic = [ + {version = ">=2.11.0,<3.0.0", markers = "python_version < \"3.14\""}, + {version = ">=2.12.0,<3.0.0", markers = "python_version >= \"3.14\""}, +] pydantic-settings = ">=2.5.2" pyjwt = {version = ">=2.10.1", extras = ["crypto"]} python-multipart = ">=0.0.9" -pywin32 = {version = ">=310", markers = "sys_platform == \"win32\""} +pywin32 = [ + {version = ">=310", markers = "sys_platform == \"win32\" and python_version < \"3.14\""}, + {version = ">=311", markers = "sys_platform == \"win32\" and python_version >= \"3.14\""}, +] sse-starlette = ">=1.6.1" -starlette = ">=0.27" +starlette = [ + {version = ">=0.27", markers = "python_version < \"3.14\""}, + {version = ">=0.48.0", markers = "python_version >= \"3.14\""}, +] typing-extensions = ">=4.9.0" typing-inspection = ">=0.4.1" uvicorn = {version = ">=0.31.1", markers = "sys_platform != \"emscripten\""} @@ -3070,7 +3260,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -3106,7 +3296,7 @@ version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" optional = false python-versions = "*" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, @@ -3273,6 +3463,32 @@ files = [ {file = "murmurhash-1.0.15.tar.gz", hash = "sha256:58e2b27b7847f9e2a6edf10b47a8c8dd70a4705f45dccb7bf76aeadacf56ba01"}, ] +[[package]] +name = "narwhals" +version = "2.25.0" +description = "Extremely lightweight compatibility layer between dataframe libraries" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "narwhals-2.25.0-py3-none-any.whl", hash = "sha256:1f0f403e8c7e4463cde9bfe78b12fdd809e3ae3dda6d9b2f802934fb9c7a6a8f"}, + {file = "narwhals-2.25.0.tar.gz", hash = "sha256:62c036c810662bf7820b7737077176313bc59350eeeefb808510f388c743e4b2"}, +] + +[package.extras] +cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] +dask = ["dask[dataframe] (>=2024.8)"] +duckdb = ["duckdb (>=1.1)"] +ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)"] +modin = ["modin (>=0.22.0)"] +pandas = ["pandas (>=1.3.4)"] +polars = ["polars (>=0.20.4)"] +pyarrow = ["pyarrow (>=13.0.0)"] +pyspark = ["pyspark (>=3.5.0)"] +pyspark-connect = ["pyspark[connect] (>=3.5.0)"] +sql = ["narwhals[duckdb]", "sqlparse (>=0.5.5)"] +sqlframe = ["sqlframe (>=3.22.0,!=3.39.3)"] + [[package]] name = "neo4j" version = "6.0.2" @@ -3299,7 +3515,7 @@ version = "3.6" description = "Python package for creating and manipulating graphs and networks" optional = false python-versions = ">=3.11" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "python_version >= \"3.14\"" files = [ {file = "networkx-3.6-py3-none-any.whl", hash = "sha256:cdb395b105806062473d3be36458d8f1459a4e4b98e236a66c3a48996e07684f"}, @@ -3323,7 +3539,7 @@ version = "3.6.1" description = "Python package for creating and manipulating graphs and networks" optional = false python-versions = "!=3.14.1,>=3.11" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "python_version <= \"3.13\"" files = [ {file = "networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762"}, @@ -3417,7 +3633,7 @@ version = "1.26.4" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" -groups = ["main", "docling-ocr"] +groups = ["main", "dev", "docling-ocr", "local-ml"] files = [ {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, @@ -3463,7 +3679,7 @@ version = "12.6.4.1" description = "CUBLAS native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08ed2686e9875d01b58e3cb379c6896df8e76c75e0d4a7f7dace3d7b6d9ef8eb"}, @@ -3477,7 +3693,7 @@ version = "12.8.4.1" description = "CUBLAS native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0"}, @@ -3491,7 +3707,7 @@ version = "12.6.80" description = "CUDA profiling tools runtime libs." optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:166ee35a3ff1587f2490364f90eeeb8da06cd867bd5b701bf7f9a02b78bc63fc"}, @@ -3507,7 +3723,7 @@ version = "12.8.90" description = "CUDA profiling tools runtime libs." optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed"}, @@ -3521,7 +3737,7 @@ version = "12.6.77" description = "NVRTC native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5847f1d6e5b757f1d2b3991a01082a44aad6f10ab3c5c0213fa3e25bddc25a13"}, @@ -3535,7 +3751,7 @@ version = "12.8.93" description = "NVRTC native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994"}, @@ -3549,7 +3765,7 @@ version = "12.6.77" description = "CUDA Runtime native Libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6116fad3e049e04791c0256a9778c16237837c08b27ed8c8401e2e45de8d60cd"}, @@ -3565,7 +3781,7 @@ version = "12.8.90" description = "CUDA Runtime native Libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d"}, @@ -3579,7 +3795,7 @@ version = "9.5.1.17" description = "cuDNN runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9fd4584468533c61873e5fda8ca41bac3a38bcb2d12350830c69b0a96a7e4def"}, @@ -3596,7 +3812,7 @@ version = "9.10.2.21" description = "cuDNN runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8"}, @@ -3613,7 +3829,7 @@ version = "11.3.0.4" description = "CUFFT native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d16079550df460376455cba121db6564089176d9bac9e4f360493ca4741b22a6"}, @@ -3632,7 +3848,7 @@ version = "11.3.3.83" description = "CUFFT native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a"}, @@ -3649,7 +3865,7 @@ version = "1.11.1.6" description = "cuFile GPUDirect libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159"}, @@ -3662,7 +3878,7 @@ version = "1.13.1.3" description = "cuFile GPUDirect libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc"}, @@ -3675,7 +3891,7 @@ version = "10.3.7.77" description = "CURAND native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:6e82df077060ea28e37f48a3ec442a8f47690c7499bff392a5938614b56c98d8"}, @@ -3691,7 +3907,7 @@ version = "10.3.9.90" description = "CURAND native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd"}, @@ -3705,7 +3921,7 @@ version = "11.7.1.2" description = "CUDA solver native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0ce237ef60acde1efc457335a2ddadfd7610b892d94efee7b776c64bb1cac9e0"}, @@ -3726,7 +3942,7 @@ version = "11.7.3.90" description = "CUDA solver native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0"}, @@ -3745,7 +3961,7 @@ version = "12.5.4.2" description = "CUSPARSE native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d25b62fb18751758fe3c93a4a08eff08effedfe4edf1c6bb5afd0890fe88f887"}, @@ -3764,7 +3980,7 @@ version = "12.5.8.93" description = "CUSPARSE native runtime libraries" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc"}, @@ -3781,7 +3997,7 @@ version = "0.6.3" description = "NVIDIA cuSPARSELt" optional = false python-versions = "*" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8371549623ba601a06322af2133c4a44350575f5a3108fb75f3ef20b822ad5f1"}, @@ -3795,7 +4011,7 @@ version = "0.7.1" description = "NVIDIA cuSPARSELt" optional = false python-versions = "*" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5"}, @@ -3809,7 +4025,7 @@ version = "2.26.2" description = "NVIDIA Collective Communication Library (NCCL) Runtime" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c196e95e832ad30fbbb50381eb3cbd1fadd5675e587a548563993609af19522"}, @@ -3822,7 +4038,7 @@ version = "2.27.5" description = "NVIDIA Collective Communication Library (NCCL) Runtime" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31432ad4d1fb1004eb0c56203dc9bc2178a1ba69d1d9e02d64a6938ab5e40e7a"}, @@ -3835,7 +4051,7 @@ version = "12.6.85" description = "Nvidia JIT LTO Library" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:eedc36df9e88b682efe4309aa16b5b4e78c2407eac59e8c10a6a47535164369a"}, @@ -3849,7 +4065,7 @@ version = "12.8.93" description = "Nvidia JIT LTO Library" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88"}, @@ -3863,7 +4079,7 @@ version = "3.3.20" description = "NVSHMEM creates a global address space that provides efficient and scalable communication for NVIDIA GPU clusters." optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0b0b960da3842212758e4fa4696b94f129090b30e5122fea3c5345916545cff0"}, @@ -3876,7 +4092,7 @@ version = "12.6.77" description = "NVIDIA Tools Extension" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f44f8d86bb7d5629988d61c8d3ae61dddb2015dee142740536bc7481b022fe4b"}, @@ -3892,7 +4108,7 @@ version = "12.8.90" description = "NVIDIA Tools Extension" optional = false python-versions = ">=3" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615"}, @@ -3901,22 +4117,19 @@ files = [ ] [[package]] -name = "ocrmac" -version = "1.0.1" -description = "A python wrapper to extract text from images on a mac system. Uses the vision framework from Apple." +name = "olefile" +version = "0.47" +description = "Python package to parse, read and write Microsoft OLE2 files (Structured Storage or Compound Document, Microsoft Office)" optional = false -python-versions = ">=3.9" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main", "docling-ocr"] -markers = "sys_platform == \"darwin\"" files = [ - {file = "ocrmac-1.0.1-py3-none-any.whl", hash = "sha256:1cef25426f7ae6bbd57fe3dc5553b25461ae8ad0d2b428a9bbadbf5907349024"}, - {file = "ocrmac-1.0.1.tar.gz", hash = "sha256:507fe5e4cbd67b2d03f6729a52bbc11f9d0b58241134eb958a5daafd4b9d93d9"}, + {file = "olefile-0.47-py2.py3-none-any.whl", hash = "sha256:543c7da2a7adadf21214938bb79c83ea12b473a4b6ee4ad4bf854e7715e13d1f"}, + {file = "olefile-0.47.zip", hash = "sha256:599383381a0bf3dfbd932ca0ca6515acd174ed48870cbf7fee123d698c192c1c"}, ] -[package.dependencies] -Click = ">=7.0" -pillow = "*" -pyobjc-framework-Vision = "*" +[package.extras] +tests = ["pytest", "pytest-cov"] [[package]] name = "omegaconf" @@ -3936,14 +4149,14 @@ PyYAML = ">=5.1.0" [[package]] name = "openai" -version = "2.5.0" +version = "2.54.0" description = "The official Python library for the openai API" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "openai-2.5.0-py3-none-any.whl", hash = "sha256:21380e5f52a71666dbadbf322dd518bdf2b9d11ed0bb3f96bea17310302d6280"}, - {file = "openai-2.5.0.tar.gz", hash = "sha256:f8fa7611f96886a0f31ac6b97e58bc0ada494b255ee2cfd51c8eb502cfcb4814"}, + {file = "openai-2.54.0-py3-none-any.whl", hash = "sha256:89089789197ccdb87f173a03145ed1598d00795220c93e96cf712b1cbf5e5f2b"}, + {file = "openai-2.54.0.tar.gz", hash = "sha256:e3e6f8bc1ba30ddf381ace1a14340eed381cb984a1a59bd0f34b5be3b5d49cfa"}, ] [package.dependencies] @@ -3954,11 +4167,13 @@ jiter = ">=0.10.0,<1" pydantic = ">=1.9.0,<3" sniffio = "*" tqdm = ">4" -typing-extensions = ">=4.11,<5" +typing-extensions = ">=4.14,<5" [package.extras] -aiohttp = ["aiohttp", "httpx-aiohttp (>=0.1.9)"] +aiohttp = ["aiohttp (>=3.14.1)", "httpx-aiohttp (>=0.1.9)"] +bedrock = ["botocore (>=1.40.0,<2)"] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +httpx2 = ["anyio (>=4.10.0,<5)", "httpx (>=0.25.1,<1)", "httpx2 (>=2.7.0,<3)"] realtime = ["websockets (>=13,<16)"] voice-helpers = ["numpy (>=2.0.2)", "sounddevice (>=0.5.1)"] @@ -4002,161 +4217,136 @@ et-xmlfile = "*" [[package]] name = "orjson" -version = "3.11.3" +version = "3.12.0" description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "orjson-3.11.3-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:29cb1f1b008d936803e2da3d7cba726fc47232c45df531b29edf0b232dd737e7"}, - {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97dceed87ed9139884a55db8722428e27bd8452817fbf1869c58b49fecab1120"}, - {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:58533f9e8266cb0ac298e259ed7b4d42ed3fa0b78ce76860626164de49e0d467"}, - {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c212cfdd90512fe722fa9bd620de4d46cda691415be86b2e02243242ae81873"}, - {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff835b5d3e67d9207343effb03760c00335f8b5285bfceefd4dc967b0e48f6a"}, - {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5aa4682912a450c2db89cbd92d356fef47e115dffba07992555542f344d301b"}, - {file = "orjson-3.11.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7d18dd34ea2e860553a579df02041845dee0af8985dff7f8661306f95504ddf"}, - {file = "orjson-3.11.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d8b11701bc43be92ea42bd454910437b355dfb63696c06fe953ffb40b5f763b4"}, - {file = "orjson-3.11.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:90368277087d4af32d38bd55f9da2ff466d25325bf6167c8f382d8ee40cb2bbc"}, - {file = "orjson-3.11.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fd7ff459fb393358d3a155d25b275c60b07a2c83dcd7ea962b1923f5a1134569"}, - {file = "orjson-3.11.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f8d902867b699bcd09c176a280b1acdab57f924489033e53d0afe79817da37e6"}, - {file = "orjson-3.11.3-cp310-cp310-win32.whl", hash = "sha256:bb93562146120bb51e6b154962d3dadc678ed0fce96513fa6bc06599bb6f6edc"}, - {file = "orjson-3.11.3-cp310-cp310-win_amd64.whl", hash = "sha256:976c6f1975032cc327161c65d4194c549f2589d88b105a5e3499429a54479770"}, - {file = "orjson-3.11.3-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9d2ae0cc6aeb669633e0124531f342a17d8e97ea999e42f12a5ad4adaa304c5f"}, - {file = "orjson-3.11.3-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:ba21dbb2493e9c653eaffdc38819b004b7b1b246fb77bfc93dc016fe664eac91"}, - {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:00f1a271e56d511d1569937c0447d7dce5a99a33ea0dec76673706360a051904"}, - {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b67e71e47caa6680d1b6f075a396d04fa6ca8ca09aafb428731da9b3ea32a5a6"}, - {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d7d012ebddffcce8c85734a6d9e5f08180cd3857c5f5a3ac70185b43775d043d"}, - {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd759f75d6b8d1b62012b7f5ef9461d03c804f94d539a5515b454ba3a6588038"}, - {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6890ace0809627b0dff19cfad92d69d0fa3f089d3e359a2a532507bb6ba34efb"}, - {file = "orjson-3.11.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9d4a5e041ae435b815e568537755773d05dac031fee6a57b4ba70897a44d9d2"}, - {file = "orjson-3.11.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2d68bf97a771836687107abfca089743885fb664b90138d8761cce61d5625d55"}, - {file = "orjson-3.11.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:bfc27516ec46f4520b18ef645864cee168d2a027dbf32c5537cb1f3e3c22dac1"}, - {file = "orjson-3.11.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f66b001332a017d7945e177e282a40b6997056394e3ed7ddb41fb1813b83e824"}, - {file = "orjson-3.11.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:212e67806525d2561efbfe9e799633b17eb668b8964abed6b5319b2f1cfbae1f"}, - {file = "orjson-3.11.3-cp311-cp311-win32.whl", hash = "sha256:6e8e0c3b85575a32f2ffa59de455f85ce002b8bdc0662d6b9c2ed6d80ab5d204"}, - {file = "orjson-3.11.3-cp311-cp311-win_amd64.whl", hash = "sha256:6be2f1b5d3dc99a5ce5ce162fc741c22ba9f3443d3dd586e6a1211b7bc87bc7b"}, - {file = "orjson-3.11.3-cp311-cp311-win_arm64.whl", hash = "sha256:fafb1a99d740523d964b15c8db4eabbfc86ff29f84898262bf6e3e4c9e97e43e"}, - {file = "orjson-3.11.3-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:8c752089db84333e36d754c4baf19c0e1437012242048439c7e80eb0e6426e3b"}, - {file = "orjson-3.11.3-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:9b8761b6cf04a856eb544acdd82fc594b978f12ac3602d6374a7edb9d86fd2c2"}, - {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b13974dc8ac6ba22feaa867fc19135a3e01a134b4f7c9c28162fed4d615008a"}, - {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f83abab5bacb76d9c821fd5c07728ff224ed0e52d7a71b7b3de822f3df04e15c"}, - {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6fbaf48a744b94091a56c62897b27c31ee2da93d826aa5b207131a1e13d4064"}, - {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc779b4f4bba2847d0d2940081a7b6f7b5877e05408ffbb74fa1faf4a136c424"}, - {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd4b909ce4c50faa2192da6bb684d9848d4510b736b0611b6ab4020ea6fd2d23"}, - {file = "orjson-3.11.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:524b765ad888dc5518bbce12c77c2e83dee1ed6b0992c1790cc5fb49bb4b6667"}, - {file = "orjson-3.11.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:84fd82870b97ae3cdcea9d8746e592b6d40e1e4d4527835fc520c588d2ded04f"}, - {file = "orjson-3.11.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:fbecb9709111be913ae6879b07bafd4b0785b44c1eb5cac8ac76da048b3885a1"}, - {file = "orjson-3.11.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9dba358d55aee552bd868de348f4736ca5a4086d9a62e2bfbbeeb5629fe8b0cc"}, - {file = "orjson-3.11.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eabcf2e84f1d7105f84580e03012270c7e97ecb1fb1618bda395061b2a84a049"}, - {file = "orjson-3.11.3-cp312-cp312-win32.whl", hash = "sha256:3782d2c60b8116772aea8d9b7905221437fdf53e7277282e8d8b07c220f96cca"}, - {file = "orjson-3.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:79b44319268af2eaa3e315b92298de9a0067ade6e6003ddaef72f8e0bedb94f1"}, - {file = "orjson-3.11.3-cp312-cp312-win_arm64.whl", hash = "sha256:0e92a4e83341ef79d835ca21b8bd13e27c859e4e9e4d7b63defc6e58462a3710"}, - {file = "orjson-3.11.3-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:af40c6612fd2a4b00de648aa26d18186cd1322330bd3a3cc52f87c699e995810"}, - {file = "orjson-3.11.3-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:9f1587f26c235894c09e8b5b7636a38091a9e6e7fe4531937534749c04face43"}, - {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:61dcdad16da5bb486d7227a37a2e789c429397793a6955227cedbd7252eb5a27"}, - {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:11c6d71478e2cbea0a709e8a06365fa63da81da6498a53e4c4f065881d21ae8f"}, - {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff94112e0098470b665cb0ed06efb187154b63649403b8d5e9aedeb482b4548c"}, - {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae8b756575aaa2a855a75192f356bbda11a89169830e1439cfb1a3e1a6dde7be"}, - {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c9416cc19a349c167ef76135b2fe40d03cea93680428efee8771f3e9fb66079d"}, - {file = "orjson-3.11.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b822caf5b9752bc6f246eb08124c3d12bf2175b66ab74bac2ef3bbf9221ce1b2"}, - {file = "orjson-3.11.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:414f71e3bdd5573893bf5ecdf35c32b213ed20aa15536fe2f588f946c318824f"}, - {file = "orjson-3.11.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:828e3149ad8815dc14468f36ab2a4b819237c155ee1370341b91ea4c8672d2ee"}, - {file = "orjson-3.11.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ac9e05f25627ffc714c21f8dfe3a579445a5c392a9c8ae7ba1d0e9fb5333f56e"}, - {file = "orjson-3.11.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e44fbe4000bd321d9f3b648ae46e0196d21577cf66ae684a96ff90b1f7c93633"}, - {file = "orjson-3.11.3-cp313-cp313-win32.whl", hash = "sha256:2039b7847ba3eec1f5886e75e6763a16e18c68a63efc4b029ddf994821e2e66b"}, - {file = "orjson-3.11.3-cp313-cp313-win_amd64.whl", hash = "sha256:29be5ac4164aa8bdcba5fa0700a3c9c316b411d8ed9d39ef8a882541bd452fae"}, - {file = "orjson-3.11.3-cp313-cp313-win_arm64.whl", hash = "sha256:18bd1435cb1f2857ceb59cfb7de6f92593ef7b831ccd1b9bfb28ca530e539dce"}, - {file = "orjson-3.11.3-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cf4b81227ec86935568c7edd78352a92e97af8da7bd70bdfdaa0d2e0011a1ab4"}, - {file = "orjson-3.11.3-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:bc8bc85b81b6ac9fc4dae393a8c159b817f4c2c9dee5d12b773bddb3b95fc07e"}, - {file = "orjson-3.11.3-cp314-cp314-manylinux_2_34_aarch64.whl", hash = "sha256:88dcfc514cfd1b0de038443c7b3e6a9797ffb1b3674ef1fd14f701a13397f82d"}, - {file = "orjson-3.11.3-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:d61cd543d69715d5fc0a690c7c6f8dcc307bc23abef9738957981885f5f38229"}, - {file = "orjson-3.11.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2b7b153ed90ababadbef5c3eb39549f9476890d339cf47af563aea7e07db2451"}, - {file = "orjson-3.11.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:7909ae2460f5f494fecbcd10613beafe40381fd0316e35d6acb5f3a05bfda167"}, - {file = "orjson-3.11.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:2030c01cbf77bc67bee7eef1e7e31ecf28649353987775e3583062c752da0077"}, - {file = "orjson-3.11.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a0169ebd1cbd94b26c7a7ad282cf5c2744fce054133f959e02eb5265deae1872"}, - {file = "orjson-3.11.3-cp314-cp314-win32.whl", hash = "sha256:0c6d7328c200c349e3a4c6d8c83e0a5ad029bdc2d417f234152bf34842d0fc8d"}, - {file = "orjson-3.11.3-cp314-cp314-win_amd64.whl", hash = "sha256:317bbe2c069bbc757b1a2e4105b64aacd3bc78279b66a6b9e51e846e4809f804"}, - {file = "orjson-3.11.3-cp314-cp314-win_arm64.whl", hash = "sha256:e8f6a7a27d7b7bec81bd5924163e9af03d49bbb63013f107b48eb5d16db711bc"}, - {file = "orjson-3.11.3-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:56afaf1e9b02302ba636151cfc49929c1bb66b98794291afd0e5f20fecaf757c"}, - {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:913f629adef31d2d350d41c051ce7e33cf0fd06a5d1cb28d49b1899b23b903aa"}, - {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e0a23b41f8f98b4e61150a03f83e4f0d566880fe53519d445a962929a4d21045"}, - {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d721fee37380a44f9d9ce6c701b3960239f4fb3d5ceea7f31cbd43882edaa2f"}, - {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73b92a5b69f31b1a58c0c7e31080aeaec49c6e01b9522e71ff38d08f15aa56de"}, - {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2489b241c19582b3f1430cc5d732caefc1aaf378d97e7fb95b9e56bed11725f"}, - {file = "orjson-3.11.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5189a5dab8b0312eadaf9d58d3049b6a52c454256493a557405e77a3d67ab7f"}, - {file = "orjson-3.11.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9d8787bdfbb65a85ea76d0e96a3b1bed7bf0fbcb16d40408dc1172ad784a49d2"}, - {file = "orjson-3.11.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:8e531abd745f51f8035e207e75e049553a86823d189a51809c078412cefb399a"}, - {file = "orjson-3.11.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8ab962931015f170b97a3dd7bd933399c1bae8ed8ad0fb2a7151a5654b6941c7"}, - {file = "orjson-3.11.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:124d5ba71fee9c9902c4a7baa9425e663f7f0aecf73d31d54fe3dd357d62c1a7"}, - {file = "orjson-3.11.3-cp39-cp39-win32.whl", hash = "sha256:22724d80ee5a815a44fc76274bb7ba2e7464f5564aacb6ecddaa9970a83e3225"}, - {file = "orjson-3.11.3-cp39-cp39-win_amd64.whl", hash = "sha256:215c595c792a87d4407cb72dd5e0f6ee8e694ceeb7f9102b533c5a9bf2a916bb"}, - {file = "orjson-3.11.3.tar.gz", hash = "sha256:1c0603b1d2ffcd43a411d64797a19556ef76958aef1c182f22dc30860152a98a"}, + {file = "orjson-3.12.0-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:747843254519dd43b93eee3153a19e5a509334320c4d2f823ec879232db5c796"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:7c2ad193c8004254f34b499f3bd2c80f043d10754aff2b38f93da574f4883f98"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:bc7a872f03522d90e0429e6c0c5cd23084f767bedcb4c58048eec19294613344"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18a87929f31d94a77f7dc93cf527e91f39ce7fe7813d588a4de2507efd32a387"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9683ee9ea0659da64f36574ef675b8a86330c34c19ea75db1fb93c3ff99e0ef"}, + {file = "orjson-3.12.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:103b5db66aa53c1f9e88c2524be4f383e831ba7dfd5f9f5af6336a177c622f11"}, + {file = "orjson-3.12.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bd57d79aefa3f84eec851d6de7a366795b9345cfaf17f82b4820430a7a5fa241"}, + {file = "orjson-3.12.0-cp310-cp310-win32.whl", hash = "sha256:3dbce9b6b3074b31a5d5dd322a9c4e5b16f206091ece4194c2e36952847a105e"}, + {file = "orjson-3.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:3bb17a06f9bd15237b3216c044209fe92597379124018cfc196fbb846cde64df"}, + {file = "orjson-3.12.0-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a94f0f0c6fcbb2b5bd9734c57a489c7584a732bbdf04a39e8c83b861e9d03e92"}, + {file = "orjson-3.12.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:a696529ec96a90d9a5f9570207efe403c8b08f8e4aa2783ee3403511e2fdfa10"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:e4ac5059baab4b3acbd99485de019ff8cda0fdf34b61fa74f7197a53db78bfe8"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8e29957429c35bbb5a185a119c523aa2428b7bbf1a293724c7b9375ed8f892a3"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dce0166feb0a737ab84f598c9a338cbc0b764a036617aa686194f53c7eba0c3e"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9caf3d09f47c3c70c4451ada20ef9bc4a4cdffa26f49862cf0a253b329aae2d5"}, + {file = "orjson-3.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b9dca132b1fda5565088e65a6b6e742285e0aeceb6fae549fa8863e16c7d3998"}, + {file = "orjson-3.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a791f793b287bbc135b8e87c34e35c8bfc693e2a8a620fab1ae682b925f9a32e"}, + {file = "orjson-3.12.0-cp311-cp311-win32.whl", hash = "sha256:31ed278a36304390adc3eec5d7f6fd593a7c3e99e5a06cd07866396c4b1b4710"}, + {file = "orjson-3.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb2539159dfe8d371914f354360fa50e4a577cc89222a3828b9650a5e5040252"}, + {file = "orjson-3.12.0-cp311-cp311-win_arm64.whl", hash = "sha256:61318b6de893c7a9d9f3e5ecbadccbfc26a7eb417ccc7bbf0771de3b4d72f868"}, + {file = "orjson-3.12.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:aa3e43a6846e91d7bde3d5a9c66090fcd8744f569a9b6cffc5e1ca38f6a461c0"}, + {file = "orjson-3.12.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:11edb4660a6680abee9788a3a9072208a2c96538cc1322bd79542065229d8e54"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d3a9da945a4d96ae758fdaaca56742e6b73b6fd554c5d8876f252a6dad70b83"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:92ffc09e07233a6ab6d4e067f7841edcbcc134cb4812155cf171ea5255a421d7"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf44e374aadde77b1f6109f1030be51433eb61984379852766b6f4e187db7b1e"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1192a7021b6d071aaf909864f6e924d6a2675ca360485b972b8401749311750b"}, + {file = "orjson-3.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:53c0c474a9d9aff9aebfc0c88de1f28f843d940e6e3a80729abdf6a20274356f"}, + {file = "orjson-3.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:532ff8cd4bd59a327a953a7dcde922c7fc25b85e29721bb8633265430d3a3873"}, + {file = "orjson-3.12.0-cp312-cp312-win32.whl", hash = "sha256:a6cf4b18e7de173f209f2084ffbd736dd72389a396326ee80a7022168be232e5"}, + {file = "orjson-3.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:010811c1b69773450a01cef97727a67b223242f350b77d4ca000e59a9ef2155a"}, + {file = "orjson-3.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:ad29eece0c601737f2a60edc2752a84e7a0785df3efb62e3012834700a5afe0d"}, + {file = "orjson-3.12.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9a36ec60f1796f9a3f13e3b98390295e17a1c7c10155b448d264098bf9ee5900"}, + {file = "orjson-3.12.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ad0422b92d5195443a39f80c3bcf731cc2e00f153bd32063a47b73b057bd0f03"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5a0fdbc216388f653d3752ff310e710f59253bd4ed6a2bfb3f4f06b84714bbd8"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2eb5c56e534127b2b8fa38d2363c8b1b8190367ee0d1d16c041517d880843b94"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784106539f4b9d4b930e0b4eb8d45168507dae001945e71b4675a367f1e5e806"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c680706fc8396d95e7c4c1f9482563f552137aef91b57237a3ad5aaf64629df"}, + {file = "orjson-3.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83445adc40cba26d6d621185a45128ce455b766af368cad2ab64b970603a7978"}, + {file = "orjson-3.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:644d005bc82f917337a95ce270c9f6f92f9834c2bed7b1477572f8db00784222"}, + {file = "orjson-3.12.0-cp313-cp313-win32.whl", hash = "sha256:d8e78d3d93705e3d27cc17cdb209e44d7a8ea203010cac6ce9c7ffc1ae1996f1"}, + {file = "orjson-3.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b85931be5b6763c31283805c9bdaae1ca03ad9f6f12a15f1cbf6745b907932c2"}, + {file = "orjson-3.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:6a31348d7dfa64cd9c78bd1f510ff44c48fe64d71094e6b90e364dba3b55949e"}, + {file = "orjson-3.12.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9e6fee342a48760e854d743e7a81534d8e2925a6f46e09f750cf56b50fd1de5d"}, + {file = "orjson-3.12.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:8c3bb86dd10f39b3fbf434b7d5dc7cac77d6fc8ac572ae30a10731ede2c4b647"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2bb3ce43203936072dd8b4917b01d3aecfc02329bfb42510cb7cfb24708adc9c"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:6a2a79c89984dc719817d388c8709e0efc2a2795a934eaa746b4882eb6045adc"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f06dd838d1e07d9b1de0932ec0485ec92c4d5f5d1ad4817a656268c3e88be1e1"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6b11be792c3d2c6a4be2af4ebf97a68d0bf5f580aca6e86a418a354f6cc846a"}, + {file = "orjson-3.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:477ecaf6b9f88f873341b91fcc736119ca81b5e002a9f7f308ff5b4f2ce2a70e"}, + {file = "orjson-3.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3c0683136acdc29afdf88a5bc2f7d3d0e34087788d1d63c0144b805a87a196f"}, + {file = "orjson-3.12.0-cp314-cp314-win32.whl", hash = "sha256:d39f3f5c3927e2dc0913fe5bbc1a2f6b1b9d1bba1de6358340d0ad0d0c00ca92"}, + {file = "orjson-3.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:0b1ac5bf6609b2716c7954011c5fef6254922df029f45d032ee4ebf5d363cbed"}, + {file = "orjson-3.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:50fae885cb073eac7556353ff3df93312b0d5137b0a5056b2bb63f97ed9a93c7"}, + {file = "orjson-3.12.0-cp315-cp315-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:01efac2074fffb4cb1ea3fab7861e9d0f2a26913854a972f5ac760525dbdaf6e"}, + {file = "orjson-3.12.0-cp315-cp315-macosx_15_0_arm64.whl", hash = "sha256:ed4ca42bd55955aa34deedcfdfd0e0c31abf51143aae158ae2bc3520b626e517"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_aarch64.whl", hash = "sha256:40f92192227505acca4e2533ce565f8e6b9535f7d0d09b0968452f18b7376b38"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_armv7l.whl", hash = "sha256:33efefcf5d88eaf400b47e2eba02f91f319bb9951be61ca500b7d536d3f2079d"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_i686.whl", hash = "sha256:8e386b0bc0ddd7cd2056f884b5a0af33592bd01ac66a7ca4b42a65a7e7774a13"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_x86_64.whl", hash = "sha256:58c58e1de0006ffb580368d6793c36c7b0b021db066479cf281bf5061e732328"}, + {file = "orjson-3.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:08231552159be266a7269555bd9f7c016aee7d9ad6dab06eb58796c5ccb7101c"}, + {file = "orjson-3.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a15f9a891bce5f5cc5d210e3ad8614d4d1b489a56448c099d6d2a7168b2d954a"}, + {file = "orjson-3.12.0-cp315-cp315-win32.whl", hash = "sha256:03091c8a64db4be38746597ceea68f33c238e27acd9bfe99fb59420224ae7a55"}, + {file = "orjson-3.12.0-cp315-cp315-win_amd64.whl", hash = "sha256:2b7bcefb9f40fa242fa6b06377232c048e655747790829609168c01162f60578"}, + {file = "orjson-3.12.0-cp315-cp315-win_arm64.whl", hash = "sha256:859fc4196855890150bb08e649b30d2c93b249b3e3edd0d3bb2231abf8aa8adc"}, + {file = "orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5"}, ] [[package]] name = "ormsgpack" -version = "1.11.0" -description = "" +version = "1.12.2" +description = "Fast, correct Python msgpack library supporting dataclasses, datetimes, and numpy" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "ormsgpack-1.11.0-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:03d4e658dd6e1882a552ce1d13cc7b49157414e7d56a4091fbe7823225b08cba"}, - {file = "ormsgpack-1.11.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bb67eb913c2b703f0ed39607fc56e50724dd41f92ce080a586b4d6149eb3fe4"}, - {file = "ormsgpack-1.11.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1e54175b92411f73a238e5653a998627f6660de3def37d9dd7213e0fd264ca56"}, - {file = "ormsgpack-1.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca2b197f4556e1823d1319869d4c5dc278be335286d2308b0ed88b59a5afcc25"}, - {file = "ormsgpack-1.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:bc62388262f58c792fe1e450e1d9dbcc174ed2fb0b43db1675dd7c5ff2319d6a"}, - {file = "ormsgpack-1.11.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c48bc10af74adfbc9113f3fb160dc07c61ad9239ef264c17e449eba3de343dc2"}, - {file = "ormsgpack-1.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a608d3a1d4fa4acdc5082168a54513cff91f47764cef435e81a483452f5f7647"}, - {file = "ormsgpack-1.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:97217b4f7f599ba45916b9c4c4b1d5656e8e2a4d91e2e191d72a7569d3c30923"}, - {file = "ormsgpack-1.11.0-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:c7be823f47d8e36648d4bc90634b93f02b7d7cc7480081195f34767e86f181fb"}, - {file = "ormsgpack-1.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68accf15d1b013812755c0eb7a30e1fc2f81eb603a1a143bf0cda1b301cfa797"}, - {file = "ormsgpack-1.11.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:805d06fb277d9a4e503c0c707545b49cde66cbb2f84e5cf7c58d81dfc20d8658"}, - {file = "ormsgpack-1.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1e57cdf003e77acc43643bda151dc01f97147a64b11cdee1380bb9698a7601c"}, - {file = "ormsgpack-1.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:37fc05bdaabd994097c62e2f3e08f66b03f856a640ede6dc5ea340bd15b77f4d"}, - {file = "ormsgpack-1.11.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:a6e9db6c73eb46b2e4d97bdffd1368a66f54e6806b563a997b19c004ef165e1d"}, - {file = "ormsgpack-1.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e9c44eae5ac0196ffc8b5ed497c75511056508f2303fa4d36b208eb820cf209e"}, - {file = "ormsgpack-1.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:11d0dfaf40ae7c6de4f7dbd1e4892e2e6a55d911ab1774357c481158d17371e4"}, - {file = "ormsgpack-1.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:0c63a3f7199a3099c90398a1bdf0cb577b06651a442dc5efe67f2882665e5b02"}, - {file = "ormsgpack-1.11.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:3434d0c8d67de27d9010222de07fb6810fb9af3bb7372354ffa19257ac0eb83b"}, - {file = "ormsgpack-1.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2da5bd097e8dbfa4eb0d4ccfe79acd6f538dee4493579e2debfe4fc8f4ca89b"}, - {file = "ormsgpack-1.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fdbaa0a5a8606a486960b60c24f2d5235d30ac7a8b98eeaea9854bffef14dc3d"}, - {file = "ormsgpack-1.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3682f24f800c1837017ee90ce321086b2cbaef88db7d4cdbbda1582aa6508159"}, - {file = "ormsgpack-1.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fcca21202bb05ccbf3e0e92f560ee59b9331182e4c09c965a28155efbb134993"}, - {file = "ormsgpack-1.11.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:c30e5c4655ba46152d722ec7468e8302195e6db362ec1ae2c206bc64f6030e43"}, - {file = "ormsgpack-1.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7138a341f9e2c08c59368f03d3be25e8b87b3baaf10d30fb1f6f6b52f3d47944"}, - {file = "ormsgpack-1.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4bd8589b78a11026d47f4edf13c1ceab9088bb12451f34396afe6497db28a27"}, - {file = "ormsgpack-1.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:e5e746a1223e70f111d4001dab9585ac8639eee8979ca0c8db37f646bf2961da"}, - {file = "ormsgpack-1.11.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:0e7b36ab7b45cb95217ae1f05f1318b14a3e5ef73cb00804c0f06233f81a14e8"}, - {file = "ormsgpack-1.11.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43402d67e03a9a35cc147c8c03f0c377cad016624479e1ee5b879b8425551484"}, - {file = "ormsgpack-1.11.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:64fd992f932764d6306b70ddc755c1bc3405c4c6a69f77a36acf7af1c8f5ada4"}, - {file = "ormsgpack-1.11.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0362fb7fe4a29c046c8ea799303079a09372653a1ce5a5a588f3bbb8088368d0"}, - {file = "ormsgpack-1.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:de2f7a65a9d178ed57be49eba3d0fc9b833c32beaa19dbd4ba56014d3c20b152"}, - {file = "ormsgpack-1.11.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:f38cfae95461466055af966fc922d06db4e1654966385cda2828653096db34da"}, - {file = "ormsgpack-1.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c88396189d238f183cea7831b07a305ab5c90d6d29b53288ae11200bd956357b"}, - {file = "ormsgpack-1.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:5403d1a945dd7c81044cebeca3f00a28a0f4248b33242a5d2d82111628043725"}, - {file = "ormsgpack-1.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:c57357b8d43b49722b876edf317bdad9e6d52071b523fdd7394c30cd1c67d5a0"}, - {file = "ormsgpack-1.11.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:d390907d90fd0c908211592c485054d7a80990697ef4dff4e436ac18e1aab98a"}, - {file = "ormsgpack-1.11.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6153c2e92e789509098e04c9aa116b16673bd88ec78fbe0031deeb34ab642d10"}, - {file = "ormsgpack-1.11.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c2b2c2a065a94d742212b2018e1fecd8f8d72f3c50b53a97d1f407418093446d"}, - {file = "ormsgpack-1.11.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:110e65b5340f3d7ef8b0009deae3c6b169437e6b43ad5a57fd1748085d29d2ac"}, - {file = "ormsgpack-1.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c27e186fca96ab34662723e65b420919910acbbc50fc8e1a44e08f26268cb0e0"}, - {file = "ormsgpack-1.11.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d56b1f877c13d499052d37a3db2378a97d5e1588d264f5040b3412aee23d742c"}, - {file = "ormsgpack-1.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c88e28cd567c0a3269f624b4ade28142d5e502c8e826115093c572007af5be0a"}, - {file = "ormsgpack-1.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:8811160573dc0a65f62f7e0792c4ca6b7108dfa50771edb93f9b84e2d45a08ae"}, - {file = "ormsgpack-1.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:23e30a8d3c17484cf74e75e6134322255bd08bc2b5b295cc9c442f4bae5f3c2d"}, - {file = "ormsgpack-1.11.0-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2905816502adfaf8386a01dd85f936cd378d243f4f5ee2ff46f67f6298dc90d5"}, - {file = "ormsgpack-1.11.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c04402fb9a0a9b9f18fbafd6d5f8398ee99b3ec619fb63952d3a954bc9d47daa"}, - {file = "ormsgpack-1.11.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a025ec07ac52056ecfd9e57b5cbc6fff163f62cb9805012b56cda599157f8ef2"}, - {file = "ormsgpack-1.11.0-cp39-cp39-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:354c6a5039faf63b63d8f42ec7915583a4a56e10b319284370a5a89c4382d985"}, - {file = "ormsgpack-1.11.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7058c85cc13dd329bc7b528e38626c6babcd0066d6e9163330a1509fe0aa4707"}, - {file = "ormsgpack-1.11.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4e15b634be324fb18dab7aa82ab929a0d57d42c12650ae3dedd07d8d31b17733"}, - {file = "ormsgpack-1.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6329e6eae9dfe600962739a6e060ea82885ec58b8338875c5ac35080da970f94"}, - {file = "ormsgpack-1.11.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:b27546c28f92b9eb757620f7f1ed89fb7b07be3b9f4ba1b7de75761ec1c4bcc8"}, - {file = "ormsgpack-1.11.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:26a17919d9144b4ac7112dbbadef07927abbe436be2cf99a703a19afe7dd5c8b"}, - {file = "ormsgpack-1.11.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5352868ee4cdc00656bf216b56bc654f72ac3008eb36e12561f6337bb7104b45"}, - {file = "ormsgpack-1.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:2ffe36f1f441a40949e8587f5aa3d3fc9f100576925aab667117403eab494338"}, - {file = "ormsgpack-1.11.0.tar.gz", hash = "sha256:7c9988e78fedba3292541eb3bb274fa63044ef4da2ddb47259ea70c05dee4206"}, + {file = "ormsgpack-1.12.2-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:c1429217f8f4d7fcb053523bbbac6bed5e981af0b85ba616e6df7cce53c19657"}, + {file = "ormsgpack-1.12.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f13034dc6c84a6280c6c33db7ac420253852ea233fc3ee27c8875f8dd651163"}, + {file = "ormsgpack-1.12.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:59f5da97000c12bc2d50e988bdc8576b21f6ab4e608489879d35b2c07a8ab51a"}, + {file = "ormsgpack-1.12.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e4459c3f27066beadb2b81ea48a076a417aafffff7df1d3c11c519190ed44f2"}, + {file = "ormsgpack-1.12.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7a1c460655d7288407ffa09065e322a7231997c0d62ce914bf3a96ad2dc6dedd"}, + {file = "ormsgpack-1.12.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:458e4568be13d311ef7d8877275e7ccbe06c0e01b39baaac874caaa0f46d826c"}, + {file = "ormsgpack-1.12.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8cde5eaa6c6cbc8622db71e4a23de56828e3d876aeb6460ffbcb5b8aff91093b"}, + {file = "ormsgpack-1.12.2-cp310-cp310-win_amd64.whl", hash = "sha256:dc7a33be14c347893edbb1ceda89afbf14c467d593a5ee92c11de4f1666b4d4f"}, + {file = "ormsgpack-1.12.2-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:bd5f4bf04c37888e864f08e740c5a573c4017f6fd6e99fa944c5c935fabf2dd9"}, + {file = "ormsgpack-1.12.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34d5b28b3570e9fed9a5a76528fc7230c3c76333bc214798958e58e9b79cc18a"}, + {file = "ormsgpack-1.12.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3708693412c28f3538fb5a65da93787b6bbab3484f6bc6e935bfb77a62400ae5"}, + {file = "ormsgpack-1.12.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43013a3f3e2e902e1d05e72c0f1aeb5bedbb8e09240b51e26792a3c89267e181"}, + {file = "ormsgpack-1.12.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7c8b1667a72cbba74f0ae7ecf3105a5e01304620ed14528b2cb4320679d2869b"}, + {file = "ormsgpack-1.12.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:df6961442140193e517303d0b5d7bc2e20e69a879c2d774316125350c4a76b92"}, + {file = "ormsgpack-1.12.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c6a4c34ddef109647c769d69be65fa1de7a6022b02ad45546a69b3216573eb4a"}, + {file = "ormsgpack-1.12.2-cp311-cp311-win_amd64.whl", hash = "sha256:73670ed0375ecc303858e3613f407628dd1fca18fe6ac57b7b7ce66cc7bb006c"}, + {file = "ormsgpack-1.12.2-cp311-cp311-win_arm64.whl", hash = "sha256:c2be829954434e33601ae5da328cccce3266b098927ca7a30246a0baec2ce7bd"}, + {file = "ormsgpack-1.12.2-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7a29d09b64b9694b588ff2f80e9826bdceb3a2b91523c5beae1fab27d5c940e7"}, + {file = "ormsgpack-1.12.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b39e629fd2e1c5b2f46f99778450b59454d1f901bc507963168985e79f09c5d"}, + {file = "ormsgpack-1.12.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:958dcb270d30a7cb633a45ee62b9444433fa571a752d2ca484efdac07480876e"}, + {file = "ormsgpack-1.12.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58d379d72b6c5e964851c77cfedfb386e474adee4fd39791c2c5d9efb53505cc"}, + {file = "ormsgpack-1.12.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8463a3fc5f09832e67bdb0e2fda6d518dc4281b133166146a67f54c08496442e"}, + {file = "ormsgpack-1.12.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:eddffb77eff0bad4e67547d67a130604e7e2dfbb7b0cde0796045be4090f35c6"}, + {file = "ormsgpack-1.12.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fcd55e5f6ba0dbce624942adf9f152062135f991a0126064889f68eb850de0dd"}, + {file = "ormsgpack-1.12.2-cp312-cp312-win_amd64.whl", hash = "sha256:d024b40828f1dde5654faebd0d824f9cc29ad46891f626272dd5bfd7af2333a4"}, + {file = "ormsgpack-1.12.2-cp312-cp312-win_arm64.whl", hash = "sha256:da538c542bac7d1c8f3f2a937863dba36f013108ce63e55745941dda4b75dbb6"}, + {file = "ormsgpack-1.12.2-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:5ea60cb5f210b1cfbad8c002948d73447508e629ec375acb82910e3efa8ff355"}, + {file = "ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3601f19afdbea273ed70b06495e5794606a8b690a568d6c996a90d7255e51c1"}, + {file = "ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29a9f17a3dac6054c0dce7925e0f4995c727f7c41859adf9b5572180f640d172"}, + {file = "ormsgpack-1.12.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39c1bd2092880e413902910388be8715f70b9f15f20779d44e673033a6146f2d"}, + {file = "ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:50b7249244382209877deedeee838aef1542f3d0fc28b8fe71ca9d7e1896a0d7"}, + {file = "ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:5af04800d844451cf102a59c74a841324868d3f1625c296a06cc655c542a6685"}, + {file = "ormsgpack-1.12.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cec70477d4371cd524534cd16472d8b9cc187e0e3043a8790545a9a9b296c258"}, + {file = "ormsgpack-1.12.2-cp313-cp313-win_amd64.whl", hash = "sha256:21f4276caca5c03a818041d637e4019bc84f9d6ca8baa5ea03e5cc8bf56140e9"}, + {file = "ormsgpack-1.12.2-cp313-cp313-win_arm64.whl", hash = "sha256:baca4b6773d20a82e36d6fd25f341064244f9f86a13dead95dd7d7f996f51709"}, + {file = "ormsgpack-1.12.2-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:bc68dd5915f4acf66ff2010ee47c8906dc1cf07399b16f4089f8c71733f6e36c"}, + {file = "ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46d084427b4132553940070ad95107266656cb646ea9da4975f85cb1a6676553"}, + {file = "ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c010da16235806cf1d7bc4c96bf286bfa91c686853395a299b3ddb49499a3e13"}, + {file = "ormsgpack-1.12.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18867233df592c997154ff942a6503df274b5ac1765215bceba7a231bea2745d"}, + {file = "ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b009049086ddc6b8f80c76b3955df1aa22a5fbd7673c525cd63bf91f23122ede"}, + {file = "ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1dcc17d92b6390d4f18f937cf0b99054824a7815818012ddca925d6e01c2e49e"}, + {file = "ormsgpack-1.12.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f04b5e896d510b07c0ad733d7fce2d44b260c5e6c402d272128f8941984e4285"}, + {file = "ormsgpack-1.12.2-cp314-cp314-win_amd64.whl", hash = "sha256:ae3aba7eed4ca7cb79fd3436eddd29140f17ea254b91604aa1eb19bfcedb990f"}, + {file = "ormsgpack-1.12.2-cp314-cp314-win_arm64.whl", hash = "sha256:118576ea6006893aea811b17429bfc561b4778fad393f5f538c84af70b01260c"}, + {file = "ormsgpack-1.12.2-cp314-cp314t-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:7121b3d355d3858781dc40dafe25a32ff8a8242b9d80c692fd548a4b1f7fd3c8"}, + {file = "ormsgpack-1.12.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ee766d2e78251b7a63daf1cddfac36a73562d3ddef68cacfb41b2af64698033"}, + {file = "ormsgpack-1.12.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:292410a7d23de9b40444636b9b8f1e4e4b814af7f1ef476e44887e52a123f09d"}, + {file = "ormsgpack-1.12.2-cp314-cp314t-win_amd64.whl", hash = "sha256:837dd316584485b72ef451d08dd3e96c4a11d12e4963aedb40e08f89685d8ec2"}, + {file = "ormsgpack-1.12.2.tar.gz", hash = "sha256:944a2233640273bee67521795a73cf1e959538e0dfb7ac635505010455e53b33"}, ] [[package]] @@ -4165,7 +4355,7 @@ version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" -groups = ["main", "docling-ocr"] +groups = ["main", "dev", "docling-ocr", "local-ml"] files = [ {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, @@ -4287,103 +4477,99 @@ numpy = "*" [[package]] name = "pillow" -version = "12.0.0" +version = "12.3.0" description = "Python Imaging Library (fork)" optional = false python-versions = ">=3.10" groups = ["main", "docling-ocr"] files = [ - {file = "pillow-12.0.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:3adfb466bbc544b926d50fe8f4a4e6abd8c6bffd28a26177594e6e9b2b76572b"}, - {file = "pillow-12.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1ac11e8ea4f611c3c0147424eae514028b5e9077dd99ab91e1bd7bc33ff145e1"}, - {file = "pillow-12.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d49e2314c373f4c2b39446fb1a45ed333c850e09d0c59ac79b72eb3b95397363"}, - {file = "pillow-12.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c7b2a63fd6d5246349f3d3f37b14430d73ee7e8173154461785e43036ffa96ca"}, - {file = "pillow-12.0.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d64317d2587c70324b79861babb9c09f71fbb780bad212018874b2c013d8600e"}, - {file = "pillow-12.0.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d77153e14b709fd8b8af6f66a3afbb9ed6e9fc5ccf0b6b7e1ced7b036a228782"}, - {file = "pillow-12.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:32ed80ea8a90ee3e6fa08c21e2e091bba6eda8eccc83dbc34c95169507a91f10"}, - {file = "pillow-12.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c828a1ae702fc712978bda0320ba1b9893d99be0badf2647f693cc01cf0f04fa"}, - {file = "pillow-12.0.0-cp310-cp310-win32.whl", hash = "sha256:bd87e140e45399c818fac4247880b9ce719e4783d767e030a883a970be632275"}, - {file = "pillow-12.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:455247ac8a4cfb7b9bc45b7e432d10421aea9fc2e74d285ba4072688a74c2e9d"}, - {file = "pillow-12.0.0-cp310-cp310-win_arm64.whl", hash = "sha256:6ace95230bfb7cd79ef66caa064bbe2f2a1e63d93471c3a2e1f1348d9f22d6b7"}, - {file = "pillow-12.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:0fd00cac9c03256c8b2ff58f162ebcd2587ad3e1f2e397eab718c47e24d231cc"}, - {file = "pillow-12.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3475b96f5908b3b16c47533daaa87380c491357d197564e0ba34ae75c0f3257"}, - {file = "pillow-12.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:110486b79f2d112cf6add83b28b627e369219388f64ef2f960fef9ebaf54c642"}, - {file = "pillow-12.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5269cc1caeedb67e6f7269a42014f381f45e2e7cd42d834ede3c703a1d915fe3"}, - {file = "pillow-12.0.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa5129de4e174daccbc59d0a3b6d20eaf24417d59851c07ebb37aeb02947987c"}, - {file = "pillow-12.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bee2a6db3a7242ea309aa7ee8e2780726fed67ff4e5b40169f2c940e7eb09227"}, - {file = "pillow-12.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:90387104ee8400a7b4598253b4c406f8958f59fcf983a6cea2b50d59f7d63d0b"}, - {file = "pillow-12.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bc91a56697869546d1b8f0a3ff35224557ae7f881050e99f615e0119bf934b4e"}, - {file = "pillow-12.0.0-cp311-cp311-win32.whl", hash = "sha256:27f95b12453d165099c84f8a8bfdfd46b9e4bda9e0e4b65f0635430027f55739"}, - {file = "pillow-12.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:b583dc9070312190192631373c6c8ed277254aa6e6084b74bdd0a6d3b221608e"}, - {file = "pillow-12.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:759de84a33be3b178a64c8ba28ad5c135900359e85fb662bc6e403ad4407791d"}, - {file = "pillow-12.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:53561a4ddc36facb432fae7a9d8afbfaf94795414f5cdc5fc52f28c1dca90371"}, - {file = "pillow-12.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:71db6b4c1653045dacc1585c1b0d184004f0d7e694c7b34ac165ca70c0838082"}, - {file = "pillow-12.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2fa5f0b6716fc88f11380b88b31fe591a06c6315e955c096c35715788b339e3f"}, - {file = "pillow-12.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:82240051c6ca513c616f7f9da06e871f61bfd7805f566275841af15015b8f98d"}, - {file = "pillow-12.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55f818bd74fe2f11d4d7cbc65880a843c4075e0ac7226bc1a23261dbea531953"}, - {file = "pillow-12.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b87843e225e74576437fd5b6a4c2205d422754f84a06942cfaf1dc32243e45a8"}, - {file = "pillow-12.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c607c90ba67533e1b2355b821fef6764d1dd2cbe26b8c1005ae84f7aea25ff79"}, - {file = "pillow-12.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:21f241bdd5080a15bc86d3466a9f6074a9c2c2b314100dd896ac81ee6db2f1ba"}, - {file = "pillow-12.0.0-cp312-cp312-win32.whl", hash = "sha256:dd333073e0cacdc3089525c7df7d39b211bcdf31fc2824e49d01c6b6187b07d0"}, - {file = "pillow-12.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe611163f6303d1619bbcb653540a4d60f9e55e622d60a3108be0d5b441017a"}, - {file = "pillow-12.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:7dfb439562f234f7d57b1ac6bc8fe7f838a4bd49c79230e0f6a1da93e82f1fad"}, - {file = "pillow-12.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0869154a2d0546545cde61d1789a6524319fc1897d9ee31218eae7a60ccc5643"}, - {file = "pillow-12.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a7921c5a6d31b3d756ec980f2f47c0cfdbce0fc48c22a39347a895f41f4a6ea4"}, - {file = "pillow-12.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1ee80a59f6ce048ae13cda1abf7fbd2a34ab9ee7d401c46be3ca685d1999a399"}, - {file = "pillow-12.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c50f36a62a22d350c96e49ad02d0da41dbd17ddc2e29750dbdba4323f85eb4a5"}, - {file = "pillow-12.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5193fde9a5f23c331ea26d0cf171fbf67e3f247585f50c08b3e205c7aeb4589b"}, - {file = "pillow-12.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bde737cff1a975b70652b62d626f7785e0480918dece11e8fef3c0cf057351c3"}, - {file = "pillow-12.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a6597ff2b61d121172f5844b53f21467f7082f5fb385a9a29c01414463f93b07"}, - {file = "pillow-12.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b817e7035ea7f6b942c13aa03bb554fc44fea70838ea21f8eb31c638326584e"}, - {file = "pillow-12.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4f1231b7dec408e8670264ce63e9c71409d9583dd21d32c163e25213ee2a344"}, - {file = "pillow-12.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6e51b71417049ad6ab14c49608b4a24d8fb3fe605e5dfabfe523b58064dc3d27"}, - {file = "pillow-12.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d120c38a42c234dc9a8c5de7ceaaf899cf33561956acb4941653f8bdc657aa79"}, - {file = "pillow-12.0.0-cp313-cp313-win32.whl", hash = "sha256:4cc6b3b2efff105c6a1656cfe59da4fdde2cda9af1c5e0b58529b24525d0a098"}, - {file = "pillow-12.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:4cf7fed4b4580601c4345ceb5d4cbf5a980d030fd5ad07c4d2ec589f95f09905"}, - {file = "pillow-12.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:9f0b04c6b8584c2c193babcccc908b38ed29524b29dd464bc8801bf10d746a3a"}, - {file = "pillow-12.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7fa22993bac7b77b78cae22bad1e2a987ddf0d9015c63358032f84a53f23cdc3"}, - {file = "pillow-12.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f135c702ac42262573fe9714dfe99c944b4ba307af5eb507abef1667e2cbbced"}, - {file = "pillow-12.0.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c85de1136429c524e55cfa4e033b4a7940ac5c8ee4d9401cc2d1bf48154bbc7b"}, - {file = "pillow-12.0.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38df9b4bfd3db902c9c2bd369bcacaf9d935b2fff73709429d95cc41554f7b3d"}, - {file = "pillow-12.0.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7d87ef5795da03d742bf49439f9ca4d027cde49c82c5371ba52464aee266699a"}, - {file = "pillow-12.0.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aff9e4d82d082ff9513bdd6acd4f5bd359f5b2c870907d2b0a9c5e10d40c88fe"}, - {file = "pillow-12.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:8d8ca2b210ada074d57fcee40c30446c9562e542fc46aedc19baf758a93532ee"}, - {file = "pillow-12.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:99a7f72fb6249302aa62245680754862a44179b545ded638cf1fef59befb57ef"}, - {file = "pillow-12.0.0-cp313-cp313t-win32.whl", hash = "sha256:4078242472387600b2ce8d93ade8899c12bf33fa89e55ec89fe126e9d6d5d9e9"}, - {file = "pillow-12.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2c54c1a783d6d60595d3514f0efe9b37c8808746a66920315bfd34a938d7994b"}, - {file = "pillow-12.0.0-cp313-cp313t-win_arm64.whl", hash = "sha256:26d9f7d2b604cd23aba3e9faf795787456ac25634d82cd060556998e39c6fa47"}, - {file = "pillow-12.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:beeae3f27f62308f1ddbcfb0690bf44b10732f2ef43758f169d5e9303165d3f9"}, - {file = "pillow-12.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d4827615da15cd59784ce39d3388275ec093ae3ee8d7f0c089b76fa87af756c2"}, - {file = "pillow-12.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:3e42edad50b6909089750e65c91aa09aaf1e0a71310d383f11321b27c224ed8a"}, - {file = "pillow-12.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:e5d8efac84c9afcb40914ab49ba063d94f5dbdf5066db4482c66a992f47a3a3b"}, - {file = "pillow-12.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:266cd5f2b63ff316d5a1bba46268e603c9caf5606d44f38c2873c380950576ad"}, - {file = "pillow-12.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58eea5ebe51504057dd95c5b77d21700b77615ab0243d8152793dc00eb4faf01"}, - {file = "pillow-12.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f13711b1a5ba512d647a0e4ba79280d3a9a045aaf7e0cc6fbe96b91d4cdf6b0c"}, - {file = "pillow-12.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6846bd2d116ff42cba6b646edf5bf61d37e5cbd256425fa089fee4ff5c07a99e"}, - {file = "pillow-12.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c98fa880d695de164b4135a52fd2e9cd7b7c90a9d8ac5e9e443a24a95ef9248e"}, - {file = "pillow-12.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa3ed2a29a9e9d2d488b4da81dcb54720ac3104a20bf0bd273f1e4648aff5af9"}, - {file = "pillow-12.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d034140032870024e6b9892c692fe2968493790dd57208b2c37e3fb35f6df3ab"}, - {file = "pillow-12.0.0-cp314-cp314-win32.whl", hash = "sha256:1b1b133e6e16105f524a8dec491e0586d072948ce15c9b914e41cdadd209052b"}, - {file = "pillow-12.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:8dc232e39d409036af549c86f24aed8273a40ffa459981146829a324e0848b4b"}, - {file = "pillow-12.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:d52610d51e265a51518692045e372a4c363056130d922a7351429ac9f27e70b0"}, - {file = "pillow-12.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1979f4566bb96c1e50a62d9831e2ea2d1211761e5662afc545fa766f996632f6"}, - {file = "pillow-12.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b2e4b27a6e15b04832fe9bf292b94b5ca156016bbc1ea9c2c20098a0320d6cf6"}, - {file = "pillow-12.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fb3096c30df99fd01c7bf8e544f392103d0795b9f98ba71a8054bcbf56b255f1"}, - {file = "pillow-12.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7438839e9e053ef79f7112c881cef684013855016f928b168b81ed5835f3e75e"}, - {file = "pillow-12.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d5c411a8eaa2299322b647cd932586b1427367fd3184ffbb8f7a219ea2041ca"}, - {file = "pillow-12.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7e091d464ac59d2c7ad8e7e08105eaf9dafbc3883fd7265ffccc2baad6ac925"}, - {file = "pillow-12.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:792a2c0be4dcc18af9d4a2dfd8a11a17d5e25274a1062b0ec1c2d79c76f3e7f8"}, - {file = "pillow-12.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:afbefa430092f71a9593a99ab6a4e7538bc9eabbf7bf94f91510d3503943edc4"}, - {file = "pillow-12.0.0-cp314-cp314t-win32.whl", hash = "sha256:3830c769decf88f1289680a59d4f4c46c72573446352e2befec9a8512104fa52"}, - {file = "pillow-12.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:905b0365b210c73afb0ebe9101a32572152dfd1c144c7e28968a331b9217b94a"}, - {file = "pillow-12.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:99353a06902c2e43b43e8ff74ee65a7d90307d82370604746738a1e0661ccca7"}, - {file = "pillow-12.0.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b22bd8c974942477156be55a768f7aa37c46904c175be4e158b6a86e3a6b7ca8"}, - {file = "pillow-12.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:805ebf596939e48dbb2e4922a1d3852cfc25c38160751ce02da93058b48d252a"}, - {file = "pillow-12.0.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cae81479f77420d217def5f54b5b9d279804d17e982e0f2fa19b1d1e14ab5197"}, - {file = "pillow-12.0.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeaefa96c768fc66818730b952a862235d68825c178f1b3ffd4efd7ad2edcb7c"}, - {file = "pillow-12.0.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09f2d0abef9e4e2f349305a4f8cc784a8a6c2f58a8c4892eea13b10a943bd26e"}, - {file = "pillow-12.0.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bdee52571a343d721fb2eb3b090a82d959ff37fc631e3f70422e0c2e029f3e76"}, - {file = "pillow-12.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:b290fd8aa38422444d4b50d579de197557f182ef1068b75f5aa8558638b8d0a5"}, - {file = "pillow-12.0.0.tar.gz", hash = "sha256:87d4f8125c9988bfbed67af47dd7a953e2fc7b0cc1e7800ec6d2080d490bb353"}, + {file = "pillow-12.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a"}, + {file = "pillow-12.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7"}, + {file = "pillow-12.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f"}, + {file = "pillow-12.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec"}, + {file = "pillow-12.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468"}, + {file = "pillow-12.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed"}, + {file = "pillow-12.3.0-cp310-cp310-win32.whl", hash = "sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1"}, + {file = "pillow-12.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb"}, + {file = "pillow-12.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f"}, + {file = "pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756"}, + {file = "pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6"}, + {file = "pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd"}, + {file = "pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd"}, + {file = "pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c"}, + {file = "pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5"}, + {file = "pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b"}, + {file = "pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a"}, + {file = "pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26"}, + {file = "pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965"}, + {file = "pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7"}, + {file = "pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9"}, + {file = "pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91"}, + {file = "pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c"}, + {file = "pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df"}, + {file = "pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f"}, + {file = "pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09"}, + {file = "pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec"}, + {file = "pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66"}, + {file = "pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35"}, + {file = "pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65"}, + {file = "pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3"}, + {file = "pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a"}, + {file = "pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e"}, + {file = "pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f"}, + {file = "pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8"}, + {file = "pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930"}, + {file = "pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8"}, + {file = "pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0"}, + {file = "pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321"}, + {file = "pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b"}, + {file = "pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198"}, + {file = "pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130"}, + {file = "pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a"}, + {file = "pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d"}, + {file = "pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838"}, + {file = "pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e"}, + {file = "pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17"}, + {file = "pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385"}, + {file = "pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c"}, + {file = "pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d"}, + {file = "pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931"}, + {file = "pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7"}, + {file = "pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c"}, + {file = "pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c"}, + {file = "pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f"}, + {file = "pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701"}, + {file = "pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace"}, + {file = "pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4"}, + {file = "pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39"}, + {file = "pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71"}, + {file = "pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827"}, + {file = "pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5"}, + {file = "pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658"}, + {file = "pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf"}, + {file = "pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64"}, + {file = "pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e"}, + {file = "pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777"}, + {file = "pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1"}, + {file = "pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9"}, + {file = "pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8"}, + {file = "pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418"}, + {file = "pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a"}, + {file = "pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce"}, ] [package.extras] @@ -4391,7 +4577,7 @@ docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-autobuild", "sphinx-copybut fpx = ["olefile"] mic = ["olefile"] test-arrow = ["arro3-compute", "arro3-core", "nanoarrow", "pyarrow"] -tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma (>=5)", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "trove-classifiers (>=2024.10.12)"] +tests = ["coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "setuptools", "trove-classifiers (>=2024.10.12)"] xmp = ["defusedxml"] [[package]] @@ -4417,7 +4603,7 @@ version = "1.6.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.9" -groups = ["main", "docling-ocr"] +groups = ["main", "dev", "docling-ocr"] files = [ {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, @@ -4607,22 +4793,22 @@ testing = ["google-api-core (>=1.31.5)"] [[package]] name = "protobuf" -version = "6.33.0" +version = "6.33.6" description = "" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "protobuf-6.33.0-cp310-abi3-win32.whl", hash = "sha256:d6101ded078042a8f17959eccd9236fb7a9ca20d3b0098bbcb91533a5680d035"}, - {file = "protobuf-6.33.0-cp310-abi3-win_amd64.whl", hash = "sha256:9a031d10f703f03768f2743a1c403af050b6ae1f3480e9c140f39c45f81b13ee"}, - {file = "protobuf-6.33.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:905b07a65f1a4b72412314082c7dbfae91a9e8b68a0cc1577515f8df58ecf455"}, - {file = "protobuf-6.33.0-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e0697ece353e6239b90ee43a9231318302ad8353c70e6e45499fa52396debf90"}, - {file = "protobuf-6.33.0-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:e0a1715e4f27355afd9570f3ea369735afc853a6c3951a6afe1f80d8569ad298"}, - {file = "protobuf-6.33.0-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:35be49fd3f4fefa4e6e2aacc35e8b837d6703c37a2168a55ac21e9b1bc7559ef"}, - {file = "protobuf-6.33.0-cp39-cp39-win32.whl", hash = "sha256:cd33a8e38ea3e39df66e1bbc462b076d6e5ba3a4ebbde58219d777223a7873d3"}, - {file = "protobuf-6.33.0-cp39-cp39-win_amd64.whl", hash = "sha256:c963e86c3655af3a917962c9619e1a6b9670540351d7af9439d06064e3317cc9"}, - {file = "protobuf-6.33.0-py3-none-any.whl", hash = "sha256:25c9e1963c6734448ea2d308cfa610e692b801304ba0908d7bfa564ac5132995"}, - {file = "protobuf-6.33.0.tar.gz", hash = "sha256:140303d5c8d2037730c548f8c7b93b20bb1dc301be280c378b82b8894589c954"}, + {file = "protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3"}, + {file = "protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326"}, + {file = "protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a"}, + {file = "protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2"}, + {file = "protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3"}, + {file = "protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593"}, + {file = "protobuf-6.33.6-cp39-cp39-win32.whl", hash = "sha256:bd56799fb262994b2c2faa1799693c95cc2e22c62f56fb43af311cae45d26f0e"}, + {file = "protobuf-6.33.6-cp39-cp39-win_amd64.whl", hash = "sha256:f443a394af5ed23672bc6c486be138628fbe5c651ccbc536873d7da23d1868cf"}, + {file = "protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901"}, + {file = "protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135"}, ] [[package]] @@ -4739,70 +4925,74 @@ files = [ [[package]] name = "pyarrow" -version = "21.0.0" +version = "23.0.1" description = "Python library for Apache Arrow" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "pyarrow-21.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e563271e2c5ff4d4a4cbeb2c83d5cf0d4938b891518e676025f7268c6fe5fe26"}, - {file = "pyarrow-21.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:fee33b0ca46f4c85443d6c450357101e47d53e6c3f008d658c27a2d020d44c79"}, - {file = "pyarrow-21.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:7be45519b830f7c24b21d630a31d48bcebfd5d4d7f9d3bdb49da9cdf6d764edb"}, - {file = "pyarrow-21.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:26bfd95f6bff443ceae63c65dc7e048670b7e98bc892210acba7e4995d3d4b51"}, - {file = "pyarrow-21.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:bd04ec08f7f8bd113c55868bd3fc442a9db67c27af098c5f814a3091e71cc61a"}, - {file = "pyarrow-21.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9b0b14b49ac10654332a805aedfc0147fb3469cbf8ea951b3d040dab12372594"}, - {file = "pyarrow-21.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:9d9f8bcb4c3be7738add259738abdeddc363de1b80e3310e04067aa1ca596634"}, - {file = "pyarrow-21.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c077f48aab61738c237802836fc3844f85409a46015635198761b0d6a688f87b"}, - {file = "pyarrow-21.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:689f448066781856237eca8d1975b98cace19b8dd2ab6145bf49475478bcaa10"}, - {file = "pyarrow-21.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:479ee41399fcddc46159a551705b89c05f11e8b8cb8e968f7fec64f62d91985e"}, - {file = "pyarrow-21.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:40ebfcb54a4f11bcde86bc586cbd0272bac0d516cfa539c799c2453768477569"}, - {file = "pyarrow-21.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8d58d8497814274d3d20214fbb24abcad2f7e351474357d552a8d53bce70c70e"}, - {file = "pyarrow-21.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:585e7224f21124dd57836b1530ac8f2df2afc43c861d7bf3d58a4870c42ae36c"}, - {file = "pyarrow-21.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:555ca6935b2cbca2c0e932bedd853e9bc523098c39636de9ad4693b5b1df86d6"}, - {file = "pyarrow-21.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:3a302f0e0963db37e0a24a70c56cf91a4faa0bca51c23812279ca2e23481fccd"}, - {file = "pyarrow-21.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:b6b27cf01e243871390474a211a7922bfbe3bda21e39bc9160daf0da3fe48876"}, - {file = "pyarrow-21.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e72a8ec6b868e258a2cd2672d91f2860ad532d590ce94cdf7d5e7ec674ccf03d"}, - {file = "pyarrow-21.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b7ae0bbdc8c6674259b25bef5d2a1d6af5d39d7200c819cf99e07f7dfef1c51e"}, - {file = "pyarrow-21.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:58c30a1729f82d201627c173d91bd431db88ea74dcaa3885855bc6203e433b82"}, - {file = "pyarrow-21.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:072116f65604b822a7f22945a7a6e581cfa28e3454fdcc6939d4ff6090126623"}, - {file = "pyarrow-21.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf56ec8b0a5c8c9d7021d6fd754e688104f9ebebf1bf4449613c9531f5346a18"}, - {file = "pyarrow-21.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e99310a4ebd4479bcd1964dff9e14af33746300cb014aa4a3781738ac63baf4a"}, - {file = "pyarrow-21.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:d2fe8e7f3ce329a71b7ddd7498b3cfac0eeb200c2789bd840234f0dc271a8efe"}, - {file = "pyarrow-21.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f522e5709379d72fb3da7785aa489ff0bb87448a9dc5a75f45763a795a089ebd"}, - {file = "pyarrow-21.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:69cbbdf0631396e9925e048cfa5bce4e8c3d3b41562bbd70c685a8eb53a91e61"}, - {file = "pyarrow-21.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:731c7022587006b755d0bdb27626a1a3bb004bb56b11fb30d98b6c1b4718579d"}, - {file = "pyarrow-21.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc56bc708f2d8ac71bd1dcb927e458c93cec10b98eb4120206a4091db7b67b99"}, - {file = "pyarrow-21.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:186aa00bca62139f75b7de8420f745f2af12941595bbbfa7ed3870ff63e25636"}, - {file = "pyarrow-21.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:a7a102574faa3f421141a64c10216e078df467ab9576684d5cd696952546e2da"}, - {file = "pyarrow-21.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:1e005378c4a2c6db3ada3ad4c217b381f6c886f0a80d6a316fe586b90f77efd7"}, - {file = "pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:65f8e85f79031449ec8706b74504a316805217b35b6099155dd7e227eef0d4b6"}, - {file = "pyarrow-21.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:3a81486adc665c7eb1a2bde0224cfca6ceaba344a82a971ef059678417880eb8"}, - {file = "pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:fc0d2f88b81dcf3ccf9a6ae17f89183762c8a94a5bdcfa09e05cfe413acf0503"}, - {file = "pyarrow-21.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6299449adf89df38537837487a4f8d3bd91ec94354fdd2a7d30bc11c48ef6e79"}, - {file = "pyarrow-21.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:222c39e2c70113543982c6b34f3077962b44fca38c0bd9e68bb6781534425c10"}, - {file = "pyarrow-21.0.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:a7f6524e3747e35f80744537c78e7302cd41deee8baa668d56d55f77d9c464b3"}, - {file = "pyarrow-21.0.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:203003786c9fd253ebcafa44b03c06983c9c8d06c3145e37f1b76a1f317aeae1"}, - {file = "pyarrow-21.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:3b4d97e297741796fead24867a8dabf86c87e4584ccc03167e4a811f50fdf74d"}, - {file = "pyarrow-21.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:898afce396b80fdda05e3086b4256f8677c671f7b1d27a6976fa011d3fd0a86e"}, - {file = "pyarrow-21.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:067c66ca29aaedae08218569a114e413b26e742171f526e828e1064fcdec13f4"}, - {file = "pyarrow-21.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0c4e75d13eb76295a49e0ea056eb18dbd87d81450bfeb8afa19a7e5a75ae2ad7"}, - {file = "pyarrow-21.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdc4c17afda4dab2a9c0b79148a43a7f4e1094916b3e18d8975bfd6d6d52241f"}, - {file = "pyarrow-21.0.0.tar.gz", hash = "sha256:5051f2dccf0e283ff56335760cbc8622cf52264d67e359d5569541ac11b6d5bc"}, + {file = "pyarrow-23.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:3fab8f82571844eb3c460f90a75583801d14ca0cc32b1acc8c361650e006fd56"}, + {file = "pyarrow-23.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:3f91c038b95f71ddfc865f11d5876c42f343b4495535bd262c7b321b0b94507c"}, + {file = "pyarrow-23.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d0744403adabef53c985a7f8a082b502a368510c40d184df349a0a8754533258"}, + {file = "pyarrow-23.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c33b5bf406284fd0bba436ed6f6c3ebe8e311722b441d89397c54f871c6863a2"}, + {file = "pyarrow-23.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ddf743e82f69dcd6dbbcb63628895d7161e04e56794ef80550ac6f3315eeb1d5"}, + {file = "pyarrow-23.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e052a211c5ac9848ae15d5ec875ed0943c0221e2fcfe69eee80b604b4e703222"}, + {file = "pyarrow-23.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:5abde149bb3ce524782d838eb67ac095cd3fd6090eba051130589793f1a7f76d"}, + {file = "pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb"}, + {file = "pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350"}, + {file = "pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd"}, + {file = "pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9"}, + {file = "pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701"}, + {file = "pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78"}, + {file = "pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919"}, + {file = "pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f"}, + {file = "pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7"}, + {file = "pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9"}, + {file = "pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05"}, + {file = "pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67"}, + {file = "pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730"}, + {file = "pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0"}, + {file = "pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8"}, + {file = "pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f"}, + {file = "pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677"}, + {file = "pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2"}, + {file = "pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37"}, + {file = "pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2"}, + {file = "pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a"}, + {file = "pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1"}, + {file = "pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500"}, + {file = "pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41"}, + {file = "pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07"}, + {file = "pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83"}, + {file = "pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125"}, + {file = "pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8"}, + {file = "pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca"}, + {file = "pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1"}, + {file = "pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb"}, + {file = "pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1"}, + {file = "pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886"}, + {file = "pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f"}, + {file = "pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5"}, + {file = "pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d"}, + {file = "pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f"}, + {file = "pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814"}, + {file = "pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d"}, + {file = "pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7"}, + {file = "pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690"}, + {file = "pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce"}, + {file = "pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019"}, ] -[package.extras] -test = ["cffi", "hypothesis", "pandas", "pytest", "pytz"] - [[package]] name = "pyasn1" -version = "0.6.1" +version = "0.6.4" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, - {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, + {file = "pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b"}, + {file = "pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81"}, ] [[package]] @@ -5048,14 +5238,14 @@ typing-extensions = ">=4.14.1" [[package]] name = "pydantic-settings" -version = "2.12.0" +version = "2.15.0" description = "Settings management using Pydantic" optional = false python-versions = ">=3.10" groups = ["main", "docling-ocr"] files = [ - {file = "pydantic_settings-2.12.0-py3-none-any.whl", hash = "sha256:fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809"}, - {file = "pydantic_settings-2.12.0.tar.gz", hash = "sha256:005538ef951e3c2a68e1c08b292b5f2e71490def8589d4221b95dab00dafcfd0"}, + {file = "pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42"}, + {file = "pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117"}, ] [package.dependencies] @@ -5064,7 +5254,7 @@ python-dotenv = ">=0.21.0" typing-inspection = ">=0.4.0" [package.extras] -aws-secrets-manager = ["boto3 (>=1.35.0)", "boto3-stubs[secretsmanager]"] +aws-secrets-manager = ["boto3 (>=1.35.0)"] azure-key-vault = ["azure-identity (>=1.16.0)", "azure-keyvault-secrets (>=4.8.0)"] gcp-secret-manager = ["google-cloud-secret-manager (>=2.23.1)"] toml = ["tomli (>=2.0.1)"] @@ -5072,14 +5262,14 @@ yaml = ["pyyaml (>=6.0.1)"] [[package]] name = "pygments" -version = "2.19.2" +version = "2.21.0" description = "Pygments is a syntax highlighting package written in Python." optional = false -python-versions = ">=3.8" -groups = ["main", "docling-ocr"] +python-versions = ">=3.9" +groups = ["main", "dev", "docling-ocr", "local-ml"] files = [ - {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, - {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, + {file = "pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9"}, + {file = "pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c"}, ] [package.extras] @@ -5087,14 +5277,14 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyjwt" -version = "2.11.0" +version = "2.13.0" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "pyjwt-2.11.0-py3-none-any.whl", hash = "sha256:94a6bde30eb5c8e04fee991062b534071fd1439ef58d2adc9ccb823e7bcd0469"}, - {file = "pyjwt-2.11.0.tar.gz", hash = "sha256:35f95c1f0fbe5d5ba6e43f00271c275f7a1a4db1dab27bf708073b75318ea623"}, + {file = "pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728"}, + {file = "pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423"}, ] [package.dependencies] @@ -5102,9 +5292,6 @@ cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"cryp [package.extras] crypto = ["cryptography (>=3.4.0)"] -dev = ["coverage[toml] (==7.10.7)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=8.4.2,<9.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] -docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] -tests = ["coverage[toml] (==7.10.7)", "pytest (>=8.4.2,<9.0.0)"] [[package]] name = "pylatexenc" @@ -5265,118 +5452,6 @@ snappy = ["python-snappy"] test = ["pytest (>=8.2)", "pytest-asyncio (>=0.24.0)"] zstd = ["zstandard"] -[[package]] -name = "pyobjc-core" -version = "12.1" -description = "Python<->ObjC Interoperability Module" -optional = false -python-versions = ">=3.10" -groups = ["main", "docling-ocr"] -markers = "sys_platform == \"darwin\"" -files = [ - {file = "pyobjc_core-12.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:93418e79c1655f66b4352168f8c85c942707cb1d3ea13a1da3e6f6a143bacda7"}, - {file = "pyobjc_core-12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c918ebca280925e7fcb14c5c43ce12dcb9574a33cccb889be7c8c17f3bcce8b6"}, - {file = "pyobjc_core-12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:818bcc6723561f207e5b5453efe9703f34bc8781d11ce9b8be286bb415eb4962"}, - {file = "pyobjc_core-12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:01c0cf500596f03e21c23aef9b5f326b9fb1f8f118cf0d8b66749b6cf4cbb37a"}, - {file = "pyobjc_core-12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:177aaca84bb369a483e4961186704f64b2697708046745f8167e818d968c88fc"}, - {file = "pyobjc_core-12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:844515f5d86395b979d02152576e7dee9cc679acc0b32dc626ef5bda315eaa43"}, - {file = "pyobjc_core-12.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:453b191df1a4b80e756445b935491b974714456ae2cbae816840bd96f86db882"}, - {file = "pyobjc_core-12.1.tar.gz", hash = "sha256:2bb3903f5387f72422145e1466b3ac3f7f0ef2e9960afa9bcd8961c5cbf8bd21"}, -] - -[[package]] -name = "pyobjc-framework-cocoa" -version = "12.1" -description = "Wrappers for the Cocoa frameworks on macOS" -optional = false -python-versions = ">=3.10" -groups = ["main", "docling-ocr"] -markers = "sys_platform == \"darwin\"" -files = [ - {file = "pyobjc_framework_cocoa-12.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:9b880d3bdcd102809d704b6d8e14e31611443aa892d9f60e8491e457182fdd48"}, - {file = "pyobjc_framework_cocoa-12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f52228bcf38da64b77328787967d464e28b981492b33a7675585141e1b0a01e6"}, - {file = "pyobjc_framework_cocoa-12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:547c182837214b7ec4796dac5aee3aa25abc665757b75d7f44f83c994bcb0858"}, - {file = "pyobjc_framework_cocoa-12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5a3dcd491cacc2f5a197142b3c556d8aafa3963011110102a093349017705118"}, - {file = "pyobjc_framework_cocoa-12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:914b74328c22d8ca261d78c23ef2befc29776e0b85555973927b338c5734ca44"}, - {file = "pyobjc_framework_cocoa-12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:03342a60fc0015bcdf9b93ac0b4f457d3938e9ef761b28df9564c91a14f0129a"}, - {file = "pyobjc_framework_cocoa-12.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6ba1dc1bfa4da42d04e93d2363491275fb2e2be5c20790e561c8a9e09b8cf2cc"}, - {file = "pyobjc_framework_cocoa-12.1.tar.gz", hash = "sha256:5556c87db95711b985d5efdaaf01c917ddd41d148b1e52a0c66b1a2e2c5c1640"}, -] - -[package.dependencies] -pyobjc-core = ">=12.1" - -[[package]] -name = "pyobjc-framework-coreml" -version = "12.1" -description = "Wrappers for the framework CoreML on macOS" -optional = false -python-versions = ">=3.10" -groups = ["main", "docling-ocr"] -markers = "sys_platform == \"darwin\"" -files = [ - {file = "pyobjc_framework_coreml-12.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:df4e9b4f97063148cc481f72e2fbe3cc53b9464d722752aa658d7c0aec9f02fd"}, - {file = "pyobjc_framework_coreml-12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:16dafcfb123f022e62f47a590a7eccf7d0cb5957a77fd5f062b5ee751cb5a423"}, - {file = "pyobjc_framework_coreml-12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:10dc8e8db53d7631ebc712cad146e3a9a9a443f4e1a037e844149a24c3c42669"}, - {file = "pyobjc_framework_coreml-12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:48fa3bb4a03fa23e0e36c93936dca2969598e4102f4b441e1663f535fc99cd31"}, - {file = "pyobjc_framework_coreml-12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:71de5b37e6a017e3ed16645c5d6533138f24708da5b56c35c818ae49d0253ee1"}, - {file = "pyobjc_framework_coreml-12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:a04a96e512ecf6999aa9e1f60ad5635cb9d1cd839be470341d8d1541797baef6"}, - {file = "pyobjc_framework_coreml-12.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:7762b3dd2de01565b7cf3049ce1e4c27341ba179d97016b0b7607448e1c39865"}, - {file = "pyobjc_framework_coreml-12.1.tar.gz", hash = "sha256:0d1a4216891a18775c9e0170d908714c18e4f53f9dc79fb0f5263b2aa81609ba"}, -] - -[package.dependencies] -pyobjc-core = ">=12.1" -pyobjc-framework-Cocoa = ">=12.1" - -[[package]] -name = "pyobjc-framework-quartz" -version = "12.1" -description = "Wrappers for the Quartz frameworks on macOS" -optional = false -python-versions = ">=3.10" -groups = ["main", "docling-ocr"] -markers = "sys_platform == \"darwin\"" -files = [ - {file = "pyobjc_framework_quartz-12.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c6f312ae79ef8b3019dcf4b3374c52035c7c7bc4a09a1748b61b041bb685a0ed"}, - {file = "pyobjc_framework_quartz-12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:19f99ac49a0b15dd892e155644fe80242d741411a9ed9c119b18b7466048625a"}, - {file = "pyobjc_framework_quartz-12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7730cdce46c7e985535b5a42c31381af4aa6556e5642dc55b5e6597595e57a16"}, - {file = "pyobjc_framework_quartz-12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:629b7971b1b43a11617f1460cd218bd308dfea247cd4ee3842eb40ca6f588860"}, - {file = "pyobjc_framework_quartz-12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:53b84e880c358ba1ddcd7e8d5ea0407d760eca58b96f0d344829162cda5f37b3"}, - {file = "pyobjc_framework_quartz-12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:42d306b07f05ae7d155984503e0fb1b701fecd31dcc5c79fe8ab9790ff7e0de0"}, - {file = "pyobjc_framework_quartz-12.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0cc08fddb339b2760df60dea1057453557588908e42bdc62184b6396ce2d6e9a"}, - {file = "pyobjc_framework_quartz-12.1.tar.gz", hash = "sha256:27f782f3513ac88ec9b6c82d9767eef95a5cf4175ce88a1e5a65875fee799608"}, -] - -[package.dependencies] -pyobjc-core = ">=12.1" -pyobjc-framework-Cocoa = ">=12.1" - -[[package]] -name = "pyobjc-framework-vision" -version = "12.1" -description = "Wrappers for the framework Vision on macOS" -optional = false -python-versions = ">=3.10" -groups = ["main", "docling-ocr"] -markers = "sys_platform == \"darwin\"" -files = [ - {file = "pyobjc_framework_vision-12.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a30c3fff926348baecc3ce1f6da8ed327d0cbd55ca1c376d018e31023b79c0ab"}, - {file = "pyobjc_framework_vision-12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1edbf2fc18ce3b31108f845901a88f2236783ae6bf0bc68438d7ece572dc2a29"}, - {file = "pyobjc_framework_vision-12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:6b3211d84f3a12aad0cde752cfd43a80d0218960ac9e6b46b141c730e7d655bd"}, - {file = "pyobjc_framework_vision-12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7273e2508db4c2e88523b4b7ff38ac54808756e7ba01d78e6c08ea68f32577d2"}, - {file = "pyobjc_framework_vision-12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:04296f0848cc8cdead66c76df6063720885cbdf24fdfd1900749a6e2297313db"}, - {file = "pyobjc_framework_vision-12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:631add775ed1dafb221a6116137cdcd78432addc16200ca434571c2a039c0e03"}, - {file = "pyobjc_framework_vision-12.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fe41a1a70cc91068aee7b5293fa09dc66d1c666a8da79fdf948900988b439df6"}, - {file = "pyobjc_framework_vision-12.1.tar.gz", hash = "sha256:a30959100e85dcede3a786c544e621ad6eb65ff6abf85721f805822b8c5fe9b0"}, -] - -[package.dependencies] -pyobjc-core = ">=12.1" -pyobjc-framework-Cocoa = ">=12.1" -pyobjc-framework-CoreML = ">=12.1" -pyobjc-framework-Quartz = ">=12.1" - [[package]] name = "pypdfium2" version = "5.5.0" @@ -5411,14 +5486,14 @@ files = [ [[package]] name = "pytest" -version = "9.0.2" +version = "9.1.1" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["main", "dev"] files = [ - {file = "pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b"}, - {file = "pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11"}, + {file = "pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c"}, + {file = "pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313"}, ] [package.dependencies] @@ -5464,14 +5539,14 @@ typing_extensions = ">=4.9.0" [[package]] name = "python-dotenv" -version = "1.1.1" +version = "1.2.3" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "docling-ocr"] files = [ - {file = "python_dotenv-1.1.1-py3-none-any.whl", hash = "sha256:31f23644fe2602f88ff55e1f5c79ba497e01224ee7737937930c448e4d0e24dc"}, - {file = "python_dotenv-1.1.1.tar.gz", hash = "sha256:a8a6399716257f45be6a007360200409fce5cda2661e3dec71d23dc15f6189ab"}, + {file = "python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9"}, + {file = "python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35"}, ] [package.extras] @@ -5479,16 +5554,33 @@ cli = ["click (>=5.0)"] [[package]] name = "python-multipart" -version = "0.0.20" +version = "0.0.32" description = "A streaming multipart parser for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104"}, - {file = "python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13"}, + {file = "python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23"}, + {file = "python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e"}, ] +[[package]] +name = "python-oxmsg" +version = "0.0.2" +description = "Extract attachments from Outlook .msg files." +optional = false +python-versions = ">=3.9" +groups = ["main", "docling-ocr"] +files = [ + {file = "python_oxmsg-0.0.2-py3-none-any.whl", hash = "sha256:22be29b14c46016bcd05e34abddfd8e05ee82082f53b82753d115da3fc7d0355"}, + {file = "python_oxmsg-0.0.2.tar.gz", hash = "sha256:a6aff4deb1b5975d44d49dab1d9384089ffeec819e19c6940bc7ffbc84775fad"}, +] + +[package.dependencies] +click = "*" +olefile = "*" +typing_extensions = ">=4.9.0" + [[package]] name = "python-pptx" version = "1.0.2" @@ -5556,7 +5648,7 @@ version = "6.0.3" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, @@ -5740,19 +5832,19 @@ all = ["numpy"] [[package]] name = "rapidocr" -version = "3.6.0" +version = "3.9.2" description = "Awesome OCR Library" optional = false -python-versions = "<4,>=3.6" +python-versions = "<4,>=3.8" groups = ["main", "docling-ocr"] files = [ - {file = "rapidocr-3.6.0-py3-none-any.whl", hash = "sha256:d16b43872fc4dfa1e60996334dcd0dc3e3f1f64161e2332bc1873b9f65754e6b"}, + {file = "rapidocr-3.9.2-py3-none-any.whl", hash = "sha256:04d6b8d151f823d930bd91910555f57bea897c0c44fa6794267b94cf9c1ef9a0"}, ] [package.dependencies] colorlog = "*" numpy = ">=1.19.5,<3.0.0" -omegaconf = "*" +omegaconf = "!=2.2.1" opencv_python = ">=4.5.1.48" Pillow = "*" pyclipper = ">=1.2.0" @@ -5801,150 +5893,149 @@ typing-extensions = {version = ">=4.4.0", markers = "python_version < \"3.13\""} [[package]] name = "regex" -version = "2025.9.18" +version = "2026.7.19" description = "Alternative regular expression module, to replace re." optional = false -python-versions = ">=3.9" -groups = ["main", "docling-ocr"] -files = [ - {file = "regex-2025.9.18-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:12296202480c201c98a84aecc4d210592b2f55e200a1d193235c4db92b9f6788"}, - {file = "regex-2025.9.18-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:220381f1464a581f2ea988f2220cf2a67927adcef107d47d6897ba5a2f6d51a4"}, - {file = "regex-2025.9.18-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:87f681bfca84ebd265278b5daa1dcb57f4db315da3b5d044add7c30c10442e61"}, - {file = "regex-2025.9.18-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34d674cbba70c9398074c8a1fcc1a79739d65d1105de2a3c695e2b05ea728251"}, - {file = "regex-2025.9.18-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:385c9b769655cb65ea40b6eea6ff763cbb6d69b3ffef0b0db8208e1833d4e746"}, - {file = "regex-2025.9.18-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8900b3208e022570ae34328712bef6696de0804c122933414014bae791437ab2"}, - {file = "regex-2025.9.18-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c204e93bf32cd7a77151d44b05eb36f469d0898e3fba141c026a26b79d9914a0"}, - {file = "regex-2025.9.18-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3acc471d1dd7e5ff82e6cacb3b286750decd949ecd4ae258696d04f019817ef8"}, - {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6479d5555122433728760e5f29edb4c2b79655a8deb681a141beb5c8a025baea"}, - {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:431bd2a8726b000eb6f12429c9b438a24062a535d06783a93d2bcbad3698f8a8"}, - {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0cc3521060162d02bd36927e20690129200e5ac9d2c6d32b70368870b122db25"}, - {file = "regex-2025.9.18-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a021217b01be2d51632ce056d7a837d3fa37c543ede36e39d14063176a26ae29"}, - {file = "regex-2025.9.18-cp310-cp310-win32.whl", hash = "sha256:4a12a06c268a629cb67cc1d009b7bb0be43e289d00d5111f86a2efd3b1949444"}, - {file = "regex-2025.9.18-cp310-cp310-win_amd64.whl", hash = "sha256:47acd811589301298c49db2c56bde4f9308d6396da92daf99cba781fa74aa450"}, - {file = "regex-2025.9.18-cp310-cp310-win_arm64.whl", hash = "sha256:16bd2944e77522275e5ee36f867e19995bcaa533dcb516753a26726ac7285442"}, - {file = "regex-2025.9.18-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:51076980cd08cd13c88eb7365427ae27f0d94e7cebe9ceb2bb9ffdae8fc4d82a"}, - {file = "regex-2025.9.18-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:828446870bd7dee4e0cbeed767f07961aa07f0ea3129f38b3ccecebc9742e0b8"}, - {file = "regex-2025.9.18-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c28821d5637866479ec4cc23b8c990f5bc6dd24e5e4384ba4a11d38a526e1414"}, - {file = "regex-2025.9.18-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:726177ade8e481db669e76bf99de0b278783be8acd11cef71165327abd1f170a"}, - {file = "regex-2025.9.18-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5cca697da89b9f8ea44115ce3130f6c54c22f541943ac8e9900461edc2b8bd4"}, - {file = "regex-2025.9.18-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dfbde38f38004703c35666a1e1c088b778e35d55348da2b7b278914491698d6a"}, - {file = "regex-2025.9.18-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2f422214a03fab16bfa495cfec72bee4aaa5731843b771860a471282f1bf74f"}, - {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a295916890f4df0902e4286bc7223ee7f9e925daa6dcdec4192364255b70561a"}, - {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5db95ff632dbabc8c38c4e82bf545ab78d902e81160e6e455598014f0abe66b9"}, - {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fb967eb441b0f15ae610b7069bdb760b929f267efbf522e814bbbfffdf125ce2"}, - {file = "regex-2025.9.18-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f04d2f20da4053d96c08f7fde6e1419b7ec9dbcee89c96e3d731fca77f411b95"}, - {file = "regex-2025.9.18-cp311-cp311-win32.whl", hash = "sha256:895197241fccf18c0cea7550c80e75f185b8bd55b6924fcae269a1a92c614a07"}, - {file = "regex-2025.9.18-cp311-cp311-win_amd64.whl", hash = "sha256:7e2b414deae99166e22c005e154a5513ac31493db178d8aec92b3269c9cce8c9"}, - {file = "regex-2025.9.18-cp311-cp311-win_arm64.whl", hash = "sha256:fb137ec7c5c54f34a25ff9b31f6b7b0c2757be80176435bf367111e3f71d72df"}, - {file = "regex-2025.9.18-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:436e1b31d7efd4dcd52091d076482031c611dde58bf9c46ca6d0a26e33053a7e"}, - {file = "regex-2025.9.18-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c190af81e5576b9c5fdc708f781a52ff20f8b96386c6e2e0557a78402b029f4a"}, - {file = "regex-2025.9.18-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e4121f1ce2b2b5eec4b397cc1b277686e577e658d8f5870b7eb2d726bd2300ab"}, - {file = "regex-2025.9.18-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:300e25dbbf8299d87205e821a201057f2ef9aa3deb29caa01cd2cac669e508d5"}, - {file = "regex-2025.9.18-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b47fcf9f5316c0bdaf449e879407e1b9937a23c3b369135ca94ebc8d74b1742"}, - {file = "regex-2025.9.18-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:57a161bd3acaa4b513220b49949b07e252165e6b6dc910ee7617a37ff4f5b425"}, - {file = "regex-2025.9.18-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f130c3a7845ba42de42f380fff3c8aebe89a810747d91bcf56d40a069f15352"}, - {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5f96fa342b6f54dcba928dd452e8d8cb9f0d63e711d1721cd765bb9f73bb048d"}, - {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0f0d676522d68c207828dcd01fb6f214f63f238c283d9f01d85fc664c7c85b56"}, - {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:40532bff8a1a0621e7903ae57fce88feb2e8a9a9116d341701302c9302aef06e"}, - {file = "regex-2025.9.18-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:039f11b618ce8d71a1c364fdee37da1012f5a3e79b1b2819a9f389cd82fd6282"}, - {file = "regex-2025.9.18-cp312-cp312-win32.whl", hash = "sha256:e1dd06f981eb226edf87c55d523131ade7285137fbde837c34dc9d1bf309f459"}, - {file = "regex-2025.9.18-cp312-cp312-win_amd64.whl", hash = "sha256:3d86b5247bf25fa3715e385aa9ff272c307e0636ce0c9595f64568b41f0a9c77"}, - {file = "regex-2025.9.18-cp312-cp312-win_arm64.whl", hash = "sha256:032720248cbeeae6444c269b78cb15664458b7bb9ed02401d3da59fe4d68c3a5"}, - {file = "regex-2025.9.18-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2a40f929cd907c7e8ac7566ac76225a77701a6221bca937bdb70d56cb61f57b2"}, - {file = "regex-2025.9.18-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c90471671c2cdf914e58b6af62420ea9ecd06d1554d7474d50133ff26ae88feb"}, - {file = "regex-2025.9.18-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1a351aff9e07a2dabb5022ead6380cff17a4f10e4feb15f9100ee56c4d6d06af"}, - {file = "regex-2025.9.18-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc4b8e9d16e20ddfe16430c23468a8707ccad3365b06d4536142e71823f3ca29"}, - {file = "regex-2025.9.18-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4b8cdbddf2db1c5e80338ba2daa3cfa3dec73a46fff2a7dda087c8efbf12d62f"}, - {file = "regex-2025.9.18-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a276937d9d75085b2c91fb48244349c6954f05ee97bba0963ce24a9d915b8b68"}, - {file = "regex-2025.9.18-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92a8e375ccdc1256401c90e9dc02b8642894443d549ff5e25e36d7cf8a80c783"}, - {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0dc6893b1f502d73037cf807a321cdc9be29ef3d6219f7970f842475873712ac"}, - {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a61e85bfc63d232ac14b015af1261f826260c8deb19401c0597dbb87a864361e"}, - {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:1ef86a9ebc53f379d921fb9a7e42b92059ad3ee800fcd9e0fe6181090e9f6c23"}, - {file = "regex-2025.9.18-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d3bc882119764ba3a119fbf2bd4f1b47bc56c1da5d42df4ed54ae1e8e66fdf8f"}, - {file = "regex-2025.9.18-cp313-cp313-win32.whl", hash = "sha256:3810a65675845c3bdfa58c3c7d88624356dd6ee2fc186628295e0969005f928d"}, - {file = "regex-2025.9.18-cp313-cp313-win_amd64.whl", hash = "sha256:16eaf74b3c4180ede88f620f299e474913ab6924d5c4b89b3833bc2345d83b3d"}, - {file = "regex-2025.9.18-cp313-cp313-win_arm64.whl", hash = "sha256:4dc98ba7dd66bd1261927a9f49bd5ee2bcb3660f7962f1ec02617280fc00f5eb"}, - {file = "regex-2025.9.18-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:fe5d50572bc885a0a799410a717c42b1a6b50e2f45872e2b40f4f288f9bce8a2"}, - {file = "regex-2025.9.18-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1b9d9a2d6cda6621551ca8cf7a06f103adf72831153f3c0d982386110870c4d3"}, - {file = "regex-2025.9.18-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:13202e4c4ac0ef9a317fff817674b293c8f7e8c68d3190377d8d8b749f566e12"}, - {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874ff523b0fecffb090f80ae53dc93538f8db954c8bb5505f05b7787ab3402a0"}, - {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d13ab0490128f2bb45d596f754148cd750411afc97e813e4b3a61cf278a23bb6"}, - {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:05440bc172bc4b4b37fb9667e796597419404dbba62e171e1f826d7d2a9ebcef"}, - {file = "regex-2025.9.18-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5514b8e4031fdfaa3d27e92c75719cbe7f379e28cacd939807289bce76d0e35a"}, - {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:65d3c38c39efce73e0d9dc019697b39903ba25b1ad45ebbd730d2cf32741f40d"}, - {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:ae77e447ebc144d5a26d50055c6ddba1d6ad4a865a560ec7200b8b06bc529368"}, - {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e3ef8cf53dc8df49d7e28a356cf824e3623764e9833348b655cfed4524ab8a90"}, - {file = "regex-2025.9.18-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9feb29817df349c976da9a0debf775c5c33fc1c8ad7b9f025825da99374770b7"}, - {file = "regex-2025.9.18-cp313-cp313t-win32.whl", hash = "sha256:168be0d2f9b9d13076940b1ed774f98595b4e3c7fc54584bba81b3cc4181742e"}, - {file = "regex-2025.9.18-cp313-cp313t-win_amd64.whl", hash = "sha256:d59ecf3bb549e491c8104fea7313f3563c7b048e01287db0a90485734a70a730"}, - {file = "regex-2025.9.18-cp313-cp313t-win_arm64.whl", hash = "sha256:dbef80defe9fb21310948a2595420b36c6d641d9bea4c991175829b2cc4bc06a"}, - {file = "regex-2025.9.18-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:c6db75b51acf277997f3adcd0ad89045d856190d13359f15ab5dda21581d9129"}, - {file = "regex-2025.9.18-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8f9698b6f6895d6db810e0bda5364f9ceb9e5b11328700a90cae573574f61eea"}, - {file = "regex-2025.9.18-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29cd86aa7cb13a37d0f0d7c21d8d949fe402ffa0ea697e635afedd97ab4b69f1"}, - {file = "regex-2025.9.18-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c9f285a071ee55cd9583ba24dde006e53e17780bb309baa8e4289cd472bcc47"}, - {file = "regex-2025.9.18-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5adf266f730431e3be9021d3e5b8d5ee65e563fec2883ea8093944d21863b379"}, - {file = "regex-2025.9.18-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1137cabc0f38807de79e28d3f6e3e3f2cc8cfb26bead754d02e6d1de5f679203"}, - {file = "regex-2025.9.18-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7cc9e5525cada99699ca9223cce2d52e88c52a3d2a0e842bd53de5497c604164"}, - {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bbb9246568f72dce29bcd433517c2be22c7791784b223a810225af3b50d1aafb"}, - {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:6a52219a93dd3d92c675383efff6ae18c982e2d7651c792b1e6d121055808743"}, - {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:ae9b3840c5bd456780e3ddf2f737ab55a79b790f6409182012718a35c6d43282"}, - {file = "regex-2025.9.18-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d488c236ac497c46a5ac2005a952c1a0e22a07be9f10c3e735bc7d1209a34773"}, - {file = "regex-2025.9.18-cp314-cp314-win32.whl", hash = "sha256:0c3506682ea19beefe627a38872d8da65cc01ffa25ed3f2e422dffa1474f0788"}, - {file = "regex-2025.9.18-cp314-cp314-win_amd64.whl", hash = "sha256:57929d0f92bebb2d1a83af372cd0ffba2263f13f376e19b1e4fa32aec4efddc3"}, - {file = "regex-2025.9.18-cp314-cp314-win_arm64.whl", hash = "sha256:6a4b44df31d34fa51aa5c995d3aa3c999cec4d69b9bd414a8be51984d859f06d"}, - {file = "regex-2025.9.18-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:b176326bcd544b5e9b17d6943f807697c0cb7351f6cfb45bf5637c95ff7e6306"}, - {file = "regex-2025.9.18-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:0ffd9e230b826b15b369391bec167baed57c7ce39efc35835448618860995946"}, - {file = "regex-2025.9.18-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec46332c41add73f2b57e2f5b642f991f6b15e50e9f86285e08ffe3a512ac39f"}, - {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b80fa342ed1ea095168a3f116637bd1030d39c9ff38dc04e54ef7c521e01fc95"}, - {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f4d97071c0ba40f0cf2a93ed76e660654c399a0a04ab7d85472239460f3da84b"}, - {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0ac936537ad87cef9e0e66c5144484206c1354224ee811ab1519a32373e411f3"}, - {file = "regex-2025.9.18-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dec57f96d4def58c422d212d414efe28218d58537b5445cf0c33afb1b4768571"}, - {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:48317233294648bf7cd068857f248e3a57222259a5304d32c7552e2284a1b2ad"}, - {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:274687e62ea3cf54846a9b25fc48a04459de50af30a7bd0b61a9e38015983494"}, - {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a78722c86a3e7e6aadf9579e3b0ad78d955f2d1f1a8ca4f67d7ca258e8719d4b"}, - {file = "regex-2025.9.18-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:06104cd203cdef3ade989a1c45b6215bf42f8b9dd705ecc220c173233f7cba41"}, - {file = "regex-2025.9.18-cp314-cp314t-win32.whl", hash = "sha256:2e1eddc06eeaffd249c0adb6fafc19e2118e6308c60df9db27919e96b5656096"}, - {file = "regex-2025.9.18-cp314-cp314t-win_amd64.whl", hash = "sha256:8620d247fb8c0683ade51217b459cb4a1081c0405a3072235ba43a40d355c09a"}, - {file = "regex-2025.9.18-cp314-cp314t-win_arm64.whl", hash = "sha256:b7531a8ef61de2c647cdf68b3229b071e46ec326b3138b2180acb4275f470b01"}, - {file = "regex-2025.9.18-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3dbcfcaa18e9480669030d07371713c10b4f1a41f791ffa5cb1a99f24e777f40"}, - {file = "regex-2025.9.18-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1e85f73ef7095f0380208269055ae20524bfde3f27c5384126ddccf20382a638"}, - {file = "regex-2025.9.18-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9098e29b3ea4ffffeade423f6779665e2a4f8db64e699c0ed737ef0db6ba7b12"}, - {file = "regex-2025.9.18-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90b6b7a2d0f45b7ecaaee1aec6b362184d6596ba2092dd583ffba1b78dd0231c"}, - {file = "regex-2025.9.18-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c81b892af4a38286101502eae7aec69f7cd749a893d9987a92776954f3943408"}, - {file = "regex-2025.9.18-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3b524d010973f2e1929aeb635418d468d869a5f77b52084d9f74c272189c251d"}, - {file = "regex-2025.9.18-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b498437c026a3d5d0be0020023ff76d70ae4d77118e92f6f26c9d0423452446"}, - {file = "regex-2025.9.18-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0716e4d6e58853d83f6563f3cf25c281ff46cf7107e5f11879e32cb0b59797d9"}, - {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:065b6956749379d41db2625f880b637d4acc14c0a4de0d25d609a62850e96d36"}, - {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:d4a691494439287c08ddb9b5793da605ee80299dd31e95fa3f323fac3c33d9d4"}, - {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ef8d10cc0989565bcbe45fb4439f044594d5c2b8919d3d229ea2c4238f1d55b0"}, - {file = "regex-2025.9.18-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:4baeb1b16735ac969a7eeecc216f1f8b7caf60431f38a2671ae601f716a32d25"}, - {file = "regex-2025.9.18-cp39-cp39-win32.whl", hash = "sha256:8e5f41ad24a1e0b5dfcf4c4e5d9f5bd54c895feb5708dd0c1d0d35693b24d478"}, - {file = "regex-2025.9.18-cp39-cp39-win_amd64.whl", hash = "sha256:50e8290707f2fb8e314ab3831e594da71e062f1d623b05266f8cfe4db4949afd"}, - {file = "regex-2025.9.18-cp39-cp39-win_arm64.whl", hash = "sha256:039a9d7195fd88c943d7c777d4941e8ef736731947becce773c31a1009cb3c35"}, - {file = "regex-2025.9.18.tar.gz", hash = "sha256:c5ba23274c61c6fef447ba6a39333297d0c247f53059dba0bca415cac511edc4"}, +python-versions = ">=3.10" +groups = ["main", "docling-ocr", "local-ml"] +files = [ + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:555497390743af1a65045fa4527782d10ff5b88970359412baa4a1e628fe393b"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:343a4504e3fb688c47cad451221ca5d4814f42b1e16c0065bde9cbf7f473bd52"}, + {file = "regex-2026.7.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ebee1ee89c39c953baac6924fcde08c5bb427c4057510862f9d7c7bdb3d8665"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:062f8cb7a9739c4835d22bd96f370c59aba89f257adcfa53be3cc209e08d3ae0"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1123ef4211d763ee771d47916a1596e2f4915794f7aabdc1adcb20e4249a6951"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6e44c0e7c5664be20aee92085153150c0a7967310a73a43c0f832b7cd35d0dd3"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98c6ac18480fcdb33f35439183f1d2e79760ab41930309c6d951cb1f8e46694c"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4458124d71339f505bf1fb94f69fd1bb8fa9d2481eebfef27c10ef4f2b9e12f6"}, + {file = "regex-2026.7.19-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbf300e2070bb35038660b3be1be4b91b0024edb41517e6996320b49b92b4175"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b2b506b1788df5fecd270a10d5e70a95fe77b87ea2b370a318043f6f5f817ee6"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:52579c60a6078be70a0e49c81d6e56d677f34cd439af281a0083b8c7bc75c095"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:2955907b7157a6660f27079edf7e0229e9c9c5325c77a2ef6a890cba91efa6f0"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:89dfee3319f5ae3f75ebd5c2445a809bb320252ba5529ffdafea4ef25d79cf1a"}, + {file = "regex-2026.7.19-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d3143f159261b1ce5b24c261c590e5913370c3200c5e9ebbb92b5aa5e111902"}, + {file = "regex-2026.7.19-cp310-cp310-win32.whl", hash = "sha256:64729333167c2dcaaa56a331d40ee097bd9c5617ffd51dabb09eaddafb1b532e"}, + {file = "regex-2026.7.19-cp310-cp310-win_amd64.whl", hash = "sha256:1c398716054621aa300b3d411f467dda903806c5da0df6945ab73982b8d115db"}, + {file = "regex-2026.7.19-cp310-cp310-win_arm64.whl", hash = "sha256:064f1760a5a4ade65c5419be23e782f29147528e8a66e0c42dd4cedb8d4e9fc6"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae"}, + {file = "regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc"}, + {file = "regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78"}, + {file = "regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327"}, + {file = "regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d"}, + {file = "regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965"}, + {file = "regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd"}, + {file = "regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68"}, + {file = "regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035"}, + {file = "regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a"}, + {file = "regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5"}, + {file = "regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312"}, + {file = "regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38"}, + {file = "regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15"}, + {file = "regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc"}, + {file = "regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2"}, + {file = "regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404"}, + {file = "regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e"}, + {file = "regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda"}, + {file = "regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a"}, + {file = "regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e"}, + {file = "regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0"}, + {file = "regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4"}, + {file = "regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974"}, + {file = "regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac"}, + {file = "regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a"}, + {file = "regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97"}, + {file = "regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4"}, + {file = "regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa"}, + {file = "regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac"}, + {file = "regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518"}, + {file = "regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276"}, + {file = "regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966"}, + {file = "regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44"}, + {file = "regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78"}, + {file = "regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2"}, + {file = "regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547"}, + {file = "regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5"}, ] [[package]] name = "requests" -version = "2.32.5" +version = "2.34.2" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "docling-ocr"] files = [ - {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, - {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, + {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, + {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, ] [package.dependencies] -certifi = ">=2017.4.17" +certifi = ">=2023.5.7" charset_normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" +urllib3 = ">=1.26,<3" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] [[package]] name = "requests-toolbelt" @@ -5967,7 +6058,7 @@ version = "14.3.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.8.0" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "rich-14.3.2-py3-none-any.whl", hash = "sha256:08e67c3e90884651da3239ea668222d19bea7b589149d8014a21c633420dbb69"}, {file = "rich-14.3.2.tar.gz", hash = "sha256:e712f11c1a562a11843306f5ed999475f09ac31ffb64281f73ab29ffdda8b3b8"}, @@ -6105,21 +6196,6 @@ files = [ {file = "rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84"}, ] -[[package]] -name = "rsa" -version = "4.9.1" -description = "Pure-Python RSA implementation" -optional = false -python-versions = "<4,>=3.6" -groups = ["main"] -files = [ - {file = "rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762"}, - {file = "rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75"}, -] - -[package.dependencies] -pyasn1 = ">=0.1.3" - [[package]] name = "rtree" version = "1.4.1" @@ -6163,7 +6239,7 @@ version = "0.7.0" description = "" optional = false python-versions = ">=3.9" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "safetensors-0.7.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c82f4d474cf725255d9e6acf17252991c3c8aac038d6ef363a4bf8be2f6db517"}, {file = "safetensors-0.7.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:94fd4858284736bb67a897a41608b5b0c2496c9bdb3bf2af1fa3409127f20d57"}, @@ -6215,7 +6291,7 @@ version = "1.8.0" description = "A set of python modules for machine learning and data mining" optional = false python-versions = ">=3.11" -groups = ["main"] +groups = ["main", "local-ml"] files = [ {file = "scikit_learn-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:146b4d36f800c013d267b29168813f7a03a43ecd2895d04861f1240b564421da"}, {file = "scikit_learn-1.8.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f984ca4b14914e6b4094c5d52a32ea16b49832c03bd17a110f004db3c223e8e1"}, @@ -6277,7 +6353,7 @@ version = "1.16.3" description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.11" -groups = ["main", "docling-ocr"] +groups = ["main", "dev", "docling-ocr", "local-ml"] files = [ {file = "scipy-1.16.3-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:40be6cf99e68b6c4321e9f8782e7d5ff8265af28ef2cd56e9c9b2638fa08ad97"}, {file = "scipy-1.16.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:8be1ca9170fcb6223cc7c27f4305d680ded114a1567c0bd2bfcbf947d1b17511"}, @@ -6368,50 +6444,58 @@ tqdm = "*" [[package]] name = "sentence-transformers" -version = "2.7.0" -description = "Multilingual text embeddings" +version = "6.0.0" +description = "Embeddings, Retrieval, and Reranking" optional = false -python-versions = ">=3.8.0" -groups = ["main"] +python-versions = ">=3.10" +groups = ["main", "local-ml"] files = [ - {file = "sentence_transformers-2.7.0-py3-none-any.whl", hash = "sha256:6a7276b05a95931581bbfa4ba49d780b2cf6904fa4a171ec7fd66c343f761c98"}, - {file = "sentence_transformers-2.7.0.tar.gz", hash = "sha256:2f7df99d1c021dded471ed2d079e9d1e4fc8e30ecb06f957be060511b36f24ea"}, + {file = "sentence_transformers-6.0.0-py3-none-any.whl", hash = "sha256:b974ac67523ea2a955afa87b1024129305472bd884367dcc969261cb086790e9"}, + {file = "sentence_transformers-6.0.0.tar.gz", hash = "sha256:9e8c2c24f3b1c7473cd5f519a3d3cff60daaeb95533b82d045ffb43ee5f2dac4"}, ] [package.dependencies] -huggingface-hub = ">=0.15.1" -numpy = "*" -Pillow = "*" -scikit-learn = "*" -scipy = "*" -torch = ">=1.11.0" -tqdm = "*" -transformers = ">=4.34.0,<5.0.0" +huggingface-hub = ">=1.3.0,<2.0.0" +numpy = ">=1.24.0" +scikit-learn = ">=1.1.0" +scipy = ">=1.0.0" +tokenizers = ">=0.19" +torch = ">=2.2" +tqdm = ">=4.0.0" +transformers = ">=5.0.0,<6.0.0" +typing_extensions = ">=4.10.0" [package.extras] -dev = ["pre-commit", "pytest", "ruff (>=0.3.0)"] +audio = ["transformers[audio]"] +dev = ["accelerate (>=1.3.0)", "datasets (>=2.16.0)", "peft", "pre-commit", "pytest", "pytest-cov", "pytest-env", "pytest-subtests", "pytest-xdist", "transformers[audio,video,vision]"] +image = ["transformers[vision]"] +onnx = ["optimum-onnx[onnxruntime]"] +onnx-gpu = ["optimum-onnx[onnxruntime-gpu]"] +openvino = ["optimum-intel[openvino] (>=2.0.0)"] +train = ["accelerate (>=1.3.0)", "datasets (>=2.16.0)"] +video = ["transformers[video]"] [[package]] name = "setuptools" -version = "80.9.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" +version = "84.0.0" +description = "Most extensible Python build backend with support for C/C++ extension modules" optional = false -python-versions = ">=3.9" -groups = ["main", "docling-ocr"] +python-versions = ">=3.10" +groups = ["main", "docling-ocr", "local-ml"] files = [ - {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, - {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, + {file = "setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670"}, + {file = "setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73"}, ] -markers = {docling-ocr = "python_version >= \"3.12\""} +markers = {docling-ocr = "python_version >= \"3.12\"", local-ml = "python_version >= \"3.12\""} [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] -core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.13.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -enabler = ["pytest-enabler (>=2.2)"] +enabler = ["pytest-enabler (>=3.4)"] test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.18.*)", "pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "shapely" @@ -6493,7 +6577,7 @@ version = "1.5.4" description = "Tool to Detect Surrounding Shell" optional = false python-versions = ">=3.7" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, @@ -6543,7 +6627,7 @@ version = "1.3.1" description = "Sniff out which async library your code is running under" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, @@ -6922,14 +7006,14 @@ uvicorn = ["uvicorn (>=0.34.0)"] [[package]] name = "starlette" -version = "0.48.0" +version = "1.6.0" description = "The little ASGI library that shines." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "starlette-0.48.0-py3-none-any.whl", hash = "sha256:0764ca97b097582558ecb498132ed0c7d942f233f365b86ba37770e026510659"}, - {file = "starlette-0.48.0.tar.gz", hash = "sha256:7e8cee469a8ab2352911528110ce9088fdc6a37d9876926e73da7ce4aa4c7a46"}, + {file = "starlette-1.6.0-py3-none-any.whl", hash = "sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c"}, + {file = "starlette-1.6.0.tar.gz", hash = "sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b"}, ] [package.dependencies] @@ -6937,7 +7021,7 @@ anyio = ">=3.6.2,<5" typing-extensions = {version = ">=4.10.0", markers = "python_version < \"3.13\""} [package.extras] -full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] +full = ["httpx (>=0.27.0,<0.29.0)", "httpx2 (>=2.0.0)", "itsdangerous", "jinja2", "python-multipart (>=0.0.18)", "pyyaml"] [[package]] name = "sympy" @@ -6945,7 +7029,7 @@ version = "1.13.3" description = "Computer algebra system (CAS) in Python" optional = false python-versions = ">=3.8" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "sympy-1.13.3-py3-none-any.whl", hash = "sha256:54612cf55a62755ee71824ce692986f23c88ffa77207b30c1368eda4a7060f73"}, {file = "sympy-1.13.3.tar.gz", hash = "sha256:b27fd2c6530e0ab39e275fc9b683895367e51d5da91baa8d3d64db2565fec4d9"}, @@ -7169,7 +7253,7 @@ version = "3.6.0" description = "threadpoolctl" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "local-ml"] files = [ {file = "threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb"}, {file = "threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e"}, @@ -7255,7 +7339,7 @@ version = "0.22.1" description = "" optional = false python-versions = ">=3.9" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "tokenizers-0.22.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:59fdb013df17455e5f950b4b834a7b3ee2e0271e6378ccb33aa74d178b513c73"}, {file = "tokenizers-0.22.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:8d4e484f7b0827021ac5f9f71d4794aaef62b979ab7608593da22b1d2e3c4edc"}, @@ -7300,7 +7384,7 @@ version = "2.7.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" optional = false python-versions = ">=3.9.0" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "python_version >= \"3.14\"" files = [ {file = "torch-2.7.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a103b5d782af5bd119b81dbcc7ffc6fa09904c423ff8db397a1e6ea8fd71508f"}, @@ -7363,7 +7447,7 @@ version = "2.9.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" optional = false python-versions = ">=3.10" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "python_version <= \"3.13\"" files = [ {file = "torch-2.9.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:1cc208435f6c379f9b8fdfd5ceb5be1e3b72a6bdf1cb46c0d2812aa73472db9e"}, @@ -7524,7 +7608,7 @@ version = "4.67.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, @@ -7542,78 +7626,114 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.57.3" -description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +version = "5.8.1" +description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false -python-versions = ">=3.9.0" -groups = ["main", "docling-ocr"] +python-versions = ">=3.10.0" +groups = ["main", "docling-ocr", "local-ml"] +markers = "sys_platform == \"darwin\"" files = [ - {file = "transformers-4.57.3-py3-none-any.whl", hash = "sha256:c77d353a4851b1880191603d36acb313411d3577f6e2897814f333841f7003f4"}, - {file = "transformers-4.57.3.tar.gz", hash = "sha256:df4945029aaddd7c09eec5cad851f30662f8bd1746721b34cc031d70c65afebc"}, + {file = "transformers-5.8.1-py3-none-any.whl", hash = "sha256:5340fb95962162cdfdae5cc91d7f8fedd92ed75216c1154c5e1f590fcf56dd0e"}, + {file = "transformers-5.8.1.tar.gz", hash = "sha256:4dd5b6de4105725104d84fd6abd74b305f4debfc251b38c648ee5dd087cf543b"}, ] [package.dependencies] -filelock = "*" -huggingface-hub = ">=0.34.0,<1.0" +huggingface-hub = ">=1.5.0,<2.0" numpy = ">=1.17" packaging = ">=20.0" pyyaml = ">=5.1" -regex = "!=2019.12.17" -requests = "*" +regex = ">=2025.10.22" safetensors = ">=0.4.3" tokenizers = ">=0.22.0,<=0.23.0" tqdm = ">=4.27" +typer = "*" [package.extras] -accelerate = ["accelerate (>=0.26.0)"] -all = ["Pillow (>=10.0.1,<=15.0)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "accelerate (>=0.26.0)", "av", "codecarbon (>=2.8.1)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "jinja2 (>=3.1.0)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "kernels (>=0.6.1,<=0.9)", "librosa", "mistral-common[opencv] (>=1.6.3)", "num2words", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm (!=1.0.18,<=1.0.19)", "tokenizers (>=0.22.0,<=0.23.0)", "torch (>=2.2)", "torchaudio", "torchvision"] -audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +accelerate = ["accelerate (>=1.1.0)"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "jmespath (>=1.0.1)", "kernels (>=0.12.0,<0.13)", "librosa", "mistral-common[image] (>=1.10.0)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] benchmark = ["optimum-benchmark (>=0.3.0)"] -chat-template = ["jinja2 (>=3.1.0)"] +chat-template = ["jinja2 (>=3.1.0)", "jmespath (>=1.0.1)"] codecarbon = ["codecarbon (>=2.8.1)"] -deepspeed = ["accelerate (>=0.26.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.26.0)", "accelerate (>=0.26.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fastapi", "libcst", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio", "pytest-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.13.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "torch (>=2.2)", "uvicorn"] -dev = ["GitPython (<3.1.19)", "GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "accelerate (>=0.26.0)", "accelerate (>=0.26.0)", "av", "beautifulsoup4", "codecarbon (>=2.8.1)", "cookiecutter (==1.7.3)", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fastapi", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "jinja2 (>=3.1.0)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "kernels (>=0.6.1,<=0.9)", "libcst", "libcst", "librosa", "mistral-common[opencv] (>=1.6.3)", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "num2words", "onnxconverter-common", "openai (>=1.98.0)", "optax (>=0.0.8,<=0.1.4)", "optuna", "pandas (<2.3.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio", "pytest-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.13.1)", "ruff (==0.13.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm (!=1.0.18,<=1.0.19)", "tokenizers (>=0.22.0,<=0.23.0)", "torch (>=2.2)", "torch (>=2.2)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] -dev-tensorflow = ["GitPython (<3.1.19)", "GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fastapi", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "libcst", "libcst", "librosa", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "onnxconverter-common", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "openai (>=1.98.0)", "pandas (<2.3.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio", "pytest-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.13.1)", "ruff (==0.13.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "tf2onnx", "timeout-decorator", "tokenizers (>=0.22.0,<=0.23.0)", "torch (>=2.2)", "urllib3 (<2.0.0)", "uvicorn"] -dev-torch = ["GitPython (<3.1.19)", "GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "accelerate (>=0.26.0)", "beautifulsoup4", "codecarbon (>=2.8.1)", "cookiecutter (==1.7.3)", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fastapi", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "kenlm", "kernels (>=0.6.1,<=0.9)", "libcst", "libcst", "librosa", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "num2words", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "openai (>=1.98.0)", "optuna", "pandas (<2.3.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio", "pytest-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.13.1)", "ruff (==0.13.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm (!=1.0.18,<=1.0.19)", "tokenizers (>=0.22.0,<=0.23.0)", "torch (>=2.2)", "torch (>=2.2)", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] -flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)", "scipy (<1.13.0)"] -flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] -ftfy = ["ftfy"] -hf-xet = ["hf_xet"] -hub-kernels = ["kernels (>=0.6.1,<=0.9)"] -integrations = ["kernels (>=0.6.1,<=0.9)", "optuna", "ray[tune] (>=2.7.0)"] +deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "jmespath (>=1.0.1)", "kernels (>=0.12.0,<0.13)", "libcst", "librosa", "mistral-common[image] (>=1.10.0)", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +docs = ["hf-doc-builder"] +integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.12.0,<0.13)", "optuna", "ray[tune] (>=2.7.0)"] ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] -mistral-common = ["mistral-common[opencv] (>=1.6.3)"] -modelcreation = ["cookiecutter (==1.7.3)"] -natten = ["natten (>=0.14.6,<0.15.0)"] +kernels = ["kernels (>=0.12.0,<0.13)"] +mistral-common = ["mistral-common[image] (>=1.10.0)"] num2words = ["num2words"] -onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] -onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] open-telemetry = ["opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "pandas (<2.3.0)", "rich", "ruff (==0.13.1)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] -ruff = ["ruff (==0.13.1)"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["accelerate (>=0.26.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "starlette", "torch (>=2.2)", "uvicorn"] -sigopt = ["sigopt"] +serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] sklearn = ["scikit-learn"] -speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "accelerate (>=0.26.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fastapi", "libcst", "mistral-common[opencv] (>=1.6.3)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "psutil", "pydantic (>=2)", "pydantic (>=2)", "pytest (>=7.2.0)", "pytest-asyncio", "pytest-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.13.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "torch (>=2.2)", "uvicorn"] -tf = ["keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] -tf-cpu = ["keras (>2.9,<2.16)", "keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow-cpu (>2.9,<2.16)", "tensorflow-probability (<0.24)", "tensorflow-text (<2.16)", "tf2onnx"] -tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] tiktoken = ["blobfile", "tiktoken"] -timm = ["timm (!=1.0.18,<=1.0.19)"] -tokenizers = ["tokenizers (>=0.22.0,<=0.23.0)"] -torch = ["accelerate (>=0.26.0)", "torch (>=2.2)"] -torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.34.0,<1.0)", "importlib_metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.22.0,<=0.23.0)", "torch (>=2.2)", "tqdm (>=4.27)"] +timm = ["timm (>=1.0.23)"] +torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] video = ["av"] -vision = ["Pillow (>=10.0.1,<=15.0)"] +vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] + +[[package]] +name = "transformers" +version = "5.12.1" +description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." +optional = false +python-versions = ">=3.10.0" +groups = ["main", "docling-ocr", "local-ml"] +markers = "sys_platform != \"darwin\"" +files = [ + {file = "transformers-5.12.1-py3-none-any.whl", hash = "sha256:2a5e109d2021265df7098ffbb738295acaf5ad256f12cbc586db2ea4dcbb1a8a"}, + {file = "transformers-5.12.1.tar.gz", hash = "sha256:679ee731c8225347889ad4fb3b2c926a62e9da3b7d284e9d12c791da7272466b"}, +] + +[package.dependencies] +huggingface-hub = ">=1.5.0,<2.0" +numpy = ">=1.17" +packaging = ">=20.0" +pyyaml = ">=5.1" +regex = ">=2025.10.22" +safetensors = ">=0.4.3" +tokenizers = ">=0.22.0,<=0.23.0" +tqdm = ">=4.60" +typer = "*" + +[package.extras] +accelerate = ["accelerate (>=1.1.0)"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "jmespath (>=1.0.1)", "kernels (>=0.12.0,<0.13)", "librosa", "mistral-common[image] (>=1.10.0)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] +benchmark = ["optimum-benchmark (>=0.3.0)"] +chat-template = ["jinja2 (>=3.1.0)", "jmespath (>=1.0.1)"] +codecarbon = ["codecarbon (>=2.8.1)"] +deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "hf-doc-builder", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "jmespath (>=1.0.1)", "kernels (>=0.12.0,<0.13)", "libcst", "librosa", "mistral-common[image] (>=1.10.0)", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "tomli", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +docs = ["hf-doc-builder"] +integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.12.0,<0.13)", "optuna", "ray[tune] (>=2.7.0)"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] +kernels = ["kernels (>=0.12.0,<0.13)"] +mistral-common = ["mistral-common[image] (>=1.10.0)"] +num2words = ["num2words"] +optuna = ["optuna"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "tomli", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)"] +ray = ["ray[tune] (>=2.7.0)"] +retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] +sagemaker = ["sagemaker (>=2.31.0)"] +sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] +serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] +sklearn = ["scikit-learn"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "hf-doc-builder", "libcst", "mistral-common[image] (>=1.10.0)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "tomli", "torch (>=2.4)", "transformers-mlinter (==0.1.1)", "ty (==0.0.20)", "urllib3 (<2.0.0)", "uvicorn"] +tiktoken = ["blobfile", "tiktoken"] +timm = ["timm (>=1.0.23)"] +torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] +video = ["av"] +vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] [[package]] name = "tree-sitter" @@ -7757,7 +7877,7 @@ version = "3.3.1" description = "A language and compiler for custom Deep Learning operations" optional = false python-versions = "*" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ {file = "triton-3.3.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b74db445b1c562844d3cfad6e9679c72e93fdfb1a90a24052b03bb5c49d1242e"}, @@ -7782,7 +7902,7 @@ version = "3.5.1" description = "A language and compiler for custom Deep Learning operations" optional = false python-versions = "<3.15,>=3.10" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] markers = "platform_machine == \"x86_64\" and platform_system == \"Linux\" and python_version <= \"3.13\"" files = [ {file = "triton-3.5.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f63e34dcb32d7bd3a1d0195f60f30d2aee8b08a69a0424189b71017e23dfc3d2"}, @@ -7808,21 +7928,21 @@ tutorials = ["matplotlib", "pandas", "tabulate"] [[package]] name = "typer" -version = "0.16.1" +version = "0.26.8" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false -python-versions = ">=3.7" -groups = ["main", "docling-ocr"] +python-versions = ">=3.10" +groups = ["main", "docling-ocr", "local-ml"] files = [ - {file = "typer-0.16.1-py3-none-any.whl", hash = "sha256:90ee01cb02d9b8395ae21ee3368421faf21fa138cb2a541ed369c08cec5237c9"}, - {file = "typer-0.16.1.tar.gz", hash = "sha256:d358c65a464a7a90f338e3bb7ff0c74ac081449e53884b12ba658cbd72990614"}, + {file = "typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c"}, + {file = "typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e"}, ] [package.dependencies] -click = ">=8.0.0" -rich = ">=10.11.0" +annotated-doc = ">=0.0.2" +colorama = {version = "*", markers = "platform_system == \"Windows\""} +rich = ">=13.8.0" shellingham = ">=1.3.0" -typing-extensions = ">=3.7.4.3" [[package]] name = "typer-slim" @@ -7849,7 +7969,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "docling-ocr"] +groups = ["main", "docling-ocr", "local-ml"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -7884,115 +8004,124 @@ files = [ [[package]] name = "ujson" -version = "5.11.0" +version = "5.13.0" description = "Ultra fast JSON encoder and decoder for Python" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "ujson-5.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:446e8c11c06048611c9d29ef1237065de0af07cabdd97e6b5b527b957692ec25"}, - {file = "ujson-5.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:16ccb973b7ada0455201808ff11d48fe9c3f034a6ab5bd93b944443c88299f89"}, - {file = "ujson-5.11.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3134b783ab314d2298d58cda7e47e7a0f7f71fc6ade6ac86d5dbeaf4b9770fa6"}, - {file = "ujson-5.11.0-cp310-cp310-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:185f93ebccffebc8baf8302c869fac70dd5dd78694f3b875d03a31b03b062cdb"}, - {file = "ujson-5.11.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d06e87eded62ff0e5f5178c916337d2262fdbc03b31688142a3433eabb6511db"}, - {file = "ujson-5.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:181fb5b15703a8b9370b25345d2a1fd1359f0f18776b3643d24e13ed9c036d4c"}, - {file = "ujson-5.11.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a4df61a6df0a4a8eb5b9b1ffd673429811f50b235539dac586bb7e9e91994138"}, - {file = "ujson-5.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6eff24e1abd79e0ec6d7eae651dd675ddbc41f9e43e29ef81e16b421da896915"}, - {file = "ujson-5.11.0-cp310-cp310-win32.whl", hash = "sha256:30f607c70091483550fbd669a0b37471e5165b317d6c16e75dba2aa967608723"}, - {file = "ujson-5.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:3d2720e9785f84312b8e2cb0c2b87f1a0b1c53aaab3b2af3ab817d54409012e0"}, - {file = "ujson-5.11.0-cp310-cp310-win_arm64.whl", hash = "sha256:85e6796631165f719084a9af00c79195d3ebf108151452fefdcb1c8bb50f0105"}, - {file = "ujson-5.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d7c46cb0fe5e7056b9acb748a4c35aa1b428025853032540bb7e41f46767321f"}, - {file = "ujson-5.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8951bb7a505ab2a700e26f691bdfacf395bc7e3111e3416d325b513eea03a58"}, - {file = "ujson-5.11.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:952c0be400229940248c0f5356514123d428cba1946af6fa2bbd7503395fef26"}, - {file = "ujson-5.11.0-cp311-cp311-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:94fcae844f1e302f6f8095c5d1c45a2f0bfb928cccf9f1b99e3ace634b980a2a"}, - {file = "ujson-5.11.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e0ec1646db172beb8d3df4c32a9d78015e671d2000af548252769e33079d9a6"}, - {file = "ujson-5.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:da473b23e3a54448b008d33f742bcd6d5fb2a897e42d1fc6e7bf306ea5d18b1b"}, - {file = "ujson-5.11.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:aa6b3d4f1c0d3f82930f4cbd7fe46d905a4a9205a7c13279789c1263faf06dba"}, - {file = "ujson-5.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4843f3ab4fe1cc596bb7e02228ef4c25d35b4bb0809d6a260852a4bfcab37ba3"}, - {file = "ujson-5.11.0-cp311-cp311-win32.whl", hash = "sha256:e979fbc469a7f77f04ec2f4e853ba00c441bf2b06720aa259f0f720561335e34"}, - {file = "ujson-5.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:683f57f0dd3acdd7d9aff1de0528d603aafcb0e6d126e3dc7ce8b020a28f5d01"}, - {file = "ujson-5.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:7855ccea3f8dad5e66d8445d754fc1cf80265a4272b5f8059ebc7ec29b8d0835"}, - {file = "ujson-5.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7895f0d2d53bd6aea11743bd56e3cb82d729980636cd0ed9b89418bf66591702"}, - {file = "ujson-5.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12b5e7e22a1fe01058000d1b317d3b65cc3daf61bd2ea7a2b76721fe160fa74d"}, - {file = "ujson-5.11.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0180a480a7d099082501cad1fe85252e4d4bf926b40960fb3d9e87a3a6fbbc80"}, - {file = "ujson-5.11.0-cp312-cp312-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:fa79fdb47701942c2132a9dd2297a1a85941d966d8c87bfd9e29b0cf423f26cc"}, - {file = "ujson-5.11.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8254e858437c00f17cb72e7a644fc42dad0ebb21ea981b71df6e84b1072aaa7c"}, - {file = "ujson-5.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1aa8a2ab482f09f6c10fba37112af5f957689a79ea598399c85009f2f29898b5"}, - {file = "ujson-5.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a638425d3c6eed0318df663df44480f4a40dc87cc7c6da44d221418312f6413b"}, - {file = "ujson-5.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7e3cff632c1d78023b15f7e3a81c3745cd3f94c044d1e8fa8efbd6b161997bbc"}, - {file = "ujson-5.11.0-cp312-cp312-win32.whl", hash = "sha256:be6b0eaf92cae8cdee4d4c9e074bde43ef1c590ed5ba037ea26c9632fb479c88"}, - {file = "ujson-5.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:b7b136cc6abc7619124fd897ef75f8e63105298b5ca9bdf43ebd0e1fa0ee105f"}, - {file = "ujson-5.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:6cd2df62f24c506a0ba322d5e4fe4466d47a9467b57e881ee15a31f7ecf68ff6"}, - {file = "ujson-5.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:109f59885041b14ee9569bf0bb3f98579c3fa0652317b355669939e5fc5ede53"}, - {file = "ujson-5.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a31c6b8004438e8c20fc55ac1c0e07dad42941db24176fe9acf2815971f8e752"}, - {file = "ujson-5.11.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78c684fb21255b9b90320ba7e199780f653e03f6c2528663768965f4126a5b50"}, - {file = "ujson-5.11.0-cp313-cp313-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:4c9f5d6a27d035dd90a146f7761c2272cf7103de5127c9ab9c4cd39ea61e878a"}, - {file = "ujson-5.11.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:837da4d27fed5fdc1b630bd18f519744b23a0b5ada1bbde1a36ba463f2900c03"}, - {file = "ujson-5.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:787aff4a84da301b7f3bac09bc696e2e5670df829c6f8ecf39916b4e7e24e701"}, - {file = "ujson-5.11.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6dd703c3e86dc6f7044c5ac0b3ae079ed96bf297974598116aa5fb7f655c3a60"}, - {file = "ujson-5.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3772e4fe6b0c1e025ba3c50841a0ca4786825a4894c8411bf8d3afe3a8061328"}, - {file = "ujson-5.11.0-cp313-cp313-win32.whl", hash = "sha256:8fa2af7c1459204b7a42e98263b069bd535ea0cd978b4d6982f35af5a04a4241"}, - {file = "ujson-5.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:34032aeca4510a7c7102bd5933f59a37f63891f30a0706fb46487ab6f0edf8f0"}, - {file = "ujson-5.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:ce076f2df2e1aa62b685086fbad67f2b1d3048369664b4cdccc50707325401f9"}, - {file = "ujson-5.11.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:65724738c73645db88f70ba1f2e6fb678f913281804d5da2fd02c8c5839af302"}, - {file = "ujson-5.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29113c003ca33ab71b1b480bde952fbab2a0b6b03a4ee4c3d71687cdcbd1a29d"}, - {file = "ujson-5.11.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c44c703842024d796b4c78542a6fcd5c3cb948b9fc2a73ee65b9c86a22ee3638"}, - {file = "ujson-5.11.0-cp314-cp314-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:e750c436fb90edf85585f5c62a35b35082502383840962c6983403d1bd96a02c"}, - {file = "ujson-5.11.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f278b31a7c52eb0947b2db55a5133fbc46b6f0ef49972cd1a80843b72e135aba"}, - {file = "ujson-5.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ab2cb8351d976e788669c8281465d44d4e94413718af497b4e7342d7b2f78018"}, - {file = "ujson-5.11.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:090b4d11b380ae25453100b722d0609d5051ffe98f80ec52853ccf8249dfd840"}, - {file = "ujson-5.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:80017e870d882d5517d28995b62e4e518a894f932f1e242cbc802a2fd64d365c"}, - {file = "ujson-5.11.0-cp314-cp314-win32.whl", hash = "sha256:1d663b96eb34c93392e9caae19c099ec4133ba21654b081956613327f0e973ac"}, - {file = "ujson-5.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:849e65b696f0d242833f1df4182096cedc50d414215d1371fca85c541fbff629"}, - {file = "ujson-5.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:e73df8648c9470af2b6a6bf5250d4744ad2cf3d774dcf8c6e31f018bdd04d764"}, - {file = "ujson-5.11.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:de6e88f62796372fba1de973c11138f197d3e0e1d80bcb2b8aae1e826096d433"}, - {file = "ujson-5.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:49e56ef8066f11b80d620985ae36869a3ff7e4b74c3b6129182ec5d1df0255f3"}, - {file = "ujson-5.11.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a325fd2c3a056cf6c8e023f74a0c478dd282a93141356ae7f16d5309f5ff823"}, - {file = "ujson-5.11.0-cp314-cp314t-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:a0af6574fc1d9d53f4ff371f58c96673e6d988ed2b5bf666a6143c782fa007e9"}, - {file = "ujson-5.11.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10f29e71ecf4ecd93a6610bd8efa8e7b6467454a363c3d6416db65de883eb076"}, - {file = "ujson-5.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1a0a9b76a89827a592656fe12e000cf4f12da9692f51a841a4a07aa4c7ecc41c"}, - {file = "ujson-5.11.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:b16930f6a0753cdc7d637b33b4e8f10d5e351e1fb83872ba6375f1e87be39746"}, - {file = "ujson-5.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:04c41afc195fd477a59db3a84d5b83a871bd648ef371cf8c6f43072d89144eef"}, - {file = "ujson-5.11.0-cp314-cp314t-win32.whl", hash = "sha256:aa6d7a5e09217ff93234e050e3e380da62b084e26b9f2e277d2606406a2fc2e5"}, - {file = "ujson-5.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:48055e1061c1bb1f79e75b4ac39e821f3f35a9b82de17fce92c3140149009bec"}, - {file = "ujson-5.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:1194b943e951092db611011cb8dbdb6cf94a3b816ed07906e14d3bc6ce0e90ab"}, - {file = "ujson-5.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:65f3c279f4ed4bf9131b11972040200c66ae040368abdbb21596bf1564899694"}, - {file = "ujson-5.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99c49400572cd77050894e16864a335225191fd72a818ea6423ae1a06467beac"}, - {file = "ujson-5.11.0-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0654a2691fc252c3c525e3d034bb27b8a7546c9d3eb33cd29ce6c9feda361a6a"}, - {file = "ujson-5.11.0-cp39-cp39-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:6b6ec7e7321d7fc19abdda3ad809baef935f49673951a8bab486aea975007e02"}, - {file = "ujson-5.11.0-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f62b9976fabbcde3ab6e413f4ec2ff017749819a0786d84d7510171109f2d53c"}, - {file = "ujson-5.11.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7f1a27ab91083b4770e160d17f61b407f587548f2c2b5fbf19f94794c495594a"}, - {file = "ujson-5.11.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ecd6ff8a3b5a90c292c2396c2d63c687fd0ecdf17de390d852524393cd9ed052"}, - {file = "ujson-5.11.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:9aacbeb23fdbc4b256a7d12e0beb9063a1ba5d9e0dbb2cfe16357c98b4334596"}, - {file = "ujson-5.11.0-cp39-cp39-win32.whl", hash = "sha256:674f306e3e6089f92b126eb2fe41bcb65e42a15432c143365c729fdb50518547"}, - {file = "ujson-5.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:c6618f480f7c9ded05e78a1938873fde68baf96cdd74e6d23c7e0a8441175c4b"}, - {file = "ujson-5.11.0-cp39-cp39-win_arm64.whl", hash = "sha256:5600202a731af24a25e2d7b6eb3f648e4ecd4bb67c4d5cf12f8fab31677469c9"}, - {file = "ujson-5.11.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:abae0fb58cc820092a0e9e8ba0051ac4583958495bfa5262a12f628249e3b362"}, - {file = "ujson-5.11.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fac6c0649d6b7c3682a0a6e18d3de6857977378dce8d419f57a0b20e3d775b39"}, - {file = "ujson-5.11.0-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b42c115c7c6012506e8168315150d1e3f76e7ba0f4f95616f4ee599a1372bbc"}, - {file = "ujson-5.11.0-pp311-pypy311_pp73-manylinux_2_24_i686.manylinux_2_28_i686.whl", hash = "sha256:86baf341d90b566d61a394869ce77188cc8668f76d7bb2c311d77a00f4bdf844"}, - {file = "ujson-5.11.0-pp311-pypy311_pp73-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4598bf3965fc1a936bd84034312bcbe00ba87880ef1ee33e33c1e88f2c398b49"}, - {file = "ujson-5.11.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:416389ec19ef5f2013592f791486bef712ebce0cd59299bf9df1ba40bb2f6e04"}, - {file = "ujson-5.11.0.tar.gz", hash = "sha256:e204ae6f909f099ba6b6b942131cee359ddda2b6e4ea39c12eb8b991fe2010e0"}, + {file = "ujson-5.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:770643b4752266c5a466149848b78c3874940926a4ecef304f518b2b6cdb432f"}, + {file = "ujson-5.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a8e8c1203cb1a27720debc334f840a9170da741503522f86999710cb4738fbe3"}, + {file = "ujson-5.13.0-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:65c1813fdd742fe3c249d9c417fa490e5b54e8a91bf343a88486ec50d175c444"}, + {file = "ujson-5.13.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5aa9bf16f0131812720dd4ae70bc1a0cf68f79e93c07c66100d328e75944b567"}, + {file = "ujson-5.13.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:82c17b904c03c2b9629486ec91a8fa46a15f10d03504284f54ed7257a917d9f1"}, + {file = "ujson-5.13.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0b5f6983b2469db00e540b68fa8297b7a0ccd0d5173c60cc3e84f336b09395f"}, + {file = "ujson-5.13.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b877fe7926107d25d54b655c5b8dd94b294b22c157233163ad29fdb54ac10cf4"}, + {file = "ujson-5.13.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f3cae1c811e787b9500e2830af8632dcb32a78dea5baf15aac51d681c59a59dd"}, + {file = "ujson-5.13.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:820b78a6a183ab6591b2ea888020032ef0fe328d852af9a5c8d8084ababb2218"}, + {file = "ujson-5.13.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b1209c985d1f4c2ae085ced7325509650cdb3533bb7294558dd1f26d48377942"}, + {file = "ujson-5.13.0-cp310-cp310-win32.whl", hash = "sha256:326553ae6c063c8246974906a6c137a0780fe46d143abebc52cd2cadda0f1814"}, + {file = "ujson-5.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:0354d6b50b0d153ed7c629845b18a953d0c727b7e768fd94a94e0602abfa1f29"}, + {file = "ujson-5.13.0-cp310-cp310-win_arm64.whl", hash = "sha256:7b4a05f61a96553995da6a4d502e07bc8aa5d07a90031df006239c6b00ce1c83"}, + {file = "ujson-5.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4b7badefa73f96bad9e295ea22bd06967b851c8aad68c74196437e3584f25de5"}, + {file = "ujson-5.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:09effd42924a80df20a63b31a1ede905e66b0ce24aafe7a4cbedb05c783f8bb3"}, + {file = "ujson-5.13.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:89d0bfc986d02b4ce76b00e0f560bc8d30dfe8c05a1bfd8529e085eb6c1a77d9"}, + {file = "ujson-5.13.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cdd9618e07b3b142a02f0ab8227fd52453688b8e8e60ac0511f13a25fa8009db"}, + {file = "ujson-5.13.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0806683e8171ec06817e6af22f14ba0cd2f16def8a2ffb22a28a10615249355d"}, + {file = "ujson-5.13.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1cf46c79498c81f088cad4165b1669a78bba7bfbeb778c7cc1aff316e062b0d"}, + {file = "ujson-5.13.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e2282039eb26f08ed1a1381360395f8a310f39a45ef7314cb0f258a7d2917ea3"}, + {file = "ujson-5.13.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2c75eb7fac0fe92925b959cf2fa18f88d9fb76b10781fd2a6ccb895d5fb89171"}, + {file = "ujson-5.13.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:12078e81def2790140583abefc1b979f3c77be15d53c524bf0e232f669822052"}, + {file = "ujson-5.13.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e12192a37c6c0e476554b62647acdf6139a47b6f13d8bad8dd2316763c4cee12"}, + {file = "ujson-5.13.0-cp311-cp311-win32.whl", hash = "sha256:ae53b3f046529c193d533ca8111492330b204d6007611cdfa20e8b764c7c1389"}, + {file = "ujson-5.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:2275bbaaea3eddd2e8ec0863e28a420f5f520b14a760fc3f1e49fd07a974448c"}, + {file = "ujson-5.13.0-cp311-cp311-win_arm64.whl", hash = "sha256:32a59e7151fe2fec8fdf9a565ee66fcf87918d48827e21cdab5e15ff9f274b78"}, + {file = "ujson-5.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bf81570ac056cb058f9117b52ca5dd800bfe9381d0076d0bb30a08a54591d654"}, + {file = "ujson-5.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7edf16359c52ed53406e216565d83e6b98c23c3cb9a0a01673f2493f8fb15edf"}, + {file = "ujson-5.13.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:24539618fb3243cfdf27dab9a850acab80798a01501e9586b61fb9ecd016a891"}, + {file = "ujson-5.13.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fdde6341d213b29f413b5fa9fad1392d5408074c75f0900ed949e97e546fa5df"}, + {file = "ujson-5.13.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:229faf041ef249ee3fd57bac1cedb123d2718ab63f6ccd50eca95ea902eb0dca"}, + {file = "ujson-5.13.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d02f31c2f59cc6a1c2c3633b377701fc2d8e876cc01950735d7a01132ccc233"}, + {file = "ujson-5.13.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ea7204e9fa7538bfbb1396e1ee8c2bbcd3818b3633ef5bb14d4fdea52994d14d"}, + {file = "ujson-5.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7c5a2478a3a1fa4421f7e035b87194eea0cf44c7971a3f32ad1b42a0dfd63c03"}, + {file = "ujson-5.13.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b535e0970c96957e999cfe5ec89361f0e8d0bb987fb5d5144f6f495cb3ed9e19"}, + {file = "ujson-5.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3d0ad1207694988498fca7e0bb28eba7564fa33261d2f9fdf66a3aaab376b803"}, + {file = "ujson-5.13.0-cp312-cp312-win32.whl", hash = "sha256:d6bc9fa43a49e403c68c7eb164eef0feee9dd29474a7c6e0d3b6267025371990"}, + {file = "ujson-5.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:6692d49ff970aaa5008f4a6fe06974bc91fd957bf13173f765e46d8ba44906ea"}, + {file = "ujson-5.13.0-cp312-cp312-win_arm64.whl", hash = "sha256:5737ffe0740a788b0e6255f0ffb281db49305fd6e6a587be44c73d9e92b554c4"}, + {file = "ujson-5.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46998fc8d11aec34a20e2010905e7059732a3d192d9a3c3fe4f9ffd146c87ec8"}, + {file = "ujson-5.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ee03ce288ba25b05cf0de87203165642277a25caa4f00a437e13152e5214e310"}, + {file = "ujson-5.13.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:cdf33b588a81b05d0b585c66f83050c49cb670623424d10e4d1ad37ba2f7eed9"}, + {file = "ujson-5.13.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4cabd73c114ce93c21d7db2e2d8e16217fd8a5b2b3ec754629eebef5c262d47f"}, + {file = "ujson-5.13.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ffc61fc756a64f4d169a78cc638d769e3c324f45fc51997626abf4e5e5dd6460"}, + {file = "ujson-5.13.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c00323c13a35822c9a67a26c0b2a0787510bf1ef490922b58009b362d1a3e21"}, + {file = "ujson-5.13.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:496e662a6b46d5f936d77fb68259cece19213bb2301ddd520dbd75ac7c90c5f4"}, + {file = "ujson-5.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7fd41b86444df14f8b4b7afaaa9f27bacfbf8c18380872317aeab6cd125dcede"}, + {file = "ujson-5.13.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bf3c2c4ea55d4187903fcdc689a9bf5b0fc72d8c0eaff39db18c1f337c8832c1"}, + {file = "ujson-5.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b6eca7751d61045a9b1e7f9a8c97ac24b164f085b60bef1c4668654bb2338011"}, + {file = "ujson-5.13.0-cp313-cp313-win32.whl", hash = "sha256:b63d3820f978bc8e98cc3f1fe26a33b0d2ea237733a23fe5e9cb5d51f466bd97"}, + {file = "ujson-5.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:17a59d5cf23ef98f7c9314524976b4b288374d83200add01d953024fb06404f9"}, + {file = "ujson-5.13.0-cp313-cp313-win_arm64.whl", hash = "sha256:b49516fbe803ff30d6caa9ccc3799ec7f968992747ce3099eae4758928577b53"}, + {file = "ujson-5.13.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:cc9dfd41fed397ab03bb9d9fe1cbd83301211c772a17536033ce7d68877ac82b"}, + {file = "ujson-5.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ca7ef2fa6c408a7c0f558e4d33d93b32ddc35ed6d3cfc505747931a64b7465d5"}, + {file = "ujson-5.13.0-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:a554b2e5bee85030369514cef8b0b913cebe1a4c2c0c13541966d50bcba22b1a"}, + {file = "ujson-5.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ea939ff629ab03ae970d03eca6d1febd8ed55ba38ca44aec64ce997537cd3fa0"}, + {file = "ujson-5.13.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b98bf2faa5e37ecfe752226ea08290031e375a0c43d425a0b955fb3e702a2a71"}, + {file = "ujson-5.13.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9a4b92344b16e414aeb609e57f62c466500e53c94f1698f5b149dc0b7223ec3e"}, + {file = "ujson-5.13.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7df805aad707507a1fa165fb716218ca3a89f142125dc4b23c9fcc08fa402d97"}, + {file = "ujson-5.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7576bdbef327c3528f011002a2d74486f6fe4e33289bdb7a042b7f1a6e9d8285"}, + {file = "ujson-5.13.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:6eee5d7cce3f32a468905f9ff61807a60287a90258d849460f6fa826e810870d"}, + {file = "ujson-5.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:144e9d8a454cfa727e0f755e1863738ed68068583bda5463052cb446835bd56c"}, + {file = "ujson-5.13.0-cp314-cp314-win32.whl", hash = "sha256:576f35c35b918d67d41b933878062ec0a5c9f4d1e9e14e04aeef35384963feae"}, + {file = "ujson-5.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:d5e206e9f849ead27e51ef8da44e52b38da7c6dbd929a7340ab44533edcda8d7"}, + {file = "ujson-5.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:dc470179775468f9a007d3a6a2734624248c94bf47c6645e808c7e50a5070d1a"}, + {file = "ujson-5.13.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:69b4e36bb7d5f413ba8c00c8006b2ec627cc5ace97301462f6aadb66ec9d2979"}, + {file = "ujson-5.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8b644d50f66de5490c1823c7176618cead5e8e8a88cba9f40a6308ca52e79267"}, + {file = "ujson-5.13.0-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:15107aaa4f559d55201165ec32abb35c283a861be1fa67229578cb7d93fcd93a"}, + {file = "ujson-5.13.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6e343c5f0c058523f1edbf6ae4eceb4e0d934205a53bbdd8d9a945c83324662a"}, + {file = "ujson-5.13.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:02200035bc80e830f076ffc1b329a94c295aee6d9de8c9043647cb9a7bd4f76f"}, + {file = "ujson-5.13.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d7f19b81b73ff28f5c5022ee794f94122bfcda07a76423078e349465d71223a1"}, + {file = "ujson-5.13.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82e1393e6dbe3c95fdfc95c6c528890e191351a1f024ef51126cf1f22543af52"}, + {file = "ujson-5.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38afcf994b28ed85ea2420e2a8d79a37d0a77348b3daf53850c16edda66f942d"}, + {file = "ujson-5.13.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:1bdf2518971586f2b413156c49d9dd8b56cc990a8647081e1bd00af60564d469"}, + {file = "ujson-5.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:751ad01042472f1c7c02f5c597c7aee79834e82a6cc384ca302173bbc8e8deb8"}, + {file = "ujson-5.13.0-cp314-cp314t-win32.whl", hash = "sha256:74f3dd61aeb01b7b2a6754e400224e819279041b3867935a55ccf57fb86a43b2"}, + {file = "ujson-5.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5c31317d5e4504dae8f98795358b6082fc0ef96e7394806db0a76a4a8717f446"}, + {file = "ujson-5.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aefd3c9c95f9b62348956396ff7b31818476f8f54dc4a4e64cbd4f0491db6fca"}, + {file = "ujson-5.13.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3e074a1f7778d58aa3b3056bab7b6251aabb3f381808018ca2b7fb8dbdeef7ab"}, + {file = "ujson-5.13.0-graalpy312-graalpy250_312_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8bb53ef95d35875262b8d0aa28506ca612ddd07058bee2a90f609938e69dc801"}, + {file = "ujson-5.13.0-graalpy312-graalpy250_312_native-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fb296a0aa480ab88d895ddaa90372604c08ccc72323f02590612c775426ab413"}, + {file = "ujson-5.13.0-graalpy312-graalpy250_312_native-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2862f81af44b3a7e74c5d80caa118d736be1991ce6f1d5c723716fa403060cc6"}, + {file = "ujson-5.13.0-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c16e07581172f08585b409246f4535dab13ee85af0e3d3cfa8684b653ca44fa8"}, + {file = "ujson-5.13.0-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:9bd0f2dd05937c3b089af316884de18c6f6182ddb8ffce597d2e7c7a9ba9f447"}, + {file = "ujson-5.13.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:96e7e019b097b4b25fccddadb369d13f412c13695fcf0680b6bf906376156151"}, + {file = "ujson-5.13.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:7892ea6dd85ede6d30fbbd22af1239b9d81dabdf9b7a8f10ca6d4464d4d9b8ab"}, + {file = "ujson-5.13.0-pp311-pypy311_pp73-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e1842e10adc8f0db0d3e8aa3a1f8b05ce0456b39e180c8553d7f36dd0bf24b6b"}, + {file = "ujson-5.13.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c8bbb1f5ab810954fb307b6c5e68af58210c31da8569f9e6498a3958c1859e72"}, + {file = "ujson-5.13.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3f6d55c68985654a84a9b47ac51f2655adac4fd264e4189f832960c2270dcf70"}, + {file = "ujson-5.13.0-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b266f182d4bce74a6a9e1f86988485e8cd422efdcc7c3f537e00cc956f52678"}, + {file = "ujson-5.13.0-pp311-pypy311_pp73-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfaa8302eb9bb7f5e231f256caf83040760585e32751d19155c0f0c0225f8de1"}, + {file = "ujson-5.13.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:43dee3081b00fe447c5b9e2fe7ebfd57f7bcc5dd25b8a439c26c8c174dd581be"}, + {file = "ujson-5.13.0.tar.gz", hash = "sha256:d62e3d7625384c08082abad81a077af587fdef2761bb14c3822f4234b8d07d75"}, ] [[package]] name = "urllib3" -version = "2.5.0" +version = "2.7.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main", "docling-ocr"] files = [ - {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, - {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, + {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, + {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, ] [package.extras] -brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "uuid-utils" @@ -8112,14 +8241,14 @@ files = [ [[package]] name = "uvicorn" -version = "0.38.0" +version = "0.52.4" description = "The lightning-fast ASGI server." optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "uvicorn-0.38.0-py3-none-any.whl", hash = "sha256:48c0afd214ceb59340075b4a052ea1ee91c16fbc2a9b1469cca0e54566977b02"}, - {file = "uvicorn-0.38.0.tar.gz", hash = "sha256:fd97093bdd120a2609fc0d3afe931d4d4ad688b6e75f0f929fde1bc36fe0e91d"}, + {file = "uvicorn-0.52.4-py3-none-any.whl", hash = "sha256:f86e41a149d7d05a9969337e3946a9c171c06a5d42680896daaba624aeac8da1"}, + {file = "uvicorn-0.52.4.tar.gz", hash = "sha256:73acfee47a0b133c5de13d219492d62d8a31e935f4fe6e41a232451a15379f86"}, ] [package.dependencies] @@ -8127,7 +8256,7 @@ click = ">=7.0" h11 = ">=0.8" [package.extras] -standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"] +standard = ["httptools (>=0.8.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.20)", "websockets (>=13.0)"] [[package]] name = "validators" @@ -8332,7 +8461,7 @@ version = "15.0.1" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "docling-ocr"] files = [ {file = "websockets-15.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d63efaa0cd96cf0c5fe4d581521d9fa87744540d4bc999ae6e08595a1014b45b"}, {file = "websockets-15.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac60e3b188ec7574cb761b08d50fcedf9d77f1530352db4eef1707fe9dee7205"}, @@ -8768,9 +8897,10 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt [extras] docling-ocr = ["accelerate", "docling"] +local-ml = ["sentence-transformers"] postgresql-backend = ["networkx", "pgvector", "psycopg2-binary"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<4.0.0" -content-hash = "f59ed72950e889463da8786073dc5d2463e0ae84e85a898c06c6079d466ad923" +content-hash = "856fe5f1e9af33a92742b5e1a2656cdde08c6db93871a2dcaca78e49cdad4cda" diff --git a/pyproject.toml b/pyproject.toml index e9d373f..596ebc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "brainapi2" -version = "2.17.0-dev" +version = "2.17.0rc1" description = "Version 2.x.x of the BrainAPI memory layer." authors = [ {name = "Christian",email = "alch.infoemail@gmail.com"} @@ -17,39 +17,43 @@ dependencies = [ "psycopg2-binary (>=2.9.6,<3.0.0)", "pgvector (>=0.3.0,<1.0.0)", "pylint (>=4.0.1,<5.0.0)", - "langchain (>=1.0.0,<2.0.0)", + "langchain (>=1.3.16,<2.0.0)", "openai (>=2.5.0,<3.0.0)", "celery (>=5.5.3,<6.0.0)", "kombu[rabbitmq] (>=5.3.0,<6.0.0)", - "fastapi (>=0.119.0,<0.120.0)", - "langchain-core (>=1.0.0,<2.0.0)", - "uvicorn (>=0.38.0,<0.39.0)", - "python-dotenv (>=1.1.1,<2.0.0)", - "langchain-openai (>=1.0.0,<2.0.0)", + "fastapi (>=0.141.1,<0.142.0)", + "langchain-core (>=1.6.0,<2.0.0)", + "uvicorn (>=0.52.4,<0.53.0)", + "python-dotenv (>=1.2.3,<2.0.0)", + "langchain-openai (>=1.6.0,<2.0.0)", "google-genai (>=1.46.0,<2.0.0)", "langchain-google-vertexai (>=3.0.1,<4.0.0)", "pymilvus (>=2.6.2,<3.0.0)", "levenshtein (>=0.27.1,<0.28.0)", - "python-multipart (>=0.0.20,<0.0.21)", - "sentence-transformers (>=2.2.0,<3.0.0)", + "python-multipart (>=0.0.32,<0.1.0)", + "pillow (>=12.3.0,<13.0.0)", "tenacity (>=8.2.3,<10.0.0)", "sympy (>=1.12.1,<1.14.0)", "numpy (>=1.26.0,<2.0.0)", "spacy (>=3.0.0,<4.0.0)", - "typer (>=0.16.0,<0.17.0)", + "typer (>=0.26.0,<0.27.0)", "spacy_langdetect (>=0.1.2,<0.2.0)", "langid (>=1.1.6,<2.0.0)", - "mcp (>=1.26.0,<2.0.0)", + "mcp (>=1.28.1,<2.0.0)", "boto3 (>=1.40.19,<2.0.0)", "langchain-aws (>=1.0.0,<2.0.0)", "watchfiles (>=1.2.0,<2.0.0)", "anthropic (>=0.57.1,<0.58.0)", + "tiktoken (>=0.12.0,<1.0.0)", ] [project.optional-dependencies] docling-ocr = [ - "docling (>=2.0.0,<3.0.0)", - "accelerate (>=1.0.0,<2.0.0)", + "docling (>=2.121.0,<3.0.0)", + "accelerate (>=1.14.0,<2.0.0)", +] +local-ml = [ + "sentence-transformers (>=6.0.0,<7.0.0)", ] postgresql-backend = [ "psycopg2-binary (>=2.9.6,<3.0.0)", @@ -70,41 +74,51 @@ networkx = ">=3.0,<4.0.0" psycopg2-binary = ">=2.9.6,<3.0.0" pgvector = ">=0.3.0,<1.0.0" pylint = ">=4.0.1,<5.0.0" -langchain = ">=1.0.0,<2.0.0" +langchain = ">=1.3.16,<2.0.0" openai = ">=2.5.0,<3.0.0" celery = ">=5.5.3,<6.0.0" kombu = {extras = ["rabbitmq"], version = ">=5.3.0,<6.0.0"} -fastapi = ">=0.119.0,<0.120.0" -langchain-core = ">=1.0.0,<2.0.0" -uvicorn = ">=0.38.0,<0.39.0" -python-dotenv = ">=1.1.1,<2.0.0" -langchain-openai = ">=1.0.0,<2.0.0" +fastapi = ">=0.141.1,<0.142.0" +langchain-core = ">=1.6.0,<2.0.0" +uvicorn = ">=0.52.4,<0.53.0" +python-dotenv = ">=1.2.3,<2.0.0" +langchain-openai = ">=1.6.0,<2.0.0" google-genai = ">=1.46.0,<2.0.0" langchain-google-vertexai = ">=3.0.1,<4.0.0" pymilvus = ">=2.6.2,<3.0.0" levenshtein = ">=0.27.1,<0.28.0" -pillow = ">=10.0.0,<12.0.0" -sentence-transformers = ">=2.2.0,<3.0.0" +pillow = ">=12.3.0,<13.0.0" tenacity = ">=8.2.3,<10.0.0" -python-multipart = ">=0.0.20,<0.0.21" +python-multipart = ">=0.0.32,<0.1.0" sympy = ">=1.12.1,<1.14.0" numpy = ">=1.26.0,<2.0.0" spacy = ">=3.0.0,<4.0.0" -typer = ">=0.16.0,<0.17.0" +typer = ">=0.26.0,<0.27.0" spacy_langdetect = ">=0.1.2,<0.2.0" langid = ">=1.1.6,<2.0.0" -mcp = ">=1.26.0,<2.0.0" +mcp = ">=1.28.1,<2.0.0" boto3 = ">=1.40.19,<2.0.0" langchain-aws = ">=1.0.0,<2.0.0" watchfiles = ">=1.2.0,<2.0.0" anthropic = ">=0.57.1,<0.58.0" +tiktoken = ">=0.12.0,<1.0.0" [tool.poetry.group.docling-ocr] optional = true [tool.poetry.group.docling-ocr.dependencies] -docling = ">=2.0.0,<3.0.0" -accelerate = ">=1.0.0,<2.0.0" +docling = ">=2.121.0,<3.0.0" +accelerate = ">=1.14.0,<2.0.0" + +[tool.poetry.group.local-ml] +optional = true + +[tool.poetry.group.local-ml.dependencies] +sentence-transformers = ">=6.0.0,<7.0.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=9.0.3,<10.0.0" +lightgbm = ">=4.3.0,<5.0.0" [tool.pylint.messages_control] @@ -123,6 +137,9 @@ max-statements = 50 [tool.pylint.basic] good-names = ["i", "j", "k", "ex", "Run", "_", "id", "db", "ai", "llm"] +[tool.pytest.ini_options] +testpaths = ["tests"] + [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/scripts/assemble_release_artifacts.py b/scripts/assemble_release_artifacts.py new file mode 100644 index 0000000..5de33c8 --- /dev/null +++ b/scripts/assemble_release_artifacts.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Combine light and heavy workflow evidence into one release-gate directory.""" + +from __future__ import annotations + +import argparse +import json +import shutil +from pathlib import Path + + +def _find(root: Path, name: str) -> Path: + matches = list(root.rglob(name)) + if len(matches) != 1: + raise RuntimeError(f"Expected one {name} under {root}, found {len(matches)}") + return matches[0] + + +def _merge_profiles(light: Path, heavy: Path, name: str) -> dict: + left = json.loads(_find(light, name).read_text(encoding="utf-8")) + right = json.loads(_find(heavy, name).read_text(encoding="utf-8")) + if set(left) != {"light"} or set(right) != {"heavy"}: + raise RuntimeError(f"Unexpected profile keys in {name}") + return {**left, **right} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--light", type=Path, required=True) + parser.add_argument("--heavy", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + for name in ("smoke.json", "latency.json"): + merged = _merge_profiles(args.light, args.heavy, name) + (output / name).write_text( + json.dumps(merged, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + for root, names in ( + ( + args.light, + ( + "restore-light.json", + "security.json", + "brainapi.spdx.json", + "backup-light-manifest.json", + "brainapi-image.tar.gz", + ), + ), + (args.heavy, ("restore-heavy.json", "backup-heavy-manifest.json")), + ): + for name in names: + shutil.copy2(_find(root, name), output / name) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_release_readiness.py b/scripts/check_release_readiness.py new file mode 100644 index 0000000..9d23b52 --- /dev/null +++ b/scripts/check_release_readiness.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Validate the recorded product, restore, security, and latency release gates.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +COUNT_KEYS = ("brains", "nodes", "edges", "chunks", "vectors", "observations") +REQUIRED_FLOWS = { + "authentication", + "brain_creation", + "plain_ingestion", + "structured_ingestion", + "task_completion", + "context_retrieval", + "search", + "console", + "mcp", +} + + +def _read(path: Path) -> dict: + if not path.is_file(): + raise RuntimeError(f"Missing release artifact: {path.name}") + return json.loads(path.read_text(encoding="utf-8")) + + +def _check_public_ledgers() -> None: + reports = _read(ROOT / "benchmarks" / "REPORTS.json")["benchmarks"] + longmem = reports["longmemeval"].get("leaderboard", []) + if not longmem: + raise RuntimeError("A representative LongMemEval result has not been published") + search = reports["search"] + representative = search.get("representative_run_id") + row = next( + (item for item in search.get("leaderboard", []) if item.get("run_id") == representative), + None, + ) + if not row or "wands" not in str(row.get("dataset", "")).lower(): + raise RuntimeError("The representative Search ledger entry must be WANDS") + + +def _check_smoke(data: dict) -> None: + for profile in ("light", "heavy"): + result = data.get(profile, {}) + if result.get("unexpected_5xx") != 0: + raise RuntimeError(f"{profile} smoke recorded unexpected 5xx responses") + passed = {name for name, ok in result.get("flows", {}).items() if ok} + missing = REQUIRED_FLOWS - passed + if missing: + raise RuntimeError(f"{profile} smoke is missing flows: {sorted(missing)}") + + +def _check_latency(data: dict) -> None: + if "light" in data or "heavy" in data: + for profile in ("light", "heavy"): + if profile not in data: + raise RuntimeError(f"Latency results are missing the {profile} profile") + _check_latency_result(profile, data[profile]) + return + _check_latency_result("release", data) + + +def _check_latency_result(profile: str, data: dict) -> None: + context = data.get("context", {}) + search = data.get("search", {}) + if float(context.get("p50_ms", float("inf"))) >= 1000: + raise RuntimeError(f"{profile} /retrieve/context p50 must be below 1000 ms") + if int(context.get("online_llm_retrieval_loops", -1)) != 0: + raise RuntimeError(f"{profile} context retrieval must not run an online LLM loop") + if float(search.get("p50_ms", float("inf"))) >= 200: + raise RuntimeError(f"{profile} default search p50 must be below 200 ms") + if search.get("excludes_embed_query") is not True: + raise RuntimeError(f"{profile} search latency must exclude embed.query") + for surface, result in (("context", context), ("search", search)): + if "p95_ms" not in result or "p99_ms" not in result: + raise RuntimeError( + f"{profile} {surface} must record non-blocking p95 and p99" + ) + + +def _check_restore(profile: str, data: dict) -> None: + before = data.get("before", {}) + after = data.get("after", {}) + for key in COUNT_KEYS: + if before.get(key) != after.get(key): + raise RuntimeError(f"{profile} restore count mismatch: {key}") + checks = data.get("retrieval_spot_checks", []) + if len(checks) < 10 or not all(item.get("match") for item in checks[:10]): + raise RuntimeError(f"{profile} restore needs ten matching retrieval spot checks") + + +def _check_security(data: dict, artifact_dir: Path) -> None: + if data.get("high") != 0 or data.get("critical") != 0: + raise RuntimeError("The exact image has high or critical advisories") + digest = str(data.get("image_digest", "")) + if not digest.startswith("sha256:"): + raise RuntimeError("The exact image digest was not recorded") + sbom = artifact_dir / str(data.get("sbom", "")) + if not sbom.is_file(): + raise RuntimeError("The exact image SBOM is missing") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact_dir", type=Path) + args = parser.parse_args() + artifact_dir = args.artifact_dir.resolve() + try: + _check_public_ledgers() + _check_smoke(_read(artifact_dir / "smoke.json")) + _check_latency(_read(artifact_dir / "latency.json")) + _check_restore("light", _read(artifact_dir / "restore-light.json")) + _check_restore("heavy", _read(artifact_dir / "restore-heavy.json")) + _check_security(_read(artifact_dir / "security.json"), artifact_dir) + except (KeyError, TypeError, ValueError, RuntimeError) as exc: + print(f"release gate failed: {exc}", file=sys.stderr) + return 1 + print("All production release gates passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/openai_ci_stub.py b/scripts/openai_ci_stub.py new file mode 100644 index 0000000..55e3154 --- /dev/null +++ b/scripts/openai_ci_stub.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Minimal deterministic OpenAI-compatible server for isolated CI smoke tests.""" + +from __future__ import annotations + +import hashlib +import json +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + +def _embedding(text: str, dimensions: int = 3072) -> list[float]: + digest = hashlib.sha256(text.encode("utf-8")).digest() + values = [((digest[index % len(digest)] / 255.0) * 2.0) - 1.0 for index in range(dimensions)] + norm = sum(value * value for value in values) ** 0.5 or 1.0 + return [value / norm for value in values] + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def _json(self, status: int, body: dict) -> None: + encoded = json.dumps(body).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def do_GET(self) -> None: # noqa: N802 + if self.path == "/health": + self._json(200, {"status": "ok"}) + else: + self._json(404, {"error": "not_found"}) + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + request = json.loads(self.rfile.read(length) or b"{}") + if self.path.rstrip("/").endswith("/embeddings"): + inputs = request.get("input", []) + if isinstance(inputs, str): + inputs = [inputs] + dimensions = int(request.get("dimensions") or 3072) + data = [ + {"object": "embedding", "index": index, "embedding": _embedding(str(text), dimensions)} + for index, text in enumerate(inputs) + ] + self._json( + 200, + { + "object": "list", + "model": request.get("model", "ci-embedding"), + "data": data, + "usage": {"prompt_tokens": 0, "total_tokens": 0}, + }, + ) + return + self._json(404, {"error": {"message": "unsupported CI stub endpoint"}}) + + def log_message(self, format: str, *args) -> None: + return + + +if __name__ == "__main__": + ThreadingHTTPServer(("0.0.0.0", 8080), Handler).serve_forever() diff --git a/scripts/production_smoke.py b/scripts/production_smoke.py new file mode 100644 index 0000000..6be9756 --- /dev/null +++ b/scripts/production_smoke.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +"""Exercise a running production profile and record release-gate artifacts.""" + +from __future__ import annotations + +import argparse +import json +import math +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + + +TERMINAL_TASK_STATES = {"completed", "failed", "partial_failed"} +COUNT_PATHS = { + "nodes": "/retrieve/entities?limit=1", + "edges": "/retrieve/relationships?limit=1", + "chunks": "/retrieve/text-chunks?limit=1", + "observations": "/retrieve/observations?limit=1", +} + + +class Client: + def __init__(self, base_url: str, system_token: str, timeout: float = 60.0): + self.base_url = base_url.rstrip("/") + self.system_token = system_token + self.timeout = timeout + self.unexpected_5xx = 0 + + def request( + self, + method: str, + path: str, + *, + token: str | None = None, + brain_id: str | None = None, + body: dict[str, Any] | None = None, + expected: set[int] | None = None, + ) -> tuple[int, Any, float]: + headers = {"Accept": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + if brain_id: + headers["X-Brain-ID"] = brain_id + payload = None + if body is not None: + payload = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + request = urllib.request.Request( + f"{self.base_url}{path}", data=payload, headers=headers, method=method + ) + started = time.perf_counter() + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + status = response.status + raw = response.read() + content_type = response.headers.get("Content-Type", "") + except urllib.error.HTTPError as exc: + status = exc.code + raw = exc.read() + content_type = exc.headers.get("Content-Type", "") + elapsed_ms = (time.perf_counter() - started) * 1000 + if status >= 500 and (expected is None or status not in expected): + self.unexpected_5xx += 1 + if expected is not None and status not in expected: + detail = raw.decode("utf-8", errors="replace")[:1000] + raise RuntimeError(f"{method} {path} returned {status}: {detail}") + if "json" in content_type: + result = json.loads(raw or b"null") + else: + result = raw.decode("utf-8", errors="replace") + return status, result, elapsed_ms + + +def _percentile(values: list[float], percentile: float) -> float: + ordered = sorted(values) + if not ordered: + raise RuntimeError("Cannot calculate a percentile without samples") + position = (len(ordered) - 1) * percentile + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return round(ordered[lower], 3) + value = ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + return round(value, 3) + + +def _latency_summary(samples: list[float]) -> dict[str, float | int]: + return { + "samples": len(samples), + "p50_ms": _percentile(samples, 0.50), + "p95_ms": _percentile(samples, 0.95), + "p99_ms": _percentile(samples, 0.99), + } + + +def _write_merged(path: Path, key: str, value: dict[str, Any]) -> None: + existing = json.loads(path.read_text()) if path.is_file() else {} + existing[key] = value + path.write_text(json.dumps(existing, indent=2, sort_keys=True) + "\n") + + +def _wait_for_task( + client: Client, task_id: str, brain_id: str, token: str, timeout: float +) -> dict[str, Any]: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + status, result, _ = client.request( + "GET", + f"/tasks/{task_id}", + token=token, + brain_id=brain_id, + expected={200, 404}, + ) + if status == 200 and result.get("status") in TERMINAL_TASK_STATES: + if result["status"] != "completed": + raise RuntimeError(f"Task {task_id} ended as {result['status']}: {result}") + return result + time.sleep(1) + raise RuntimeError(f"Task {task_id} did not complete within {timeout:g}s") + + +def _stage_ms(report: dict[str, Any] | None, name: str) -> float: + for stage in (report or {}).get("stages", []): + if stage.get("stage") == name: + return float(stage.get("wall_ms", 0)) + raise RuntimeError(f"Profiled response did not include {name}") + + +def _online_llm_stages(report: dict[str, Any] | None) -> int: + return sum( + 1 + for stage in (report or {}).get("stages", []) + if "llm" in str(stage.get("stage", "")).lower() + ) + + +def _capture_counts(client: Client, brain_id: str, token: str) -> dict[str, int]: + _, brains, _ = client.request( + "GET", "/system/brains-list", token=client.system_token, expected={200} + ) + counts = {"brains": len(brains)} + for key, path in COUNT_PATHS.items(): + _, result, _ = client.request( + "GET", path, token=token, brain_id=brain_id, expected={200} + ) + counts[key] = int(result.get("total", result.get("count", 0))) + _, stores, _ = client.request( + "GET", "/retrieve/vectors/stores", token=token, brain_id=brain_id, expected={200} + ) + vector_count = 0 + for store in stores.get("stores", []): + name = store["name"] + _, vectors, _ = client.request( + "GET", + f"/retrieve/vectors/{name}?limit=1", + token=token, + brain_id=brain_id, + expected={200}, + ) + vector_count += int(vectors.get("total", 0)) + counts["vectors"] = vector_count + return counts + + +def _search_check(client: Client, brain_id: str, token: str, query: str) -> dict[str, Any]: + _, response, _ = client.request( + "POST", + "/retrieve/search", + token=token, + brain_id=brain_id, + body={ + "query": query, + "brain_id": brain_id, + "k": 5, + "channels": ["passages"], + "profile_stages": True, + }, + expected={200}, + ) + snippets = [str(hit.get("snippet", "")) for hit in response.get("hits", [])] + return {"query": query, "match": any(query.lower() in item.lower() for item in snippets)} + + +def exercise(args: argparse.Namespace) -> None: + artifacts = args.artifact_dir.resolve() + artifacts.mkdir(parents=True, exist_ok=True) + client = Client(args.base_url, args.system_token, args.request_timeout) + brain_id = args.brain_id or f"productiongate{int(time.time())}" + markers = [f"releaseproof{i:02d}x" for i in range(10)] + flows: dict[str, bool] = {} + + client.request( + "GET", + "/system/brains-list", + token="malformed-token", + expected={401}, + ) + flows["authentication"] = True + + _, brain, _ = client.request( + "POST", + "/system/brains", + token=args.system_token, + body={"brain_id": brain_id}, + expected={200}, + ) + brain_token = brain["pat"] + flows["brain_creation"] = brain.get("name_key") == brain_id + + source_text = "BrainAPI deterministic production proof: " + " ".join(markers) + _, accepted, _ = client.request( + "POST", + "/ingest/", + token=brain_token, + brain_id=brain_id, + body={ + "brain_id": brain_id, + "data": {"data_type": "text", "text_data": source_text}, + "skip_enrichment": True, + }, + expected={202}, + ) + _wait_for_task(client, accepted["task_id"], brain_id, brain_token, args.task_timeout) + flows["plain_ingestion"] = True + + _, accepted, _ = client.request( + "POST", + "/ingest/structured", + token=brain_token, + brain_id=brain_id, + body={ + "brain_id": brain_id, + "mode": "deterministic", + "data": [ + { + "subject": {"name": "Release runner", "type": "AGENT"}, + "subj_event": { + "name": "VALIDATED", + "description": "validated the release profile", + }, + "object": {"name": "BrainAPI", "type": "PRODUCT"}, + } + ], + }, + expected={202}, + ) + _wait_for_task(client, accepted["task_id"], brain_id, brain_token, args.task_timeout) + flows["structured_ingestion"] = True + flows["task_completion"] = True + + context_samples: list[float] = [] + search_samples: list[float] = [] + for index in range(args.warmups + args.samples): + _, context, context_ms = client.request( + "POST", + "/retrieve/context", + token=brain_token, + brain_id=brain_id, + body={ + "text": markers[index % len(markers)], + "brain_id": brain_id, + "profile_stages": True, + "apply_fact_filter": False, + }, + expected={200}, + ) + _, search, search_ms = client.request( + "POST", + "/retrieve/search", + token=brain_token, + brain_id=brain_id, + body={ + "query": markers[index % len(markers)], + "brain_id": brain_id, + "k": 5, + "channels": ["passages"], + "profile_stages": True, + }, + expected={200}, + ) + embed_ms = _stage_ms(search.get("stage_timings"), "embed.query") + if index >= args.warmups: + context_samples.append(context_ms) + search_samples.append(max(0.0, search_ms - embed_ms)) + flows["context_retrieval"] = bool(context.get("text_context") is not None) + flows["search"] = bool(search.get("hits")) + + _, console, _ = client.request("GET", "/console/", expected={200}) + flows["console"] = " None: + artifacts = args.artifact_dir.resolve() + state_path = artifacts / f"state-{args.profile}.json" + state = json.loads(state_path.read_text()) + client = Client(args.base_url, args.system_token, args.request_timeout) + brain_id = state["brain_id"] + brain_token = args.system_token + checks = [ + _search_check(client, brain_id, brain_token, item["query"]) + for item in state["retrieval_spot_checks"][:10] + ] + result = { + "before": state["before"], + "after": _capture_counts(client, brain_id, brain_token), + "retrieval_spot_checks": checks, + } + output = artifacts / f"restore-{args.profile}.json" + output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + if result["before"] != result["after"] or not all(item["match"] for item in checks): + raise RuntimeError(f"{args.profile} restore verification failed; see {output}") + print(output) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("exercise", "verify-restore")) + parser.add_argument("--profile", required=True, choices=("light", "heavy")) + parser.add_argument("--base-url", default="http://localhost") + parser.add_argument("--system-token", required=True) + parser.add_argument("--artifact-dir", type=Path, required=True) + parser.add_argument("--brain-id") + parser.add_argument("--samples", type=int, default=10) + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--task-timeout", type=float, default=300) + parser.add_argument("--request-timeout", type=float, default=60) + args = parser.parse_args() + if args.command == "exercise": + exercise(args) + else: + verify_restore(args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/seed_public_demo.py b/scripts/seed_public_demo.py new file mode 100644 index 0000000..5dec1ba --- /dev/null +++ b/scripts/seed_public_demo.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Idempotently seed the dedicated public demo brain from published V2 docs.""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import sys +import time + +import requests + + +DEFAULT_DOCS_URL = "https://brainapi.lumen-labs.ai/docs/llms-full.txt" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--api-url", default=os.getenv("BRAINAPI_URL", "http://localhost:8000")) + parser.add_argument("--docs-url", default=DEFAULT_DOCS_URL) + parser.add_argument("--brain", default=os.getenv("PUBLIC_DEMO_BRAIN_ID", "agentdemo")) + parser.add_argument("--timeout", type=int, default=600) + args = parser.parse_args() + + token = os.getenv("BRAINPAT_TOKEN") + if not token: + print("BRAINPAT_TOKEN is required", file=sys.stderr) + return 2 + if not args.brain.isalnum() or args.brain == "system": + print("Demo brain must be a non-system alphanumeric identifier", file=sys.stderr) + return 2 + + api_url = args.api_url.rstrip("/") + docs_response = requests.get(args.docs_url, timeout=30) + docs_response.raise_for_status() + docs_text = docs_response.text + digest = hashlib.sha256(docs_text.encode("utf-8")).hexdigest() + marker = f"BRAINAPI_PUBLIC_DEMO_SHA256_{digest}" + headers = {"BrainPAT": token, "X-Brain-ID": args.brain} + + existing = requests.get( + f"{api_url}/retrieve/text-chunks", + headers=headers, + params={"query_text": marker, "limit": 1}, + timeout=30, + ) + if existing.status_code == 200 and existing.json().get("total", 0) > 0: + print(f"Public demo brain {args.brain!r} is already current ({digest[:12]}).") + return 0 + if existing.status_code not in (200, 406): + existing.raise_for_status() + + payload = { + "data": { + "data_type": "text", + "text_data": f"{marker}\nSource: {args.docs_url}\n\n{docs_text}", + }, + "brain_id": args.brain, + "skip_enrichment": True, + } + accepted = requests.post( + f"{api_url}/ingest/", + headers={ + **headers, + "Content-Type": "application/json", + "Task-Identifier": f"public-demo-{digest[:24]}", + }, + json=payload, + timeout=30, + ) + accepted.raise_for_status() + task_id = accepted.json()["task_id"] + + deadline = time.monotonic() + args.timeout + while time.monotonic() < deadline: + task_response = requests.get( + f"{api_url}/tasks/{task_id}", + headers=headers, + timeout=30, + ) + task_response.raise_for_status() + task = task_response.json() + status = str(task.get("status", "unknown")).lower() + if status in {"completed", "success", "succeeded"}: + print(f"Seeded public demo brain {args.brain!r} ({digest[:12]}).") + return 0 + if status in {"failed", "error", "revoked"}: + print(f"Public demo seed task failed: {task}", file=sys.stderr) + return 1 + time.sleep(2) + + print(f"Timed out waiting for seed task {task_id}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/write_security_artifact.py b/scripts/write_security_artifact.py new file mode 100644 index 0000000..993b506 --- /dev/null +++ b/scripts/write_security_artifact.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Summarize an exact image's Trivy scan, digest, and SBOM for release gates.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--trivy", type=Path, required=True) + parser.add_argument("--sbom", type=Path, required=True) + parser.add_argument("--image", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + scan = json.loads(args.trivy.read_text(encoding="utf-8")) + counts = {"HIGH": 0, "CRITICAL": 0} + for result in scan.get("Results") or []: + for vulnerability in result.get("Vulnerabilities") or []: + severity = str(vulnerability.get("Severity", "")).upper() + if severity in counts: + counts[severity] += 1 + digest = subprocess.run( + ["docker", "image", "inspect", args.image, "--format", "{{.Id}}"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + result = { + "high": counts["HIGH"], + "critical": counts["CRITICAL"], + "image_digest": digest, + "sbom": args.sbom.name, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/adapters/data.py b/src/adapters/data.py index dc12088..d0ac6bc 100644 --- a/src/adapters/data.py +++ b/src/adapters/data.py @@ -49,6 +49,11 @@ def search(self, text: str, brain_id: str = "default") -> SearchResult: """ return self.data.search(text=text, brain_id=brain_id) + def search_bm25( + self, text: str, brain_id: str = "default", limit: int = 10 + ) -> List[Tuple[TextChunk, float]]: + return self.data.search_bm25(text=text, brain_id=brain_id, limit=limit) + def get_text_chunks_by_ids( self, ids: List[str], with_observations: bool, brain_id: str = "default" ) -> Tuple[List[TextChunk], List[Observation]]: diff --git a/src/adapters/graph.py b/src/adapters/graph.py index cc2cc5c..b720706 100644 --- a/src/adapters/graph.py +++ b/src/adapters/graph.py @@ -355,6 +355,26 @@ def search_entities( brain_id, limit, skip, node_labels, node_uuids, query_text ) + def search_nodes_bm25( + self, + query_text: str, + brain_id: str = "default", + *, + limit: int = 10, + node_labels: Optional[list[str]] = None, + node_uuids: Optional[list[str]] = None, + ): + method = getattr(self.graph, "search_nodes_bm25", None) + if method is None: + return [] + return method( + query_text, + brain_id, + limit=limit, + node_labels=node_labels, + node_uuids=node_uuids, + ) + def deprecate_relationship( self, subject: Node, diff --git a/src/adapters/interfaces/data.py b/src/adapters/interfaces/data.py index 80cb8da..cf99abd 100644 --- a/src/adapters/interfaces/data.py +++ b/src/adapters/interfaces/data.py @@ -53,6 +53,13 @@ def search(self, text: str, brain_id: str) -> SearchResult: """ raise NotImplementedError("search method not implemented") + def search_bm25( + self, text: str, brain_id: str, limit: int = 10 + ) -> List[Tuple[TextChunk, float]]: + raise NotImplementedError( + "search_bm25 requires SEARCH_ENABLED=true and DATA_DB=postgresql" + ) + @abstractmethod def get_text_chunks_by_ids( self, ids: List[str], with_observations: bool, brain_id: str diff --git a/src/config.py b/src/config.py index 74bcc03..00eca61 100644 --- a/src/config.py +++ b/src/config.py @@ -71,6 +71,56 @@ def validate_ingest_architect_prior_context( ) +def validate_search_fusion(value: str | None) -> Literal["rrf", "cc"]: + normalized = (value or "rrf").strip().lower() + if normalized in ("rrf", "cc"): + return normalized # type: ignore[return-value] + raise ValueError( + f"Invalid SEARCH_FUSION: {value!r}. Expected 'rrf' or 'cc'." + ) + + +def validate_context_passage_mode( + value: str | None, +) -> Literal["hybrid", "bm25", "dense", "ilike"]: + normalized = (value or "hybrid").strip().lower() + if normalized in ("hybrid", "bm25", "dense", "ilike"): + return normalized # type: ignore[return-value] + raise ValueError( + f"Invalid CONTEXT_PASSAGE_MODE: {value!r}. " + "Expected 'hybrid', 'bm25', 'dense', or 'ilike'." + ) + + +def validate_search_config( + *, + enabled: bool, + use_dense: bool, + use_bm25: bool, + data_db: str, + bm25_k1: float, + bm25_b: float, +) -> None: + if bm25_k1 <= 0: + raise ValueError(f"Invalid SEARCH_BM25_K1: {bm25_k1!r}. Must be > 0.") + if not 0.0 <= bm25_b <= 1.0: + raise ValueError( + f"Invalid SEARCH_BM25_B: {bm25_b!r}. Must be between 0 and 1." + ) + if not enabled: + return + if use_bm25 and (data_db or "").strip().lower() != "postgresql": + raise ValueError( + "SEARCH_USE_BM25=true requires DATA_DB=postgresql " + "(BM25 indexes are Postgres-only)." + ) + if not use_dense and not use_bm25: + raise ValueError( + "SEARCH_ENABLED=true requires SEARCH_USE_DENSE=true " + "and/or SEARCH_USE_BM25=true." + ) + + class AzureConfig: """ Configuration class for the Azure configuration. @@ -253,6 +303,7 @@ def __init__(self): self.host = os.getenv("REDIS_HOST") port_str = os.getenv("REDIS_PORT") self.port = int(port_str) if port_str else None + self.password = os.getenv("REDIS_PASSWORD") or None if [self.host, self.port].count(None) > 0: raise ValueError("Redis configuration is not complete") @@ -452,6 +503,7 @@ def __init__(self): ) +SEARCH_FTS_REGCONFIGS = frozenset({"italian", "spanish", "simple"}) _MODES = ("local", "remote") _PROVIDERS = ( "ollama", @@ -645,6 +697,55 @@ def __init__(self): self.agentic_architecture: Literal["custom", "langchain"] = os.getenv( "AGENTIC_ARCHITECTURE", "custom" ) + self.search_enabled = os.getenv("SEARCH_ENABLED", "false") == "true" + self.search_use_dense = os.getenv("SEARCH_USE_DENSE", "true") == "true" + self.search_use_bm25 = os.getenv("SEARCH_USE_BM25", "true") == "true" + community_raw = os.getenv("SEARCH_COMMUNITY_LABELS", "TYPE,CLASS,TOPIC") + community_labels = [ + part.strip() for part in community_raw.split(",") if part.strip() + ] + self.search_community_labels = community_labels or [ + "TYPE", + "CLASS", + "TOPIC", + ] + self.search_neighbor_fanout = int(os.getenv("SEARCH_NEIGHBOR_FANOUT", "50")) + self.search_fusion: Literal["rrf", "cc"] = validate_search_fusion( + os.getenv("SEARCH_FUSION", "rrf") + ) + self.search_fusion_alpha = float(os.getenv("SEARCH_FUSION_ALPHA", "0.5")) + self.search_literal_fill = os.getenv("SEARCH_LITERAL_FILL", "false") == "true" + self.search_bm25_k1 = float(os.getenv("SEARCH_BM25_K1", "1.2")) + self.search_bm25_b = float(os.getenv("SEARCH_BM25_B", "0.75")) + self.search_fts_regconfig = os.getenv("SEARCH_FTS_REGCONFIG", "").strip().lower() + self.search_fts_brains = frozenset( + part.strip() + for part in os.getenv("SEARCH_FTS_BRAINS", "").split(",") + if part.strip() + ) + self.context_passage_mode: Literal[ + "hybrid", "bm25", "dense", "ilike" + ] = validate_context_passage_mode( + os.getenv("CONTEXT_PASSAGE_MODE", "hybrid") + ) + validate_search_config( + enabled=self.search_enabled, + use_dense=self.search_use_dense, + use_bm25=self.search_use_bm25, + data_db=self.data_db, + bm25_k1=self.search_bm25_k1, + bm25_b=self.search_bm25_b, + ) + + def search_fts_regconfig_for_brain(self, brain_id: str) -> str | None: + bid = (brain_id or "").strip() + if not bid.lower().startswith("searchbench"): + return None + if bid not in self.search_fts_brains: + return None + if self.search_fts_regconfig not in SEARCH_FTS_REGCONFIGS: + return None + return self.search_fts_regconfig config = Config() diff --git a/src/constants/tasks/ingestion.py b/src/constants/tasks/ingestion.py index fee5335..3941d11 100644 --- a/src/constants/tasks/ingestion.py +++ b/src/constants/tasks/ingestion.py @@ -85,3 +85,10 @@ class IngestionTaskArgs(BaseModel): "Used to resolve relative dates like 'yesterday' into absolute dates." ), ) + skip_enrichment: bool = Field( + default=False, + description=( + "Persist the text chunk and embedding only. Skip observations and " + "knowledge-graph enrichment (Scout/Architect)." + ), + ) diff --git a/src/core/agents/architect_agent.py b/src/core/agents/architect_agent.py index e07cf34..be744b3 100644 --- a/src/core/agents/architect_agent.py +++ b/src/core/agents/architect_agent.py @@ -73,6 +73,7 @@ stable_relationship_id, ) from src.core.saving.ingestion_manager import IngestionManager +from src.core.search.catalog_graph import is_static_has_triple from src.services.api.constants.requests import IngestionTripleSet from src.utils.cleanup import strip_properties from src.utils.dates import normalize_date_string @@ -184,8 +185,28 @@ def _triple_entity(node) -> ArchitectAgentEntity: if not cr.subject or not cr.subj_event: continue subject = _triple_entity(cr.subject) - event = _triple_entity(cr.event) obj = _triple_entity(cr.object) + if cr.event is None or is_static_has_triple(cr): + pred_name = cr.subj_event.name or "HAS" + rel_uuid = cr.subj_event.uuid or stable_relationship_id( + subject.uuid, pred_name, obj.uuid + ) + triple_relationships.append( + ArchitectAgentRelationship( + tail=subject, + name=pred_name, + tip=obj, + description=cr.subj_event.description, + amount=cr.subj_event.amount, + properties=cr.subj_event.properties or {}, + uuid=rel_uuid, + flow_key=rel_uuid, + ) + ) + continue + if not cr.event or not cr.event_obj: + continue + event = _triple_entity(cr.event) flow_key = stable_flow_key( event_uuid=event.uuid, event_name=event.name, @@ -224,6 +245,8 @@ def _triple_entity(node) -> ArchitectAgentEntity: ] ) for pt in partial_triples: + if not pt.event or not pt.event_obj: + continue event = _triple_entity(pt.event) obj = _triple_entity(pt.object) if pt.subject: diff --git a/src/core/plugins/celery_discovery.py b/src/core/plugins/celery_discovery.py index 6805aaf..6e27444 100644 --- a/src/core/plugins/celery_discovery.py +++ b/src/core/plugins/celery_discovery.py @@ -32,22 +32,27 @@ def discover_plugin_celery(plugins_dir: Path) -> tuple[tuple, dict]: continue plugin_dir_str = str(child) - if plugin_dir_str not in sys.path: + added_plugin_path = plugin_dir_str not in sys.path + if added_plugin_path: sys.path.insert(0, plugin_dir_str) - - safe_name = child.name.replace("-", "_") - celery_mod = _load_module( - child, - "workers/celery.py", - f"brainapi_plugin_{safe_name}_workers_celery", - ) - _load_module( - child, - "workers/tasks.py", - f"brainapi_plugin_{safe_name}_workers_tasks", - ) - queues.extend(getattr(celery_mod, "QUEUES", ())) - routes.update(getattr(celery_mod, "ROUTES", {})) + try: + safe_name = child.name.replace("-", "_") + celery_mod = _load_module( + child, + "workers/celery.py", + f"brainapi_plugin_{safe_name}_workers_celery", + ) + _load_module( + child, + "workers/tasks.py", + f"brainapi_plugin_{safe_name}_workers_tasks", + ) + queues.extend(getattr(celery_mod, "QUEUES", ())) + routes.update(getattr(celery_mod, "ROUTES", {})) + finally: + if added_plugin_path and plugin_dir_str in sys.path: + sys.path.remove(plugin_dir_str) + sys.path_importer_cache.pop(plugin_dir_str, None) return tuple(queues), routes diff --git a/src/core/plugins/context.py b/src/core/plugins/context.py index 0bf5e7e..04a8f70 100644 --- a/src/core/plugins/context.py +++ b/src/core/plugins/context.py @@ -53,6 +53,8 @@ def __init__( self.config = config self._routers: list[tuple["APIRouter", dict[str, Any]]] = [] self._event_handlers: dict[str, list[Callable]] = {} + self._search_retrievers: dict[str, Callable] = {} + self._search_rerankers: dict[str, Callable] = {} @classmethod def _build_adapters(cls) -> PluginAdapters: @@ -133,3 +135,15 @@ def register_mcp_tool(self, fn: Callable, **kwargs: Any) -> None: def add_event_handler(self, event: str, handler: Callable) -> None: self._event_handlers.setdefault(event, []).append(handler) + + def register_search_retriever(self, name: str, fn: Callable) -> None: + from src.core.search.hooks import register_search_retriever + + register_search_retriever(name, fn) + self._search_retrievers[name] = fn + + def register_search_reranker(self, name: str, fn: Callable) -> None: + from src.core.search.hooks import register_search_reranker + + register_search_reranker(name, fn) + self._search_rerankers[name] = fn diff --git a/src/core/plugins/loader.py b/src/core/plugins/loader.py index 7d4cc5b..9a7eab2 100644 --- a/src/core/plugins/loader.py +++ b/src/core/plugins/loader.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib import importlib.util import logging import shutil @@ -80,11 +81,17 @@ def load(self, manifest: PluginManifest) -> bool: return False plugin_dir_str = str(plugin_dir) - added_plugin_path = False - if plugin_dir_str not in sys.path: - sys.path.insert(0, plugin_dir_str) - added_plugin_path = True + original_sys_path = list(sys.path) + sys.path[:] = [ + plugin_dir_str, + *[ + entry + for entry in original_sys_path + if entry != plugin_dir_str and not self._is_plugin_path(entry) + ], + ] self._clear_local_plugin_namespaces() + importlib.invalidate_caches() module_name = self._entry_point_resolver.resolve( entry_point=manifest.entry_point, @@ -103,30 +110,41 @@ def load(self, manifest: PluginManifest) -> bool: ) except Exception as exc: logger.error("Failed to import plugin '%s' (module: %s): %s", manifest.name, module_name, exc) - if added_plugin_path and plugin_dir_str in sys.path: - sys.path.remove(plugin_dir_str) + self._cleanup_plugin_imports(plugin_dir_str, original_sys_path) return False register_fn = getattr(module, "register", None) if register_fn is None or not callable(register_fn): logger.error("Plugin '%s' has no callable 'register' function in '%s'", manifest.name, module_name) - if added_plugin_path and plugin_dir_str in sys.path: - sys.path.remove(plugin_dir_str) + self._cleanup_plugin_imports(plugin_dir_str, original_sys_path) return False try: register_fn(self.context) self._loaded[manifest.name] = manifest logger.info("Plugin '%s' v%s loaded successfully", manifest.name, manifest.version) - if added_plugin_path and plugin_dir_str in sys.path: - sys.path.remove(plugin_dir_str) + self._cleanup_plugin_imports(plugin_dir_str, original_sys_path) return True except Exception as exc: logger.error("Plugin '%s' register() raised an exception: %s", manifest.name, exc, exc_info=True) - if added_plugin_path and plugin_dir_str in sys.path: - sys.path.remove(plugin_dir_str) + self._cleanup_plugin_imports(plugin_dir_str, original_sys_path) return False + @staticmethod + def _is_plugin_path(path_entry: str) -> bool: + try: + return "plugins" in Path(path_entry).parts + except (TypeError, ValueError): + return False + + def _cleanup_plugin_imports( + self, plugin_dir: str, original_sys_path: list[str] + ) -> None: + sys.path[:] = original_sys_path + self._clear_local_plugin_namespaces() + sys.path_importer_cache.pop(plugin_dir, None) + importlib.invalidate_caches() + def _clear_local_plugin_namespaces(self) -> None: namespace_roots = ( "routes", diff --git a/src/core/saving/ingestion_manager.py b/src/core/saving/ingestion_manager.py index b3a9887..d8be498 100644 --- a/src/core/saving/ingestion_manager.py +++ b/src/core/saving/ingestion_manager.py @@ -12,14 +12,16 @@ from src.adapters.embeddings import EmbeddingsAdapter from src.adapters.graph import GraphAdapter from src.adapters.embeddings import VectorStoreAdapter +from src.constants.agents import ArchitectAgentRelationship +from src.constants.embeddings import Vector from src.core.agents.scout_agent import ScoutEntity -from src.core.agents.architect_agent import ArchitectAgentRelationship from src.core.saving.identity import ( stable_flow_key, stable_node_id, stable_relationship_id, stable_uuid, ) +from src.core.search.catalog_graph import node_embed_text __all__ = [ "IngestionManager", @@ -53,6 +55,16 @@ def __init__( self.kg = graph_adapter self.resolved_cache = {} self.metadata = {} + self._text_embed_cache: dict[str, list[float]] = {} + + def _embed_text_cached(self, text: str) -> Vector: + cached = self._text_embed_cache.get(text) + if cached is not None: + return Vector(id="", embeddings=list(cached), metadata={}) + vector = self.embeddings.embed_text(text) + if vector.embeddings: + self._text_embed_cache[text] = list(vector.embeddings) + return vector def process_node_vectors(self, node_data: ScoutEntity, brain_id): """ @@ -70,12 +82,19 @@ def process_node_vectors(self, node_data: ScoutEntity, brain_id): Returns: The node's UUID. """ - if node_data.name in self.resolved_cache: + text, cache_key = node_embed_text(node_data) + if cache_key in self.resolved_cache: return node_data.uuid - v_sub = self.embeddings.embed_text(node_data.name) + v_sub = self._embed_text_cached(text) + labels = [node_data.type] + extra = (node_data.properties or {}).get("catalog_labels") or [] + for item in extra: + label = str(item).strip() + if label and label not in labels: + labels.append(label) v_sub.metadata = { - "labels": [node_data.type], + "labels": labels, "name": node_data.name, "uuid": node_data.uuid, } @@ -87,7 +106,7 @@ def process_node_vectors(self, node_data: ScoutEntity, brain_id): **(node_data.properties or {}), "v_id": v_ids[0], } - self.resolved_cache[node_data.name] = v_ids[0] + self.resolved_cache[cache_key] = v_ids[0] return node_data.uuid else: print("[ ! ]Node not embedded:", node_data) @@ -118,7 +137,7 @@ def process_rel_vectors(self, rel_data: ArchitectAgentRelationship, brain_id): ) text_to_embed = (rel_data.description or "").strip() or rel_data.name if text_to_embed: - v_rel = self.embeddings.embed_text(text_to_embed) + v_rel = self._embed_text_cached(text_to_embed) v_rel.metadata = { **(self.metadata or {}), "uuid": rel_data.uuid, diff --git a/src/core/search/bm25.py b/src/core/search/bm25.py new file mode 100644 index 0000000..1ae9dd2 --- /dev/null +++ b/src/core/search/bm25.py @@ -0,0 +1,43 @@ +import math + + +def okapi_idf(n_docs: float, df: float) -> float: + n_docs = max(float(n_docs), 0.0) + df = max(float(df), 0.0) + return math.log(1.0 + (n_docs - df + 0.5) / (df + 0.5)) + + +def okapi_bm25_term( + tf: float, + dl: float, + avgdl: float, + idf: float, + *, + k1: float = 1.2, + b: float = 0.75, +) -> float: + if tf <= 0 or idf == 0: + return 0.0 + avgdl = avgdl if avgdl > 0 else 1.0 + dl = dl if dl > 0 else 1.0 + denom = tf + k1 * (1.0 - b + b * dl / avgdl) + if denom == 0: + return 0.0 + return idf * (tf * (k1 + 1.0)) / denom + + +def okapi_bm25_document( + term_tfs: dict[str, float], + *, + dl: float, + avgdl: float, + n_docs: float, + dfs: dict[str, float], + k1: float = 1.2, + b: float = 0.75, +) -> float: + score = 0.0 + for term, tf in term_tfs.items(): + idf = okapi_idf(n_docs, dfs.get(term, 0.0)) + score += okapi_bm25_term(tf, dl, avgdl, idf, k1=k1, b=b) + return score diff --git a/src/core/search/catalog_graph.py b/src/core/search/catalog_graph.py new file mode 100644 index 0000000..7bdde46 --- /dev/null +++ b/src/core/search/catalog_graph.py @@ -0,0 +1,772 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any, Sequence + +from src.core.search.recommend import behavior_weight + +HAS_EVENT_NAME = "HAS" +ENTITY_TYPE = "ENTITY" +CLASS_LABEL = "CLASS" +TYPE_LABEL = "TYPE" +ATTR_LABEL = "ATTR" +EVENT_LABEL = "EVENT" +USER_TYPE = "USER" +OPTION_ALLOWLIST = frozenset( + {"color", "material", "category", "brand", "size", "style"} +) +_BEHAVIOR_EVENT = { + "view": "View", + "viewed": "View", + "click": "View", + "clicked": "View", + "cart": "AddToCart", + "add_to_cart": "AddToCart", + "added_to_cart": "AddToCart", + "addtocart": "AddToCart", + "purchase": "Purchase", + "purchased": "Purchase", + "buy": "Purchase", + "bought": "Purchase", + "favorite": "Favorite", + "favourite": "Favorite", + "favorites": "Favorite", + "favourites": "Favorite", + "wishlist": "Wishlist", + "add_to_favorite": "Favorite", + "addtofavorite": "Favorite", + "add_to_favourite": "Favorite", + "addtofavourite": "Favorite", + "add_to_wishlist": "Wishlist", + "addtowishlist": "Wishlist", + "follow": "Follow", + "flw": "Follow", +} +FEATURE_KEY_CAP = 16 +FEATURE_KEY_PRIORITY = ( + "style", + "mood", + "color", + "colour", + "material", + "shape", + "finish", + "fabric", + "design", + "woodtone", +) +FEATURE_SKIP_TOKENS = ( + "width", + "height", + "weight", + "length", + "depth", + "capacity", + "temperature", + "rating", + "price", + "count", + "warranty", + "origin", + "assembly", + "thickness", + "diameter", + "clearance", + "sku", +) +CONTINUOUS_PROPERTY_KEYS = ( + "price", + "rating", + "average_rating", + "rating_count", + "review_count", + "width", + "height", + "depth", + "weight", + "length", +) +BOOLEAN_NOISE = { + "", + "yes", + "no", + "true", + "false", + "y", + "n", + "0", + "1", + "none", + "n/a", + "na", + "unknown", + "null", +} +_LABEL_RE = re.compile(r"[^A-Za-z0-9]+") +_TEXT_FIELD_RE = re.compile( + r"^(Title|Brand|Color|Class|Category|Features|Locale|Hierarchy|Description|Price|Rating)\s*:\s*(.+)$", + re.IGNORECASE, +) +_HIER_SPLIT = re.compile(r"\s*[>/|]+\s*") + + +def entity_uuid(doc_id: str) -> str: + return str(doc_id) + + +def hub_uuid(kind: str, value: str) -> str: + return f"hub:{kind}:{value.strip().lower()}" + + +def event_uuid(owner: str, kind: str, seq: int) -> str: + return f"evt:{owner}:{kind}:{seq}" + + +def rel_uuid(owner: str, kind: str, seq: int) -> str: + return f"rel:{owner}:{kind}:{seq}" + + +def sanitize_label(raw: str) -> str: + cleaned = _LABEL_RE.sub("_", (raw or "").strip().upper()).strip("_") + if not cleaned or cleaned[0].isdigit(): + return ATTR_LABEL + return cleaned[:40] + + +_DOC_ID_RE = re.compile(r"DOCID\s+(\S+)") + + +def compose_search_text(*parts: str | None) -> str: + chunks = [str(part).strip() for part in parts if str(part or "").strip()] + return " ".join(chunks) + + +def doc_id_from_text(text: str | None) -> str | None: + match = _DOC_ID_RE.search(str(text or "")) + if not match: + return None + doc_id = match.group(1).strip().rstrip(".") + return doc_id or None + + +def node_id_from_passage_text(text: str | None) -> str | None: + doc_id = doc_id_from_text(text) + if not doc_id: + return None + return entity_uuid(doc_id) + + +def entity_search_text(doc: dict[str, Any], fields: dict[str, Any] | None = None) -> str: + raw = str(doc.get("text") or "").strip() + if raw: + return raw + resolved = fields if fields is not None else doc_fields(doc) + return compose_search_text( + resolved.get("title"), + resolved.get("class"), + resolved.get("hierarchy"), + resolved.get("brand"), + resolved.get("color"), + resolved.get("features"), + resolved.get("description"), + ) + + +def node_embed_text(node_data: Any) -> tuple[str, str]: + props = getattr(node_data, "properties", None) or {} + if not isinstance(props, dict): + props = {} + search_text = str(props.get("search_text") or "").strip() + name = str(getattr(node_data, "name", "") or "").strip() + if search_text: + uuid = str(getattr(node_data, "uuid", "") or "").strip() + return search_text, f"uuid:{uuid}" if uuid else search_text + return name, name + + +def split_hierarchy(raw: str | None) -> list[str]: + text = str(raw or "").strip() + if not text: + return [] + parts = [item.strip() for item in _HIER_SPLIT.split(text) if item.strip()] + seen: set[str] = set() + out: list[str] = [] + for part in parts: + key = part.lower() + if key in seen: + continue + seen.add(key) + out.append(part) + return out + + +def _feature_key_rank(key: str) -> int: + lowered = key.lower().replace(" ", "") + for index, token in enumerate(FEATURE_KEY_PRIORITY): + if token in lowered: + return index + return len(FEATURE_KEY_PRIORITY) + + +def _skip_feature_key(key: str) -> bool: + lowered = key.lower().replace(" ", "") + if not lowered: + return True + return any(token in lowered for token in FEATURE_SKIP_TOKENS) + + +def parse_feature_string( + raw: str | None, *, cap: int = FEATURE_KEY_CAP +) -> list[tuple[str, str]]: + text = str(raw or "").strip() + if not text: + return [] + pairs: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + parts = [part.strip() for part in text.split("|") if part.strip()] + if ":" in text and not parts: + parts = [text] + for part in parts: + if ":" not in part: + continue + key, value = part.split(":", 1) + key = key.strip() + value = value.strip() + if not key or not value: + continue + if _skip_feature_key(key): + continue + if value.lower() in BOOLEAN_NOISE: + continue + item = (key, value) + if item in seen: + continue + seen.add(item) + pairs.append(item) + pairs.sort(key=lambda item: (_feature_key_rank(item[0]), item[0].lower())) + if cap <= 0: + return pairs + kept: list[tuple[str, str]] = [] + keys: set[str] = set() + for key, value in pairs: + key_id = key.lower() + if key_id not in keys and len(keys) >= cap: + continue + keys.add(key_id) + kept.append((key, value)) + return kept + + +def _fields_from_text(text: str) -> dict[str, str]: + out: dict[str, str] = {} + for line in (text or "").splitlines(): + match = _TEXT_FIELD_RE.match(line.strip()) + if not match: + continue + key = match.group(1).strip().lower() + value = match.group(2).strip() + if value: + out[key] = value + return out + + +def doc_fields(doc: dict[str, Any]) -> dict[str, Any]: + text_fields = _fields_from_text(str(doc.get("text") or "")) + title = str(doc.get("title") or text_fields.get("title") or "").strip() + class_name = str( + doc.get("class") + or doc.get("product_class") + or text_fields.get("class") + or "" + ).strip() + hierarchy = str( + doc.get("hierarchy") + or doc.get("category_hierarchy") + or text_fields.get("hierarchy") + or text_fields.get("category") + or "" + ).strip() + if not class_name: + class_name = str(text_fields.get("category") or "").strip() + brand = str(doc.get("brand") or text_fields.get("brand") or "").strip() + color = str(doc.get("color") or text_fields.get("color") or "").strip() + locale = str(doc.get("locale") or text_fields.get("locale") or "").strip() + features = str(doc.get("features") or text_fields.get("features") or "").strip() + description = str( + doc.get("description") or text_fields.get("description") or "" + ).strip() + dataset = str(doc.get("dataset") or "").strip() + properties: dict[str, str] = {} + for key in CONTINUOUS_PROPERTY_KEYS: + value = str(doc.get(key) or text_fields.get(key) or "").strip() + if value: + properties[key] = value + return { + "title": title, + "class": class_name, + "hierarchy": hierarchy, + "brand": brand, + "color": color, + "locale": locale, + "features": features, + "description": description, + "dataset": dataset, + "properties": properties, + } + + +def _triple_pred_name(value: Any) -> str: + if value is None: + return "" + if isinstance(value, dict): + return str(value.get("name") or "").strip().upper() + return str(getattr(value, "name", "") or "").strip().upper() + + +def is_static_has_triple(triple: Any) -> bool: + if isinstance(triple, dict): + event = triple.get("event") + subj_event = triple.get("subj_event") + event_obj = triple.get("event_obj") + else: + event = getattr(triple, "event", None) + subj_event = getattr(triple, "subj_event", None) + event_obj = getattr(triple, "event_obj", None) + happened = None + event_name = "" + if isinstance(event, dict): + happened = event.get("happened_at") + event_name = str(event.get("name") or "").strip().upper() + elif event is not None: + happened = getattr(event, "happened_at", None) + event_name = str(getattr(event, "name", "") or "").strip().upper() + if happened: + return False + pred = _triple_pred_name(subj_event) or HAS_EVENT_NAME + if event is None: + return pred == HAS_EVENT_NAME + other = _triple_pred_name(event_obj) or HAS_EVENT_NAME + return event_name == HAS_EVENT_NAME and pred == HAS_EVENT_NAME and other == HAS_EVENT_NAME + + +def has_triple( + *, + subject: dict[str, Any], + object_node: dict[str, Any], + seq: int, + kind: str, +) -> dict[str, Any]: + owner = str(subject.get("uuid") or subject.get("name") or "node") + return { + "subject": subject, + "subj_event": { + "name": HAS_EVENT_NAME, + "uuid": rel_uuid(owner, f"{kind}_has", seq), + }, + "object": object_node, + } + + +def _entity_node( + doc_id: str, + name: str, + *, + description: str | None = None, + properties: dict[str, Any] | None = None, + search_text: str | None = None, +) -> dict[str, Any]: + props = dict(properties or {}) + blob = str(search_text or "").strip() or compose_search_text(name, description) + if blob: + props["search_text"] = blob + node: dict[str, Any] = { + "name": name or doc_id, + "type": ENTITY_TYPE, + "uuid": entity_uuid(doc_id), + "labels": [ENTITY_TYPE], + "properties": props, + } + if description: + node["description"] = description + return node + + +def _hub_node(kind: str, value: str, extra_label: str | None = None) -> dict[str, Any]: + labels = [kind] + if extra_label: + sanitized = sanitize_label(extra_label) + if sanitized and sanitized not in labels: + labels.append(sanitized) + search_text = compose_search_text(extra_label or kind, value) + return { + "name": value, + "type": kind, + "uuid": hub_uuid(kind.lower(), value), + "labels": labels, + "description": search_text, + "properties": { + "search_text": search_text, + "catalog_labels": labels, + }, + } + + +def doc_to_triples(doc: dict[str, Any]) -> list[dict[str, Any]]: + doc_id = str(doc.get("doc_id") or "").strip() + if not doc_id: + return [] + fields = doc_fields(doc) + name = fields["title"] or doc_id + subject = _entity_node( + doc_id, + name, + description=fields["description"] or None, + properties=dict(fields["properties"] or {}), + search_text=entity_search_text(doc, fields), + ) + triples: list[dict[str, Any]] = [] + seq = 0 + class_name = fields["class"] + if class_name: + triples.append( + has_triple( + subject=subject, + object_node=_hub_node(CLASS_LABEL, class_name), + seq=seq, + kind="class", + ) + ) + seq += 1 + class_key = class_name.strip().lower() + for part in split_hierarchy(fields["hierarchy"]): + if part.strip().lower() == class_key: + continue + triples.append( + has_triple( + subject=subject, + object_node=_hub_node(TYPE_LABEL, part, extra_label="CATEGORY"), + seq=seq, + kind="hierarchy", + ) + ) + seq += 1 + if fields["brand"]: + triples.append( + has_triple( + subject=subject, + object_node=_hub_node(ATTR_LABEL, fields["brand"], extra_label="BRAND"), + seq=seq, + kind="brand", + ) + ) + seq += 1 + if fields["color"]: + triples.append( + has_triple( + subject=subject, + object_node=_hub_node(ATTR_LABEL, fields["color"], extra_label="COLOR"), + seq=seq, + kind="color", + ) + ) + seq += 1 + if fields["locale"]: + triples.append( + has_triple( + subject=subject, + object_node=_hub_node(TYPE_LABEL, fields["locale"], extra_label="LOCALE"), + seq=seq, + kind="locale", + ) + ) + seq += 1 + for key, value in parse_feature_string(fields["features"]): + triples.append( + has_triple( + subject=subject, + object_node=_hub_node(ATTR_LABEL, value, extra_label=key), + seq=seq, + kind=f"feat:{sanitize_label(key).lower()}", + ) + ) + seq += 1 + + if not triples: + dataset = fields["dataset"] or "ITEM" + triples.append( + has_triple( + subject=subject, + object_node=_hub_node(TYPE_LABEL, dataset), + seq=seq, + kind="type", + ) + ) + return triples + + +def docs_to_triples(docs: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + triples: list[dict[str, Any]] = [] + for doc in docs: + triples.extend(doc_to_triples(doc)) + return triples + + +def catalog_entity_backfill_rows(docs: Sequence[dict[str, Any]]) -> list[dict[str, str]]: + rows: list[dict[str, str]] = [] + for doc in docs: + doc_id = str(doc.get("doc_id") or "").strip() + if not doc_id: + continue + fields = doc_fields(doc) + rows.append( + { + "uuid": entity_uuid(doc_id), + "name": str(fields.get("title") or doc_id), + "search_text": entity_search_text(doc, fields), + } + ) + return rows + + +def format_happened_at(timestamp: str | None) -> str | None: + if not timestamp: + return None + raw = str(timestamp).strip() + if not raw: + return None + from datetime import datetime + + if len(raw) == 10 and raw[2] == "/" and raw[5] == "/": + return raw + for fmt in ( + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%dT%H:%M:%S.%f%z", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d", + "%m/%d/%Y", + "%d/%m/%Y", + ): + try: + parsed = datetime.strptime( + raw.replace("Z", "+0000") if fmt.endswith("%z") and raw.endswith("Z") else raw, + fmt, + ) + return parsed.strftime("%m/%d/%Y") + except ValueError: + continue + try: + parsed = datetime.fromisoformat(raw.replace("Z", "+00:00")) + return parsed.strftime("%m/%d/%Y") + except ValueError: + return raw + + +def _interaction_event_name(behavior: str) -> str: + key = (behavior or "").strip().lower().replace("-", "_").replace(" ", "_") + if key in _BEHAVIOR_EVENT: + return _BEHAVIOR_EVENT[key] + compact = key.replace("_", "") + for alias, name in _BEHAVIOR_EVENT.items(): + if alias.replace("_", "") == compact: + return name + label = (behavior or "Interaction").strip() or "Interaction" + return label[:1].upper() + label[1:] + + +def _option_pairs(row: dict[str, Any]) -> list[tuple[str, str]]: + pairs: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + for src in (row.get("options"), row.get("attributes")): + if not isinstance(src, dict): + continue + for key, raw in src.items(): + facet = str(key).strip().lower() + if facet not in OPTION_ALLOWLIST or raw is None: + continue + value = str(raw).strip() + if not value: + continue + pair = (facet, value) + if pair in seen: + continue + seen.add(pair) + pairs.append(pair) + return pairs + + +def prefers_triple( + *, + user_id: str, + facet: str, + value: str, + seq: int, + amount: float, +) -> dict[str, Any]: + extra = "CATEGORY" if facet == "category" else facet.upper() + kind = CLASS_LABEL if facet == "category" else ATTR_LABEL + return { + "subject": { + "name": user_id, + "type": USER_TYPE, + "uuid": f"user:{user_id}", + "labels": [USER_TYPE], + }, + "subj_event": { + "name": "PREFERS", + "uuid": rel_uuid(user_id, f"prefers:{facet}:{value}", seq), + "amount": amount, + "properties": {"facet": facet, "value": value, "weight": amount}, + }, + "object": _hub_node(kind, value, extra_label=extra), + } + + +def interaction_to_triples( + row: dict[str, Any] | Sequence[dict[str, Any]], *, seq: int = 1 +) -> list[dict[str, Any]]: + if not isinstance(row, dict): + return interactions_to_triples(row, seq_start=seq) + user_id = str(row.get("user_id") or "").strip() + item_id = str(row.get("item_id") or "").strip() + if not user_id or not item_id: + return [] + behavior = str(row.get("behavior") or "interaction").strip() or "interaction" + event_name = _interaction_event_name(behavior) + happened_at = format_happened_at(row.get("timestamp") or row.get("ts")) + item_name = str(row.get("title") or row.get("item_name") or item_id) + event_node: dict[str, Any] = { + "name": event_name, + "type": EVENT_LABEL, + "uuid": event_uuid(user_id, f"{item_id}:{behavior}", seq), + "labels": [EVENT_LABEL], + } + if happened_at: + event_node["happened_at"] = happened_at + user_node = { + "name": user_id, + "type": USER_TYPE, + "uuid": f"user:{user_id}", + "labels": [USER_TYPE], + } + triples = [ + { + "subject": user_node, + "subj_event": { + "name": "MADE", + "uuid": rel_uuid(user_id, f"made:{item_id}", seq), + }, + "event": event_node, + "event_obj": { + "name": "TARGETED", + "uuid": rel_uuid(user_id, f"tgt:{item_id}", seq), + }, + "object": _entity_node(item_id, item_name), + } + ] + weight = behavior_weight(event_name) + for facet, value in _option_pairs(row): + triples.append( + prefers_triple( + user_id=user_id, + facet=facet, + value=value, + seq=seq, + amount=weight, + ) + ) + catalog_doc = { + "doc_id": item_id, + "title": item_name, + "class": row.get("category") or row.get("class"), + "brand": row.get("brand"), + "color": row.get("color"), + "features": row.get("features"), + "dataset": "interaction", + } + existing = { + str((t.get("event") or {}).get("uuid") or "") + for t in triples + if (t.get("event") or {}).get("uuid") + } + for triple in doc_to_triples(catalog_doc): + event_id = str((triple.get("event") or {}).get("uuid") or "") + if event_id and event_id in existing: + continue + triples.append(triple) + return triples + + +def interactions_to_triples( + rows: Sequence[dict[str, Any]], + *, + seq_start: int = 1, +) -> list[dict[str, Any]]: + triples: list[dict[str, Any]] = [] + for i, row in enumerate(rows): + triples.extend(interaction_to_triples(row, seq=seq_start + i)) + return triples + + +def load_interaction_rows(path: Path) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + text = line.strip() + if not text: + continue + obj = json.loads(text) + if not isinstance(obj, dict): + continue + if obj.get("user_id") and obj.get("item_id"): + rows.append(obj) + return rows + + +def structured_ingest_body( + triples: Sequence[dict[str, Any]], + *, + brain_id: str, + mode: str = "deterministic", +) -> dict[str, Any]: + return { + "mode": mode, + "brain_id": brain_id, + "data": list(triples), + } + + +__all__ = [ + "HAS_EVENT_NAME", + "ENTITY_TYPE", + "CLASS_LABEL", + "TYPE_LABEL", + "ATTR_LABEL", + "EVENT_LABEL", + "USER_TYPE", + "FEATURE_KEY_CAP", + "compose_search_text", + "doc_fields", + "doc_id_from_text", + "doc_to_triples", + "entity_search_text", + "node_id_from_passage_text", + "docs_to_triples", + "catalog_entity_backfill_rows", + "entity_uuid", + "format_happened_at", + "has_triple", + "is_static_has_triple", + "hub_uuid", + "interaction_to_triples", + "prefers_triple", + "interactions_to_triples", + "load_interaction_rows", + "node_embed_text", + "parse_feature_string", + "sanitize_label", + "split_hierarchy", + "structured_ingest_body", +] diff --git a/src/core/search/entity_info.py b/src/core/search/entity_info.py index c459f62..881644b 100644 --- a/src/core/search/entity_info.py +++ b/src/core/search/entity_info.py @@ -56,7 +56,10 @@ def _recency_score(node: Node) -> float: f"on node {node.uuid}, recency left neutral" ) return 1.0 - days_ago = max(0, (datetime.now() - to_naive_utc(happened_at)).days) + days_ago = max( + 0, + (datetime.now().date() - to_naive_utc(happened_at).date()).days, + ) if days_ago <= 0: return 1.0 return 1 / (1 + np.log1p(days_ago)) diff --git a/src/core/search/graph_channels.py b/src/core/search/graph_channels.py new file mode 100644 index 0000000..ea84004 --- /dev/null +++ b/src/core/search/graph_channels.py @@ -0,0 +1,903 @@ +from __future__ import annotations + +import math +import re +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Iterable + +from src.core.search.hybrid import dense_similarity, fuse_passage_lists, passage_snippet +from src.utils.dates import parse_date_string, to_naive_utc + +PASSAGES_CHANNEL = "passages" +ENTITIES_CHANNEL = "entities" +EVENTS_CHANNEL = "events" +COMMUNITIES_CHANNEL = "communities" +NEIGHBORS_CHANNEL = "neighbors" + +CORE_CHANNELS = frozenset( + { + PASSAGES_CHANNEL, + ENTITIES_CHANNEL, + EVENTS_CHANNEL, + COMMUNITIES_CHANNEL, + } +) +GRAPH_CHANNELS = frozenset( + {ENTITIES_CHANNEL, EVENTS_CHANNEL, COMMUNITIES_CHANNEL} +) + +DEFAULT_COMMUNITY_LABELS = ("TYPE", "CLASS", "TOPIC") +DEFAULT_NEIGHBOR_FANOUT = 50 +EVENT_LABEL = "EVENT" +ITEM_LABEL = "ENTITY" +HUB_ID_PREFIX = "hub:" +HUB_LABELS = frozenset({"ATTR", "TYPE", "CLASS", "TOPIC"}) +EVENT_RECENCY_HALF_LIFE_DAYS = 365.0 +_TOKEN_RE = re.compile(r"[A-Za-z0-9]{3,}") +_MAX_NAME_TOKENS = 6 +_TOKEN_STOPWORDS = frozenset( + { + "the", + "and", + "for", + "with", + "from", + "that", + "this", + "are", + "was", + "not", + "without", + "your", + "you", + "any", + "all", + } +) + + +@dataclass +class GraphHit: + id: str + channel: str + score: float + snippet: str + labels: list[str] = field(default_factory=list) + extras: dict[str, Any] | None = None + + +def parse_community_labels(raw: str | None) -> list[str]: + items = [part.strip() for part in str(raw or "").split(",") if part.strip()] + return items or list(DEFAULT_COMMUNITY_LABELS) + + +def selected_graph_channels(channels: list[str] | None) -> list[str]: + selected: list[str] = [] + seen: set[str] = set() + for item in channels or []: + lowered = str(item or "").strip().lower() + if lowered in GRAPH_CHANNELS and lowered not in seen: + selected.append(lowered) + seen.add(lowered) + return selected + + +def _upper_set(values: Iterable[str] | None) -> set[str]: + return {str(item).strip().upper() for item in (values or []) if str(item).strip()} + + +def _node_labels(node: Any) -> list[str]: + raw = getattr(node, "labels", None) or [] + return [str(item) for item in raw if str(item).strip()] + + +def _node_uuid(node: Any) -> str: + return str(getattr(node, "uuid", "") or "").strip() + + +def _node_name(node: Any) -> str: + name = getattr(node, "name", None) + if name: + return str(name) + return _node_uuid(node) + + +def _node_search_text(node: Any) -> str: + props = getattr(node, "properties", None) or {} + search_text = "" + if isinstance(props, dict): + search_text = str(props.get("search_text") or "").strip() + description = str(getattr(node, "description", None) or "").strip() + return search_text or compose_parts(_node_name(node), description) + + +def compose_parts(*parts: str) -> str: + return " ".join(item for item in parts if item) + + +def _hub_kind(labels: Iterable[str] | None) -> str | None: + have = _upper_set(labels) + for kind in ("CLASS", "ATTR", "TYPE", "TOPIC"): + if kind in have: + return kind + return None + + +def _happened_at(node: Any) -> str | None: + raw = getattr(node, "happened_at", None) + if raw: + text = str(raw).strip() + if text: + return text + props = getattr(node, "properties", None) or {} + if isinstance(props, dict): + value = props.get("happened_at") + if value: + text = str(value).strip() + if text: + return text + return None + + +def _labels_match(node_labels: list[str], wanted: list[str] | None) -> bool: + if not wanted: + return True + have = _upper_set(node_labels) + need = _upper_set(wanted) + return bool(have & need) + + +def is_item_entity(uuid: str, labels: Iterable[str] | None = None) -> bool: + node_id = str(uuid or "").strip() + if not node_id or node_id.startswith(HUB_ID_PREFIX): + return False + have = _upper_set(labels) + if EVENT_LABEL in have and ITEM_LABEL not in have: + return False + if have & HUB_LABELS and ITEM_LABEL not in have: + return False + return True + + +def _entity_search_labels(node_labels: list[str] | None) -> list[str]: + wanted = [str(item).strip() for item in (node_labels or []) if str(item).strip()] + return wanted or [ITEM_LABEL] + + +def _name_match_score(name: str, query: str) -> float: + text = (name or "").strip().lower() + needle = (query or "").strip().lower() + if not text: + return 0.0 + score = 0.0 + if needle: + if text == needle: + score += 5.0 + elif needle in text: + score += 3.0 + tokens = _query_tokens(query) + if tokens: + matched = [token for token in tokens if token in text] + score += float(len(matched)) + if matched: + score += len(matched) / len(tokens) + for left, right in zip(tokens, tokens[1:]): + if f"{left} {right}" in text: + score += 0.75 + return score + + +def _search_tokens(query: str) -> list[str]: + query_lower = (query or "").strip().lower() + tokens: list[str] = [] + for token in _query_tokens(query): + if token in _TOKEN_STOPWORDS or token == query_lower: + continue + tokens.append(token) + return tokens + + +def _degree_idf(degree: int) -> float: + return 1.0 / math.log2(2.0 + max(0, int(degree))) + + +def _parse_when(raw: str | None) -> datetime | None: + if not raw: + return None + text = str(raw).strip() + if not text: + return None + candidates = [text] + if text.endswith("Z") and "+0000" not in text: + candidates.append(text[:-1] + "+0000") + for item in candidates: + for fmt in ( + "%Y-%m-%dT%H:%M:%S%z", + "%Y-%m-%dT%H:%M:%SZ", + "%Y-%m-%dT%H:%M:%S", + "%m/%d/%Y", + "%Y-%m-%d", + ): + try: + parsed = datetime.strptime(item, fmt) + return parsed + except ValueError: + continue + return parse_date_string(text) + + +def recency_weight( + happened_at: str | None, + *, + half_life_days: float = EVENT_RECENCY_HALF_LIFE_DAYS, + now: datetime | None = None, +) -> float: + if not happened_at or half_life_days <= 0: + return 1.0 + parsed = _parse_when(happened_at) + if parsed is None: + return 1.0 + ref = now or datetime.now() + age_days = max( + 0.0, + (to_naive_utc(ref) - to_naive_utc(parsed)).total_seconds() / 86400.0, + ) + return float(0.5 ** (age_days / float(half_life_days))) + + +def _query_tokens(query: str) -> list[str]: + seen: set[str] = set() + tokens: list[str] = [] + for match in _TOKEN_RE.finditer(query or ""): + token = match.group(0).lower() + if token in seen: + continue + seen.add(token) + tokens.append(token) + if len(tokens) >= _MAX_NAME_TOKENS: + break + return tokens + + +def _hit_from_node( + node: Any, + *, + channel: str, + score: float, + extras: dict[str, Any] | None = None, +) -> GraphHit | None: + node_id = _node_uuid(node) + if not node_id: + return None + name = _node_name(node) + payload = dict(extras or {}) + happened = _happened_at(node) + if happened and "happened_at" not in payload: + payload["happened_at"] = happened + return GraphHit( + id=node_id, + channel=channel, + score=float(score), + snippet=passage_snippet(name), + labels=_node_labels(node), + extras=payload or None, + ) + + +def _merge_hit(bucket: dict[str, GraphHit], hit: GraphHit | None) -> None: + if hit is None or not hit.id: + return + current = bucket.get(hit.id) + if current is None or hit.score > current.score: + bucket[hit.id] = hit + + +def _ranked_hits(bucket: dict[str, GraphHit], k: int) -> list[GraphHit]: + ordered = sorted(bucket.values(), key=lambda item: (-item.score, item.id)) + if k <= 0: + return ordered + return ordered[:k] + + +def _search_entities( + graph: Any, + *, + brain_id: str, + query_text: str | None, + node_labels: list[str] | None, + limit: int, +) -> list[Any]: + if graph is None or limit <= 0: + return [] + try: + result = graph.search_entities( + brain_id=brain_id, + limit=limit, + skip=0, + node_labels=node_labels, + query_text=query_text, + ) + except Exception: + return [] + raw = getattr(result, "results", None) or [] + if not isinstance(raw, (list, tuple)): + return [] + return list(raw) + + +def _hydrate_nodes(graph: Any, uuids: list[str], brain_id: str) -> dict[str, Any]: + if graph is None or not uuids: + return {} + unique = [] + seen: set[str] = set() + for item in uuids: + key = str(item or "").strip() + if not key or key in seen: + continue + seen.add(key) + unique.append(key) + if not unique: + return {} + try: + nodes = graph.get_by_uuids(unique, brain_id=brain_id) + except Exception: + nodes = [] + for key in unique: + try: + node = graph.get_by_uuid(key, brain_id=brain_id) + except Exception: + continue + if node is not None: + nodes.append(node) + found: dict[str, Any] = {} + for node in nodes or []: + node_id = _node_uuid(node) + if node_id: + found[node_id] = node + return found + + +def collect_entity_hits( + *, + query: str, + brain_id: str, + k: int, + graph: Any, + vector_search: Any = None, + query_vector: list[float] | None = None, + node_labels: list[str] | None = None, + channel: str = ENTITIES_CHANNEL, +) -> list[GraphHit]: + if k <= 0 or graph is None: + return [] + bucket: dict[str, GraphHit] = {} + wanted = _entity_search_labels(node_labels) + restrict_items = not node_labels + fetch_k = max(k * 8, 40) + + def _keep(node_id: str, labels: list[str]) -> bool: + if not _labels_match(labels, wanted): + return False + if restrict_items and not is_item_entity(node_id, labels): + return False + return True + + def _add_node(node: Any, extra: float = 0.0) -> None: + node_id = _node_uuid(node) + labels = _node_labels(node) + if not node_id or not _keep(node_id, labels): + return + score = _name_match_score(_node_search_text(node), query) + extra + _merge_hit( + bucket, + _hit_from_node(node, channel=channel, score=score), + ) + + nodes = _search_entities( + graph, + brain_id=brain_id, + query_text=query, + node_labels=wanted, + limit=fetch_k, + ) + for node in nodes: + _add_node(node) + + for token in _search_tokens(query): + token_nodes = _search_entities( + graph, + brain_id=brain_id, + query_text=token, + node_labels=wanted, + limit=fetch_k, + ) + for node in token_nodes: + _add_node(node) + + bm25_method = getattr(graph, "search_nodes_bm25", None) + if callable(bm25_method): + try: + bm25_hits = bm25_method( + query, + brain_id, + limit=fetch_k, + node_labels=wanted, + ) + except TypeError: + try: + bm25_hits = bm25_method(query, brain_id) + except Exception: + bm25_hits = [] + except Exception: + bm25_hits = [] + for item in bm25_hits or []: + node = item[0] if isinstance(item, (tuple, list)) and item else item + score = 0.0 + if isinstance(item, (tuple, list)) and len(item) >= 2: + try: + score = float(item[1]) + except (TypeError, ValueError): + score = 0.0 + extra = min(3.0, max(0.0, score)) + _add_node(node, extra=extra) + + if vector_search is not None and query_vector: + try: + vectors = vector_search.search_nodes( + query_vector, brain_id=brain_id, k=fetch_k + ) + except Exception: + vectors = [] + ann_ids: list[str] = [] + ann_scores: dict[str, float] = {} + ann_meta: dict[str, dict[str, Any]] = {} + for vector in vectors or []: + meta = getattr(vector, "metadata", None) or {} + node_id = str(meta.get("uuid") or getattr(vector, "id", "") or "").strip() + if not node_id: + continue + ann_ids.append(node_id) + distance = getattr(vector, "distance", None) + ann_scores[node_id] = dense_similarity( + float(distance) if distance is not None else float("inf") + ) + if isinstance(meta, dict): + ann_meta[node_id] = meta + hydrated = _hydrate_nodes(graph, ann_ids, brain_id) + for node_id in ann_ids: + node = hydrated.get(node_id) + labels = _node_labels(node) if node is not None else [] + extra = 0.5 * float(ann_scores.get(node_id, 0.0)) + if node is None: + meta = ann_meta.get(node_id) or {} + raw_labels = meta.get("labels") or [] + if isinstance(raw_labels, str): + raw_labels = [raw_labels] + labels = [str(item) for item in raw_labels if str(item).strip()] + if not labels and meta.get("type"): + labels = [str(meta.get("type"))] + if not _keep(node_id, labels): + continue + name = str(meta.get("name") or node_id) + _merge_hit( + bucket, + GraphHit( + id=node_id, + channel=channel, + score=_name_match_score(name, query) + extra, + snippet=passage_snippet(name), + labels=labels, + extras=None, + ), + ) + continue + _add_node(node, extra=extra) + + return _ranked_hits(bucket, k) + + +def collect_event_hits( + *, + query: str, + brain_id: str, + k: int, + graph: Any, + vector_search: Any = None, + query_vector: list[float] | None = None, +) -> list[GraphHit]: + hits = collect_entity_hits( + query=query, + brain_id=brain_id, + k=k, + graph=graph, + vector_search=vector_search, + query_vector=query_vector, + node_labels=[EVENT_LABEL], + channel=EVENTS_CHANNEL, + ) + scored: list[GraphHit] = [] + for hit in hits: + happened = None + if hit.extras: + happened = hit.extras.get("happened_at") + weight = recency_weight(str(happened) if happened else None) + extras = dict(hit.extras or {}) + if happened: + extras["recency"] = weight + scored.append( + GraphHit( + id=hit.id, + channel=EVENTS_CHANNEL, + score=float(hit.score) * weight, + snippet=hit.snippet, + labels=list(hit.labels or []), + extras=extras or None, + ) + ) + return _ranked_hits({item.id: item for item in scored}, k) + + +def _event_centric_others( + graph: Any, + uuid: str, + brain_id: str, + *, + skip_labels: set[str], + fanout: int, +) -> list[Any]: + found: list[Any] = [] + seen = {uuid} + try: + triples = graph.get_event_centric_neighbors([uuid], brain_id=brain_id) + except Exception: + triples = [] + for triple in triples or []: + if not isinstance(triple, (tuple, list)) or len(triple) < 5: + continue + n, _p1, m, _p2, b = triple[:5] + for node in (n, m, b): + node_id = _node_uuid(node) + if not node_id or node_id in seen: + continue + labels = _upper_set(_node_labels(node)) + if skip_labels and labels & skip_labels: + continue + seen.add(node_id) + found.append(node) + if len(found) >= fanout: + return found + return found + + +def _direct_others( + graph: Any, + uuid: str, + brain_id: str, + *, + skip_labels: set[str], + fanout: int, + skip_event_wrappers: bool, +) -> list[Any]: + found: list[Any] = [] + seen = {uuid} + try: + neighbors = graph.get_neighbors([uuid], brain_id=brain_id) + except Exception: + return found + rows = [] + if isinstance(neighbors, dict): + rows = neighbors.get(uuid) or [] + for entry in rows: + node = entry[1] if isinstance(entry, (tuple, list)) and len(entry) >= 2 else entry + node_id = _node_uuid(node) + if not node_id or node_id in seen: + continue + labels = _upper_set(_node_labels(node)) + if skip_event_wrappers and EVENT_LABEL in labels: + try: + inner = graph.get_neighbors([node_id], brain_id=brain_id) + except Exception: + inner = {} + inner_rows = [] + if isinstance(inner, dict): + inner_rows = inner.get(node_id) or [] + for inner_entry in inner_rows: + tip = ( + inner_entry[1] + if isinstance(inner_entry, (tuple, list)) and len(inner_entry) >= 2 + else inner_entry + ) + tip_id = _node_uuid(tip) + if not tip_id or tip_id in seen: + continue + tip_labels = _upper_set(_node_labels(tip)) + if skip_labels and tip_labels & skip_labels: + continue + if EVENT_LABEL in tip_labels: + continue + seen.add(tip_id) + found.append(tip) + if len(found) >= fanout: + return found + continue + if skip_labels and labels & skip_labels: + continue + seen.add(node_id) + found.append(node) + if len(found) >= fanout: + return found + return found + + +def adjacent_nodes( + graph: Any, + uuid: str, + brain_id: str, + *, + skip_labels: Iterable[str] | None = None, + fanout: int = DEFAULT_NEIGHBOR_FANOUT, + skip_event_wrappers: bool = True, +) -> list[Any]: + if graph is None or not uuid or fanout <= 0: + return [] + skip = _upper_set(skip_labels) + found = _event_centric_others( + graph, uuid, brain_id, skip_labels=skip, fanout=fanout + ) + if len(found) >= fanout: + return found[:fanout] + seen = {uuid, *(_node_uuid(node) for node in found)} + for node in _direct_others( + graph, + uuid, + brain_id, + skip_labels=skip, + fanout=fanout, + skip_event_wrappers=skip_event_wrappers, + ): + node_id = _node_uuid(node) + if not node_id or node_id in seen: + continue + seen.add(node_id) + found.append(node) + if len(found) >= fanout: + break + return found[:fanout] + + +def _member_hits_for_hub( + *, + hub: GraphHit, + graph: Any, + brain_id: str, + query: str, + skip: set[str], + fanout: int, +) -> tuple[list[GraphHit], dict[str, Any]]: + members = adjacent_nodes( + graph, + hub.id, + brain_id, + skip_labels=skip, + fanout=fanout, + skip_event_wrappers=True, + ) + degree = max(len(members), 1) + idf = _degree_idf(degree) + scored: list[GraphHit] = [] + nodes: dict[str, Any] = {} + for member in members: + labels_of = _node_labels(member) + member_id = _node_uuid(member) + if not is_item_entity(member_id, labels_of): + continue + lexical = _name_match_score(_node_search_text(member), query) + hit = _hit_from_node( + member, + channel=COMMUNITIES_CHANNEL, + score=float(hub.score) * idf * (1.0 + lexical), + extras={"hub_id": hub.id, "hub_name": hub.snippet}, + ) + if hit is None: + continue + scored.append(hit) + nodes[member_id] = member + scored.sort(key=lambda item: (-item.score, item.id)) + return scored, nodes + + +def _hybrid_rank_community_hits( + hits: list[GraphHit], + *, + query: str, + brain_id: str, + k: int, + graph: Any, + vector_search: Any = None, + query_vector: list[float] | None = None, +) -> list[GraphHit]: + if not hits: + return [] + by_id = {hit.id: hit for hit in hits} + candidate_ids = list(by_id.keys()) + lexical_ids: list[str] = [] + bm25_method = getattr(graph, "search_nodes_bm25", None) + if callable(bm25_method): + try: + bm25_hits = bm25_method( + query, + brain_id, + limit=max(k * 8, len(candidate_ids)), + node_uuids=candidate_ids, + ) + except Exception: + bm25_hits = [] + for item in bm25_hits or []: + node = item[0] if isinstance(item, (tuple, list)) and item else item + node_id = _node_uuid(node) + if node_id in by_id and node_id not in lexical_ids: + lexical_ids.append(node_id) + dense_ids: list[str] = [] + if vector_search is not None and query_vector: + try: + vectors = vector_search.search_nodes( + query_vector, brain_id=brain_id, k=max(k * 8, len(candidate_ids)) + ) + except Exception: + vectors = [] + for vector in vectors or []: + meta = getattr(vector, "metadata", None) or {} + node_id = str(meta.get("uuid") or getattr(vector, "id", "") or "").strip() + if node_id in by_id and node_id not in dense_ids: + dense_ids.append(node_id) + fused = fuse_passage_lists(dense_ids, lexical_ids) + if not fused: + return _ranked_hits(by_id, k) + bucket: dict[str, GraphHit] = {} + for node_id, score in fused: + hit = by_id.get(node_id) + if hit is None: + continue + bucket[node_id] = GraphHit( + id=hit.id, + channel=hit.channel, + score=float(score), + snippet=hit.snippet, + labels=list(hit.labels or []), + extras=dict(hit.extras or {}) or None, + ) + for hit in hits: + if hit.id not in bucket: + bucket[hit.id] = GraphHit( + id=hit.id, + channel=hit.channel, + score=0.0, + snippet=hit.snippet, + labels=list(hit.labels or []), + extras=dict(hit.extras or {}) or None, + ) + return _ranked_hits(bucket, k) + + +def collect_community_hits( + *, + query: str, + brain_id: str, + k: int, + graph: Any, + vector_search: Any = None, + query_vector: list[float] | None = None, + community_labels: list[str] | None = None, + fanout: int = DEFAULT_NEIGHBOR_FANOUT, +) -> list[GraphHit]: + labels = [item for item in (community_labels or []) if str(item).strip()] + if not labels: + labels = list(DEFAULT_COMMUNITY_LABELS) + hub_labels = list(labels) + if "ATTR" not in _upper_set(hub_labels): + hub_labels.append("ATTR") + hubs = collect_entity_hits( + query=query, + brain_id=brain_id, + k=k, + graph=graph, + vector_search=vector_search, + query_vector=query_vector, + node_labels=hub_labels, + channel=COMMUNITIES_CHANNEL, + ) + skip = _upper_set(hub_labels) | HUB_LABELS | {EVENT_LABEL} + cap = max(1, int(fanout)) + hub_cap = max(1, min(len(hubs), 8)) + kinds = { + kind + for kind in (_hub_kind(hub.labels) for hub in hubs[:hub_cap]) + if kind + } + expand_cap = max(cap, 200) if len(kinds) >= 2 else cap + members_by_kind: dict[str, set[str]] = {} + union_bucket: dict[str, GraphHit] = {} + for hub in hubs[:hub_cap]: + kind = _hub_kind(hub.labels) or "ATTR" + scored, _nodes = _member_hits_for_hub( + hub=hub, + graph=graph, + brain_id=brain_id, + query=query, + skip=skip, + fanout=expand_cap, + ) + kind_ids = members_by_kind.setdefault(kind, set()) + for hit in scored[:expand_cap]: + kind_ids.add(hit.id) + _merge_hit(union_bucket, hit) + kind_sets = [ids for ids in members_by_kind.values() if ids] + selected_ids: set[str] | None = None + if len(kind_sets) >= 2: + selected_ids = set(kind_sets[0]) + for ids in kind_sets[1:]: + selected_ids &= ids + if not selected_ids: + selected_ids = None + if selected_ids is None: + candidates = list(union_bucket.values()) + else: + candidates = [ + hit for hit in union_bucket.values() if hit.id in selected_ids + ] + return _hybrid_rank_community_hits( + candidates, + query=query, + brain_id=brain_id, + k=k, + graph=graph, + vector_search=vector_search, + query_vector=query_vector, + ) + + +def expand_neighbor_hits( + seeds: list[GraphHit], + *, + brain_id: str, + k: int, + graph: Any, + community_labels: list[str] | None = None, + fanout: int = DEFAULT_NEIGHBOR_FANOUT, +) -> list[GraphHit]: + if not seeds or graph is None or k <= 0: + return [] + skip = _upper_set(community_labels) + seed_ids = {hit.id for hit in seeds} + bucket: dict[str, GraphHit] = {} + cap = max(1, int(fanout)) + for seed in seeds: + skip_events = seed.channel != EVENTS_CHANNEL + members = adjacent_nodes( + graph, + seed.id, + brain_id, + skip_labels=skip, + fanout=cap, + skip_event_wrappers=skip_events, + ) + degree = max(len(members), 1) + idf = _degree_idf(degree) + scored: list[GraphHit] = [] + for member in members: + node_id = _node_uuid(member) + if not node_id or node_id in seed_ids: + continue + hit = _hit_from_node( + member, + channel=NEIGHBORS_CHANNEL, + score=float(seed.score) * idf, + extras={"seed_id": seed.id, "seed_channel": seed.channel}, + ) + if hit is not None: + scored.append(hit) + scored.sort(key=lambda item: (-item.score, item.id)) + for hit in scored[:cap]: + _merge_hit(bucket, hit) + return _ranked_hits(bucket, k) diff --git a/src/core/search/hooks.py b/src/core/search/hooks.py new file mode 100644 index 0000000..28dfde6 --- /dev/null +++ b/src/core/search/hooks.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from typing import Any, Callable, Optional + +from src.core.search.graph_channels import CORE_CHANNELS + +RERANK_MAX_K = 10 +CATALOG_RERANK_MAX_K = 50 +CATALOG_RETRIEVE_MIN_K = 50 +CATALOG_RETRIEVE_MAX_K = 200 + + +def retrieve_k_for_mode(mode: str | None, k: int) -> int: + if (mode or "default") == "catalog": + return min(CATALOG_RETRIEVE_MAX_K, max(int(k), CATALOG_RETRIEVE_MIN_K)) + return int(k) + + +def rerank_max_k_for_mode(mode: str | None) -> int: + if (mode or "default") == "catalog": + return CATALOG_RERANK_MAX_K + return RERANK_MAX_K + + +RetrieveFn = Callable[ + [str, str, int], + tuple[list[str], dict[str, float], dict[str, str]], +] +RerankFn = Callable[[str, list[dict[str, Any]], int], list[dict[str, Any]]] + +_retrievers: dict[str, RetrieveFn] = {} +_rerankers: dict[str, RerankFn] = {} + + +class SearchPluginError(ValueError): + pass + + +def reset_search_plugins() -> None: + _retrievers.clear() + _rerankers.clear() + + +def register_search_retriever(name: str, fn: RetrieveFn) -> None: + key = _clean_name(name) + if not key: + raise SearchPluginError("Retriever name is required.") + _retrievers[key] = fn + + +def register_search_reranker(name: str, fn: RerankFn) -> None: + key = _clean_name(name) + if not key: + raise SearchPluginError("Reranker name is required.") + _rerankers[key] = fn + + +def get_search_retriever(name: str) -> Optional[RetrieveFn]: + return _retrievers.get(_clean_name(name)) + + +def get_search_reranker(name: str) -> Optional[RerankFn]: + return _rerankers.get(_clean_name(name)) + + +def listed_retrievers() -> list[str]: + return sorted(_retrievers) + + +def listed_rerankers() -> list[str]: + return sorted(_rerankers) + + +def _clean_name(name: str) -> str: + return str(name or "").strip().lower() + + +def parse_rerank(value: Optional[str]) -> Optional[str]: + raw = (value or "").strip() + if not raw or raw.lower() == "none": + return None + lowered = raw.lower() + if lowered == "linear": + raise SearchPluginError( + "rerank=linear is not implemented. Use none or plugin:." + ) + if not lowered.startswith("plugin:"): + raise SearchPluginError( + f"Invalid rerank={raw!r}. Expected none or plugin:." + ) + name = _clean_name(raw.split(":", 1)[1]) + if not name: + raise SearchPluginError("rerank=plugin: requires a plugin name.") + return name + + +def parse_plugin_channels(channels: Optional[list[str]]) -> list[str]: + names: list[str] = [] + for item in channels or []: + raw = str(item or "").strip() + if not raw: + continue + lowered = raw.lower() + if lowered in CORE_CHANNELS: + continue + if lowered.startswith("plugin:"): + name = _clean_name(raw.split(":", 1)[1]) + if not name: + raise SearchPluginError("channels plugin: requires a plugin name.") + names.append(name) + continue + if get_search_retriever(lowered) is not None: + names.append(lowered) + continue + raise SearchPluginError(f"Unknown search channel {raw!r}.") + return names + + +def resolve_reranker(value: Optional[str]) -> Optional[RerankFn]: + name = parse_rerank(value) + if name is None: + return None + fn = get_search_reranker(name) + if fn is None: + available = ", ".join(listed_rerankers()) or "(none loaded)" + raise SearchPluginError( + f"Unknown search rerank plugin {name!r}. Loaded: {available}." + ) + return fn + + +def resolve_retrievers(channels: Optional[list[str]]) -> list[tuple[str, RetrieveFn]]: + resolved: list[tuple[str, RetrieveFn]] = [] + for name in parse_plugin_channels(channels): + fn = get_search_retriever(name) + if fn is None: + available = ", ".join(listed_retrievers()) or "(none loaded)" + raise SearchPluginError( + f"Unknown search retriever plugin {name!r}. Loaded: {available}." + ) + resolved.append((name, fn)) + return resolved diff --git a/src/core/search/hybrid.py b/src/core/search/hybrid.py new file mode 100644 index 0000000..005a852 --- /dev/null +++ b/src/core/search/hybrid.py @@ -0,0 +1,339 @@ +import re +from typing import Any, Optional + +from src.core.search.fact_filter import reciprocal_rank_fusion + +_TOKEN_RE = re.compile(r"[^\W_]+", re.UNICODE) +_LITERAL_STOP = frozenset( + { + "the", + "and", + "for", + "with", + "from", + "this", + "that", + "are", + "was", + } +) + +_EXTRAS_SKIP = frozenset( + {"resource_id", "uuid", "id", "inserted_at", "brain_version"} +) + + +def passage_snippet(text: str, max_len: int = 240) -> str: + body = (text or "").strip() + if len(body) <= max_len: + return body + return body[: max_len - 1].rstrip() + "…" + + +def dense_similarity(distance: float) -> float: + return 1.0 - float(distance) + + +def extras_from_metadata(meta: Any) -> dict[str, str] | None: + if not isinstance(meta, dict) or not meta: + return None + out: dict[str, str] = {} + for key, value in meta.items(): + name = str(key or "").strip() + if not name or name.lower() in _EXTRAS_SKIP: + continue + if value is None or isinstance(value, (dict, list, tuple, set)): + continue + text = str(value).strip() + if not text: + continue + out[name] = text + return out or None + + +def merge_hit_extras(*parts: dict[str, Any] | None) -> dict[str, Any] | None: + merged: dict[str, Any] = {} + for part in parts: + if not part: + continue + for key, value in part.items(): + if value is None: + continue + merged[str(key)] = value + return merged or None + + +def hit_matches_extras( + have: dict[str, Any] | None, + wanted: dict[str, str] | None, +) -> bool: + if not wanted: + return True + lookup = { + str(key).strip().lower(): str(value).strip().lower() + for key, value in (have or {}).items() + if value is not None + } + for key, value in wanted.items(): + if lookup.get(str(key).strip().lower()) != str(value).strip().lower(): + return False + return True + + +def facet_counts_from_extras( + extras_list: list[dict[str, Any] | None], +) -> dict[str, dict[str, int]] | None: + buckets: dict[str, dict[str, int]] = {} + for extras in extras_list: + if not extras: + continue + for key, value in extras.items(): + name = str(key or "").strip() + if not name or name.lower() in _EXTRAS_SKIP: + continue + if value is None or isinstance(value, (dict, list, tuple, set)): + continue + text = str(value).strip() + if not text: + continue + inner = buckets.setdefault(name, {}) + inner[text] = inner.get(text, 0) + 1 + return buckets or None + + +def collect_dense_passages( + vector_search: Any, + query_vector: list[float], + brain_id: str, + k: int, + extras_out: Optional[dict[str, dict[str, str]]] = None, +) -> tuple[list[str], dict[str, float]]: + if not query_vector or k <= 0: + return [], {} + hits = vector_search.search_data( + query_vector, brain_id=brain_id, k=k + ) + ids: list[str] = [] + distances: dict[str, float] = {} + for vector in hits: + meta = vector.metadata or {} + resource_id = meta.get("resource_id") or getattr(vector, "id", None) + if not resource_id: + continue + resource_id = str(resource_id) + ids.append(resource_id) + distances[resource_id] = ( + float(vector.distance) if vector.distance is not None else float("inf") + ) + if extras_out is not None: + parsed = extras_from_metadata(meta) + if parsed: + extras_out[resource_id] = parsed + return ids, distances + + +def collect_bm25_passages( + data_adapter: Any, + query: str, + brain_id: str, + k: int, + extras_out: Optional[dict[str, dict[str, str]]] = None, +) -> tuple[list[str], dict[str, float], dict[str, str]]: + if not query or k <= 0: + return [], {}, {} + ranked = data_adapter.search_bm25(query, brain_id, limit=k) + ids: list[str] = [] + scores: dict[str, float] = {} + texts: dict[str, str] = {} + for chunk, score in ranked or []: + chunk_id = str(getattr(chunk, "id", "") or "") + if not chunk_id: + continue + ids.append(chunk_id) + scores[chunk_id] = float(score) + texts[chunk_id] = getattr(chunk, "text", "") or "" + if extras_out is not None: + parsed = extras_from_metadata(getattr(chunk, "metadata", None)) + if parsed: + extras_out[chunk_id] = parsed + return ids, scores, texts + + +def collect_ilike_passages( + data_adapter: Any, + query: str, + brain_id: str, +) -> tuple[list[str], dict[str, str]]: + ids: list[str] = [] + texts: dict[str, str] = {} + search_result = data_adapter.search(query, brain_id) + for chunk in getattr(search_result, "text_chunks", None) or []: + chunk_id = str(getattr(chunk, "id", "") or "") + if not chunk_id: + continue + ids.append(chunk_id) + texts[chunk_id] = getattr(chunk, "text", "") or "" + return ids, texts + + +def query_tokens(text: str) -> list[str]: + found: list[str] = [] + seen: set[str] = set() + for raw in _TOKEN_RE.findall(text or ""): + token = raw.lower() + if len(token) < 3 or token in _LITERAL_STOP or token in seen: + continue + seen.add(token) + found.append(token) + return found + + +def literal_overlap_ids( + query: str, + texts_by_id: dict[str, str], + *, + k: int = 50, +) -> list[str]: + qtoks = set(query_tokens(query)) + if not qtoks or k <= 0: + return [] + scored: list[tuple[int, str]] = [] + for doc_id, body in texts_by_id.items(): + key = str(doc_id or "") + if not key: + continue + overlap = len(qtoks & set(query_tokens(body or ""))) + if overlap <= 0: + continue + scored.append((overlap, key)) + scored.sort(key=lambda item: (-item[0], item[1])) + return [doc_id for _, doc_id in scored[:k]] + + +def collect_literal_residual( + data_adapter: Any, + query: str, + brain_id: str, + k: int, +) -> tuple[list[str], dict[str, str]]: + getter = getattr(data_adapter, "get_text_chunks", None) + if getter is None or not query or k <= 0: + return [], {} + texts: dict[str, str] = {} + for token in query_tokens(query): + try: + chunks, _total = getter(brain_id, limit=k, query_text=token) + except TypeError: + try: + chunks, _total = getter(brain_id, k, 0, token) + except Exception: + continue + except Exception: + continue + for chunk in chunks or []: + chunk_id = str(getattr(chunk, "id", "") or "") + if not chunk_id: + continue + texts[chunk_id] = getattr(chunk, "text", "") or "" + return literal_overlap_ids(query, texts, k=k), texts + + +def frozen_head_merge( + ranked_ids: list[str], + extra_id_lists: Optional[list[list[str]]] = None, + *, + head_k: int = 10, + k: int = 50, + prefer_ids: Optional[set[str]] = None, +) -> list[str]: + ranked = [str(item) for item in ranked_ids if item] + head = ranked[: max(int(head_k), 0)] + ranked_k = ranked[: max(int(k), 0)] + passages_k = set(ranked_k) + extras: list[str] = [] + seen_extra: set[str] = set() + for extra in extra_id_lists or []: + for hid in extra[: max(int(k), 0)]: + key = str(hid) + if not key or key in seen_extra or key in passages_k: + continue + if prefer_ids is not None and key not in prefer_ids: + continue + seen_extra.add(key) + extras.append(key) + rest = [item for item in ranked_k[len(head) :] if item not in set(head)] + if prefer_ids is not None: + fill = ( + extras + + [item for item in rest if item in prefer_ids] + + [item for item in rest if item not in prefer_ids] + ) + else: + fill = extras + rest + tail: list[str] = [] + seen = set(head) + limit = max(int(k), 0) + for hid in fill: + if hid in seen: + continue + seen.add(hid) + tail.append(hid) + if len(head) + len(tail) >= limit: + break + return (head + tail)[:limit] + + +def _minmax(values: dict[str, float]) -> dict[str, float]: + if not values: + return {} + lo = min(values.values()) + hi = max(values.values()) + if hi <= lo: + return {key: 1.0 for key in values} + scale = hi - lo + return {key: (value - lo) / scale for key, value in values.items()} + + +def convex_combination( + dense_scores: dict[str, float], + bm25_scores: dict[str, float], + alpha: float, +) -> list[tuple[str, float]]: + alpha = min(1.0, max(0.0, float(alpha))) + dense_n = _minmax(dense_scores) + bm25_n = _minmax(bm25_scores) + ids = set(dense_n) | set(bm25_n) + fused = { + item: alpha * dense_n.get(item, 0.0) + (1.0 - alpha) * bm25_n.get(item, 0.0) + for item in ids + } + return sorted(fused.items(), key=lambda pair: pair[1], reverse=True) + + +def fuse_passage_lists( + dense_ids: list[str], + lexical_ids: list[str], + *, + fusion: str = "rrf", + alpha: float = 0.5, + dense_similarities: Optional[dict[str, float]] = None, + bm25_scores: Optional[dict[str, float]] = None, + extra_id_lists: Optional[list[list[str]]] = None, +) -> list[tuple[str, float]]: + extra = [ids for ids in (extra_id_lists or []) if ids] + lists = [ids for ids in (dense_ids, lexical_ids) if ids] + if fusion == "cc" and (dense_ids or lexical_ids): + core = convex_combination( + dense_similarities or {}, + bm25_scores or {}, + alpha, + ) + if not extra: + return core + return reciprocal_rank_fusion([[item for item, _ in core], *extra]) + lists.extend(extra) + if not lists: + return [] + if len(lists) == 1: + return [(item, 1.0 / (rank + 1)) for rank, item in enumerate(lists[0])] + return reciprocal_rank_fusion(lists) diff --git a/src/core/search/personalize.py b/src/core/search/personalize.py new file mode 100644 index 0000000..210d82a --- /dev/null +++ b/src/core/search/personalize.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +import re +from collections import defaultdict +from typing import Dict, Iterable, List, Optional + +from src.constants.kg import Node +from src.core.search.entity_sibilings import _predicate_currently_valid +from src.core.search.recommend import ( + _is_attr_node, + _prefers_weight, + behavior_weight, + recency_decay, +) +from src.services.kg_agent.main import graph_adapter + +SHORT_TERM_HALF_LIFE_DAYS = 14.0 +ITEM_LABELS = frozenset({"ENTITY", "PRODUCT"}) +PREF_PREDICATES = frozenset({"PREFERS", "PREFER", "LIKES"}) +_TOKEN_RE = re.compile(r"[A-Za-z0-9]+") +_STOP = frozenset( + { + "the", + "and", + "for", + "with", + "from", + "this", + "that", + "are", + "was", + "a", + "an", + "of", + "to", + "in", + "on", + } +) + + +def query_personalize_lambda(query: str) -> float: + tokens = [ + token.lower() + for token in _TOKEN_RE.findall(query or "") + if token.lower() not in _STOP + ] + if not tokens: + return 0.0 + if any(any(char.isdigit() for char in token) for token in tokens): + return 0.0 + count = len(tokens) + if count == 1: + return 0.85 + if count == 2: + return 0.5 + if count == 3: + return 0.25 + return 0.1 + + +def _minmax(values: Dict[str, float]) -> Dict[str, float]: + if not values: + return {} + lo = min(values.values()) + hi = max(values.values()) + if hi <= lo: + return {key: 0.0 for key in values} + span = hi - lo + return {key: (value - lo) / span for key, value in values.items()} + + +def blend_ranked( + ids: List[str], + retrieve_scores: Dict[str, float], + pref_scores: Dict[str, float], + lam: float, +) -> tuple[list[str], dict[str, float]]: + ordered_ids = list(ids) + if not ordered_ids: + return [], {} + lam = max(0.0, min(1.0, float(lam))) + retrieve_out = { + item_id: float(retrieve_scores.get(item_id, 0.0)) for item_id in ordered_ids + } + if lam <= 0.0 or not any( + float(pref_scores.get(item_id, 0.0)) for item_id in ordered_ids + ): + return ordered_ids, retrieve_out + retrieve_norm = _minmax(retrieve_out) + pref_norm = _minmax( + {item_id: float(pref_scores.get(item_id, 0.0)) for item_id in ordered_ids} + ) + blended = { + item_id: (1.0 - lam) * retrieve_norm[item_id] + lam * pref_norm[item_id] + for item_id in ordered_ids + } + ranked = sorted( + ordered_ids, + key=lambda item_id: (-blended[item_id], ordered_ids.index(item_id)), + ) + return ranked, blended + + +def _is_item_node(node: Node) -> bool: + labels = {str(label).upper() for label in (node.labels or [])} + return bool(labels & ITEM_LABELS) + + +def _neighbors(uuid: str, brain_id: str, of_types: Optional[list[str]] = None): + try: + payload = graph_adapter.get_neighbors( + [uuid], of_types=of_types, brain_id=brain_id + ) + except TypeError: + payload = graph_adapter.get_neighbors([uuid], brain_id=brain_id) + except Exception: + return [] + return payload.get(uuid, []) if isinstance(payload, dict) else [] + + +def _attr_nodes_from_item(item: Node, brain_id: str) -> List[Node]: + attrs: List[Node] = [] + for pred, neighbor in _neighbors(item.uuid, brain_id): + if not _predicate_currently_valid(pred): + continue + if _is_attr_node(neighbor): + attrs.append(neighbor) + continue + if "EVENT" in {str(label).upper() for label in (neighbor.labels or [])}: + for pred2, tip in _neighbors(neighbor.uuid, brain_id): + if not _predicate_currently_valid(pred2): + continue + if _is_attr_node(tip): + attrs.append(tip) + return attrs + + +def resolve_user_node(target: str, brain_id: str) -> Optional[Node]: + text = (target or "").strip() + if not text: + return None + candidates = [text] + if not text.startswith("user:"): + candidates.append(f"user:{text}") + try: + nodes = graph_adapter.get_by_uuids(candidates, brain_id=brain_id) or [] + except Exception: + nodes = [] + for node in nodes: + if node is not None and getattr(node, "uuid", None): + return node + for uuid in candidates: + try: + node = graph_adapter.get_by_uuid(uuid, brain_id=brain_id) + except Exception: + node = None + if node is not None and getattr(node, "uuid", None): + return node + return None + + +def user_pref_weights( + seed: Node | None, + brain_id: str, + *, + short_half_life_days: float = SHORT_TERM_HALF_LIFE_DAYS, + behavior_weights: Optional[Dict[str, float]] = None, +) -> Dict[str, float]: + scores: Dict[str, float] = defaultdict(float) + if seed is None or not getattr(seed, "uuid", None): + return {} + for pred, neighbor in _neighbors(seed.uuid, brain_id): + if not _predicate_currently_valid(pred): + continue + pred_name = (pred.name or "").upper() + if pred_name in PREF_PREDICATES or "PREFER" in pred_name or "LIKE" in pred_name: + if _is_attr_node(neighbor): + scores[neighbor.uuid] += _prefers_weight(pred) + for pred, event_node in _neighbors(seed.uuid, brain_id, of_types=["EVENT"]): + if not _predicate_currently_valid(pred): + continue + if "EVENT" not in {str(label).upper() for label in (event_node.labels or [])}: + continue + weight = behavior_weight(event_node.name, behavior_weights) + decay = recency_decay(event_node.happened_at, short_half_life_days) + for pred2, tip in _neighbors(event_node.uuid, brain_id): + if not _predicate_currently_valid(pred2): + continue + if tip.uuid == seed.uuid: + continue + if not _is_item_node(tip): + continue + contrib = float(weight) * float(decay) + for attr in _attr_nodes_from_item(tip, brain_id): + scores[attr.uuid] += contrib + return {uuid: float(value) for uuid, value in scores.items() if value} + + +def score_nodes_for_user( + node_ids: Iterable[str], + prefs: Dict[str, float], + brain_id: str, +) -> Dict[str, float]: + if not prefs: + return {str(node_id): 0.0 for node_id in node_ids if node_id} + out: Dict[str, float] = {} + for node_id in node_ids: + key = str(node_id or "") + if not key: + continue + total = 0.0 + try: + node = graph_adapter.get_by_uuid(key, brain_id=brain_id) + except Exception: + node = None + if node is None: + out[key] = 0.0 + continue + seen: set[str] = set() + for attr in _attr_nodes_from_item(node, brain_id): + if attr.uuid in seen: + continue + seen.add(attr.uuid) + total += float(prefs.get(attr.uuid, 0.0)) + out[key] = total + return out + + +def personalize_ranked_ids( + *, + query: str, + ranked_ids: List[str], + retrieve_scores: Dict[str, float], + node_id_by_hit: Dict[str, Optional[str]], + target: Optional[str], + brain_id: str, +) -> tuple[list[str], dict[str, float]]: + ids = list(ranked_ids) + if not target or not ids: + return ids, {} + seed = resolve_user_node(target, brain_id) + if seed is None: + return ids, {} + prefs = user_pref_weights(seed, brain_id) + if not prefs: + return ids, {} + unique_nodes = [] + seen_nodes: set[str] = set() + for hit_id in ids: + node_id = node_id_by_hit.get(hit_id) + if not node_id or node_id in seen_nodes: + continue + seen_nodes.add(node_id) + unique_nodes.append(node_id) + node_prefs = score_nodes_for_user(unique_nodes, prefs, brain_id) + pref_by_hit = { + hit_id: float(node_prefs.get(node_id_by_hit.get(hit_id) or "", 0.0)) + for hit_id in ids + } + lam = query_personalize_lambda(query) + return blend_ranked(ids, retrieve_scores, pref_by_hit, lam) + + +__all__ = [ + "SHORT_TERM_HALF_LIFE_DAYS", + "blend_ranked", + "personalize_ranked_ids", + "query_personalize_lambda", + "resolve_user_node", + "score_nodes_for_user", + "user_pref_weights", +] diff --git a/src/core/search/recommend.py b/src/core/search/recommend.py index 80a0e27..5197f7d 100644 --- a/src/core/search/recommend.py +++ b/src/core/search/recommend.py @@ -14,6 +14,8 @@ from src.utils.dates import parse_date_string, to_naive_utc from src.services.kg_agent.main import graph_adapter +UNKNOWN_BEHAVIOR_WEIGHT = 0.2 + DEFAULT_BEHAVIOR_WEIGHTS: Dict[str, float] = { "view": 0.2, "click": 0.2, @@ -24,6 +26,19 @@ "buy": 1.0, "purchased": 1.0, "bought": 1.0, + "favorite": 0.7, + "favourite": 0.7, + "favorites": 0.7, + "favourites": 0.7, + "wishlist": 0.7, + "add_to_favorite": 0.7, + "addtofavorite": 0.7, + "add_to_favourite": 0.7, + "addtofavourite": 0.7, + "add_to_wishlist": 0.7, + "addtowishlist": 0.7, + "follow": 0.5, + "flw": 0.5, } ATTR_LABELS = frozenset( @@ -46,7 +61,7 @@ def behavior_weight( compact = alias.replace("-", "").replace("_", "").replace(" ", "") if compact == key: return float(value) - return 1.0 + return UNKNOWN_BEHAVIOR_WEIGHT def recency_decay( diff --git a/src/lib/postgresql/data.py b/src/lib/postgresql/data.py index 93affcd..48e83ca 100644 --- a/src/lib/postgresql/data.py +++ b/src/lib/postgresql/data.py @@ -16,7 +16,7 @@ import psycopg2.extras from src.adapters.interfaces.data import DataClient, SearchResult -from src.config import config +from src.config import SEARCH_FTS_REGCONFIGS, config from src.constants.data import ( Brain, KGChanges, @@ -100,6 +100,126 @@ ON data_kg_changes (timestamp DESC); """ +_SEARCH_DDL = """ +ALTER TABLE data_text_chunks + ADD COLUMN IF NOT EXISTS search_tsv tsvector + GENERATED ALWAYS AS (to_tsvector('english', coalesce(text, ''))) STORED; +ALTER TABLE data_text_chunks + ADD COLUMN IF NOT EXISTS search_len integer + GENERATED ALWAYS AS ( + GREATEST(length(to_tsvector('english', coalesce(text, ''))), 0) + ) STORED; +CREATE INDEX IF NOT EXISTS idx_data_text_chunks_search_tsv + ON data_text_chunks USING gin (search_tsv); +""" + + +def _search_alt_ddl(regconfig: str) -> str: + if regconfig not in SEARCH_FTS_REGCONFIGS: + raise ValueError(f"Unsupported FTS regconfig {regconfig!r}") + return f""" +ALTER TABLE data_text_chunks + ADD COLUMN IF NOT EXISTS search_tsv_alt tsvector + GENERATED ALWAYS AS (to_tsvector('{regconfig}', coalesce(text, ''))) STORED; +ALTER TABLE data_text_chunks + ADD COLUMN IF NOT EXISTS search_len_alt integer + GENERATED ALWAYS AS ( + GREATEST(length(to_tsvector('{regconfig}', coalesce(text, ''))), 0) + ) STORED; +CREATE INDEX IF NOT EXISTS idx_data_text_chunks_search_tsv_alt + ON data_text_chunks USING gin (search_tsv_alt); +""" + + +def _bm25_sql( + regconfig: str, + tsv_col: str, + len_col: str, + *, + query_match: str | None = None, +) -> str: + if tsv_col not in {"search_tsv", "search_tsv_alt"}: + raise ValueError(f"Unsupported tsv column {tsv_col!r}") + if len_col not in {"search_len", "search_len_alt"}: + raise ValueError(f"Unsupported len column {len_col!r}") + if regconfig not in SEARCH_FTS_REGCONFIGS and regconfig != "english": + raise ValueError(f"Unsupported FTS regconfig {regconfig!r}") + if query_match is None: + query_match = "or" if tsv_col == "search_tsv_alt" else "and" + if query_match not in {"and", "or"}: + raise ValueError(f"Unsupported BM25 query_match {query_match!r}") + if query_match == "and": + match_sql = f"c.{tsv_col} @@ plainto_tsquery('{regconfig}', %s)" + else: + match_sql = "TRUE" + return f""" + WITH query_lexemes AS ( + SELECT unnest(tsvector_to_array(to_tsvector('{regconfig}', %s))) + AS lexeme + ), + coll AS ( + SELECT + COUNT(*)::double precision AS n_docs, + COALESCE( + AVG(GREATEST({len_col}, 1)), + 1 + )::double precision AS avgdl + FROM data_text_chunks + ), + idf AS ( + SELECT + s.word AS lexeme, + ln( + 1.0 + (coll.n_docs - s.ndoc + 0.5) + / (s.ndoc + 0.5) + ) AS idf + FROM ts_stat( + 'SELECT {tsv_col} FROM data_text_chunks' + ) AS s + CROSS JOIN coll + WHERE s.word IN (SELECT lexeme FROM query_lexemes) + ), + docs AS ( + SELECT + c.id, + c.document, + c.text, + GREATEST(c.{len_col}, 1)::double precision AS dl, + lex.lexeme, + COALESCE( + array_length(lex.positions, 1), + 1 + )::double precision AS tf + FROM data_text_chunks c + CROSS JOIN LATERAL unnest(c.{tsv_col}) + AS lex(lexeme, positions) + WHERE {match_sql} + AND lex.lexeme IN (SELECT lexeme FROM query_lexemes) + ) + SELECT + d.id, + d.document, + d.text, + SUM( + COALESCE(i.idf, 0) + * (d.tf * (%s + 1.0)) + / ( + d.tf + + %s * ( + 1.0 - %s + + %s * d.dl + / NULLIF(c.avgdl, 0) + ) + ) + ) AS bm25 + FROM docs d + CROSS JOIN coll c + LEFT JOIN idf i ON i.lexeme = d.lexeme + GROUP BY d.id, d.document, d.text + ORDER BY bm25 DESC, d.id ASC + LIMIT %s + """ + def _ilike_pattern(query_text: str) -> str: escaped = query_text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") @@ -130,6 +250,8 @@ def __init__(self): self._lock = threading.Lock() self._system_initialized = False self._initialized_brains: set[str] = set() + self._search_ready_brains: set[str] = set() + self._search_alt_brains: set[str] = set() @contextmanager def _system_connection(self) -> Iterator[psycopg2.extensions.connection]: @@ -159,17 +281,47 @@ def _ensure_system_schema(self) -> None: self._system_initialized = True def _ensure_brain_schema(self, brain_id: str) -> None: - if brain_id in self._initialized_brains: + alt_regconfig = config.search_fts_regconfig_for_brain(brain_id) + needs_base = brain_id not in self._initialized_brains + needs_search = ( + config.search_enabled and brain_id not in self._search_ready_brains + ) + needs_alt = ( + config.search_enabled + and alt_regconfig is not None + and brain_id not in self._search_alt_brains + ) + if not needs_base and not needs_search and not needs_alt: return with self._lock: - if brain_id in self._initialized_brains: + alt_regconfig = config.search_fts_regconfig_for_brain(brain_id) + needs_base = brain_id not in self._initialized_brains + needs_search = ( + config.search_enabled and brain_id not in self._search_ready_brains + ) + needs_alt = ( + config.search_enabled + and alt_regconfig is not None + and brain_id not in self._search_alt_brains + ) + if not needs_base and not needs_search and not needs_alt: return ensure_brain_database(brain_id) with borrow(get_brain_pool(brain_id)) as conn: with conn.cursor() as cur: - cur.execute(_BRAIN_DDL) + if needs_base: + cur.execute(_BRAIN_DDL) + if needs_search: + cur.execute(_SEARCH_DDL) + if needs_alt and alt_regconfig is not None: + cur.execute(_search_alt_ddl(alt_regconfig)) conn.commit() - self._initialized_brains.add(brain_id) + if needs_base: + self._initialized_brains.add(brain_id) + if needs_search: + self._search_ready_brains.add(brain_id) + if needs_alt: + self._search_alt_brains.add(brain_id) def save_text_chunk(self, text_chunk: TextChunk, brain_id: str) -> TextChunk: document = text_chunk.model_dump(mode="json") @@ -279,6 +431,39 @@ def search( ] return SearchResult(text_chunks=text_chunks, observations=observations) + def search_bm25( + self, + text: str, + brain_id: str, + collection: str = "text_chunks", + limit: int = 10, + ) -> List[Tuple[TextChunk, float]]: + if not text or not text.strip() or limit <= 0: + return [] + if collection not in ("*", "text_chunks"): + return [] + k1 = float(config.search_bm25_k1) + b = float(config.search_bm25_b) + alt_regconfig = config.search_fts_regconfig_for_brain(brain_id) + if alt_regconfig: + sql = _bm25_sql(alt_regconfig, "search_tsv_alt", "search_len_alt") + params = (text, k1, k1, b, b, limit) + else: + sql = _bm25_sql("english", "search_tsv", "search_len") + params = (text, text, k1, k1, b, b, limit) + with self._brain_connection(brain_id) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + sql, + params, + ) + rows = cur.fetchall() + results: List[Tuple[TextChunk, float]] = [] + for row in rows: + chunk = TextChunk.model_validate(row["document"]) + results.append((chunk, float(row["bm25"] or 0.0))) + return results + def get_text_chunks_by_ids( self, ids: List[str], with_observations: bool = False, brain_id: str = "default" ) -> Tuple[List[TextChunk], List[Observation]]: diff --git a/src/lib/postgresql/graph_store.py b/src/lib/postgresql/graph_store.py index 0e7b88d..b0fdabf 100644 --- a/src/lib/postgresql/graph_store.py +++ b/src/lib/postgresql/graph_store.py @@ -194,25 +194,64 @@ class PostgreSQLGraphStore: ON kg_topic_sessions(session_id); """ + _SEARCH_DOCUMENT_SQL = ( + "coalesce(data->>'name', '') || ' ' || " + "coalesce(data->>'description', '') || ' ' || " + "coalesce(data->>'search_text', '')" + ) + _SEARCH_DDL = f""" + ALTER TABLE kg_nodes + ADD COLUMN IF NOT EXISTS search_tsv tsvector + GENERATED ALWAYS AS ( + to_tsvector('english'::regconfig, {_SEARCH_DOCUMENT_SQL}) + ) STORED; + ALTER TABLE kg_nodes + ADD COLUMN IF NOT EXISTS search_len integer + GENERATED ALWAYS AS ( + GREATEST( + length(to_tsvector('english'::regconfig, {_SEARCH_DOCUMENT_SQL})), + 0 + ) + ) STORED; + CREATE INDEX IF NOT EXISTS idx_kg_nodes_search_tsv + ON kg_nodes USING gin (search_tsv); + """ + def __init__(self) -> None: config.postgresql.validate_credentials() self._brains: dict[str, _BrainGraph] = {} self._schema_ready: set[str] = set() + self._search_ready_brains: set[str] = set() self._schema_lock = threading.Lock() self._brains_lock = threading.RLock() def _ensure_brain_schema(self, brain_id: str) -> None: - if brain_id in self._schema_ready: + needs_base = brain_id not in self._schema_ready + needs_search = ( + bool(config.search_enabled) and brain_id not in self._search_ready_brains + ) + if not needs_base and not needs_search: return with self._schema_lock: - if brain_id in self._schema_ready: + needs_base = brain_id not in self._schema_ready + needs_search = ( + bool(config.search_enabled) + and brain_id not in self._search_ready_brains + ) + if not needs_base and not needs_search: return ensure_brain_database(brain_id) with borrow(get_brain_pool(brain_id)) as conn: - with conn.cursor() as cur: - cur.execute(self._DDL) - conn.commit() - self._schema_ready.add(brain_id) + if needs_base: + with conn.cursor() as cur: + cur.execute(self._DDL) + conn.commit() + self._schema_ready.add(brain_id) + if needs_search: + with conn.cursor() as cur: + cur.execute(self._SEARCH_DDL) + conn.commit() + self._search_ready_brains.add(brain_id) @contextmanager def _connection(self, brain_id: str): @@ -223,6 +262,122 @@ def _connection(self, brain_id: str): def _ensure_brain_row(self, brain_id: str) -> None: self._ensure_brain_schema(brain_id) + def search_nodes_bm25( + self, + text: str, + brain_id: str, + *, + limit: int = 10, + node_labels: list[str] | None = None, + node_uuids: list[str] | None = None, + ) -> list[tuple[str, float, dict]]: + if not config.search_enabled or not text or not str(text).strip() or limit <= 0: + return [] + self._ensure_brain_schema(brain_id) + k1 = float(config.search_bm25_k1) + b = float(config.search_bm25_b) + labels = [ + str(item).strip().upper() + for item in (node_labels or []) + if str(item).strip() + ] + uuids = [str(item).strip() for item in (node_uuids or []) if str(item).strip()] + sql = """ + WITH query_lexemes AS ( + SELECT unnest(tsvector_to_array(to_tsvector('english', %s))) + AS lexeme + ), + coll AS ( + SELECT + COUNT(*)::double precision AS n_docs, + COALESCE(AVG(GREATEST(search_len, 1)), 1)::double precision AS avgdl + FROM kg_nodes + ), + idf AS ( + SELECT + s.word AS lexeme, + ln( + 1.0 + (coll.n_docs - s.ndoc + 0.5) + / (s.ndoc + 0.5) + ) AS idf + FROM ts_stat('SELECT search_tsv FROM kg_nodes') AS s + CROSS JOIN coll + WHERE s.word IN (SELECT lexeme FROM query_lexemes) + ), + docs AS ( + SELECT + n.uuid, + n.data, + GREATEST(n.search_len, 1)::double precision AS dl, + lex.lexeme, + COALESCE(array_length(lex.positions, 1), 1)::double precision AS tf + FROM kg_nodes n + CROSS JOIN LATERAL unnest(n.search_tsv) AS lex(lexeme, positions) + WHERE n.search_tsv @@ plainto_tsquery('english', %s) + AND lex.lexeme IN (SELECT lexeme FROM query_lexemes) + AND (%s::text[] IS NULL OR n.uuid = ANY(%s)) + AND ( + %s::text[] IS NULL + OR EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text( + coalesce(n.data->'labels', '[]'::jsonb) + ) AS lbl + WHERE upper(lbl) = ANY(%s) + ) + ) + ) + SELECT + d.uuid, + d.data, + SUM( + COALESCE(i.idf, 0) + * (d.tf * (%s + 1.0)) + / ( + d.tf + + %s * ( + 1.0 - %s + + %s * d.dl / NULLIF(c.avgdl, 0) + ) + ) + ) AS bm25 + FROM docs d + CROSS JOIN coll c + LEFT JOIN idf i ON i.lexeme = d.lexeme + GROUP BY d.uuid, d.data + ORDER BY bm25 DESC, d.uuid ASC + LIMIT %s + """ + label_param = labels or None + uuid_param = uuids or None + params = ( + text, + text, + uuid_param, + uuid_param, + label_param, + label_param, + k1, + k1, + b, + b, + limit, + ) + with self._connection(brain_id) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute(sql, params) + rows = cur.fetchall() + out: list[tuple[str, float, dict]] = [] + for row in rows: + node_id = str(row.get("uuid") or "").strip() + if not node_id: + continue + data = row.get("data") or {} + if not isinstance(data, dict): + data = {} + out.append((node_id, float(row.get("bm25") or 0.0), data)) + return out + def get_brain(self, brain_id: str) -> _BrainGraph: return self._load_brain(brain_id) diff --git a/src/lib/postgresql/networkx_client.py b/src/lib/postgresql/networkx_client.py index 2dd14e6..8a20c65 100644 --- a/src/lib/postgresql/networkx_client.py +++ b/src/lib/postgresql/networkx_client.py @@ -711,6 +711,40 @@ def search_entities( ) return SearchEntitiesResult(results=nodes, total=total) + def search_nodes_bm25( + self, + query_text: str, + brain_id: str, + *, + limit: int = 10, + node_labels: Optional[list[str]] = None, + node_uuids: Optional[list[str]] = None, + ) -> list[tuple[Node, float]]: + rows = self._store.search_nodes_bm25( + query_text, + brain_id, + limit=limit, + node_labels=node_labels, + node_uuids=node_uuids, + ) + hits: list[tuple[Node, float]] = [] + for node_uuid, score, data in rows: + labels = [str(item) for item in (data.get("labels") or []) if str(item).strip()] + properties = {k: v for k, v in data.items() if k != "labels"} + hits.append( + ( + Node( + uuid=str(node_uuid), + name=str(data.get("name") or node_uuid), + labels=labels, + description=data.get("description"), + properties=properties, + ), + float(score), + ) + ) + return hits + def deprecate_relationship( self, subject: Node, diff --git a/src/lib/postgresql/vectors.py b/src/lib/postgresql/vectors.py index e8b5402..5e77a32 100644 --- a/src/lib/postgresql/vectors.py +++ b/src/lib/postgresql/vectors.py @@ -20,6 +20,7 @@ from pgvector.psycopg2 import register_vector from src.adapters.interfaces.embeddings import VectorStoreClient +from src.config import config from src.constants.embeddings import EMBEDDING_STORES_SIZES, Vector from ._provisioning import borrow, ensure_brain_database, get_brain_pool @@ -57,14 +58,34 @@ def _table_name(store: str) -> str: return f"vectors_{_safe_store(store)}" -def _vector_index_ddl(table: str, dimension: int) -> str: +def _vector_index_ddl( + table: str, dimension: int, *, search_enabled: bool | None = None +) -> str: dim = int(dimension) - if dim > 2000: - return "" - return f""" + if search_enabled is None: + search_enabled = bool(config.search_enabled) + if dim <= 2000: + return f""" CREATE INDEX IF NOT EXISTS idx_{table}_embeddings ON {table} USING hnsw (embeddings vector_cosine_ops); """ + if not search_enabled: + return "" + return f""" + CREATE INDEX IF NOT EXISTS idx_{table}_embeddings_halfvec + ON {table} USING hnsw + ((embeddings::halfvec({dim})) halfvec_cosine_ops); + """ + + +def _uses_halfvec_ann(dimension: int, *, search_enabled: bool | None = None) -> bool: + if search_enabled is None: + search_enabled = bool(config.search_enabled) + try: + dim = int(dimension) + except (TypeError, ValueError): + return False + return search_enabled and dim > 2000 def _table_vector_dimension(cur: psycopg2.extensions.cursor, table: str) -> Optional[int]: @@ -132,6 +153,7 @@ def _ensure_store(self, store: str, brain_id: str) -> None: return dimension = EMBEDDING_STORES_SIZES[store] table = _table_name(store) + index_ddl = _vector_index_ddl(table, dimension) ddl = f""" CREATE TABLE IF NOT EXISTS {table} ( id BIGINT PRIMARY KEY, @@ -141,7 +163,7 @@ def _ensure_store(self, store: str, brain_id: str) -> None: ); CREATE INDEX IF NOT EXISTS idx_{table}_uuid ON {table} (uuid); - {_vector_index_ddl(table, dimension)} + {index_ddl} """ with self._connection(brain_id) as conn: with conn.cursor() as cur: @@ -155,7 +177,17 @@ def _ensure_store(self, store: str, brain_id: str) -> None: dimension, ) cur.execute(f"DROP TABLE IF EXISTS {table} CASCADE") - cur.execute(ddl) + try: + cur.execute(ddl) + except Exception as exc: + if _uses_halfvec_ann(dimension): + raise RuntimeError( + "SEARCH_ENABLED=true requires a halfvec HNSW index " + f"on {table} (dimension {dimension} > 2000). " + "The vector backend could not create it: " + f"{exc}" + ) from exc + raise conn.commit() self._initialized_stores.add(key) @@ -212,6 +244,15 @@ def search_vectors( return [] table = _table_name(store) fetch_k = ann_overfetch_k(k) + dimension = EMBEDDING_STORES_SIZES[store] + use_halfvec = _uses_halfvec_ann(dimension) + if use_halfvec: + order_sql = ( + f"embeddings::halfvec({dimension}) " + f"<=> (%s::vector)::halfvec({dimension})" + ) + else: + order_sql = "embeddings <=> %s::vector" with self._connection(brain_id) as conn: with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute( @@ -223,7 +264,7 @@ def search_vectors( SELECT id, uuid, metadata, (embeddings <=> %s::vector) AS distance FROM {table} - ORDER BY embeddings <=> %s::vector, uuid ASC, id ASC + ORDER BY {order_sql}, uuid ASC, id ASC LIMIT %s """, (data_vector, data_vector, fetch_k), diff --git a/src/lib/redis/client.py b/src/lib/redis/client.py index dff8fa8..f256477 100644 --- a/src/lib/redis/client.py +++ b/src/lib/redis/client.py @@ -28,6 +28,7 @@ def __init__(self): pool = ConnectionPool( host=config.redis.host, port=config.redis.port, + password=config.redis.password, socket_connect_timeout=5, socket_timeout=5, retry_on_timeout=True, diff --git a/src/services/api/app.py b/src/services/api/app.py index 331c3c2..ae7377f 100644 --- a/src/services/api/app.py +++ b/src/services/api/app.py @@ -1,5 +1,7 @@ import logging import os +import re +from hashlib import sha1 from contextlib import asynccontextmanager from pathlib import Path @@ -11,15 +13,20 @@ from fastapi import FastAPI, Response from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import RedirectResponse +from fastapi.routing import APIRoute from uvicorn import run +from src.constants.data import BRAIN_VERSION from src.services.api.console_static import SPAStaticFiles +from src.services.api.errors import install_error_handlers +from src.services.api.openapi import install_openapi_contract from src.services.api.middlewares.auth import BrainPATMiddleware from src.services.api.middlewares.brains import BrainMiddleware from src.services.api.routes.ingest import ingest_router from src.services.api.routes.meta import meta_router from src.services.api.routes.model import model_router +from src.services.api.routes.public import public_router from src.services.api.routes.retrieve import retrieve_router from src.services.api.routes.system import system_router from src.services.api.routes.tasks import tasks_router @@ -36,6 +43,50 @@ def _console_enabled() -> bool: return os.getenv("CONSOLE_ENABLED", "true").strip().lower() != "false" +def _production_mode() -> bool: + return os.getenv("ENV", "production").strip().lower() != "development" + + +def _cors_allowed_origins() -> list[str]: + configured = [ + origin.strip() + for origin in os.getenv("CORS_ALLOWED_ORIGINS", "").split(",") + if origin.strip() + ] + if not configured and not _production_mode(): + return ["*"] + if "*" in configured and _production_mode(): + raise RuntimeError( + "CORS_ALLOWED_ORIGINS='*' is allowed only when ENV=development" + ) + return configured + + +def _enforce_plugin_results(results: dict[str, bool]) -> None: + failed = sorted(name for name, loaded in results.items() if not loaded) + if not failed: + return + default_policy = "fail" if _production_mode() else "warn" + policy = os.getenv("PLUGIN_FAILURE_POLICY", default_policy).strip().lower() + if policy not in {"fail", "warn"}: + raise RuntimeError("PLUGIN_FAILURE_POLICY must be 'fail' or 'warn'") + if policy == "fail": + raise RuntimeError(f"Required plugins failed to load: {', '.join(failed)}") + + +def stable_operation_id(route: APIRoute) -> str: + """Generate function-calling-safe IDs that are stable across schema exports.""" + + tag = route.tags[0] if route.tags else "api" + method = sorted(route.methods or {"get"})[0].lower() + raw = f"{tag}_{route.name}_{method}".lower() + normalized = re.sub(r"[^a-z0-9_]+", "_", raw).strip("_") + if len(normalized) <= 64: + return normalized + digest = sha1(normalized.encode("utf-8")).hexdigest()[:8] + return f"{normalized[:55].rstrip('_')}_{digest}" + + @asynccontextmanager async def lifespan(app: FastAPI): from src.core.plugins.context import PluginContext @@ -47,6 +98,7 @@ async def lifespan(app: FastAPI): results = loader.load_all() _log_plugin_banner(loader, results) + _enforce_plugin_results(results) for event_name, handlers in ctx._event_handlers.items(): if event_name == "startup": @@ -118,16 +170,21 @@ def _log_plugin_banner(loader, results: dict[str, bool]): app = FastAPI( + title="BrainAPI", + version=BRAIN_VERSION, + description="Knowledge, memory, search, recommendation, and agent API.", debug=os.getenv("ENV") == "development", lifespan=lifespan, - redirect_slashes=True, + redirect_slashes=False, + generate_unique_id_function=stable_operation_id, ) +install_error_handlers(app) app.add_middleware(BrainPATMiddleware) app.add_middleware(BrainMiddleware) app.add_middleware( CORSMiddleware, - allow_origins=["*"], + allow_origins=_cors_allowed_origins(), allow_credentials=False, allow_methods=["*"], allow_headers=["*"], @@ -140,9 +197,11 @@ def _log_plugin_banner(loader, results: dict[str, bool]): app.include_router(model_router) app.include_router(system_router) app.include_router(tasks_router) +app.include_router(public_router) +install_openapi_contract(app) -@app.get("/") +@app.get("/", include_in_schema=False) async def root(): return Response(content="ok", status_code=200) diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.py index f71ea48..855c736 100644 --- a/src/services/api/constants/requests.py +++ b/src/services/api/constants/requests.py @@ -107,10 +107,30 @@ class IngestionTripleSet(BaseModel): None, description="The optional event predicate of the triple.", ) - event: RequestPartialNode - event_obj: RequestPartialPredicate + event: Optional[RequestPartialNode] = Field( + None, + description="The optional event node. Omit for a direct subject-predicate-object edge.", + ) + event_obj: Optional[RequestPartialPredicate] = Field( + None, + description="The optional event-to-object predicate. Paired with event when present.", + ) object: RequestPartialNode + @model_validator(mode="after") + def require_event_pair_or_direct_edge(self): + has_event = self.event is not None + has_event_obj = self.event_obj is not None + if has_event != has_event_obj: + raise ValueError("event and event_obj must both be set or both omitted") + if has_event: + return self + if not self.subject or not self.subj_event: + raise ValueError( + "direct triples require subject, subj_event, and object" + ) + return self + class PartialNodeFilter(BaseModel): """ @@ -469,3 +489,74 @@ class GetContextResponse(BaseModel): paths: Optional[List[dict[str, Any]]] = None topics: Optional[List[dict[str, Any]]] = None stage_timings: Optional[dict[str, Any]] = None + + +class SearchRequestBody(BaseModel): + query: str + brain_id: str = "default" + k: int = Field(10, ge=1, le=200) + channels: List[str] = Field(default_factory=lambda: ["passages"]) + node_labels: Optional[List[str]] = None + community_labels: Optional[List[str]] = None + expand: Literal["none", "neighbors"] = "none" + fusion: Optional[Literal["rrf", "cc"]] = None + fusion_alpha: Optional[float] = Field(default=None, ge=0.0, le=1.0) + rerank: Optional[str] = Field( + default=None, + description="none or plugin:. Unknown plugin names return 400.", + ) + mode: Literal["default", "catalog"] = "default" + profile_stages: bool = False + extras: Optional[Dict[str, str]] = None + target: Optional[str] = None + + @field_validator("target", mode="before") + @classmethod + def empty_target(cls, value): + if value is None: + return None + text = str(value).strip() + return text or None + + @field_validator("extras", mode="before") + @classmethod + def coerce_extras_filter(cls, value): + if value is None or value == {}: + return None + if not isinstance(value, dict): + raise ValueError("extras must be an object of string keys and values") + return { + str(key): str(item) + for key, item in value.items() + if item is not None + } or None + + +class SearchHitScores(BaseModel): + bm25: Optional[float] = None + dense: Optional[float] = None + rrf: Optional[float] = None + cc: Optional[float] = None + rerank: Optional[float] = None + plugin: Optional[dict[str, float]] = None + graph: Optional[float] = None + personalize: Optional[float] = None + + +class SearchHit(BaseModel): + id: str + channel: str + score: float + scores: SearchHitScores + snippet: str + labels: List[str] = Field(default_factory=list) + extras: Optional[dict[str, Any]] = None + node_id: Optional[str] = None + + +class SearchResponse(BaseModel): + hits: List[SearchHit] + stage_timings: Optional[dict[str, Any]] = None + channel_lists: Optional[dict[str, list[str]]] = None + facets: Optional[dict[str, dict[str, int]]] = None + node_ids: List[str] = Field(default_factory=list) diff --git a/src/services/api/constants/responses.py b/src/services/api/constants/responses.py new file mode 100644 index 0000000..4585049 --- /dev/null +++ b/src/services/api/constants/responses.py @@ -0,0 +1,145 @@ +"""Typed response envelopes for the public OpenAPI contract.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, JsonValue, RootModel + +from src.constants.data import KGChanges, Observation, StructuredData, TextChunk +from src.constants.embeddings import Vector +from src.constants.kg import Node, Predicate, Triple + + +class IngestionAcceptedResponse(BaseModel): + message: str + task_id: str + + +class LoginInfoResponse(BaseModel): + is_system_pat: bool + brain_id: str + + +class StringListResponse(RootModel[list[str]]): + pass + + +class FlatHopNode(BaseModel): + uuid: str + labels: list[str] + name: str + + +class FlatHopPredicate(BaseModel): + uuid: str + name: str + direction: str | None = None + + +HopNode = Node | FlatHopNode +HopPredicate = Predicate | FlatHopPredicate +SecondDegreeHop = tuple[ + HopNode, + list[tuple[HopPredicate, HopNode, list[tuple[HopPredicate, HopNode]]]], +] + + +class HopListResponse(RootModel[list[SecondDegreeHop]]): + pass + + +class RelationshipListResponse(BaseModel): + message: str + relationships: list[Triple] + total: int + + +class EntityListResponse(BaseModel): + message: str + entities: list[Node] + total: int + + +class StructuredDataItemResponse(BaseModel): + message: str + data: StructuredData + + +class StructuredDataListResponse(BaseModel): + message: str + data: list[StructuredData] + count: int + total: int + + +class TypeListResponse(BaseModel): + message: str + types: list[str] + count: int + + +class TextChunkListResponse(BaseModel): + message: str + data: list[TextChunk] + total: int + + +class ObservationItemResponse(BaseModel): + message: str + observation: Observation + + +class ObservationListResponse(BaseModel): + message: str + observations: list[Observation] + count: int + + +class LabelListResponse(BaseModel): + message: str + labels: list[str] + count: int + + +class ChangelogItemResponse(BaseModel): + message: str + changelog: KGChanges + + +class ChangelogListResponse(BaseModel): + message: str + changelogs: list[KGChanges] + count: int + + +class VectorStoreInfo(BaseModel): + name: str + dimension: int + + +class VectorStoreListResponse(BaseModel): + stores: list[VectorStoreInfo] + + +class VectorListResponse(BaseModel): + message: str + store: str + vectors: list[Vector] + total: int + + +class TaskStateResponse(BaseModel): + """Task payload with stable state fields and pipeline-specific extensions.""" + + model_config = ConfigDict(extra="allow") + + task_id: str | None = None + id: str | None = None + status: str + stage: str | None = None + error: str | None = None + result: JsonValue | None = None + + +class TaskListResponse(BaseModel): + tasks: list[TaskStateResponse] + diff --git a/src/services/api/controllers/entities.py b/src/services/api/controllers/entities.py index 995c55f..23cce19 100644 --- a/src/services/api/controllers/entities.py +++ b/src/services/api/controllers/entities.py @@ -11,6 +11,7 @@ from typing import List, Literal, Optional +from fastapi import HTTPException from starlette.responses import JSONResponse from src.core.search.entity_context import EntityContext from src.core.search.entity_info import EventSynergyRetriever @@ -114,9 +115,9 @@ async def get_entity_sibilings( ) ) if target_node is None: - return JSONResponse( + raise HTTPException( status_code=404, - content={"detail": "No entity found matching the target."}, + detail="No entity found matching the target.", ) return JSONResponse( content={ @@ -173,9 +174,9 @@ async def get_recommendations( behavior_weights=behavior_weights, ) if target_node is None: - return JSONResponse( + raise HTTPException( status_code=404, - content={"detail": "No entity found matching the target."}, + detail="No entity found matching the target.", ) items = [ RecommendItem( diff --git a/src/services/api/controllers/retrieve.py b/src/services/api/controllers/retrieve.py index a62434f..7cd4bc5 100644 --- a/src/services/api/controllers/retrieve.py +++ b/src/services/api/controllers/retrieve.py @@ -26,6 +26,12 @@ personalized_pagerank, reciprocal_rank_fusion, ) +from src.core.search.hybrid import ( + collect_bm25_passages, + collect_dense_passages, + collect_ilike_passages, + fuse_passage_lists, +) from src.core.search.relationships import search_relationships from src.lib.tracing.profiler import profile_request, profile_stage from src.utils.vector_search import VectorSearchFacade @@ -43,6 +49,7 @@ from src.utils.dates import parse_date_string, to_naive_utc from src.utils.similarity.vectors import cosine_similarity from src.utils.nlp.ner import _entity_extractor +from src.config import config vector_search = VectorSearchFacade(vector_store_adapter) @@ -857,10 +864,12 @@ def _hub_completeness_score(candidate: dict[str, Any]) -> float: return score -def _fact_recency_score(candidate: dict[str, Any]) -> float: +def _fact_recency_score( + candidate: dict[str, Any], *, now: datetime | None = None +) -> float: triple = candidate.get("triple") or () best = 0.0 - now = datetime.now(tz=None) + now = now or datetime.now(tz=None) for node in triple[0::2] if triple else (): raw = getattr(node, "happened_at", None) if not raw: @@ -875,12 +884,14 @@ def _fact_recency_score(candidate: dict[str, Any]) -> float: return best -def _relevance_score(candidate: dict[str, Any]) -> float: +def _relevance_score( + candidate: dict[str, Any], *, now: datetime | None = None +) -> float: distance = _candidate_distance(candidate) distance_term = 1.0 / (1.0 + max(0.0, distance)) ppr_term = float(candidate.get("ppr_mass") or 0.0) completeness = _hub_completeness_score(candidate) / 3.8 - recency = _fact_recency_score(candidate) + recency = _fact_recency_score(candidate, now=now) topic_boost = 0.08 if candidate.get("topic_preferred") else 0.0 return ( 0.45 * distance_term @@ -939,12 +950,14 @@ def _bridge_reserve_ok(candidate: dict[str, Any]) -> bool: return True -def _prepare_diversify_item(candidate: dict[str, Any]) -> dict[str, Any]: +def _prepare_diversify_item( + candidate: dict[str, Any], *, now: datetime | None = None +) -> dict[str, Any]: key = candidate.get("key") or ("", "") is_bridge = _is_bridge_candidate(candidate) return { "candidate": candidate, - "relevance": _relevance_score(candidate), + "relevance": _relevance_score(candidate, now=now), "hub": _event_hub_id(candidate), "sessions": frozenset(_candidate_session_ids(candidate)), "kind": _hub_leg_kind(candidate), @@ -963,11 +976,20 @@ def _mmr_pick_index( best_idx = 0 best_mmr = None best_tie = None + selected_kinds_by_hub: dict[str, set[str]] = defaultdict(set) + for selected in selected_meta: + selected_kinds_by_hub[selected["hub"]].add(selected["kind"]) for idx, item in enumerate(remaining): relevance = item["relevance"] hub = item["hub"] sessions = item["sessions"] - complementary = _is_complementary_hub_leg(selected_meta, item) + complementary = ( + item["kind"] in {"spine", "context"} + and bool( + selected_kinds_by_hub.get(item["hub"], set()) + & ({"spine", "context"} - {item["kind"]}) + ) + ) if complementary: hub_pen = 0.0 sess_pen = 0.0 @@ -1066,7 +1088,10 @@ def _diversify_facts( if len(ranked) <= max_facts: return list(ranked) - prepared = [_prepare_diversify_item(candidate) for candidate in ranked] + ranking_now = datetime.now(tz=None) + prepared = [ + _prepare_diversify_item(candidate, now=ranking_now) for candidate in ranked + ] prepared.sort(key=lambda item: (-item["relevance"], item["tie"])) selected: list[dict[str, Any]] = [] @@ -1681,45 +1706,52 @@ def _retrieve_passages( text: str, brain_id: str, *, limit: int ) -> list[tuple[str, float, str]]: """Return (chunk_id, score, text) ranked passages via vector + keyword fusion.""" + mode = ( + config.context_passage_mode if config.search_enabled else "ilike" + ) + use_dense = mode in ("hybrid", "dense", "ilike") + use_bm25 = mode in ("hybrid", "bm25") + use_ilike = mode == "ilike" with profile_stage("passages.retrieve", blocking=True, queries=1): - with profile_stage("embed.query"): - text_embeddings = embeddings_adapter.embed_text(text) - with profile_stage("vector.search_data", k=max(limit, _PASSAGE_K)): - vector_hits = vector_search.search_data( - text_embeddings.embeddings, - brain_id=brain_id, - k=max(limit, _PASSAGE_K), - ) vector_ids: list[str] = [] - id_to_text: dict[str, str] = {} id_to_distance: dict[str, float] = {} - for vector in vector_hits: - meta = vector.metadata or {} - resource_id = meta.get("resource_id") or getattr(vector, "id", None) - if not resource_id: - continue - resource_id = str(resource_id) - vector_ids.append(resource_id) - id_to_distance[resource_id] = ( - float(vector.distance) - if vector.distance is not None - else float("inf") - ) + if use_dense: + with profile_stage("embed.query"): + text_embeddings = embeddings_adapter.embed_text(text) + with profile_stage("vector.search_data", k=max(limit, _PASSAGE_K)): + vector_ids, id_to_distance = collect_dense_passages( + vector_search, + text_embeddings.embeddings, + brain_id, + max(limit, _PASSAGE_K), + ) keyword_ids: list[str] = [] - with profile_stage("data.keyword_search"): - try: - search_result = data_adapter.search(text, brain_id) - for chunk in getattr(search_result, "text_chunks", None) or []: - chunk_id = str(getattr(chunk, "id", "") or "") - if not chunk_id: - continue - keyword_ids.append(chunk_id) - id_to_text[chunk_id] = getattr(chunk, "text", "") or "" - except Exception: - pass + id_to_text: dict[str, str] = {} + if use_bm25: + with profile_stage("data.bm25_search"): + try: + keyword_ids, _, id_to_text = collect_bm25_passages( + data_adapter, + text, + brain_id, + max(limit, _PASSAGE_K), + ) + except Exception: + pass + elif use_ilike: + with profile_stage("data.keyword_search"): + try: + keyword_ids, id_to_text = collect_ilike_passages( + data_adapter, text, brain_id + ) + except Exception: + pass - fused = reciprocal_rank_fusion([vector_ids, keyword_ids]) + fused = fuse_passage_lists( + vector_ids if use_dense else [], + keyword_ids, + ) ranked_ids = [item for item, _ in fused[:limit]] missing = [cid for cid in ranked_ids if cid not in id_to_text] if missing: @@ -2069,7 +2101,7 @@ async def _get_historical_context(): except Exception as exc: topic_detail["error"] = str(exc)[:120] - with profile_stage("facts.diversify", max_facts=max_facts) as detail: + with profile_stage("fact_filter") as detail: if request.apply_fact_filter and fact_filter_adapter is not None and ranked: detail["filter_applied"] = True keep = filter_relevant_facts( @@ -2084,18 +2116,21 @@ async def _get_historical_context(): else: detail["filter_applied"] = False filtered = ranked + + with profile_stage("facts.diversify", max_facts=max_facts) as detail: filtered = _rank_facts_with_completeness(filtered) curated = _diversify_facts(filtered, max_facts=max_facts) detail["candidates"] = len(filtered) detail["curated"] = len(curated) - temporal_conflicts = _temporal_conflict_meta(curated) - - text_lines, triples, graph_session_ids = _build_fact_channel( - curated, chunk_sessions - ) - - source_passages = source_passages[:max_passages] + with profile_stage("context.assemble") as detail: + temporal_conflicts = _temporal_conflict_meta(curated) + text_lines, triples, graph_session_ids = _build_fact_channel( + curated, chunk_sessions + ) + source_passages = source_passages[:max_passages] + detail["facts"] = len(text_lines) + detail["passages"] = len(source_passages) insufficient = False if request.sufficiency_retry: diff --git a/src/services/api/controllers/search.py b/src/services/api/controllers/search.py new file mode 100644 index 0000000..65e4459 --- /dev/null +++ b/src/services/api/controllers/search.py @@ -0,0 +1,514 @@ +import asyncio +from typing import Any, Optional + +from fastapi import HTTPException + +from src.config import config +from src.core.search.catalog_graph import node_id_from_passage_text +from src.core.search.personalize import personalize_ranked_ids +from src.core.search.graph_channels import ( + COMMUNITIES_CHANNEL, + ENTITIES_CHANNEL, + EVENTS_CHANNEL, + GraphHit, + NEIGHBORS_CHANNEL, + PASSAGES_CHANNEL, + collect_community_hits, + collect_entity_hits, + collect_event_hits, + expand_neighbor_hits, + is_item_entity, + selected_graph_channels, +) +from src.core.search.hooks import ( + SearchPluginError, + rerank_max_k_for_mode, + resolve_reranker, + resolve_retrievers, + retrieve_k_for_mode, +) +from src.core.search.hybrid import ( + collect_bm25_passages, + collect_dense_passages, + collect_literal_residual, + dense_similarity, + extras_from_metadata, + facet_counts_from_extras, + frozen_head_merge, + fuse_passage_lists, + hit_matches_extras, + merge_hit_extras, + passage_snippet, +) +from src.lib.tracing.profiler import profile_request, profile_stage +from src.services.api.constants.requests import ( + SearchHit, + SearchHitScores, + SearchRequestBody, + SearchResponse, +) +from src.services.data.main import data_adapter +from src.services.kg_agent.main import embeddings_adapter, graph_adapter, vector_store_adapter +from src.utils.vector_search import VectorSearchFacade + +vector_search = VectorSearchFacade(vector_store_adapter) + + +def _require_search_enabled() -> None: + if not config.search_enabled: + raise HTTPException( + status_code=404, + detail="Search is disabled. Set SEARCH_ENABLED=true to enable /retrieve/search.", + ) + + +def _plugin_http(exc: SearchPluginError) -> HTTPException: + return HTTPException(status_code=400, detail=str(exc)) + + +def _fetch_snippets(ids: list[str], known: dict[str, str], brain_id: str) -> dict[str, str]: + texts, _ = _fetch_chunk_fields(ids, known, {}, brain_id) + return texts + + +def _fetch_chunk_fields( + ids: list[str], + known_texts: dict[str, str], + known_extras: dict[str, dict[str, Any]], + brain_id: str, +) -> tuple[dict[str, str], dict[str, dict[str, Any]]]: + texts = dict(known_texts) + extras: dict[str, dict[str, Any]] = dict(known_extras) + missing = [ + chunk_id + for chunk_id in ids + if chunk_id not in texts or chunk_id not in extras + ] + if not missing: + return texts, extras + try: + chunks, _ = data_adapter.get_text_chunks_by_ids(missing, False, brain_id) + except Exception: + return texts, extras + if not isinstance(chunks, (list, tuple)): + return texts, extras + for chunk in chunks: + chunk_id = str(getattr(chunk, "id", "") or "") + if not chunk_id: + continue + if chunk_id not in texts: + texts[chunk_id] = getattr(chunk, "text", "") or "" + parsed = extras_from_metadata(getattr(chunk, "metadata", None)) + if parsed: + extras[chunk_id] = merge_hit_extras(extras.get(chunk_id), parsed) or parsed + elif chunk_id not in extras: + extras[chunk_id] = {} + return texts, extras + + +def _run_dense( + query_vector: list[float], + brain_id: str, + k: int, + extras_out: dict[str, dict[str, str]] | None = None, +): + with profile_stage("search.dense"): + return collect_dense_passages( + vector_search, query_vector, brain_id, k, extras_out=extras_out + ) + + +def _run_bm25( + query: str, + brain_id: str, + k: int, + extras_out: dict[str, dict[str, str]] | None = None, +): + with profile_stage("search.bm25"): + return collect_bm25_passages( + data_adapter, query, brain_id, k, extras_out=extras_out + ) + + +def _want_passages(channels: list[str] | None) -> bool: + if not channels: + return True + return any(str(item).strip().lower() == PASSAGES_CHANNEL for item in channels) + + +def _collect_graph_hits( + request: SearchRequestBody, + *, + query_vector: list[float] | None, + graph_channels: list[str], +) -> list[GraphHit]: + k = request.k + brain_id = request.brain_id + fanout = int(getattr(config, "search_neighbor_fanout", 50) or 50) + community_labels = request.community_labels + if community_labels is None: + community_labels = list(config.search_community_labels) + hits: list[GraphHit] = [] + if ENTITIES_CHANNEL in graph_channels: + hits.extend( + collect_entity_hits( + query=request.query, + brain_id=brain_id, + k=k, + graph=graph_adapter, + vector_search=vector_search, + query_vector=query_vector, + node_labels=request.node_labels, + ) + ) + if EVENTS_CHANNEL in graph_channels: + hits.extend( + collect_event_hits( + query=request.query, + brain_id=brain_id, + k=k, + graph=graph_adapter, + vector_search=vector_search, + query_vector=query_vector, + ) + ) + if COMMUNITIES_CHANNEL in graph_channels: + hits.extend( + collect_community_hits( + query=request.query, + brain_id=brain_id, + k=k, + graph=graph_adapter, + vector_search=vector_search, + query_vector=query_vector, + community_labels=community_labels, + fanout=fanout, + ) + ) + if request.expand == "neighbors" and hits: + hits.extend( + expand_neighbor_hits( + hits, + brain_id=brain_id, + k=k, + graph=graph_adapter, + community_labels=community_labels, + fanout=fanout, + ) + ) + return hits + + +async def search(request: SearchRequestBody) -> SearchResponse: + _require_search_enabled() + with profile_request( + "retrieve.search", enabled=request.profile_stages + ) as profiler: + response = await _search(request) + if profiler is not None: + response.stage_timings = profiler.last_report + return response + + +async def _search(request: SearchRequestBody) -> SearchResponse: + try: + reranker = resolve_reranker(request.rerank) + plugin_retrievers = resolve_retrievers(request.channels) + except SearchPluginError as exc: + raise _plugin_http(exc) from exc + + k = request.k + k_ret = retrieve_k_for_mode(request.mode, k) + rerank_max = rerank_max_k_for_mode(request.mode) + brain_id = request.brain_id + fusion = request.fusion or config.search_fusion + fusion_alpha = ( + request.fusion_alpha + if request.fusion_alpha is not None + else config.search_fusion_alpha + ) + want_passages = _want_passages(request.channels) + graph_channels = selected_graph_channels(request.channels) + want_graph = bool(graph_channels) + use_dense = want_passages and config.search_use_dense + use_bm25 = want_passages and config.search_use_bm25 + query_vector: Optional[list[float]] = None + if use_dense or want_graph: + with profile_stage("embed.query", blocking=False): + embedded = await asyncio.to_thread( + embeddings_adapter.embed_text, request.query + ) + query_vector = list(embedded.embeddings or []) + + dense_extras: dict[str, dict[str, str]] = {} + bm25_extras: dict[str, dict[str, str]] = {} + retrieve_jobs = [] + if use_dense: + retrieve_jobs.append( + asyncio.to_thread( + _run_dense, query_vector or [], brain_id, k_ret, dense_extras + ) + ) + if use_bm25: + retrieve_jobs.append( + asyncio.to_thread( + _run_bm25, request.query, brain_id, k_ret, bm25_extras + ) + ) + + dense_ids: list[str] = [] + dense_distances: dict[str, float] = {} + bm25_ids: list[str] = [] + bm25_scores: dict[str, float] = {} + texts: dict[str, str] = {} + plugin_id_lists: list[list[str]] = [] + plugin_scores: dict[str, dict[str, float]] = {} + plugin_channels: dict[str, str] = {} + graph_hits: list[GraphHit] = [] + literal_ids: list[str] = [] + with profile_stage("search.retrieve", blocking=False): + gathered = await asyncio.gather(*retrieve_jobs) if retrieve_jobs else [] + offset = 0 + if use_dense: + dense_ids, dense_distances = gathered[offset] + offset += 1 + if use_bm25: + bm25_ids, bm25_scores, texts = gathered[offset] + for name, retriever in plugin_retrievers: + with profile_stage(f"search.plugin.{name}"): + ids, scores, plugin_texts = await asyncio.to_thread( + retriever, request.query, brain_id, k_ret + ) + plugin_id_lists.append(ids) + plugin_scores[name] = scores + texts.update(plugin_texts) + for item_id in ids: + plugin_channels.setdefault(str(item_id), f"plugin:{name}") + if want_graph: + try: + graph_hits = await asyncio.to_thread( + _collect_graph_hits, + request, + query_vector=query_vector, + graph_channels=graph_channels, + ) + except Exception: + graph_hits = [] + if getattr(config, "search_literal_fill", False): + lit_ids, literal_texts = collect_literal_residual( + data_adapter, request.query, brain_id, k_ret + ) + literal_ids = list(lit_ids) + texts.update(literal_texts) + for item_id in literal_ids: + plugin_channels.setdefault(str(item_id), PASSAGES_CHANNEL) + + passage_extras = {**dense_extras, **bm25_extras} + dense_sims = { + chunk_id: dense_similarity(distance) + for chunk_id, distance in dense_distances.items() + } + extra_id_lists: list[list[str]] = [] + graph_by_channel: dict[str, list[str]] = {} + graph_by_id: dict[str, GraphHit] = {} + for hit in graph_hits: + graph_by_channel.setdefault(hit.channel, []).append(hit.id) + current = graph_by_id.get(hit.id) + if current is None or hit.score > current.score: + graph_by_id[hit.id] = hit + + def _item_ids(channel_name: str) -> list[str]: + ids = graph_by_channel.get(channel_name) or [] + if channel_name != EVENTS_CHANNEL: + ids = [ + item_id + for item_id in ids + if is_item_entity( + item_id, + graph_by_id[item_id].labels if item_id in graph_by_id else [], + ) + ] + return ids + + graph_channel_ids = { + ENTITIES_CHANNEL: _item_ids(ENTITIES_CHANNEL), + EVENTS_CHANNEL: _item_ids(EVENTS_CHANNEL), + COMMUNITIES_CHANNEL: _item_ids(COMMUNITIES_CHANNEL), + NEIGHBORS_CHANNEL: _item_ids(NEIGHBORS_CHANNEL), + } + for ids in graph_channel_ids.values(): + if ids: + extra_id_lists.append(ids) + for item_id in ids: + texts.setdefault(item_id, graph_by_id[item_id].snippet) + fused = fuse_passage_lists( + dense_ids if use_dense else [], + bm25_ids if use_bm25 else [], + fusion=fusion, + alpha=fusion_alpha, + dense_similarities=dense_sims, + bm25_scores=bm25_scores, + extra_id_lists=extra_id_lists, + ) + ranked_ids = [item for item, _ in fused] + sidecar_lists = [ids for ids in plugin_id_lists if ids] + if literal_ids: + sidecar_lists.append(literal_ids) + if sidecar_lists: + ranked_ids = frozen_head_merge( + ranked_ids, + sidecar_lists, + head_k=10, + k=k_ret, + ) + else: + ranked_ids = ranked_ids[:k_ret] + rerank_scores: dict[str, float] = {} + if reranker is not None and ranked_ids: + head_n = min(rerank_max, len(ranked_ids)) + head_ids = ranked_ids[:head_n] + tail_ids = ranked_ids[head_n:] + with profile_stage("search.rerank", blocking=False, chunks=head_n): + texts = await asyncio.to_thread( + _fetch_snippets, head_ids, texts, brain_id + ) + candidates = [ + { + "id": chunk_id, + "text": texts.get(chunk_id) or "", + "score": float(dict(fused).get(chunk_id, 0.0)), + } + for chunk_id in head_ids + ] + reranked = await asyncio.to_thread( + reranker, request.query, candidates, head_n + ) + ordered: list[str] = [] + seen: set[str] = set() + for item in reranked or []: + chunk_id = str((item or {}).get("id") or "") + if not chunk_id or chunk_id not in head_ids or chunk_id in seen: + continue + ordered.append(chunk_id) + seen.add(chunk_id) + if item.get("score") is not None: + rerank_scores[chunk_id] = float(item["score"]) + for chunk_id in head_ids: + if chunk_id not in seen: + ordered.append(chunk_id) + ranked_ids = ordered + tail_ids + ranked_ids = ranked_ids[:k_ret] + fused_map = dict(fused) + + extras_by_id: dict[str, dict[str, Any]] = dict(passage_extras) + for item_id, graph_hit in graph_by_id.items(): + extras_by_id[item_id] = ( + merge_hit_extras(extras_by_id.get(item_id), graph_hit.extras) or {} + ) + + with profile_stage("search.snippets", blocking=False, chunks=len(ranked_ids)): + texts, extras_by_id = await asyncio.to_thread( + _fetch_chunk_fields, ranked_ids, texts, extras_by_id, brain_id + ) + + if request.extras: + ranked_ids = [ + item_id + for item_id in ranked_ids + if hit_matches_extras(extras_by_id.get(item_id), request.extras) + ] + node_id_by_hit: dict[str, str | None] = {} + for item_id in ranked_ids: + graph_hit = graph_by_id.get(item_id) + if graph_hit is not None and is_item_entity(graph_hit.id, graph_hit.labels): + node_id_by_hit[item_id] = graph_hit.id + else: + node_id_by_hit[item_id] = node_id_from_passage_text( + texts.get(item_id) or "" + ) + personalize_scores: dict[str, float] = {} + if request.target: + retrieve_for_blend = { + item_id: float( + rerank_scores.get(item_id, fused_map.get(item_id, 0.0)) + ) + for item_id in ranked_ids + } + ranked_ids, personalize_scores = personalize_ranked_ids( + query=request.query, + ranked_ids=ranked_ids, + retrieve_scores=retrieve_for_blend, + node_id_by_hit=node_id_by_hit, + target=request.target, + brain_id=brain_id, + ) + ranked_ids = ranked_ids[:k] + + channel_of: dict[str, str] = {} + for chunk_id in list(dense_ids) + list(bm25_ids): + channel_of.setdefault(chunk_id, PASSAGES_CHANNEL) + for hit in graph_hits: + channel_of.setdefault(hit.id, hit.channel) + for chunk_id, plugin_channel in plugin_channels.items(): + channel_of.setdefault(chunk_id, plugin_channel) + + hits: list[SearchHit] = [] + for chunk_id in ranked_ids: + body = texts.get(chunk_id) or "" + rrf = fused_map.get(chunk_id) + plugin = { + name: scores[chunk_id] + for name, scores in plugin_scores.items() + if chunk_id in scores + } + graph_hit = graph_by_id.get(chunk_id) + extras = merge_hit_extras(extras_by_id.get(chunk_id), graph_hit.extras if graph_hit else None) + scores = SearchHitScores( + bm25=bm25_scores.get(chunk_id), + dense=dense_sims.get(chunk_id), + rrf=rrf if fusion == "rrf" else None, + cc=rrf if fusion == "cc" else None, + rerank=rerank_scores.get(chunk_id), + plugin=plugin or None, + graph=graph_hit.score if graph_hit is not None else None, + personalize=personalize_scores.get(chunk_id), + ) + node_id = node_id_by_hit.get(chunk_id) + hits.append( + SearchHit( + id=chunk_id, + channel=channel_of.get(chunk_id, PASSAGES_CHANNEL), + score=float( + personalize_scores.get( + chunk_id, + rerank_scores.get(chunk_id, fused_map.get(chunk_id, 0.0)), + ) + ), + scores=scores, + snippet=passage_snippet(body), + labels=list(graph_hit.labels) if graph_hit is not None else [], + extras=extras, + node_id=node_id, + ) + ) + node_ids: list[str] = [] + seen_nodes: set[str] = set() + for hit in hits: + if not hit.node_id or hit.node_id in seen_nodes: + continue + seen_nodes.add(hit.node_id) + node_ids.append(hit.node_id) + return SearchResponse( + hits=hits, + facets=facet_counts_from_extras([hit.extras for hit in hits]), + channel_lists={ + "dense": list(dense_ids), + "bm25": list(bm25_ids), + "entities": list(graph_channel_ids[ENTITIES_CHANNEL]), + "events": list(graph_channel_ids[EVENTS_CHANNEL]), + "communities": list(graph_channel_ids[COMMUNITIES_CHANNEL]), + "neighbors": list(graph_channel_ids[NEIGHBORS_CHANNEL]), + "literal": list(literal_ids), + }, + node_ids=node_ids, + ) diff --git a/src/services/api/errors.py b/src/services/api/errors.py new file mode 100644 index 0000000..a6ea207 --- /dev/null +++ b/src/services/api/errors.py @@ -0,0 +1,181 @@ +"""Structured, agent-readable HTTP error responses.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any +from uuid import uuid4 + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from pydantic import BaseModel, Field, JsonValue +from starlette.exceptions import HTTPException as StarletteHTTPException + + +class ErrorInfo(BaseModel): + """Stable machine-readable information attached to every API error.""" + + code: str = Field(description="Stable error code for programmatic handling.") + message: str = Field(description="Short human-readable error summary.") + resolution: str = Field(description="Concrete guidance for resolving the error.") + request_id: str = Field(description="Identifier shared with the X-Request-ID header.") + + +class ErrorResponse(BaseModel): + """Additive error envelope that preserves FastAPI's legacy detail field.""" + + detail: JsonValue = Field(description="Original FastAPI error detail value.") + error: ErrorInfo + + +_STATUS_DEFAULTS: dict[int, tuple[str, str, str]] = { + 400: ( + "INVALID_REQUEST", + "The request is invalid.", + "Correct the request parameters and retry.", + ), + 401: ( + "AUTH_INVALID", + "Authentication failed.", + "Send a valid BrainPAT or Bearer token for the selected brain.", + ), + 403: ( + "FORBIDDEN", + "The operation is not permitted.", + "Use credentials with permission for this operation.", + ), + 404: ( + "RESOURCE_NOT_FOUND", + "The requested resource was not found.", + "Check the path, identifier, and selected brain before retrying.", + ), + 405: ( + "METHOD_NOT_ALLOWED", + "The HTTP method is not allowed for this resource.", + "Use one of the methods documented in the OpenAPI specification.", + ), + 406: ( + "BRAIN_NOT_FOUND", + "The selected brain could not be resolved.", + "Create the brain or provide the correct X-Brain-ID value.", + ), + 422: ( + "VALIDATION_ERROR", + "Request validation failed.", + "Correct the fields listed in detail and retry.", + ), + 429: ( + "RATE_LIMITED", + "The request rate limit was exceeded.", + "Wait before retrying and use bounded exponential backoff.", + ), + 500: ( + "INTERNAL_ERROR", + "The service could not complete the request.", + "Retry only when safe; contact support with the request ID if it persists.", + ), + 501: ( + "not_implemented", + "The operation is not implemented.", + "Use a supported operation or consult the release notes for availability.", + ), + 503: ( + "SERVICE_UNAVAILABLE", + "A required service is temporarily unavailable.", + "Retry with bounded exponential backoff after the dependency recovers.", + ), +} + + +def _request_id(request: Request) -> str: + return ( + request.headers.get("X-Request-ID") + or request.headers.get("X-Trace-ID") + or str(uuid4()) + ) + + +def error_response( + request: Request, + *, + status_code: int, + detail: Any, + code: str | None = None, + message: str | None = None, + resolution: str | None = None, + extra: dict[str, Any] | None = None, + headers: Mapping[str, str] | None = None, +) -> JSONResponse: + """Create the common JSON envelope while retaining the legacy detail value.""" + + default_code, default_message, default_resolution = _STATUS_DEFAULTS.get( + status_code, + _STATUS_DEFAULTS[500], + ) + request_id = _request_id(request) + content: dict[str, Any] = { + "detail": detail, + "error": { + "code": code or default_code, + "message": message or default_message, + "resolution": resolution or default_resolution, + "request_id": request_id, + }, + } + if extra: + content.update(extra) + response_headers = dict(headers or {}) + response_headers["X-Request-ID"] = request_id + return JSONResponse( + status_code=status_code, + content=content, + headers=response_headers, + ) + + +def install_error_handlers(app: FastAPI) -> None: + """Install handlers for framework, validation, and unexpected exceptions.""" + + @app.exception_handler(StarletteHTTPException) + async def http_exception_handler(request: Request, exc: StarletteHTTPException): + return error_response( + request, + status_code=exc.status_code, + detail=exc.detail, + headers=exc.headers, + ) + + @app.exception_handler(RequestValidationError) + async def validation_exception_handler( + request: Request, + exc: RequestValidationError, + ): + return error_response( + request, + status_code=422, + detail=exc.errors(), + ) + + @app.exception_handler(Exception) + async def unexpected_exception_handler(request: Request, _exc: Exception): + return error_response( + request, + status_code=500, + detail="Internal server error", + ) + + +COMMON_ERROR_RESPONSES = { + 400: {"model": ErrorResponse, "description": "Invalid request"}, + 401: {"model": ErrorResponse, "description": "Authentication failed"}, + 403: {"model": ErrorResponse, "description": "Operation forbidden"}, + 404: {"model": ErrorResponse, "description": "Resource not found"}, + 405: {"model": ErrorResponse, "description": "Method not allowed"}, + 406: {"model": ErrorResponse, "description": "Brain not found"}, + 422: {"model": ErrorResponse, "description": "Validation failed"}, + 429: {"model": ErrorResponse, "description": "Rate limit exceeded"}, + 500: {"model": ErrorResponse, "description": "Internal service error"}, + 501: {"model": ErrorResponse, "description": "Operation not implemented"}, + 503: {"model": ErrorResponse, "description": "Dependency unavailable"}, +} diff --git a/src/services/api/middlewares/auth.py b/src/services/api/middlewares/auth.py index 5e6c65c..fa01289 100644 --- a/src/services/api/middlewares/auth.py +++ b/src/services/api/middlewares/auth.py @@ -10,18 +10,22 @@ import os from fastapi import Request -from fastapi.responses import JSONResponse from starlette.middleware.base import BaseHTTPMiddleware from starlette import status from src.services.api.console_static import is_console_path +from src.services.api.errors import error_response from src.services.kg_agent.main import cache_adapter from src.services.data.main import data_adapter class BrainPATMiddleware(BaseHTTPMiddleware): - excluded_prefixes: set[str] = {"/console"} - auth_exempt_paths: set[str] = {"/meta/login-info"} + excluded_prefixes: set[str] = {"/console", "/docs", "/redoc", "/demo"} + auth_exempt_paths: set[str] = { + "/health", + "/openapi.json", + "/meta/login-info", + } async def dispatch(self, request: Request, call_next): if request.method == "OPTIONS": @@ -41,46 +45,64 @@ async def dispatch(self, request: Request, call_next): if not brainpat: brainpat = request.headers.get("Authorization") if brainpat: - brainpat = brainpat.split(" ")[1] - if brainpat: - brainpat = brainpat.rstrip() + scheme, _, token = brainpat.partition(" ") + brainpat = token.rstrip() if scheme.lower() == "bearer" else None + if not brainpat: + return error_response( + request, + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or missing BrainPAT header", + ) system_pat = os.getenv("BRAINPAT_TOKEN") if request.url.path.startswith("/system") or request.url.path == "/": if brainpat == system_pat: return await call_next(request) - return JSONResponse( + return error_response( + request, status_code=status.HTTP_401_UNAUTHORIZED, - content={"detail": "Invalid or missing BrainPAT header"}, + detail="Invalid or missing BrainPAT header", ) brain_id = getattr(request.state, "brain_id", None) if not brain_id: - return JSONResponse( + return error_response( + request, status_code=status.HTTP_400_BAD_REQUEST, - content={"detail": "Brain ID is required."}, + detail="Brain ID is required.", + code="BRAIN_ID_REQUIRED", + message="A brain identifier is required.", + resolution="Send X-Brain-ID with the intended alphanumeric brain name.", ) cachepat_key = f"brainpat:{brain_id}" - cached_brainpat = cache_adapter.get(key=cachepat_key, brain_id="system") - - # Logic -------------------------------------------------- if brainpat == system_pat: return await call_next(request) + try: + cached_brainpat = cache_adapter.get(key=cachepat_key, brain_id="system") - if not cached_brainpat: - stored_brain = data_adapter.get_brain(name_key=brain_id) - if not stored_brain or stored_brain.pat != brainpat: - return JSONResponse( + # Logic -------------------------------------------------- + if not cached_brainpat: + stored_brain = data_adapter.get_brain(name_key=brain_id) + if not stored_brain or stored_brain.pat != brainpat: + return error_response( + request, + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or missing BrainPAT header", + ) + cached_brainpat = stored_brain.pat + cache_adapter.set( + key=cachepat_key, value=stored_brain.pat, brain_id="system" + ) + elif cached_brainpat != brainpat: + return error_response( + request, status_code=status.HTTP_401_UNAUTHORIZED, - content={"detail": "Invalid or missing BrainPAT header"}, + detail="Invalid or missing BrainPAT header", ) - cached_brainpat = stored_brain.pat - cache_adapter.set( - key=cachepat_key, value=stored_brain.pat, brain_id="system" - ) - elif cached_brainpat != brainpat: - return JSONResponse( - status_code=status.HTTP_401_UNAUTHORIZED, - content={"detail": "Invalid or missing BrainPAT header"}, + except Exception: + return error_response( + request, + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Authentication store unavailable", ) response = await call_next(request) diff --git a/src/services/api/middlewares/brains.py b/src/services/api/middlewares/brains.py index 527f964..c30fd1e 100644 --- a/src/services/api/middlewares/brains.py +++ b/src/services/api/middlewares/brains.py @@ -14,12 +14,11 @@ from email.parser import BytesParser from fastapi import Request -from fastapi.responses import JSONResponse -from pymongo.errors import OperationFailure, ServerSelectionTimeoutError from starlette.middleware.base import BaseHTTPMiddleware from starlette import status from src.services.api.console_static import is_console_path +from src.services.api.errors import error_response from src.services.data.main import data_adapter from src.services.kg_agent.main import cache_adapter @@ -44,8 +43,12 @@ def _brain_id_from_multipart(body: bytes, content_type: str) -> str | None: class BrainMiddleware(BaseHTTPMiddleware): - excluded_prefixes: set[str] = {"/console"} - brain_exempt_paths: set[str] = {"/meta/login-info"} + excluded_prefixes: set[str] = {"/console", "/docs", "/redoc", "/demo"} + brain_exempt_paths: set[str] = { + "/health", + "/openapi.json", + "/meta/login-info", + } async def dispatch(self, request: Request, call_next): if is_console_path(request.url.path): @@ -57,6 +60,13 @@ async def dispatch(self, request: Request, call_next): if request.url.path in self.brain_exempt_paths: return await call_next(request) + brainpat = request.headers.get("BrainPAT") + authorization = request.headers.get("Authorization", "") + if not brainpat and not authorization.lower().startswith("bearer "): + # Authentication middleware owns the missing-credential response. Avoid + # touching a backing store before it can return a structured 401. + return await call_next(request) + async def _get_brain_id(): brain_id = None @@ -103,7 +113,6 @@ async def receive(): brain_id = await _get_brain_id() brain_creation_allowed = os.getenv("BRAIN_CREATION_ALLOWED") == "true" default_brain_fallback = os.getenv("DEFAULT_BRAIN_FALLBACK") == "true" - cached_brain_id = cache_adapter.get(key=f"brain:{brain_id}", brain_id="system") # Bypassing system routes -------------------------------- if request.url.path.startswith("/system") or request.url.path == "/": @@ -111,17 +120,29 @@ async def receive(): # Cleanup checks ----------------------------------------- if brain_id == "system": - return JSONResponse( + return error_response( + request, status_code=status.HTTP_400_BAD_REQUEST, - content={"detail": "System brain is not allowed to be used."}, + detail="System brain is not allowed to be used.", + code="BRAIN_ID_RESERVED", + message="The system brain is reserved.", + resolution="Select a non-system application brain.", ) if brain_id and not brain_id.isalnum(): - return JSONResponse( + return error_response( + request, status_code=status.HTTP_400_BAD_REQUEST, - content={"detail": "Brain ID must be alphanumeric.", "value": brain_id}, + detail="Brain ID must be alphanumeric.", + code="BRAIN_ID_INVALID", + message="The brain identifier is invalid.", + resolution="Use an alphanumeric X-Brain-ID value.", + extra={"value": brain_id}, ) try: + cached_brain_id = cache_adapter.get( + key=f"brain:{brain_id}", brain_id="system" + ) if brain_id and not cached_brain_id: stored_brain = data_adapter.get_brain(name_key=brain_id) @@ -157,19 +178,18 @@ async def receive(): brain_id="system", ) request.state.brain_id = "default" - except (OperationFailure, ServerSelectionTimeoutError) as e: - return JSONResponse( + except Exception: + return error_response( + request, status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - content={ - "detail": "Database unavailable. Check MongoDB connection and credentials (e.g. MONGO_* or MONGO_CONNECTION_STRING).", - "error": str(e), - }, + detail="Database unavailable. Check MongoDB connection and credentials (e.g. MONGO_* or MONGO_CONNECTION_STRING).", ) if getattr(request.state, "brain_id", None) is None: - return JSONResponse( + return error_response( + request, status_code=status.HTTP_406_NOT_ACCEPTABLE, - content={"detail": "Brain not found or creation is not allowed."}, + detail="Brain not found or creation is not allowed.", ) return await call_next(request) diff --git a/src/services/api/openapi.py b/src/services/api/openapi.py new file mode 100644 index 0000000..0e52e37 --- /dev/null +++ b/src/services/api/openapi.py @@ -0,0 +1,177 @@ +"""OpenAPI guarantees for discovery and LLM function-calling clients.""" + +from __future__ import annotations + +from typing import Any + +from fastapi import FastAPI +from fastapi.openapi.utils import get_openapi + + +_HTTP_METHODS = {"get", "post", "put", "patch", "delete", "options", "head"} +_CORE_TAGS = {"public", "ingest", "retrieve", "model", "meta", "tasks", "system"} +_LIST_PATHS = { + "/meta/entity-labels", + "/meta/entity-properties", + "/meta/relationships-properties", + "/retrieve/hops", + "/system/brains-list", +} + + +def _error_response(description: str) -> dict[str, Any]: + return { + "description": description, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/ErrorResponse"} + } + }, + } + + +def install_openapi_contract(app: FastAPI) -> None: + """Install a deterministic, fully-described OpenAPI schema builder.""" + + def custom_openapi() -> dict[str, Any]: + if app.openapi_schema is not None: + return app.openapi_schema + + schema = get_openapi( + title=app.title, + version=app.version, + openapi_version=app.openapi_version, + description=app.description, + routes=app.routes, + ) + components = schema.setdefault("components", {}) + component_schemas = components.setdefault("schemas", {}) + security_schemes = components.setdefault("securitySchemes", {}) + security_schemes.update( + { + "BrainPAT": { + "type": "apiKey", + "in": "header", + "name": "BrainPAT", + "description": "BrainAPI personal access token.", + }, + "BearerAuth": { + "type": "http", + "scheme": "bearer", + "description": "BrainAPI PAT sent as a Bearer token.", + }, + } + ) + + protected_errors = { + "400": _error_response("Invalid request"), + "401": _error_response("Authentication failed"), + "403": _error_response("Operation forbidden"), + "404": _error_response("Resource not found"), + "405": _error_response("Method not allowed"), + "406": _error_response("Brain not found"), + "422": _error_response("Validation failed"), + "429": _error_response("Rate limit exceeded"), + "500": _error_response("Internal service error"), + "503": _error_response("Dependency unavailable"), + } + + for path, path_item in schema.get("paths", {}).items(): + for method, operation in path_item.items(): + if method.lower() not in _HTTP_METHODS or not isinstance(operation, dict): + continue + summary = operation.get("summary") or operation.get("operationId") + if not operation.get("description"): + operation["description"] = ( + f"{summary}. See the BrainAPI developer documentation for " + "request, response, authentication, and retry semantics." + ) + + for status_code, response in operation.get("responses", {}).items(): + if not str(status_code).startswith("2") or not isinstance(response, dict): + continue + for media in response.get("content", {}).values(): + if isinstance(media, dict) and media.get("schema") == {}: + if _CORE_TAGS.intersection(operation.get("tags", [])): + raise ValueError( + f"Core operation {method.upper()} {path} must define " + "a typed success response model" + ) + model = ( + "GenericListResponse" + if path in _LIST_PATHS + else "GenericObjectResponse" + ) + component_schemas.setdefault( + model, + ( + { + "type": "array", + "description": "A list of operation-specific JSON values.", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "additionalProperties": True, + }, + ] + }, + } + if model == "GenericListResponse" + else { + "type": "object", + "description": ( + "An operation-specific JSON object from an " + "installed extension." + ), + "additionalProperties": True, + } + ), + ) + media["schema"] = { + "$ref": f"#/components/schemas/{model}" + } + + is_public = path in {"/health", "/demo/search"} + if is_public: + operation["security"] = [] + public_errors = { + "405": _error_response("Method not allowed"), + "500": _error_response("Internal service error"), + "503": _error_response("Dependency unavailable"), + } + if path == "/demo/search": + public_errors.update( + { + "404": _error_response("Public demo disabled"), + "422": _error_response("Validation failed"), + } + ) + responses = operation.setdefault("responses", {}) + for code, response in public_errors.items(): + if code == "422" or code not in responses: + responses[code] = response + else: + operation["security"] = [{"BrainPAT": []}, {"BearerAuth": []}] + responses = operation.setdefault("responses", {}) + for code, response in protected_errors.items(): + if code == "422" or code not in responses: + responses[code] = response + + if path == "/demo/search": + operation.setdefault("responses", {})["429"] = _error_response( + "Public demo rate limit exceeded" + ) + operation["x-rate-limit"] = { + "requests": 30, + "window": "60 seconds", + } + + schema["servers"] = [ + {"url": "http://localhost:8000", "description": "Local BrainAPI"} + ] + app.openapi_schema = schema + return schema + + app.openapi = custom_openapi diff --git a/src/services/api/routes/ingest.py b/src/services/api/routes/ingest.py index 51ed4a8..6eb936c 100644 --- a/src/services/api/routes/ingest.py +++ b/src/services/api/routes/ingest.py @@ -25,6 +25,7 @@ IngestionRequestBody, IngestionStructuredRequestBody, ) +from src.services.api.constants.responses import IngestionAcceptedResponse from src.services.input.agents import cache_adapter from src.workers.tasks.ingestion import ingest_data as ingest_data_task from src.workers.tasks.ingestion import ingest_file as ingest_file_task @@ -39,7 +40,9 @@ ingest_router = APIRouter(prefix="/ingest", tags=["ingest"]) -@ingest_router.post(path="/", status_code=202) +@ingest_router.post( + path="/", status_code=202, response_model=IngestionAcceptedResponse +) async def ingest_data( data: IngestionRequestBody, request: Request, @@ -81,7 +84,9 @@ async def ingest_data( ) -@ingest_router.post(path="/structured", status_code=202) +@ingest_router.post( + path="/structured", status_code=202, response_model=IngestionAcceptedResponse +) async def ingest_structured_data( data: IngestionStructuredRequestBody, brain_id: str = Depends(get_brain_id), @@ -119,7 +124,9 @@ async def ingest_structured_data( ) -@ingest_router.post(path="/file") +@ingest_router.post( + path="/file", status_code=202, response_model=IngestionAcceptedResponse +) async def ingest_file( request: Request, file: Annotated[UploadFile, File()], @@ -179,14 +186,17 @@ async def ingest_file( headers={"Authorization": f"Bearer {config.docparser_token}"}, ) if response.status_code != 200: - raise HTTPException(status_code=500, detail=response.text) + raise HTTPException( + status_code=503, + detail="Document parser rejected the ingestion request", + ) response_content = { "message": "File ingestion accepted", "task_id": task, } except HTTPException: raise - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + except Exception as exc: + raise HTTPException(status_code=500, detail="File ingestion failed") from exc return JSONResponse(status_code=202, content=response_content) diff --git a/src/services/api/routes/meta.py b/src/services/api/routes/meta.py index fd89411..f3a9d37 100644 --- a/src/services/api/routes/meta.py +++ b/src/services/api/routes/meta.py @@ -17,17 +17,18 @@ get_relationships_properties as get_relationships_properties_controller, get_entity_properties as get_entity_properties_controller, ) +from src.services.api.constants.responses import LoginInfoResponse, StringListResponse meta_router = APIRouter(prefix="/meta", tags=["meta"]) -@meta_router.get(path="/login-info") +@meta_router.get(path="/login-info", response_model=LoginInfoResponse) async def get_login_info(request: Request): """ Resolve whether a BrainPAT is the system token or scoped to a single brain. """ return await get_login_info_controller(request) -@meta_router.get(path="/relationships-properties") +@meta_router.get(path="/relationships-properties", response_model=StringListResponse) async def get_relationships_properties( brain_id: str = Depends(get_brain_id), ): @@ -42,7 +43,7 @@ async def get_relationships_properties( """ return await get_relationships_properties_controller(brain_id) -@meta_router.get(path="/entity-labels") +@meta_router.get(path="/entity-labels", response_model=StringListResponse) async def get_entities_labels( brain_id: str = Depends(get_brain_id), ): @@ -57,7 +58,7 @@ async def get_entities_labels( """ return await get_entities_labels_controller(brain_id) -@meta_router.get(path="/entity-properties") +@meta_router.get(path="/entity-properties", response_model=StringListResponse) async def get_entity_properties( brain_id: str = Depends(get_brain_id), ): @@ -70,4 +71,4 @@ async def get_entity_properties( Returns: list[str]: A list of property key names present on entities in the graph. """ - return await get_entity_properties_controller(brain_id) \ No newline at end of file + return await get_entity_properties_controller(brain_id) diff --git a/src/services/api/routes/model.py b/src/services/api/routes/model.py index c676509..a915796 100644 --- a/src/services/api/routes/model.py +++ b/src/services/api/routes/model.py @@ -23,11 +23,12 @@ add_relationship as add_relationship_controller, update_relationship as update_relationship_controller, ) +from src.constants.kg import Node, Predicate model_router = APIRouter(prefix="/model", tags=["model"]) -@model_router.post(path="/entity") +@model_router.post(path="/entity", response_model=list[Node] | str) async def add_entity( request: AddEntityRequest, brain_id: str = Depends(get_brain_id), @@ -54,7 +55,7 @@ async def add_entity( ) -@model_router.put(path="/entity") +@model_router.put(path="/entity", response_model=Node | None) async def update_entity( request: UpdateEntityRequest, brain_id: str = Depends(get_brain_id), @@ -76,7 +77,7 @@ async def update_entity( ) -@model_router.post(path="/relationship") +@model_router.post(path="/relationship", response_model=str) async def add_relationship( request: AddRelationshipRequest, brain_id: str = Depends(get_brain_id), @@ -97,7 +98,7 @@ async def add_relationship( ) -@model_router.put(path="/relationship") +@model_router.put(path="/relationship", response_model=Predicate | None) async def update_relationship( request: UpdateRelationshipRequest, brain_id: str = Depends(get_brain_id), diff --git a/src/services/api/routes/public.py b/src/services/api/routes/public.py new file mode 100644 index 0000000..91003cc --- /dev/null +++ b/src/services/api/routes/public.py @@ -0,0 +1,111 @@ +"""Public, read-only API surfaces intended for discovery and evaluation.""" + +from __future__ import annotations + +import os + +from fastapi import APIRouter, HTTPException, Query +from pydantic import BaseModel, Field + +from src.config import config +from src.constants.data import BRAIN_VERSION +from src.services.api.constants.requests import SearchRequestBody, SearchResponse +from src.services.api.controllers.search import search as search_controller +from src.services.api.errors import ErrorResponse + + +class HealthResponse(BaseModel): + status: str = Field(description="Service health status.") + service: str = Field(description="Service name.") + version: str = Field(description="Running BrainAPI version.") + + +PUBLIC_ERROR_RESPONSES = { + 404: {"model": ErrorResponse, "description": "Public demo is disabled"}, + 422: {"model": ErrorResponse, "description": "Invalid query"}, + 500: {"model": ErrorResponse, "description": "Internal service error"}, + 503: {"model": ErrorResponse, "description": "Demo search is unavailable"}, +} + +public_router = APIRouter(tags=["public"]) + + +@public_router.get( + "/health", + response_model=HealthResponse, + openapi_extra={"security": []}, +) +async def health() -> HealthResponse: + """Return public liveness metadata without accessing customer data.""" + + return HealthResponse(status="ok", service="brainapi", version=BRAIN_VERSION) + + +@public_router.get( + "/demo/search", + response_model=SearchResponse, + responses=PUBLIC_ERROR_RESPONSES, + openapi_extra={"security": []}, +) +async def demo_search( + query: str = Query( + ..., + min_length=1, + max_length=500, + description="Search query over the public BrainAPI documentation corpus.", + ), + k: int = Query( + 10, + ge=1, + le=10, + description="Number of public documentation passages to return (maximum 10).", + ), +) -> SearchResponse: + """Search the fixed public demo brain using passages-only BM25 retrieval. + + This endpoint never accepts a caller-selected brain and exposes no write path. + Production applies an additional per-client rate limit at the edge. + """ + + if os.getenv("PUBLIC_DEMO_ENABLED", "false").strip().lower() != "true": + raise HTTPException(status_code=404, detail="Public demo search is disabled") + + brain_id = os.getenv("PUBLIC_DEMO_BRAIN_ID", "agentdemo").strip() + if not brain_id or not brain_id.isalnum() or brain_id == "system": + raise HTTPException( + status_code=503, + detail="Public demo brain is not configured correctly", + ) + + try: + configured_max = int(os.getenv("PUBLIC_DEMO_MAX_K", "10")) + except ValueError as exc: + raise HTTPException( + status_code=503, + detail="PUBLIC_DEMO_MAX_K must be an integer", + ) from exc + max_k = min(max(configured_max, 1), 10) + if k > max_k: + raise HTTPException( + status_code=422, + detail=f"k must be less than or equal to {max_k}", + ) + + if ( + not config.search_enabled + or not config.search_use_bm25 + or config.search_use_dense + ): + raise HTTPException( + status_code=503, + detail="Public demo requires BM25-only Search configuration", + ) + + return await search_controller( + SearchRequestBody( + query=query, + k=k, + channels=["passages"], + brain_id=brain_id, + ) + ) diff --git a/src/services/api/routes/retrieve.py b/src/services/api/routes/retrieve.py index b984f48..71fef98 100644 --- a/src/services/api/routes/retrieve.py +++ b/src/services/api/routes/retrieve.py @@ -14,13 +14,36 @@ from fastapi.responses import JSONResponse from src.services.api.dependencies import get_brain_id from src.services.api.constants.requests import ( + GetEntityContextResponse, + GetEntityInfoResponse, + GetEntitySibilingsResponse, + GetEntityStatusResponse, GetContextRequestBody, GetContextResponse, + RecommendResponse, RecommendRequestBody, RetrieveNeighborsAiModeRequestBody, RetrieveNeighborsWithIdentificationParamsRequestBody, RetrieveRequestResponse, RetrieveNeighborsRequestResponse, + SearchRequestBody, + SearchResponse, +) +from src.services.api.constants.responses import ( + ChangelogItemResponse, + ChangelogListResponse, + EntityListResponse, + HopListResponse, + LabelListResponse, + ObservationItemResponse, + ObservationListResponse, + RelationshipListResponse, + StructuredDataItemResponse, + StructuredDataListResponse, + TextChunkListResponse, + TypeListResponse, + VectorListResponse, + VectorStoreListResponse, ) from src.services.api.controllers.retrieve import ( retrieve_neighbors as retrieve_neighbors_controller, @@ -33,6 +56,7 @@ from src.services.api.controllers.retrieve import ( retrieve_data as retrieve_data_controller, ) +from src.services.api.controllers.search import search as search_controller from src.services.api.controllers.structured_data import ( get_structured_data_by_id as get_structured_data_by_id_controller, get_structured_data_list as get_structured_data_list_controller, @@ -119,7 +143,9 @@ async def get_neighbors_with_identification_params( ) -@retrieve_router.post("/entities/neighbors/ai-mode") +@retrieve_router.post( + "/entities/neighbors/ai-mode", response_model=RetrieveNeighborsRequestResponse +) async def get_neighbors_ai_mode( request: RetrieveNeighborsAiModeRequestBody = Body( ..., @@ -143,7 +169,7 @@ async def get_neighbors_ai_mode( ) -@retrieve_router.post("/context") +@retrieve_router.post("/context", response_model=GetContextResponse) async def get_context( request: GetContextRequestBody, brain_id: str = Depends(get_brain_id), @@ -163,7 +189,7 @@ async def get_context( return await retrieve_get_context_controller(request) -@retrieve_router.get(path="/relationships") +@retrieve_router.get(path="/relationships", response_model=RelationshipListResponse) async def get_relationships( limit: int = 10, skip: int = 0, @@ -208,7 +234,7 @@ async def get_relationships( ) -@retrieve_router.get(path="/entities") +@retrieve_router.get(path="/entities", response_model=EntityListResponse) async def get_entities( limit: int = 10, skip: int = 0, @@ -234,7 +260,7 @@ async def get_entities( ) -@retrieve_router.get(path="/structured-data/types") +@retrieve_router.get(path="/structured-data/types", response_model=TypeListResponse) async def get_structured_data_types( brain_id: str = Depends(get_brain_id), ): @@ -244,7 +270,9 @@ async def get_structured_data_types( return await get_structured_data_types_controller(brain_id) -@retrieve_router.get(path="/structured-data/{id}") +@retrieve_router.get( + path="/structured-data/{id}", response_model=StructuredDataItemResponse +) async def get_structured_data_by_id( id: str, brain_id: str = Depends(get_brain_id), @@ -255,7 +283,7 @@ async def get_structured_data_by_id( return await get_structured_data_by_id_controller(id, brain_id) -@retrieve_router.get(path="/structured-data") +@retrieve_router.get(path="/structured-data", response_model=StructuredDataListResponse) async def get_structured_data_list( limit: int = 10, skip: int = 0, @@ -273,7 +301,7 @@ async def get_structured_data_list( ) -@retrieve_router.get(path="/text-chunks") +@retrieve_router.get(path="/text-chunks", response_model=TextChunkListResponse) async def get_text_chunks( brain_id: str = Depends(get_brain_id), limit: int = 10, @@ -325,7 +353,7 @@ async def get_text_chunks( ) -@retrieve_router.get(path="/observations/labels") +@retrieve_router.get(path="/observations/labels", response_model=LabelListResponse) async def get_observation_labels( brain_id: str = Depends(get_brain_id), ): @@ -335,7 +363,9 @@ async def get_observation_labels( return await get_observation_labels_controller(brain_id) -@retrieve_router.get(path="/observations/{id}") +@retrieve_router.get( + path="/observations/{id}", response_model=ObservationItemResponse +) async def get_observation_by_id( id: str, brain_id: str = Depends(get_brain_id), @@ -346,7 +376,7 @@ async def get_observation_by_id( return await get_observation_by_id_controller(id, brain_id) -@retrieve_router.get(path="/observations") +@retrieve_router.get(path="/observations", response_model=ObservationListResponse) async def get_observations_list( limit: int = 10, skip: int = 0, @@ -377,7 +407,7 @@ async def get_observations_list( ) -@retrieve_router.get(path="/changelogs/types") +@retrieve_router.get(path="/changelogs/types", response_model=TypeListResponse) async def get_changelog_types( brain_id: str = Depends(get_brain_id), ): @@ -393,7 +423,7 @@ async def get_changelog_types( return await get_changelog_types_controller(brain_id) -@retrieve_router.get(path="/changelogs/{id}") +@retrieve_router.get(path="/changelogs/{id}", response_model=ChangelogItemResponse) async def get_changelog_by_id( id: str, brain_id: str = Depends(get_brain_id), @@ -411,7 +441,7 @@ async def get_changelog_by_id( return await get_changelog_by_id_controller(id, brain_id) -@retrieve_router.get(path="/changelogs") +@retrieve_router.get(path="/changelogs", response_model=ChangelogListResponse) async def get_changelogs_list( limit: int = 10, skip: int = 0, @@ -436,7 +466,7 @@ async def get_changelogs_list( ) -@retrieve_router.get(path="/hops") +@retrieve_router.get(path="/hops", response_model=HopListResponse) async def get_hops( query: str, degrees: Literal[2] = 2, @@ -458,7 +488,7 @@ async def get_hops( return await get_hops_controller(query, degrees, flattened, brain_id) -@retrieve_router.get(path="/entity/info") +@retrieve_router.get(path="/entity/info", response_model=GetEntityInfoResponse) async def get_entity_info( target: str, query: str, @@ -480,7 +510,7 @@ async def get_entity_info( return await get_entity_info_controller(target, query, max_depth, brain_id) -@retrieve_router.get(path="/entity/context") +@retrieve_router.get(path="/entity/context", response_model=GetEntityContextResponse) async def get_entity_context( target: str, context_depth: int = 3, @@ -500,7 +530,7 @@ async def get_entity_context( return await get_entity_context_controller(target, context_depth, brain_id) -@retrieve_router.get(path="/entity/synergies") +@retrieve_router.get(path="/entity/synergies", response_model=GetEntitySibilingsResponse) async def get_entity_synergies( target: str, polarity: Literal["same", "opposite"] = "same", @@ -532,7 +562,12 @@ async def get_entity_synergies( ) -@retrieve_router.get(path="/recommend") +@retrieve_router.get( + path="/recommend", + response_model=RecommendResponse, + summary="Recommend entities (Preview)", + openapi_extra={"x-stability": "preview"}, +) async def get_recommend( target: str, polarity: Literal["same", "opposite"] = "same", @@ -549,7 +584,7 @@ async def get_recommend( brain_id: str = Depends(get_brain_id), ): """ - Ranked event-graph recommendations composing synergies, asymmetric + Preview: ranked event-graph recommendations composing synergies, asymmetric complementary walks, multi-interest medoids, and optional attribute preferences (retrieval-time only; no training). """ @@ -571,11 +606,17 @@ async def get_recommend( ) -@retrieve_router.post(path="/recommend") +@retrieve_router.post( + path="/recommend", + response_model=RecommendResponse, + summary="Recommend entities from a request body (Preview)", + openapi_extra={"x-stability": "preview"}, +) async def post_recommend( body: RecommendRequestBody, brain_id: str = Depends(get_brain_id), ): + """Preview recommendation surface; request and response compatibility is kept.""" return await get_recommendations_controller( body.target, body.polarity, @@ -594,7 +635,84 @@ async def post_recommend( ) -@retrieve_router.get(path="/entity/status") +@retrieve_router.get(path="/search", response_model=SearchResponse) +async def get_search( + query: str = Query(..., description="The search query."), + k: int = Query(10, ge=1, le=200, description="Number of hits to return."), + channels: Optional[str] = Query( + "passages", + description=( + "Comma-separated channels: passages, entities, events, " + "communities, and/or plugin:." + ), + ), + node_labels: Optional[str] = Query( + None, + description="Comma-separated node labels to filter the entities channel.", + ), + community_labels: Optional[str] = Query( + None, + description="Comma-separated hub labels for the communities channel.", + ), + expand: Literal["none", "neighbors"] = Query( + "none", + description="Optional 1-hop expansion from graph channel seeds.", + ), + fusion: Optional[Literal["rrf", "cc"]] = Query( + None, description="Fusion override. Default is SEARCH_FUSION." + ), + rerank: Optional[str] = Query( + None, + description="none or plugin:. Unknown plugin names return 400.", + ), + target: Optional[str] = Query( + None, + description="Optional USER uuid or id for query-gated rerank of retrieved hits.", + ), + profile_stages: bool = Query(False), + brain_id: str = Depends(get_brain_id), +): + channel_list = [ + item.strip() + for item in (channels or "passages").split(",") + if item.strip() + ] or ["passages"] + node_label_list = [ + item.strip() + for item in (node_labels or "").split(",") + if item.strip() + ] or None + community_label_list = [ + item.strip() + for item in (community_labels or "").split(",") + if item.strip() + ] or None + request = SearchRequestBody( + query=query, + k=k, + channels=channel_list, + node_labels=node_label_list, + community_labels=community_label_list, + expand=expand, + fusion=fusion, + rerank=rerank, + target=target, + profile_stages=profile_stages, + brain_id=brain_id, + ) + return await search_controller(request) + + +@retrieve_router.post(path="/search", response_model=SearchResponse) +async def post_search( + body: SearchRequestBody, + brain_id: str = Depends(get_brain_id), +): + body.brain_id = brain_id or body.brain_id + return await search_controller(body) + + +@retrieve_router.get(path="/entity/status", response_model=GetEntityStatusResponse) async def get_entity_status( target: str, types: Optional[List[str]] = None, @@ -614,7 +732,7 @@ async def get_entity_status( return await get_entity_status_controller(target, types or [], brain_id) -@retrieve_router.get(path="/vectors/stores") +@retrieve_router.get(path="/vectors/stores", response_model=VectorStoreListResponse) async def get_vector_stores(): """ List available vector store names and dimensions. @@ -622,7 +740,7 @@ async def get_vector_stores(): return await get_vector_stores_controller() -@retrieve_router.get(path="/vectors/{store}") +@retrieve_router.get(path="/vectors/{store}", response_model=VectorListResponse) async def get_vectors_list( store: str, limit: int = 10, diff --git a/src/services/api/routes/system.py b/src/services/api/routes/system.py index 71b9d3a..9e136e5 100644 --- a/src/services/api/routes/system.py +++ b/src/services/api/routes/system.py @@ -8,17 +8,19 @@ ----- """ -from fastapi import APIRouter +from fastapi import APIRouter, Request from src.services.api.constants.requests import CreateBrainRequest from src.services.api.controllers.system import ( get_brains_list as get_brains_list_controller, create_new_brain as create_new_brain_controller, ) +from src.constants.data import Brain +from src.services.api.errors import error_response system_router = APIRouter(prefix="/system", tags=["system"]) -@system_router.get(path="/brains-list") +@system_router.get(path="/brains-list", response_model=list[Brain]) async def get_brains_list(): """ Get the list of brains. @@ -26,7 +28,7 @@ async def get_brains_list(): return await get_brains_list_controller() -@system_router.post(path="/brains") +@system_router.post(path="/brains", response_model=Brain) async def create_brain(request: CreateBrainRequest): """ Create a new brain @@ -34,25 +36,39 @@ async def create_brain(request: CreateBrainRequest): return await create_new_brain_controller(request) -@system_router.get(path="/brains/{brain_id}/reset") # TODO -async def reset(brain_id: str): +@system_router.get(path="/brains/{brain_id}/reset", include_in_schema=False) # TODO +async def reset(brain_id: str, request: Request): """ Resets the brain """ - raise NotImplementedError("Not implemented") + return _not_implemented(request, brain_id, "reset") -@system_router.get(path="/brains/{brain_id}/delete") # TODO -async def delete(brain_id: str): +@system_router.get(path="/brains/{brain_id}/delete", include_in_schema=False) # TODO +async def delete(brain_id: str, request: Request): """ Deletes the brain and all its data """ - raise NotImplementedError("Not implemented") + return _not_implemented(request, brain_id, "delete") -@system_router.post(path="/brains/{brain_id}/create-backup") # TODO -async def create_backup(brain_id: str): +@system_router.post( + path="/brains/{brain_id}/create-backup", include_in_schema=False +) # TODO +async def create_backup(brain_id: str, request: Request): """ Creates a backup of the brain and returns a task ID """ - raise NotImplementedError("Not implemented") + return _not_implemented(request, brain_id, "create-backup") + + +def _not_implemented(request: Request, brain_id: str, operation: str): + return error_response( + request, + status_code=501, + detail="Not implemented", + code="not_implemented", + message=f"Brain {operation} is not implemented.", + resolution="Use the profile-aware deploy backup tooling where applicable.", + extra={"brain_id": brain_id, "operation": operation}, + ) diff --git a/src/services/api/routes/tasks.py b/src/services/api/routes/tasks.py index 536d2ab..63c950d 100644 --- a/src/services/api/routes/tasks.py +++ b/src/services/api/routes/tasks.py @@ -13,11 +13,12 @@ from fastapi import APIRouter, Depends, HTTPException from src.services.api.dependencies import get_brain_id from src.services.kg_agent.main import cache_adapter +from src.services.api.constants.responses import TaskListResponse, TaskStateResponse tasks_router = APIRouter(prefix="/tasks", tags=["tasks"]) -@tasks_router.get("/") +@tasks_router.get("/", response_model=TaskListResponse) async def get_tasks(brain_id: str = Depends(get_brain_id)): try: task_keys = cache_adapter.get_task_keys(brain_id) @@ -38,13 +39,13 @@ async def get_tasks(brain_id: str = Depends(get_brain_id)): return {"tasks": results} except Exception as e: log(f"Error in get_tasks: {type(e).__name__}: {str(e)}") - return { - "status": "error", - "result": {"error": str(e), "error_type": type(e).__name__}, - } + raise HTTPException( + status_code=500, + detail="Could not retrieve tasks", + ) from e -@tasks_router.get("/{task_id}") +@tasks_router.get("/{task_id}", response_model=TaskStateResponse) async def get_task(task_id: str, brain_id: str = Depends(get_brain_id)): """ Get the result of a task by its ID. @@ -66,8 +67,7 @@ async def get_task(task_id: str, brain_id: str = Depends(get_brain_id)): raise except Exception as e: log(f"Error in get_task: {type(e).__name__}: {str(e)}") - return { - "task_id": task_id, - "status": "error", - "result": {"error": str(e), "error_type": type(e).__name__}, - } + raise HTTPException( + status_code=500, + detail="Could not retrieve task", + ) from e diff --git a/src/services/mcp/app.py b/src/services/mcp/app.py index 1a55df6..6b59daf 100644 --- a/src/services/mcp/app.py +++ b/src/services/mcp/app.py @@ -12,8 +12,6 @@ _project_root = Path(__file__).resolve().parent.parent.parent.parent dotenv.load_dotenv(_project_root / ".env") -from contextlib import asynccontextmanager - from src.lib.tracing.middleware import TraceMiddleware from src.lib.tracing.runtime import start_runtime_monitoring, stop_runtime_monitoring from src.services.mcp.main import auth_token_var, mcp, oauth_provider @@ -31,6 +29,15 @@ def _load_mcp_plugins(): loader = PluginLoader(plugins_dir=PLUGINS_DIR, context=ctx) results = loader.load_all() _log_plugin_banner(loader, results) + failed = sorted(name for name, loaded in results.items() if not loaded) + default_policy = ( + "warn" if os.getenv("ENV", "production").lower() == "development" else "fail" + ) + policy = os.getenv("PLUGIN_FAILURE_POLICY", default_policy).strip().lower() + if policy not in {"fail", "warn"}: + raise RuntimeError("PLUGIN_FAILURE_POLICY must be 'fail' or 'warn'") + if failed and policy == "fail": + raise RuntimeError(f"Required plugins failed to load: {', '.join(failed)}") def _log_plugin_banner(loader, results: dict[str, bool]): @@ -150,12 +157,6 @@ async def _mcp_info(_request): return JSONResponse(body, status_code=200) -@asynccontextmanager -async def _lifespan(app): - async with _mcp_app.router.lifespan_context(_mcp_app): - yield - - _custom_routes = [ Route("/", _health, methods=["GET"]), Route("/mcp", _mcp_info, methods=["GET"]), diff --git a/src/workers/app.py b/src/workers/app.py index e4d33b4..6bfb7fe 100644 --- a/src/workers/app.py +++ b/src/workers/app.py @@ -24,6 +24,7 @@ ) from src.workers.kombu_redis_patch import apply_kombu_redis_unblocked_patch +from src.workers.redis_url import redis_connection_url apply_kombu_redis_unblocked_patch() @@ -42,18 +43,23 @@ REDIS_HOST = os.getenv("REDIS_HOST", "localhost") REDIS_PORT = os.getenv("REDIS_PORT", "6379") +REDIS_URL = redis_connection_url( + REDIS_HOST, + REDIS_PORT, + os.getenv("REDIS_PASSWORD"), +) ingestion_app = Celery( "ingestion_app", broker=( "amqp://kalo:kalo@localhost:5672/" if os.getenv("CELERY_BACKEND") == "rabbitmq" - else f"redis://{REDIS_HOST}:{REDIS_PORT}/0" + else REDIS_URL ), backend=( "rpc://" if os.getenv("CELERY_BACKEND") == "rabbitmq" - else f"redis://{REDIS_HOST}:{REDIS_PORT}/0" + else REDIS_URL ), include=["src.workers.tasks.ingestion"], ) diff --git a/src/workers/redis_url.py b/src/workers/redis_url.py new file mode 100644 index 0000000..58a4dbe --- /dev/null +++ b/src/workers/redis_url.py @@ -0,0 +1,18 @@ +"""Redis connection URL helpers for worker transports.""" + +from __future__ import annotations + +from urllib.parse import quote + + +def redis_connection_url( + host: str, + port: str | int, + password: str | None, + *, + database: int = 0, +) -> str: + """Build a Redis URL, percent-encoding credentials when authentication is enabled.""" + + auth = f":{quote(password, safe='')}@" if password else "" + return f"redis://{auth}{host}:{port}/{database}" diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.py index 069d2ec..c14c864 100644 --- a/src/workers/tasks/ingestion.py +++ b/src/workers/tasks/ingestion.py @@ -656,6 +656,16 @@ def ingest_data(self, args: dict): with track_stage(cost_ledger, "embed"): cost_ledger.embed.add_usage(calls=1) + if payload.skip_enrichment: + set_ingestion_task_status( + self.request.id, + payload.brain_id, + "completed", + stage="completed", + cost=cost_ledger.to_dict(), + ) + return self.request.id + if config.pipeline_mode == "accurate" and config.run_observations: with track_stage(cost_ledger, "observations"): observations = observations_agent.observe( @@ -967,19 +977,26 @@ def process_architect_relationships(self, args: dict): print(f"> Processing node {node_data.name}") try: future.result(timeout=180) + props = { + k: v + for k, v in (node_data.properties or {}).items() + if v is not None + } + labels = [node_data.type] + extra = props.get("catalog_labels") or [] + if isinstance(extra, str): + extra = [extra] + for item in extra: + label = str(item).strip() + if label and label not in labels: + labels.append(label) graph_nodes.append( Node( uuid=node_data.uuid, - labels=[node_data.type], + labels=labels, name=node_data.name, description=node_data.description, - properties={ - k: v - for k, v in ( - node_data.properties or {} - ).items() - if v is not None - }, + properties=props, polarity=( node_data.polarity if node_data.polarity diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e1a3ac9 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,73 @@ +"""Process-wide isolation for the BrainAPI test suite.""" + +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path + +import pytest + + +_REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +if str(_REPOSITORY_ROOT) not in sys.path: + sys.path.insert(0, str(_REPOSITORY_ROOT)) + +# Installed development plugins must never change core test collection or imports. +# Plugin-specific tests pass their own temporary directories directly to loaders. +_PLUGIN_SANDBOX = tempfile.TemporaryDirectory(prefix="brainapi-test-plugins-") +os.environ["PLUGINS_DIR"] = _PLUGIN_SANDBOX.name + +# Configuration is instantiated during module import. Establish deterministic +# non-secret test values before collection so file ordering cannot change it. +_TEST_ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "test-password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "POSTGRES_HOST": "localhost", + "POSTGRES_PORT": "5432", + "POSTGRES_USERNAME": "postgres", + "POSTGRES_PASSWORD": "test-password", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", +} +for _key, _value in _TEST_ENV_DEFAULTS.items(): + os.environ.setdefault(_key, _value) + + +@pytest.fixture(autouse=True) +def _restore_retrieval_globals_after_test(): + """Undo process-global mocks leaked by overlapping async patch contexts.""" + + module = sys.modules.get("src.services.api.controllers.retrieve") + if module is None: + yield + return + + passage_retriever = getattr(module, "_retrieve_passages", None) + extractor = getattr(module, "_entity_extractor", None) + extract_elements = getattr(extractor, "extract_elements", None) + yield + if passage_retriever is not None: + module._retrieve_passages = passage_retriever + if extractor is not None and extract_elements is not None: + extractor.extract_elements = extract_elements diff --git a/tests/fixtures/jdsearch/product_meta_data.txt b/tests/fixtures/jdsearch/product_meta_data.txt new file mode 100644 index 0000000..f47d95e --- /dev/null +++ b/tests/fixtures/jdsearch/product_meta_data.txt @@ -0,0 +1,3 @@ +wid name brand_id brand_name cate_id_1 cate_name_1 cate_id_2 cate_name_2 cate_id_3 cate_name_3 cate_id_4 cate_name_4 shop_id +100 111222 b1 brandX c1 cat1 c2 cat2 c3 cat3 c4 cat4 s1 +200 333 b2 brandY c1 cat1 c2 cat2 c3 cat3 c4 cat4 s2 diff --git a/tests/fixtures/jdsearch/user_behavior_data.txt b/tests/fixtures/jdsearch/user_behavior_data.txt new file mode 100644 index 0000000..d8d53b7 --- /dev/null +++ b/tests/fixtures/jdsearch/user_behavior_data.txt @@ -0,0 +1,3 @@ +query candidate_wid_list candidate_label_list history_qry_list history_wid_list history_type_list history_time_list +111222 100_200_999 1.0_0.0_0.0 -1 200 CLICK 0_120 +333 200_100 2.0_1.0 111 100_200 CART_FLW 0_50_10 diff --git a/tests/plugins_path_helper.py b/tests/plugins_path_helper.py index 2bc4fb7..31e3b5b 100644 --- a/tests/plugins_path_helper.py +++ b/tests/plugins_path_helper.py @@ -4,6 +4,7 @@ import importlib.util import sys +import unittest from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -11,6 +12,9 @@ def _load(name: str, path: Path): + if not path.is_file(): + raise unittest.SkipTest(f"optional plugin artifact is not installed: {path}") + # Ensure plugin root is on path so `from models.X` works inside modules. plugin_str = str(PLUGIN) if plugin_str not in sys.path: diff --git a/tests/test_agent_readiness_api.py b/tests/test_agent_readiness_api.py new file mode 100644 index 0000000..9c894f0 --- /dev/null +++ b/tests/test_agent_readiness_api.py @@ -0,0 +1,368 @@ +import os +import re +import unittest +from unittest.mock import AsyncMock, Mock, patch + +ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", + "PIPELINE_MODE": "accurate", +} +for key, value in ENV_DEFAULTS.items(): + os.environ.setdefault(key, value) + +from fastapi import FastAPI, HTTPException, Query +from fastapi.testclient import TestClient + +from src.services.api.constants.requests import SearchResponse +from src.services.api.errors import install_error_handlers +from src.services.api.middlewares.auth import BrainPATMiddleware +from src.services.api.middlewares.brains import BrainMiddleware +from src.services.api.routes.public import public_router + + +def _public_app() -> FastAPI: + app = FastAPI() + install_error_handlers(app) + app.include_router(public_router) + return app + + +class StructuredErrorTests(unittest.TestCase): + def test_http_error_preserves_detail_and_request_id(self): + app = FastAPI() + install_error_handlers(app) + + @app.get("/bad") + async def bad(): + raise HTTPException(status_code=404, detail="missing") + + response = TestClient(app).get( + "/bad", + headers={"X-Request-ID": "request-123"}, + ) + self.assertEqual(response.status_code, 404) + self.assertEqual(response.json()["detail"], "missing") + self.assertEqual(response.json()["error"]["code"], "RESOURCE_NOT_FOUND") + self.assertTrue(response.json()["error"]["resolution"]) + self.assertEqual(response.json()["error"]["request_id"], "request-123") + self.assertEqual(response.headers["X-Request-ID"], "request-123") + + def test_validation_error_keeps_structured_detail(self): + app = FastAPI() + install_error_handlers(app) + + @app.get("/validate") + async def validate(value: int = Query(..., ge=1)): + return {"value": value} + + response = TestClient(app).get("/validate?value=0") + self.assertEqual(response.status_code, 422) + self.assertIsInstance(response.json()["detail"], list) + self.assertEqual(response.json()["error"]["code"], "VALIDATION_ERROR") + + def test_unexpected_error_is_sanitized(self): + app = FastAPI() + install_error_handlers(app) + + @app.get("/explode") + async def explode(): + raise RuntimeError("secret backend detail") + + response = TestClient(app, raise_server_exceptions=False).get("/explode") + self.assertEqual(response.status_code, 500) + self.assertEqual(response.json()["detail"], "Internal server error") + self.assertNotIn("secret backend detail", response.text) + + def test_unknown_route_and_brain_failure_use_stable_codes(self): + app = FastAPI() + install_error_handlers(app) + + @app.get("/brain") + async def brain_failure(): + raise HTTPException(status_code=406, detail="unknown brain") + + missing = TestClient(app).get("/missing") + brain = TestClient(app).get("/brain") + self.assertEqual(missing.status_code, 404) + self.assertEqual(missing.json()["error"]["code"], "RESOURCE_NOT_FOUND") + self.assertEqual(brain.status_code, 406) + self.assertEqual(brain.json()["error"]["code"], "BRAIN_NOT_FOUND") + + def test_auth_middleware_returns_structured_json(self): + app = FastAPI() + install_error_handlers(app) + app.add_middleware(BrainPATMiddleware) + + @app.get("/system/test") + async def protected(): + return {"ok": True} + + with patch.dict(os.environ, {"BRAINPAT_TOKEN": "system-token"}): + response = TestClient(app).get("/system/test") + self.assertEqual(response.status_code, 401) + self.assertEqual(response.json()["error"]["code"], "AUTH_INVALID") + + def test_brain_middleware_validates_before_backing_store_access(self): + app = FastAPI() + install_error_handlers(app) + app.add_middleware(BrainMiddleware) + + @app.get("/protected") + async def protected(): + return {"ok": True} + + response = TestClient(app).get( + "/protected", + headers={"BrainPAT": "invalid", "X-Brain-ID": "not-valid!"}, + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()["error"]["code"], "BRAIN_ID_INVALID") + + def test_brain_store_failure_is_structured_service_unavailable(self): + app = FastAPI() + install_error_handlers(app) + app.add_middleware(BrainMiddleware) + + @app.get("/protected") + async def protected(): + return {"ok": True} + + with patch( + "src.services.api.middlewares.brains.cache_adapter.get", + side_effect=ConnectionError("redis secret"), + ): + response = TestClient(app).get( + "/protected", + headers={"BrainPAT": "invalid", "X-Brain-ID": "validbrain"}, + ) + self.assertEqual(response.status_code, 503) + self.assertEqual(response.json()["error"]["code"], "SERVICE_UNAVAILABLE") + self.assertNotIn("redis secret", response.text) + + +class PublicApiTests(unittest.TestCase): + def test_health_is_typed_and_public(self): + response = TestClient(_public_app()).get("/health") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["status"], "ok") + self.assertEqual(response.json()["service"], "brainapi") + self.assertTrue(response.json()["version"]) + + def test_full_app_exempts_machine_routes_and_authenticates_protected_routes(self): + from src.services.api.app import app + + client = TestClient(app) + health = client.get("/health", headers={"Origin": "https://agent.example"}) + openapi = client.get("/openapi.json") + protected = client.get("/tasks/", headers={"X-Request-ID": "auth-check"}) + + self.assertEqual(health.status_code, 200) + self.assertNotIn("Access-Control-Allow-Origin", health.headers) + self.assertEqual(openapi.status_code, 200) + self.assertEqual(openapi.json()["openapi"], "3.1.0") + self.assertEqual(protected.status_code, 401) + self.assertEqual(protected.json()["error"]["code"], "AUTH_INVALID") + self.assertEqual(protected.headers["X-Request-ID"], "auth-check") + + def test_demo_is_disabled_by_default(self): + with patch.dict(os.environ, {"PUBLIC_DEMO_ENABLED": "false"}): + response = TestClient(_public_app()).get("/demo/search?query=memory") + self.assertEqual(response.status_code, 404) + self.assertEqual(response.json()["error"]["code"], "RESOURCE_NOT_FOUND") + + def test_demo_forces_brain_passages_and_k_limit(self): + captured = [] + + async def fake_search(request): + captured.append(request) + return SearchResponse(hits=[]) + + env = { + "PUBLIC_DEMO_ENABLED": "true", + "PUBLIC_DEMO_BRAIN_ID": "agentdemo", + "PUBLIC_DEMO_MAX_K": "5", + } + with ( + patch.dict(os.environ, env), + patch("src.services.api.routes.public.config.search_enabled", True), + patch("src.services.api.routes.public.config.search_use_bm25", True), + patch("src.services.api.routes.public.config.search_use_dense", False), + patch( + "src.services.api.routes.public.search_controller", + new=AsyncMock(side_effect=fake_search), + ), + ): + response = TestClient(_public_app()).get( + "/demo/search?query=memory&k=5&brain_id=customerbrain" + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(len(captured), 1) + self.assertEqual(captured[0].brain_id, "agentdemo") + self.assertEqual(captured[0].channels, ["passages"]) + self.assertEqual(captured[0].k, 5) + + def test_demo_rejects_configured_limit_and_long_query(self): + env = { + "PUBLIC_DEMO_ENABLED": "true", + "PUBLIC_DEMO_BRAIN_ID": "agentdemo", + "PUBLIC_DEMO_MAX_K": "3", + } + with patch.dict(os.environ, env): + too_many = TestClient(_public_app()).get("/demo/search?query=x&k=4") + too_long = TestClient(_public_app()).get( + f"/demo/search?query={'x' * 501}" + ) + self.assertEqual(too_many.status_code, 422) + self.assertEqual(too_long.status_code, 422) + + def test_demo_reports_unavailable_search_dependency(self): + env = { + "PUBLIC_DEMO_ENABLED": "true", + "PUBLIC_DEMO_BRAIN_ID": "agentdemo", + "PUBLIC_DEMO_MAX_K": "10", + } + with ( + patch.dict(os.environ, env), + patch("src.services.api.routes.public.config.search_enabled", False), + ): + response = TestClient(_public_app()).get("/demo/search?query=memory") + + self.assertEqual(response.status_code, 503) + self.assertEqual(response.json()["error"]["code"], "SERVICE_UNAVAILABLE") + self.assertTrue(response.json()["error"]["resolution"]) + + def test_demo_controller_failures_are_structured_and_writes_are_absent(self): + env = { + "PUBLIC_DEMO_ENABLED": "true", + "PUBLIC_DEMO_BRAIN_ID": "agentdemo", + "PUBLIC_DEMO_MAX_K": "10", + } + with ( + patch.dict(os.environ, env), + patch("src.services.api.routes.public.config.search_enabled", True), + patch("src.services.api.routes.public.config.search_use_bm25", True), + patch("src.services.api.routes.public.config.search_use_dense", False), + patch( + "src.services.api.routes.public.search_controller", + new=AsyncMock(side_effect=RuntimeError("backend secret")), + ), + ): + failed = TestClient( + _public_app(), raise_server_exceptions=False + ).get("/demo/search?query=memory") + write = TestClient(_public_app()).post("/demo/search", json={"query": "x"}) + + self.assertEqual(failed.status_code, 500) + self.assertEqual(failed.json()["error"]["code"], "INTERNAL_ERROR") + self.assertNotIn("backend secret", failed.text) + self.assertEqual(write.status_code, 405) + self.assertEqual(write.json()["error"]["code"], "METHOD_NOT_ALLOWED") + self.assertEqual(write.headers["Allow"], "GET") + + +class OpenApiContractTests(unittest.TestCase): + def test_operations_are_function_calling_ready(self): + from src.services.api.app import app + + app.openapi_schema = None + schema = app.openapi() + operations = [] + for path, path_item in schema["paths"].items(): + for method, operation in path_item.items(): + if method not in {"get", "post", "put", "patch", "delete"}: + continue + operations.append((path, method, operation)) + + operation_ids = [operation["operationId"] for _, _, operation in operations] + self.assertEqual(len(operation_ids), len(set(operation_ids))) + for path, method, operation in operations: + operation_id = operation["operationId"] + self.assertRegex(operation_id, re.compile(r"^[a-z][a-z0-9_]{0,63}$")) + self.assertTrue(operation.get("description"), (method, path)) + for parameter in operation.get("parameters", []): + self.assertTrue( + "$ref" in parameter or "schema" in parameter, + (method, path, parameter), + ) + success_schemas = [ + media.get("schema") + for status, response in operation["responses"].items() + if str(status).startswith("2") + for media in response.get("content", {}).values() + ] + self.assertTrue(success_schemas, (method, path)) + self.assertTrue(all(item for item in success_schemas), (method, path)) + expected_security = [] if path in {"/health", "/demo/search"} else [ + {"BrainPAT": []}, + {"BearerAuth": []}, + ] + self.assertEqual(operation.get("security"), expected_security) + if path not in {"/health", "/demo/search"}: + self.assertEqual( + operation["responses"]["422"]["content"]["application/json"][ + "schema" + ]["$ref"], + "#/components/schemas/ErrorResponse", + ) + + self.assertNotIn("/system/brains/{brain_id}/reset", schema["paths"]) + self.assertIn("202", schema["paths"]["/ingest/file"]["post"]["responses"]) + + generic_refs = [ + response["content"]["application/json"]["schema"].get("$ref", "") + for _, _, operation in operations + for status, response in operation["responses"].items() + if str(status).startswith("2") + and "application/json" in response.get("content", {}) + ] + self.assertFalse(any("Generic" in ref for ref in generic_refs)) + + +class PublicDemoSeedTests(unittest.TestCase): + def test_seed_command_skips_an_up_to_date_brain(self): + from scripts import seed_public_demo + + docs = Mock() + docs.text = "public docs" + docs.raise_for_status.return_value = None + existing = Mock(status_code=200) + existing.json.return_value = {"total": 1} + post = Mock() + + with ( + patch.dict(os.environ, {"BRAINPAT_TOKEN": "secret", "PUBLIC_DEMO_BRAIN_ID": "agentdemo"}), + patch("sys.argv", ["seed_public_demo.py"]), + patch.object(seed_public_demo.requests, "get", side_effect=[docs, existing]), + patch.object(seed_public_demo.requests, "post", post), + ): + result = seed_public_demo.main() + + self.assertEqual(result, 0) + post.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_benchmark_contract_guardrails.py b/tests/test_benchmark_contract_guardrails.py index 69cd877..2aecc31 100644 --- a/tests/test_benchmark_contract_guardrails.py +++ b/tests/test_benchmark_contract_guardrails.py @@ -70,6 +70,36 @@ def test_bench_clients_call_core_ingest(self): msg=f"{path} should still target core ingest", ) + def test_search_client_scores_via_search_not_context(self): + client = ROOT / "benchmarks" / "search" / "client.py" + self.assertTrue(client.exists(), msg="benchmarks/search/client.py is required") + text = client.read_text(encoding="utf-8") + self.assertIn("/retrieve/search", text) + self.assertIn("/ingest/", text) + self.assertIn("skip_enrichment", text) + self.assertNotIn( + "/retrieve/context", + text, + msg="search harness must not score via /retrieve/context", + ) + self.assertNotIn( + "/retrieve/recommend", + text, + msg="search harness must not score via /retrieve/recommend", + ) + for needle in ("locomoconv", "demorecsys"): + self.assertNotIn( + needle, + text, + msg=f"search client must not default to {needle}", + ) + + def test_search_config_defaults_to_searchbench(self): + config = ROOT / "benchmarks" / "search" / "config.py" + text = config.read_text(encoding="utf-8") + self.assertIn('DEFAULT_BRAIN_ID = "searchbenchsmoke"', text) + self.assertIn('REQUIRED_PREFIX = "searchbench"', text) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_catalog_graph_search.py b/tests/test_catalog_graph_search.py new file mode 100644 index 0000000..ebedcf4 --- /dev/null +++ b/tests/test_catalog_graph_search.py @@ -0,0 +1,102 @@ +import os +import unittest +from unittest.mock import MagicMock + +ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", +} +for key, value in ENV_DEFAULTS.items(): + os.environ.setdefault(key, value) + +from src.constants.embeddings import Vector +from src.core.agents.scout_agent import ScoutEntity +from src.core.search.catalog_graph import node_embed_text +from src.core.saving.ingestion_manager import IngestionManager +from src.lib.postgresql.graph_store import PostgreSQLGraphStore +from src.services.api.constants.requests import SearchRequestBody + + +class CatalogGraphSearchUnitTests(unittest.TestCase): + def test_search_request_body_stays_passages_without_catalog_fields(self): + body = SearchRequestBody(query="modern sofas") + self.assertEqual(body.channels, ["passages"]) + fields = set(SearchRequestBody.model_fields) + self.assertIn("target", fields) + for banned in ("product_id", "sku", "brand", "user_id"): + self.assertNotIn(banned, fields) + + def test_node_embed_text_uses_search_text_only_when_set(self): + catalog = ScoutEntity( + type="ENTITY", + name="sofa", + uuid="sofa-1", + properties={"search_text": "velvet sofa navy modern"}, + ) + text, cache_key = node_embed_text(catalog) + self.assertEqual(text, "velvet sofa navy modern") + self.assertEqual(cache_key, "uuid:sofa-1") + memory = ScoutEntity(type="PERSON", name="Ada", uuid="p1") + mem_text, mem_key = node_embed_text(memory) + self.assertEqual(mem_text, "Ada") + self.assertEqual(mem_key, "Ada") + + def test_process_node_vectors_embeds_search_text(self): + embeddings = MagicMock() + embeddings.embed_text.return_value = Vector( + id="v1", embeddings=[0.1, 0.2], metadata={} + ) + store = MagicMock() + store.add_vectors.return_value = ["vid-1"] + manager = IngestionManager(embeddings, store, MagicMock()) + node = ScoutEntity( + type="ENTITY", + name="sofa", + uuid="sofa-1", + properties={"search_text": "velvet sofa for small rooms"}, + ) + manager.process_node_vectors(node, "searchbenchsmoke") + embeddings.embed_text.assert_called_once_with("velvet sofa for small rooms") + meta = embeddings.embed_text.return_value.metadata + self.assertEqual(meta["uuid"], "sofa-1") + self.assertEqual(meta["name"], "sofa") + memory = ScoutEntity(type="PERSON", name="Ada", uuid="p1") + manager.process_node_vectors(memory, "searchbenchsmoke") + embeddings.embed_text.assert_called_with("Ada") + + def test_node_search_ddl_is_english_and_gated(self): + ddl = PostgreSQLGraphStore._SEARCH_DDL + self.assertIn("kg_nodes", ddl) + self.assertIn("search_tsv", ddl) + self.assertIn("to_tsvector", ddl) + self.assertIn("'english'", ddl) + self.assertNotIn("'italian'", ddl) + self.assertIn("data->>'search_text'", ddl) + self.assertNotIn("concat_ws", ddl) + self.assertIn("::regconfig", ddl) + self.assertIn("||", ddl) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_chatbot_inference_plugin.py b/tests/test_chatbot_inference_plugin.py index 65ad112..43d0d1b 100644 --- a/tests/test_chatbot_inference_plugin.py +++ b/tests/test_chatbot_inference_plugin.py @@ -32,6 +32,9 @@ os.environ.setdefault(key, value) CHATBOT_PLUGIN_DIR = Path(__file__).resolve().parent.parent / "plugins" / "chatbot" +if not (CHATBOT_PLUGIN_DIR / "main.py").is_file(): + raise unittest.SkipTest("optional chatbot plugin is not installed") + if str(CHATBOT_PLUGIN_DIR) not in sys.path: sys.path.insert(0, str(CHATBOT_PLUGIN_DIR)) diff --git a/tests/test_context_retrieval.py b/tests/test_context_retrieval.py index b57f106..d7c4709 100644 --- a/tests/test_context_retrieval.py +++ b/tests/test_context_retrieval.py @@ -3,6 +3,7 @@ import sys import types import unittest +from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -41,6 +42,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from src.constants.embeddings import Vector +from src.constants.data import TextChunk from src.constants.kg import Node, Predicate from src.core.search.entity_info import EventSynergyRetriever from src.services.api.constants.requests import GetContextRequestBody @@ -483,5 +485,99 @@ def test_work_cap_stops_exploration(self): self.assertLessEqual(work[0], 3) +class ContextPassageModeTests(unittest.TestCase): + def test_search_off_keeps_ilike_lexical_leg(self): + from src.services.api.controllers import retrieve as retrieve_mod + + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.1], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_data.return_value = [ + Vector(id="d", metadata={"resource_id": "dense-1"}, distance=0.2) + ] + ilike_chunk = TextChunk(id="ilike-1", text="ilike hit about license") + mock_data = MagicMock() + mock_data.search.return_value = SimpleNamespace(text_chunks=[ilike_chunk]) + mock_data.get_text_chunks_by_ids.return_value = ( + [TextChunk(id="dense-1", text="dense body")], + [], + ) + + with ( + patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings), + patch.object(retrieve_mod, "vector_search", mock_vs), + patch.object(retrieve_mod, "data_adapter", mock_data), + patch.object(retrieve_mod.config, "search_enabled", False), + patch.object(retrieve_mod.config, "context_passage_mode", "hybrid"), + ): + hits = retrieve_mod._retrieve_passages("license", "brain-a", limit=8) + + mock_data.search.assert_called() + mock_data.search_bm25.assert_not_called() + self.assertTrue(any(item[0] == "ilike-1" for item in hits)) + + def test_search_on_hybrid_uses_bm25(self): + from src.services.api.controllers import retrieve as retrieve_mod + + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.1], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_data.return_value = [ + Vector(id="d", metadata={"resource_id": "dense-1"}, distance=0.2) + ] + bm25_chunk = TextChunk(id="bm25-1", text="bm25 hit about license") + mock_data = MagicMock() + mock_data.search_bm25.return_value = [(bm25_chunk, 5.0)] + mock_data.get_text_chunks_by_ids.return_value = ( + [TextChunk(id="dense-1", text="dense body")], + [], + ) + + with ( + patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings), + patch.object(retrieve_mod, "vector_search", mock_vs), + patch.object(retrieve_mod, "data_adapter", mock_data), + patch.object(retrieve_mod.config, "search_enabled", True), + patch.object(retrieve_mod.config, "context_passage_mode", "hybrid"), + ): + hits = retrieve_mod._retrieve_passages("license", "brain-a", limit=8) + + mock_data.search_bm25.assert_called() + mock_data.search.assert_not_called() + ids = [item[0] for item in hits] + self.assertIn("bm25-1", ids) + self.assertIn("dense-1", ids) + + def test_search_on_ilike_freezes_old_lexical_leg(self): + from src.services.api.controllers import retrieve as retrieve_mod + + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.1], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_data.return_value = [] + ilike_chunk = TextChunk(id="ilike-2", text="legacy ilike") + mock_data = MagicMock() + mock_data.search.return_value = SimpleNamespace(text_chunks=[ilike_chunk]) + + with ( + patch.object(retrieve_mod, "embeddings_adapter", mock_embeddings), + patch.object(retrieve_mod, "vector_search", mock_vs), + patch.object(retrieve_mod, "data_adapter", mock_data), + patch.object(retrieve_mod.config, "search_enabled", True), + patch.object(retrieve_mod.config, "context_passage_mode", "ilike"), + ): + hits = retrieve_mod._retrieve_passages("license", "brain-a", limit=8) + + mock_data.search.assert_called() + mock_data.search_bm25.assert_not_called() + self.assertEqual(hits[0][0], "ilike-2") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_deploy_production_contract.py b/tests/test_deploy_production_contract.py new file mode 100644 index 0000000..b2bcfa0 --- /dev/null +++ b/tests/test_deploy_production_contract.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +from deploy.backup_restore import verify_backup +from scripts.check_release_readiness import _check_latency +from scripts.openai_ci_stub import _embedding +from scripts.production_smoke import _latency_summary + + +ROOT = Path(__file__).resolve().parents[1] +DEPLOY = ROOT / "deploy" + + +@pytest.mark.parametrize("profile", ["light", "heavy"]) +def test_only_nginx_publishes_ports(profile): + data = yaml.safe_load( + (DEPLOY / f"docker-compose.{profile}.yaml").read_text(encoding="utf-8") + ) + published = { + service: config["ports"] + for service, config in data["services"].items() + if config.get("ports") + } + assert published == {"nginx": ["80:80", "443:443"]} + assert "version" not in data + + +def _compose_env(profile: str) -> dict[str, str]: + env = { + **os.environ, + "BRAINAPI_IMAGE": "brainapi:test", + "BRAINAPI_ENV_FILE": str(DEPLOY / f"env.{profile}.example"), + "REDIS_PASSWORD": "redis-secret", + } + if profile == "light": + env["POSTGRES_PASSWORD"] = "postgres-secret" + else: + env.update( + { + "NEO4J_PASSWORD": "neo-secret", + "MONGO_PASSWORD": "mongo-secret", + "MINIO_ACCESS_KEY": "minio-access", + "MINIO_SECRET_KEY": "minio-secret", + "MILVUS_TOKEN": "root:milvus-secret", + "MILVUS_ROOT_PASSWORD": "milvus-secret", + } + ) + return env + + +@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker unavailable") +@pytest.mark.parametrize("profile", ["light", "heavy"]) +def test_compose_profiles_render_with_escaped_healthchecks(profile): + result = subprocess.run( + [ + "docker", + "compose", + "-f", + str(DEPLOY / f"docker-compose.{profile}.yaml"), + "config", + "--format", + "json", + ], + env=_compose_env(profile), + text=True, + capture_output=True, + check=True, + ) + config = json.loads(result.stdout) + assert config["services"]["redis"]["healthcheck"]["test"][1].find( + "REDIS_PASSWORD" + ) >= 0 + assert set(config["services"]["nginx"]["ports"][0]) >= {"target", "published"} + + +@pytest.mark.skipif(shutil.which("docker") is None, reason="Docker unavailable") +def test_compose_rejects_missing_required_secret(): + env = _compose_env("light") + env.pop("REDIS_PASSWORD") + result = subprocess.run( + ["docker", "compose", "-f", str(DEPLOY / "docker-compose.light.yaml"), "config", "-q"], + env=env, + text=True, + capture_output=True, + ) + assert result.returncode != 0 + assert "REDIS_PASSWORD is required" in result.stderr + + +def test_nginx_routes_console_api_and_mcp(): + config = (DEPLOY / "nginx" / "conf.d" / "brainapi.conf").read_text( + encoding="utf-8" + ) + assert "location /mcp" in config + assert "location /" in config + assert "brainapi_api" in config + assert "brainapi_mcp" in config + + +def test_backup_manifest_verification_detects_tampering(tmp_path): + artifact = tmp_path / "postgres.dump" + artifact.write_bytes(b"consistent") + manifest = { + "format_version": 1, + "profile": "light", + "checksums": { + artifact.name: hashlib.sha256(artifact.read_bytes()).hexdigest() + }, + } + (tmp_path / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + verify_backup(tmp_path, "light") + artifact.write_bytes(b"tampered") + with pytest.raises(RuntimeError, match="Checksum mismatch"): + verify_backup(tmp_path, "light") + + +def test_light_backup_includes_per_brain_postgres_databases(): + source = (DEPLOY / "backup_restore.py").read_text(encoding="utf-8") + assert "postgres-brains.tar.gz" in source + assert "brain_*.dump" in source + + +def test_recommendation_openapi_surface_is_preview(): + from src.services.api.app import app + + schema = app.openapi() + assert schema["paths"]["/retrieve/recommend"]["get"]["x-stability"] == "preview" + assert schema["paths"]["/retrieve/recommend"]["post"]["x-stability"] == "preview" + + +def test_production_smoke_percentiles_are_deterministic(): + assert _latency_summary([1, 2, 3, 4, 5]) == { + "samples": 5, + "p50_ms": 3, + "p95_ms": 4.8, + "p99_ms": 4.96, + } + + +def test_product_state_artifact_never_persists_a_brain_token(): + source = (ROOT / "scripts" / "production_smoke.py").read_text(encoding="utf-8") + state_block = source.split('state = {', 1)[1].split('}', 1)[0] + assert '"brain_token"' not in state_block + + +def test_release_latency_gate_requires_both_profiles(): + result = { + "context": { + "p50_ms": 999, + "p95_ms": 1200, + "p99_ms": 1500, + "online_llm_retrieval_loops": 0, + }, + "search": { + "p50_ms": 199, + "p95_ms": 250, + "p99_ms": 300, + "excludes_embed_query": True, + }, + } + _check_latency({"light": result, "heavy": result}) + with pytest.raises(RuntimeError, match="missing the heavy"): + _check_latency({"light": result}) + + +def test_ci_embedding_stub_is_deterministic_and_normalized(): + first = _embedding("same input", dimensions=8) + assert first == _embedding("same input", dimensions=8) + assert first != _embedding("different input", dimensions=8) + assert sum(value * value for value in first) == pytest.approx(1.0) + + +def test_release_workflow_enforces_artifact_gate_before_publish(): + workflow = (ROOT / ".github" / "workflows" / "release.yaml").read_text( + encoding="utf-8" + ) + assert "check_release_readiness.py gate-artifacts" in workflow + assert "needs: [validate-tag, verify-required-checks]" in workflow + + +def test_mcp_has_only_one_lifespan_definition(): + source = (ROOT / "src" / "services" / "mcp" / "app.py").read_text( + encoding="utf-8" + ) + assert source.count("async def _lifespan") == 1 + + +def test_ci_profiles_enable_dense_search_without_online_llm(): + override = yaml.safe_load((DEPLOY / "docker-compose.ci.yaml").read_text()) + environment = override["x-ci-model-environment"] + assert environment["SEARCH_ENABLED"] == "true" + assert environment["SEARCH_USE_DENSE"] == "true" + assert environment["SEARCH_USE_BM25"] == "false" + assert environment["CONTEXT_PASSAGE_MODE"] == "dense" diff --git a/tests/test_halfvec_ann.py b/tests/test_halfvec_ann.py new file mode 100644 index 0000000..5ee34c1 --- /dev/null +++ b/tests/test_halfvec_ann.py @@ -0,0 +1,182 @@ +import os +import unittest +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", +} +for key, value in ENV_DEFAULTS.items(): + os.environ.setdefault(key, value) + +from src.constants.embeddings import Vector +from src.lib.postgresql import vectors as vectors_mod +from src.utils.vector_search import stable_top_k_vectors + + +def _postgres_available() -> bool: + try: + import psycopg2 + + from src.config import config + + pg = config.postgresql + pg.validate_credentials() + conn = psycopg2.connect( + host=pg.host, + port=pg.port, + user=pg.username, + password=pg.password, + dbname=pg.maintenance_database or "postgres", + connect_timeout=2, + ) + conn.close() + return True + except Exception: + return False + + +class HalfvecAnnTests(unittest.TestCase): + def test_3072_creates_halfvec_hnsw_when_search_on(self): + ddl = vectors_mod._vector_index_ddl( + "vectors_data", 3072, search_enabled=True + ) + self.assertIn("halfvec(3072)", ddl) + self.assertIn("hnsw", ddl) + self.assertIn("halfvec_cosine_ops", ddl) + + def test_3072_skips_index_when_search_off(self): + ddl = vectors_mod._vector_index_ddl( + "vectors_data", 3072, search_enabled=False + ) + self.assertEqual(ddl.strip(), "") + + def test_under_2000_keeps_float32_hnsw(self): + ddl = vectors_mod._vector_index_ddl( + "vectors_data", 1536, search_enabled=True + ) + self.assertIn("vector_cosine_ops", ddl) + self.assertNotIn("halfvec", ddl) + + def test_overfetch_float32_rerank_matches_exact_topk(self): + ann_order = [ + Vector(id="b", metadata={"uuid": "b", "resource_id": "b"}, distance=0.05), + Vector(id="a", metadata={"uuid": "a", "resource_id": "a"}, distance=0.01), + Vector(id="c", metadata={"uuid": "c", "resource_id": "c"}, distance=0.20), + ] + reranked = stable_top_k_vectors(ann_order, 2) + self.assertEqual([v.id for v in reranked], ["a", "b"]) + + def test_search_vectors_orders_by_halfvec_and_reranks_float32(self): + client = vectors_mod.PostgreSQLVectorStoreClient.__new__( + vectors_mod.PostgreSQLVectorStoreClient + ) + executed: list[str] = [] + + class Cursor: + def execute(self, sql, params=None): + executed.append(str(sql)) + + def fetchall(self): + return [ + { + "id": 2, + "uuid": "b", + "metadata": {"resource_id": "b"}, + "distance": 0.2, + }, + { + "id": 1, + "uuid": "a", + "metadata": {"resource_id": "a"}, + "distance": 0.05, + }, + ] + + @contextmanager + def fake_connection(_brain_id): + conn = MagicMock() + cur = Cursor() + + @contextmanager + def fake_cursor(**_kwargs): + yield cur + + conn.cursor.side_effect = lambda **kwargs: fake_cursor() + yield conn + + with ( + patch.object(client, "_ensure_store"), + patch.object(client, "_connection", fake_connection), + patch.object(vectors_mod, "EMBEDDING_STORES_SIZES", {"data": 3072}), + patch.object(vectors_mod.config, "search_enabled", True), + ): + hits = client.search_vectors([0.0] * 3072, "brain", "data", k=1) + + joined = "\n".join(executed) + self.assertIn("halfvec(3072)", joined) + self.assertIn("embeddings <=> %s::vector", joined) + self.assertEqual([h.id for h in hits], ["a"]) + + @unittest.skipUnless(_postgres_available(), "postgres not available") + def test_postgres_creates_halfvec_hnsw_for_3072(self): + import psycopg2.extras + + from src.lib.postgresql._naming import brain_db_name + from src.lib.postgresql._provisioning import borrow, get_brain_pool + + brain_id = "searchbench_halfvec_unit" + sizes = { + "nodes": 3072, + "triplets": 3072, + "observations": 3072, + "data": 3072, + "relationships": 3072, + } + client = vectors_mod.PostgreSQLVectorStoreClient() + with ( + patch.object(vectors_mod.config, "search_enabled", True), + patch.object(vectors_mod, "EMBEDDING_STORES_SIZES", sizes), + ): + client._initialized_stores.clear() + client._ensure_store("data", brain_id) + with borrow(get_brain_pool(brain_id)) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + SELECT indexdef + FROM pg_indexes + WHERE tablename = 'vectors_data' + AND indexname = 'idx_vectors_data_embeddings_halfvec' + """ + ) + row = cur.fetchone() + self.assertIsNotNone(row) + self.assertIn("halfvec", row["indexdef"]) + self.assertIn("hnsw", row["indexdef"].lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ingestion_standard.py b/tests/test_ingestion_standard.py index 47a483c..be2c2c5 100644 --- a/tests/test_ingestion_standard.py +++ b/tests/test_ingestion_standard.py @@ -48,6 +48,13 @@ def test_source_text_from_text_payload(self): data={"data_type": "text", "text_data": "hello world"}, brain_id="brain1", ) + self.assertFalse(payload.skip_enrichment) + skipped = IngestionTaskArgs( + data={"data_type": "text", "text_data": "catalog row"}, + brain_id="brain1", + skip_enrichment=True, + ) + self.assertTrue(skipped.skip_enrichment) self.assertEqual(source_text_from_payload(payload), "hello world") def test_source_text_from_json_payload(self): @@ -115,7 +122,7 @@ def fake_observe(*, text, observate_for, context=None): ) return ["obs-1"] - def fake_enrich(text, targeting=None, brain_id="default"): + def fake_enrich(text, targeting=None, brain_id="default", **kwargs): from src.core.saving.auto_kg import EnrichmentOrchestrationResult enrich_calls.append({"text": text, "brain_id": brain_id}) @@ -143,6 +150,7 @@ def fake_enrich(text, targeting=None, brain_id="default"): patch.object(ingestion_mod, "observations_agent") as observations_agent, patch.object(ingestion_mod, "enrich_kg_from_input", side_effect=fake_enrich), patch.object(ingestion_mod.config, "pipeline_mode", "accurate"), + patch.object(ingestion_mod.config, "run_observations", True), patch.object(ingestion_mod, "set_ingestion_task_status"), patch("celery.chain") as chain_mock, ): @@ -171,6 +179,65 @@ def fake_enrich(text, targeting=None, brain_id="default"): self.assertEqual(enrich_calls[0]["text"], expected_text) self.assertEqual(enrich_calls[0]["brain_id"], "tenant-a") + def test_skip_enrichment_writes_chunk_without_kg(self): + from src.constants.embeddings import Vector + from src.workers.tasks import ingestion as ingestion_mod + + enrich_calls = [] + observe_calls = [] + + def fake_save_text_chunk(chunk, brain_id="default"): + return chunk + + def fake_enrich(*args, **kwargs): + enrich_calls.append(kwargs) + raise AssertionError("enrichment should be skipped") + + def fake_observe(**kwargs): + observe_calls.append(kwargs) + raise AssertionError("observations should be skipped") + + task = ingestion_mod.ingest_data + request = self._make_request() + args = { + "data": {"data_type": "text", "text_data": "DOCID p1. Title: kettle"}, + "brain_id": "searchbenchesci", + "skip_enrichment": True, + } + + with ( + patch.object(ingestion_mod, "data_adapter") as data_adapter, + patch.object(ingestion_mod, "embeddings_adapter") as embeddings_adapter, + patch.object(ingestion_mod, "vector_store_adapter") as vector_store_adapter, + patch.object(ingestion_mod, "cache_adapter"), + patch.object(ingestion_mod, "observations_agent") as observations_agent, + patch.object(ingestion_mod, "enrich_kg_from_input", side_effect=fake_enrich), + patch.object(ingestion_mod.config, "pipeline_mode", "accurate"), + patch.object(ingestion_mod, "set_ingestion_task_status") as set_status, + ): + data_adapter.save_text_chunk.side_effect = fake_save_text_chunk + embeddings_adapter.embed_text.return_value = Vector( + id="v1", embeddings=[0.1, 0.2], metadata={} + ) + observations_agent.observe.side_effect = fake_observe + vector_store_adapter.add_vectors.return_value = ["v1"] + result = task.run.__func__( + type("Bound", (), {"request": request})(), + args, + ) + + self.assertEqual(result, "task-123") + self.assertEqual(observe_calls, []) + self.assertEqual(enrich_calls, []) + self.assertTrue(data_adapter.save_text_chunk.called) + self.assertTrue(embeddings_adapter.embed_text.called) + completed = [ + call + for call in set_status.call_args_list + if call.args and call.args[2] == "completed" + ] + self.assertTrue(completed) + def test_invalid_pipeline_mode_fails_ingest(self): from src.workers.tasks import ingestion as ingestion_mod diff --git a/tests/test_ingestion_structured_contract.py b/tests/test_ingestion_structured_contract.py index 0c8124a..ff55362 100644 --- a/tests/test_ingestion_structured_contract.py +++ b/tests/test_ingestion_structured_contract.py @@ -51,6 +51,31 @@ def test_triple_only_without_anchor_or_text_validates(self): self.assertIsNone(body.anchor) self.assertIsNone(body.text) + def test_direct_has_triple_validates(self): + from src.services.api.constants.requests import IngestionTripleSet + + triple = IngestionTripleSet( + subject={"name": "bed", "type": "ENTITY", "uuid": "0"}, + subj_event={"name": "HAS", "uuid": "rel-1"}, + object={"name": "navy", "type": "ATTR", "uuid": "hub-navy"}, + ) + self.assertIsNone(triple.event) + self.assertIsNone(triple.event_obj) + prefers = IngestionTripleSet( + subject={"name": "u01", "type": "USER", "uuid": "user:u01"}, + subj_event={"name": "PREFERS", "uuid": "rel-pref"}, + object={"name": "70s", "type": "ATTR", "uuid": "hub:attr:70s"}, + ) + self.assertIsNone(prefers.event) + self.assertEqual(prefers.subj_event.name, "PREFERS") + with self.assertRaises(ValidationError): + IngestionTripleSet( + subject={"name": "bed", "type": "ENTITY", "uuid": "0"}, + subj_event={"name": "HAS"}, + event={"name": "HAS", "type": "EVENT"}, + object={"name": "navy", "type": "ATTR"}, + ) + def test_string_anchor_is_rejected(self): from src.services.api.constants.requests import IngestionStructuredRequestBody diff --git a/tests/test_production_hardening.py b/tests/test_production_hardening.py new file mode 100644 index 0000000..2ff7f5a --- /dev/null +++ b/tests/test_production_hardening.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import os +from unittest.mock import patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from src.services.api.errors import install_error_handlers +from src.services.api.middlewares.auth import BrainPATMiddleware +from src.services.api.routes.system import system_router +from src.workers.redis_url import redis_connection_url + + +def test_malformed_authorization_fails_closed_with_401(): + app = FastAPI() + install_error_handlers(app) + app.add_middleware(BrainPATMiddleware) + + @app.get("/system/test") + async def protected(): + return {"ok": True} + + with patch.dict(os.environ, {"BRAINPAT_TOKEN": "system-token"}): + response = TestClient(app).get( + "/system/test", headers={"Authorization": "malformed"} + ) + + assert response.status_code == 401 + assert response.json()["error"]["code"] == "AUTH_INVALID" + + +@pytest.mark.parametrize( + ("path", "method", "operation"), + [ + ("/system/brains/demo/reset", "get", "reset"), + ("/system/brains/demo/delete", "get", "delete"), + ("/system/brains/demo/create-backup", "post", "create-backup"), + ], +) +def test_unfinished_system_routes_return_structured_501(path, method, operation): + app = FastAPI() + install_error_handlers(app) + app.include_router(system_router) + + response = getattr(TestClient(app), method)(path) + + assert response.status_code == 501 + assert response.json()["error"]["code"] == "not_implemented" + assert response.json()["operation"] == operation + + +def test_cors_defaults_and_production_wildcard_guard(): + from src.services.api.app import _cors_allowed_origins + + with patch.dict(os.environ, {"ENV": "production", "CORS_ALLOWED_ORIGINS": ""}): + assert _cors_allowed_origins() == [] + with patch.dict(os.environ, {"ENV": "development", "CORS_ALLOWED_ORIGINS": ""}): + assert _cors_allowed_origins() == ["*"] + with patch.dict( + os.environ, {"ENV": "production", "CORS_ALLOWED_ORIGINS": "*"} + ): + with pytest.raises(RuntimeError, match="development"): + _cors_allowed_origins() + + +def test_plugin_failures_are_fatal_by_default_in_production(): + from src.services.api.app import _enforce_plugin_results + + with patch.dict(os.environ, {"ENV": "production"}, clear=False): + with pytest.raises(RuntimeError, match="demo"): + _enforce_plugin_results({"demo": False}) + with patch.dict( + os.environ, + {"ENV": "production", "PLUGIN_FAILURE_POLICY": "warn"}, + clear=False, + ): + _enforce_plugin_results({"demo": False}) + + +def test_worker_redis_url_propagates_encoded_password(): + assert ( + redis_connection_url("redis", 6379, "secret:/@ value") + == "redis://:secret%3A%2F%40%20value@redis:6379/0" + ) + assert redis_connection_url("redis", "6379", None) == "redis://redis:6379/0" diff --git a/tests/test_recommend_recsys.py b/tests/test_recommend_recsys.py index 45a6b16..1485226 100644 --- a/tests/test_recommend_recsys.py +++ b/tests/test_recommend_recsys.py @@ -76,8 +76,22 @@ def test_purchase_higher_than_view(self): self.assertEqual(behavior_weight("View"), 0.2) self.assertEqual(behavior_weight("AddToCart"), 0.5) - def test_unknown_defaults_to_one(self): - self.assertEqual(behavior_weight("WeirdEvent"), 1.0) + def test_unknown_defaults_to_view_weight(self): + self.assertEqual(behavior_weight("WeirdEvent"), 0.2) + + def test_favorite_between_cart_and_purchase(self): + self.assertEqual(behavior_weight("Favorite"), 0.7) + self.assertEqual(behavior_weight("wishlist"), 0.7) + self.assertEqual(behavior_weight("add_to_favorite"), 0.7) + self.assertGreater(behavior_weight("Favorite"), behavior_weight("AddToCart")) + self.assertLess(behavior_weight("Favorite"), behavior_weight("Purchase")) + + def test_follow_is_half(self): + self.assertEqual(behavior_weight("Follow"), 0.5) + self.assertEqual(behavior_weight("FLW"), 0.5) + self.assertEqual(behavior_weight("follow"), 0.5) + self.assertGreater(behavior_weight("Follow"), behavior_weight("WeirdEvent")) + self.assertLess(behavior_weight("Follow"), behavior_weight("Favorite")) def test_custom_table(self): self.assertEqual(behavior_weight("View", {"view": 0.9}), 0.9) diff --git a/tests/test_recsys_lightgcn.py b/tests/test_recsys_lightgcn.py index ffffd6c..d250ee2 100644 --- a/tests/test_recsys_lightgcn.py +++ b/tests/test_recsys_lightgcn.py @@ -9,6 +9,8 @@ def test_train_and_infer_toy_edges(self): root = Path(__file__).resolve().parents[1] plugin = root / "plugins" / "recsys-gnn" + if not (plugin / "models" / "lightgcn.py").is_file(): + self.skipTest("optional recsys-gnn plugin is not installed") if str(plugin) not in sys.path: sys.path.insert(0, str(plugin)) diff --git a/tests/test_search_api.py b/tests/test_search_api.py new file mode 100644 index 0000000..07feb93 --- /dev/null +++ b/tests/test_search_api.py @@ -0,0 +1,393 @@ +import os +import unittest +from unittest.mock import MagicMock, patch + +ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", +} +for key, value in ENV_DEFAULTS.items(): + os.environ.setdefault(key, value) + +from fastapi import HTTPException + +from src.constants.embeddings import Vector +from src.constants.data import TextChunk +from src.services.api.constants.requests import SearchRequestBody + + +class SearchApiTests(unittest.IsolatedAsyncioTestCase): + async def test_404_when_search_disabled(self): + from src.services.api.controllers import search as search_mod + + with patch.object(search_mod.config, "search_enabled", False): + with self.assertRaises(HTTPException) as ctx: + await search_mod.search(SearchRequestBody(query="license")) + self.assertEqual(ctx.exception.status_code, 404) + + async def test_fused_hits_when_both_channels_on(self): + from src.services.api.controllers import search as search_mod + + dense_vec = Vector( + id="dense-1", + metadata={"resource_id": "chunk-dense"}, + distance=0.1, + ) + bm25_chunk = TextChunk(id="chunk-bm25", text="counseling license renewal") + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.1, 0.2], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_data.return_value = [dense_vec] + mock_data = MagicMock() + mock_data.search_bm25.return_value = [(bm25_chunk, 4.2)] + mock_data.get_text_chunks_by_ids.return_value = ( + [TextChunk(id="chunk-dense", text="dense passage about alice")], + [], + ) + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", True), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod.config, "search_fusion", "rrf"), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody(query="license", k=10, profile_stages=True) + ) + + ids = [hit.id for hit in response.hits] + self.assertIn("chunk-dense", ids) + self.assertIn("chunk-bm25", ids) + self.assertTrue(any(hit.scores.rrf for hit in response.hits)) + mock_embeddings.embed_text.assert_called_once_with("license") + self.assertIsNotNone(response.stage_timings) + stage_names = [s["stage"] for s in response.stage_timings.get("stages", [])] + self.assertIn("embed.query", stage_names) + self.assertTrue( + any(name.startswith("search.") for name in stage_names), + stage_names, + ) + + async def test_dense_only_skips_bm25(self): + from src.services.api.controllers import search as search_mod + + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.1], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_data.return_value = [ + Vector(id="d1", metadata={"resource_id": "only-dense"}, distance=0.2) + ] + mock_data = MagicMock() + mock_data.get_text_chunks_by_ids.return_value = ( + [TextChunk(id="only-dense", text="vector hit")], + [], + ) + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", True), + patch.object(search_mod.config, "search_use_bm25", False), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search(SearchRequestBody(query="alice", k=5)) + + self.assertEqual([hit.id for hit in response.hits], ["only-dense"]) + mock_data.search_bm25.assert_not_called() + self.assertIsNotNone(response.hits[0].scores.dense) + + async def test_bm25_only_skips_embed(self): + from src.services.api.controllers import search as search_mod + + mock_embeddings = MagicMock() + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + (TextChunk(id="lex", text="license board"), 3.1) + ] + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search(SearchRequestBody(query="license", k=5)) + + self.assertEqual([hit.id for hit in response.hits], ["lex"]) + mock_embeddings.embed_text.assert_not_called() + mock_vs.search_data.assert_not_called() + self.assertEqual(response.hits[0].scores.bm25, 3.1) + self.assertIsNone(response.hits[0].node_id) + self.assertEqual(response.node_ids, []) + + async def test_passage_docid_sets_node_id(self): + from src.services.api.controllers import search as search_mod + + mock_embeddings = MagicMock() + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + ( + TextChunk( + id="chunk-bed", + text="DOCID 0.\nTitle: solid wood platform bed\nClass: Beds", + ), + 5.0, + ) + ] + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody(query="platform bed", k=5) + ) + + self.assertEqual([hit.id for hit in response.hits], ["chunk-bed"]) + self.assertEqual(response.hits[0].channel, "passages") + self.assertEqual(response.hits[0].node_id, "0") + self.assertEqual(response.node_ids, ["0"]) + + def test_default_search_mode_is_default(self): + body = SearchRequestBody(query="license") + self.assertEqual(body.mode, "default") + self.assertEqual(body.k, 10) + self.assertIsNone(body.extras) + + async def test_extras_equality_filter_and_facets(self): + from src.services.api.controllers import search as search_mod + from src.core.search.hybrid import ( + facet_counts_from_extras, + hit_matches_extras, + ) + + self.assertTrue(hit_matches_extras({"locale": "it"}, {"locale": "IT"})) + self.assertFalse(hit_matches_extras({"color": "argento"}, {"color": "nope"})) + self.assertEqual( + facet_counts_from_extras( + [ + {"locale": "it", "color": "argento", "brand": "CasaLuce"}, + {"locale": "it", "color": "blu navy", "brand": "AtelierNord"}, + None, + ] + ), + { + "locale": {"it": 2}, + "color": {"argento": 1, "blu navy": 1}, + "brand": {"CasaLuce": 1, "AtelierNord": 1}, + }, + ) + self.assertIsNone(facet_counts_from_extras([None, {}])) + self.assertEqual( + facet_counts_from_extras( + [{"locale": "it", "uuid": "skip-me", "resource_id": "chunk"}] + ), + {"locale": {"it": 1}}, + ) + + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + ( + TextChunk( + id="it-1", + text="bollitore acciaio", + metadata={"locale": "it", "color": "argento", "brand": "CasaLuce"}, + ), + 3.0, + ), + ( + TextChunk( + id="it-2", + text="divano velluto", + metadata={"locale": "it", "color": "blu navy", "brand": "AtelierNord"}, + ), + 2.0, + ), + ] + mock_data.get_text_chunks_by_ids.return_value = ([], []) + mock_embeddings = MagicMock() + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + kept = await search_mod.search( + SearchRequestBody(query="divano", k=10, extras={"locale": "it"}) + ) + missed = await search_mod.search( + SearchRequestBody(query="divano", k=10, extras={"color": "nope"}) + ) + unfiltered = await search_mod.search( + SearchRequestBody(query="divano", k=10) + ) + + self.assertEqual({hit.id for hit in kept.hits}, {"it-1", "it-2"}) + self.assertEqual(kept.hits[0].extras["locale"], "it") + self.assertEqual(kept.facets["locale"]["it"], 2) + self.assertEqual(kept.facets["brand"]["CasaLuce"], 1) + self.assertEqual(missed.hits, []) + self.assertIsNone(missed.facets) + self.assertEqual({hit.id for hit in unfiltered.hits}, {"it-1", "it-2"}) + self.assertEqual(unfiltered.hits[0].channel, "passages") + mock_embeddings.embed_text.assert_not_called() + + def test_literal_overlap_and_frozen_head_helpers(self): + from src.core.search.hybrid import ( + frozen_head_merge, + fuse_passage_lists, + literal_overlap_ids, + ) + + ranked = frozen_head_merge( + [f"h{index}" for index in range(12)], + [["extra", "h0"]], + head_k=10, + k=15, + ) + self.assertEqual(ranked[:10], [f"h{index}" for index in range(10)]) + self.assertEqual(ranked[10], "extra") + ids = literal_overlap_ids( + "navy velvet sofa", + { + "hit": "Title: Navy velvet sofa two seats", + "miss": "plastic kettle lid", + "partial": "velvet throw pillow", + }, + k=10, + ) + self.assertEqual(ids[0], "hit") + self.assertIn("partial", ids) + self.assertNotIn("miss", ids) + fused = fuse_passage_lists( + ["d1", "d2"], + ["b1", "d1"], + fusion="cc", + alpha=0.7, + dense_similarities={"d1": 1.0, "d2": 0.2}, + bm25_scores={"b1": 4.0, "d1": 1.0}, + ) + self.assertTrue(fused) + self.assertEqual(fused[0][0], "d1") + + async def test_fusion_alpha_request_overrides_config(self): + from src.services.api.controllers import search as search_mod + + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + (TextChunk(id="lex", text="license board"), 3.1) + ] + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.1], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_data.return_value = [ + Vector(id="d1", metadata={"resource_id": "dense-1"}, distance=0.1) + ] + mock_data.get_text_chunks_by_ids.return_value = ( + [TextChunk(id="dense-1", text="vector hit")], + [], + ) + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", True), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod.config, "search_fusion", "rrf"), + patch.object(search_mod.config, "search_fusion_alpha", 0.1), + patch.object(search_mod.config, "search_literal_fill", False), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody( + query="license", + k=5, + fusion="cc", + fusion_alpha=0.9, + ) + ) + + ids = [hit.id for hit in response.hits] + self.assertIn("dense-1", ids) + self.assertTrue(any(hit.scores.cc is not None for hit in response.hits)) + + async def test_literal_fill_uses_frozen_head(self): + from src.services.api.controllers import search as search_mod + + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + (TextChunk(id=f"h{index}", text=f"t{index}"), float(20 - index)) + for index in range(12) + ] + mock_data.get_text_chunks.return_value = ( + [TextChunk(id="title-hit", text="navy velvet sofa")], + 1, + ) + mock_embeddings = MagicMock() + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod.config, "search_literal_fill", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody(query="navy velvet sofa", k=15) + ) + + ids = [hit.id for hit in response.hits] + self.assertEqual(ids[:10], [f"h{index}" for index in range(10)]) + self.assertIn("title-hit", ids[10:]) + self.assertEqual(response.channel_lists["literal"], ["title-hit"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search_bench.py b/tests/test_search_bench.py new file mode 100644 index 0000000..7c04ac7 --- /dev/null +++ b/tests/test_search_bench.py @@ -0,0 +1,1691 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +class SearchBenchTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + import sys + + root = Path(__file__).resolve().parents[1] + bench = root / "benchmarks" + if str(bench) not in sys.path: + sys.path.insert(0, str(bench)) + + def require_local_artifact(self, path: Path) -> Path: + if not path.is_file(): + self.skipTest(f"optional local benchmark artifact is absent: {path}") + return path + + def test_brain_id_guard(self): + from search.config import DEFAULT_BRAIN_ID, validate_brain_id + + self.assertEqual(validate_brain_id("searchbenchsmoke"), "searchbenchsmoke") + self.assertEqual(validate_brain_id(DEFAULT_BRAIN_ID), DEFAULT_BRAIN_ID) + self.assertEqual(validate_brain_id("searchbench1"), "searchbench1") + self.assertEqual(validate_brain_id("searchbenchjdslice"), "searchbenchjdslice") + for bad in ( + "locomoconv26", + "locomoconv26clean", + "beam1m1clean", + "beam100k1", + "demorecsys", + "lme-s-smoke", + "default", + "search", + ): + with self.assertRaises(SystemExit): + validate_brain_id(bad) + + def test_metrics_ranking_and_latency(self): + from search.metrics import ( + aggregate_query_metrics, + mrr, + ndcg_at_k, + percentile, + recall_at_k, + retrieve_latency_ms, + ) + + ranked = ["a", "b", "gold", "d"] + gold = {"gold"} + self.assertEqual(recall_at_k(ranked, gold, 2), 0.0) + self.assertEqual(recall_at_k(ranked, gold, 3), 1.0) + self.assertEqual(mrr(ranked, gold), 1.0 / 3) + self.assertGreater(ndcg_at_k(ranked, gold, 10), 0.0) + self.assertLess(ndcg_at_k(ranked, gold, 10), 1.0) + self.assertEqual(ndcg_at_k(["gold"], gold, 10), 1.0) + self.assertGreater( + ndcg_at_k( + ["exact", "sub"], + {"exact", "sub"}, + 10, + grades={"exact": 1.0, "sub": 0.1}, + ), + ndcg_at_k( + ["sub", "exact"], + {"exact", "sub"}, + 10, + grades={"exact": 1.0, "sub": 0.1}, + ), + ) + wands_gold = {"exact", "partial"} + wands_grades = {"exact": 1.0, "partial": 0.5} + self.assertGreater( + ndcg_at_k(["exact", "partial"], wands_gold, 10, grades=wands_grades), + ndcg_at_k(["partial", "exact"], wands_gold, 10, grades=wands_grades), + ) + self.assertAlmostEqual( + ndcg_at_k(["exact", "partial"], wands_gold, 10, grades=wands_grades), + ndcg_at_k( + ["exact", "partial"], + wands_gold, + 10, + grades={"exact": 2.0, "partial": 1.0}, + ), + ) + import math + + from search.metrics import _dcg + + self.assertAlmostEqual( + _dcg([1.0, 0.5], 2), + 1.0 / math.log2(2) + 0.5 / math.log2(3), + ) + self.assertEqual(recall_at_k(["exact", "other"], wands_gold, 10), 0.5) + self.assertEqual(percentile([10.0, 20.0, 30.0], 50), 20.0) + + retrieve, embed = retrieve_latency_ms( + { + "stages": [ + {"stage": "embed.query", "wall_ms": 80.0}, + {"stage": "search.retrieve", "wall_ms": 12.5}, + ] + }, + 200.0, + ) + self.assertEqual(retrieve, 12.5) + self.assertEqual(embed, 80.0) + retrieve_fallback, embed_fallback = retrieve_latency_ms( + {"stages": [{"stage": "embed.query", "wall_ms": 40.0}]}, + 90.0, + ) + self.assertEqual(retrieve_fallback, 50.0) + self.assertEqual(embed_fallback, 40.0) + + rows = [ + { + "slice": "keyword", + "metrics": { + "recall@5": 1.0, + "recall@10": 1.0, + "recall@20": 1.0, + "ndcg@10": 1.0, + "ndcg@20": 1.0, + "ndcg": 1.0, + "mrr": 1.0, + }, + "retrieve_ms": 10.0, + "embed_ms": 50.0, + "client_wall_ms": 70.0, + }, + { + "slice": "paraphrase", + "metrics": { + "recall@5": 0.0, + "recall@10": 0.0, + "recall@20": 1.0, + "ndcg@10": 0.0, + "ndcg@20": 0.5, + "ndcg": 0.5, + "mrr": 0.0, + }, + "retrieve_ms": 30.0, + "embed_ms": 50.0, + "client_wall_ms": 90.0, + }, + ] + metrics = aggregate_query_metrics(rows) + self.assertEqual(metrics["recall@10"], 0.5) + self.assertEqual(metrics["ndcg@10"], 0.5) + self.assertEqual(metrics["ndcg@20"], 0.75) + self.assertEqual(metrics["by_slice"]["keyword"]["recall@10"], 1.0) + self.assertEqual(metrics["p50_retrieve_ms"], 20.0) + + from search.metrics import ndcg_at_k + + ranked_pool = ["exact", "irr", "sub", "other"] + grades_pool = {"exact": 1.0, "sub": 0.1, "irr": 0.0} + gold_pool = {"exact", "sub"} + manual = ndcg_at_k(ranked_pool, gold_pool, 20, grades=grades_pool) + self.assertGreater(manual, 0.0) + self.assertEqual( + ndcg_at_k(ranked_pool, gold_pool, 20, grades=grades_pool), + ndcg_at_k(ranked_pool, gold_pool, 4, grades=grades_pool), + ) + + def test_doc_marker_mapping(self): + from search.dataset import dataset_stats, load_records, map_doc_ids_to_chunks + + root = Path(__file__).resolve().parents[1] + fixture = self.require_local_artifact( + root / "benchmarks" / "data" / "search_toy.jsonl" + ) + rows = load_records(fixture) + stats = dataset_stats(rows) + self.assertGreaterEqual(stats["n_docs"], 6) + self.assertGreaterEqual(stats["n_queries"], 8) + self.assertIn("keyword", stats["slices"]) + self.assertIn("paraphrase", stats["slices"]) + mapping = map_doc_ids_to_chunks( + [{"doc_id": "license-alice"}], + [ + {"id": "c1", "text": "DOCID license-alice. Alice completed her license."}, + {"id": "c2", "text": "unrelated"}, + ], + ) + self.assertEqual(mapping["license-alice"], {"c1"}) + + def test_gold_matches_chunk_or_node_uuid(self): + from search.evaluate import gold_hit_grades, gold_hit_ids + from search.metrics import ndcg_at_k + + query = {"gold_doc_ids": ["sku-1"], "gold_grades": {"sku-1": 1.0}} + mapped = {"sku-1": {"chunk-1"}} + gold = gold_hit_ids(query, mapped) + self.assertEqual(gold, {"sku-1"}) + grades = gold_hit_grades(query, mapped) + self.assertEqual(ndcg_at_k(["sku-1"], gold, 10, grades=grades), 1.0) + from search.evaluate import canonicalize_hit_ids, invert_doc_chunks + + ranked_chunk = canonicalize_hit_ids(["chunk-1"], invert_doc_chunks(mapped)) + self.assertEqual(ndcg_at_k(ranked_chunk, gold, 10, grades=grades), 1.0) + self.assertEqual(ndcg_at_k(["other"], gold, 10, grades=grades), 0.0) + + def test_wands_recall_gold_is_exact_and_partial(self): + from search.evaluate import gold_hit_ids + from search.metrics import recall_at_k + + query = { + "gold_doc_ids": ["exact", "partial"], + "gold_grades": {"exact": 1.0, "partial": 0.5}, + } + gold = gold_hit_ids(query, {}) + self.assertEqual(gold, {"exact", "partial"}) + self.assertEqual(recall_at_k(["exact", "irr"], gold, 10), 0.5) + self.assertEqual(recall_at_k(["exact", "partial"], gold, 10), 1.0) + unlabeled = gold_hit_ids( + { + "gold_doc_ids": ["exact"], + "gold_grades": {"exact": 1.0, "partial": 0.5}, + }, + {}, + ) + self.assertEqual(unlabeled, {"exact", "partial"}) + irr_excluded = gold_hit_ids( + { + "gold_doc_ids": ["exact"], + "gold_grades": {"exact": 1.0, "irr": 0.0}, + }, + {}, + ) + self.assertEqual(irr_excluded, {"exact"}) + + def test_replay_fusion_compact_hub_drop(self): + from search.replay_fusion import replay + + root = Path(__file__).resolve().parents[1] + graph_path = ( + root / "benchmarks" / "runs" / "search-esci-slice-allcols" / "eval.json" + ) + passages_path = ( + root / "benchmarks" / "runs" / "search-esci-slice-passages" / "eval.json" + ) + if not graph_path.exists() or not passages_path.exists(): + self.skipTest("missing paired ESCI slice eval.json files") + result = replay( + json.loads(graph_path.read_text(encoding="utf-8")), + json.loads(passages_path.read_text(encoding="utf-8")), + ) + self.assertEqual(result["n_queries"], 11) + self.assertAlmostEqual(result["hub_drop"]["recall@10"], 0.4758, places=3) + self.assertAlmostEqual(result["hub_drop"]["mrr"], 0.8485, places=3) + + def test_unique_doc_counts_and_channel_lists(self): + from search.evaluate import channel_id_lists, unique_doc_counts + + raw = ["chunk-1", "sku-1", "chunk-2", "hub:attr:red"] + mapping = {"chunk-1": "sku-1", "chunk-2": "sku-2"} + n_raw, n_canon = unique_doc_counts(raw, mapping, k=20) + self.assertEqual(n_raw, 4) + self.assertEqual(n_canon, 3) + lists = channel_id_lists( + { + "channel_lists": { + "dense": ["chunk-1"], + "bm25": ["chunk-2"], + "entities": ["sku-1"], + "communities": ["sku-3"], + } + }, + [{"id": "chunk-1", "channel": "passages", "doc_id": "sku-1"}], + ) + self.assertEqual(lists["dense_ids"], ["chunk-1"]) + self.assertEqual(lists["entity_ids"], ["sku-1"]) + self.assertEqual(lists["community_ids"], ["sku-3"]) + self.assertEqual(lists["passage_ids"], ["chunk-1"]) + + def test_replay_collapse_and_gated_arms(self): + from search.replay_fusion import pick_offline_winner, replay_offline + + gold = [f"d{i}" for i in range(1, 21)] + grades = {doc: 1.0 for doc in gold} + passages_hits = [ + {"id": f"chunk-{item}", "channel": "passages", "doc_id": item} + for item in gold + ] + passages = { + "queries": [ + { + "qid": "q1", + "gold_doc_ids": gold, + "gold_grades": grades, + "hit_ids": gold, + "dense_ids": [f"chunk-{item}" for item in gold], + "bm25_ids": [f"chunk-{item}" for item in gold], + "passage_ids": [f"chunk-{item}" for item in gold], + "hits": passages_hits, + "metrics": { + "ndcg@10": 1.0, + "recall@10": 0.5, + "recall@20": 1.0, + "mrr": 1.0, + }, + } + ] + } + fused_raw = [f"chunk-{item}" for item in gold[:8]] + gold[:8] + gold[8:12] + graph = { + "queries": [ + { + "qid": "q1", + "gold_doc_ids": gold, + "gold_grades": grades, + "hit_ids": gold[:12], + "entity_ids": gold[:5], + "community_ids": ["d21"], + "hits": [ + { + "id": hid, + "channel": "passages" if hid.startswith("chunk-") else "entities", + "doc_id": hid.replace("chunk-", ""), + } + for hid in fused_raw + ], + "metrics": { + "ndcg@10": 0.6, + "recall@10": 0.4, + "recall@20": 0.6, + "mrr": 1.0, + }, + } + ] + } + result = replay_offline(graph, passages) + self.assertEqual(result["n_queries"], 1) + self.assertGreaterEqual( + result["arms"]["collapse-rrf"]["recall@20"], + result["arms"]["passages"]["recall@20"] - 1e-9, + ) + self.assertEqual(result["arms"]["passages"]["unique_docs@20"], 20.0) + self.assertEqual(result["arms"]["confirmation"]["unique_docs@20"], 20.0) + self.assertEqual(result["arms"]["expansion-n10"]["unique_docs@20"], 20.0) + winner = pick_offline_winner( + { + "passages": { + "ndcg@10": 0.758, + "recall@20": 0.847, + }, + "collapse-rrf": { + "ndcg@10": 0.640, + "recall@20": 0.847, + }, + "expansion-n10": { + "ndcg@10": 0.758, + "recall@20": 0.847, + }, + } + ) + self.assertEqual(winner, "expansion-n10") + self.assertEqual( + pick_offline_winner( + { + "passages": {"ndcg@10": 0.758, "recall@20": 0.847}, + "collapse-rrf": {"ndcg@10": 0.640, "recall@20": 0.847}, + "expansion-n10": {"ndcg@10": 0.758, "recall@20": 0.815}, + } + ), + "G08", + ) + self.assertEqual( + pick_offline_winner( + { + "passages": {"ndcg@10": 0.758, "recall@20": 0.847}, + "collapse-rrf": {"ndcg@10": 0.640, "recall@20": 0.847}, + } + ), + "G08", + ) + + def test_score_search_dumps_channel_and_doc_id(self): + from search.client import TimedResult + from search.evaluate import score_search_result + from search.metrics import ndcg_at_k, recall_at_k + + result = TimedResult( + data={ + "hits": [ + {"id": "chunk-1", "channel": "passages", "node_id": "sku-1"}, + {"id": "sku-1", "channel": "entities", "node_id": "sku-1"}, + {"id": "hub:attr:red", "channel": "communities"}, + ], + "stage_timings": { + "stages": [{"stage": "search.retrieve", "wall_ms": 11.0}] + }, + }, + latency_ms=40.0, + ) + gold = {"sku-1"} + grades = {"sku-1": 1.0} + scored = score_search_result( + result, + gold=gold, + ks=(5, 10, 20, 50), + grades=grades, + chunk_to_doc={"chunk-1": "sku-1"}, + ) + self.assertEqual(scored["hit_ids"], ["sku-1", "hub:attr:red"]) + self.assertEqual( + scored["hits"], + [ + { + "id": "chunk-1", + "channel": "passages", + "doc_id": "sku-1", + "node_id": "sku-1", + }, + { + "id": "sku-1", + "channel": "entities", + "doc_id": "sku-1", + "node_id": "sku-1", + }, + { + "id": "hub:attr:red", + "channel": "communities", + "doc_id": "hub:attr:red", + "node_id": None, + }, + ], + ) + self.assertEqual( + scored["metrics"]["recall@10"], + recall_at_k(scored["hit_ids"], gold, 10), + ) + self.assertEqual( + scored["metrics"]["ndcg@10"], + ndcg_at_k(scored["hit_ids"], gold, 10, grades=grades), + ) + self.assertEqual( + [row["doc_id"] for row in scored["hits"][:1]], + scored["hit_ids"][:1], + ) + self.assertEqual(scored["n_unique_docs_raw"], 3) + self.assertEqual(scored["n_unique_docs_canonical"], 2) + self.assertEqual(scored["unique_docs_k"], 50) + self.assertEqual(scored["n_unique_docs_retrieve_canonical"], 2) + self.assertIn("ndcg@50", scored["metrics"]) + self.assertEqual(scored["entity_ids"], ["sku-1"]) + self.assertEqual(scored["community_ids"], ["hub:attr:red"]) + self.assertEqual(scored["passage_ids"], ["chunk-1"]) + + def test_rank_pool_filters_and_keeps_irrelevant_grades(self): + from search.client import TimedResult + from search.evaluate import ( + candidate_pool_grades, + candidate_pool_ids, + filter_ranked_to_pool, + score_search_result, + ) + from search.metrics import ndcg_at_k + + query = { + "gold_doc_ids": ["p-e"], + "gold_grades": {"p-e": 1.0, "p-s": 0.1}, + "candidate_doc_ids": ["p-e", "p-s", "p-i"], + "candidate_grades": {"p-e": 1.0, "p-s": 0.1, "p-i": 0.0}, + } + self.assertEqual(candidate_pool_ids(query), ["p-e", "p-s", "p-i"]) + self.assertEqual(candidate_pool_grades(query)["p-i"], 0.0) + self.assertEqual( + filter_ranked_to_pool(["p-x", "p-e", "p-i", "p-y"], query["candidate_doc_ids"]), + ["p-e", "p-i"], + ) + result = TimedResult( + data={ + "hits": [ + {"id": "outside", "channel": "passages"}, + {"id": "p-e", "channel": "passages"}, + {"id": "p-i", "channel": "passages"}, + ], + "stage_timings": { + "stages": [{"stage": "search.retrieve", "wall_ms": 9.0}] + }, + }, + latency_ms=12.0, + ) + scored = score_search_result( + result, + gold={"p-e", "p-s"}, + ks=(5, 10, 20), + grades=candidate_pool_grades(query), + pool_ids=candidate_pool_ids(query), + ) + self.assertEqual(scored["hit_ids"], ["p-e", "p-i"]) + expected = ndcg_at_k( + ["p-e", "p-i"], + {"p-e", "p-s"}, + 20, + grades=candidate_pool_grades(query), + ) + self.assertEqual(scored["metrics"]["ndcg@20"], expected) + + def test_rank_pool_ce_uses_injected_predict(self): + import sys + + root = Path(__file__).resolve().parents[1] + plugin = root / "plugins" / "search-rerank" + self.require_local_artifact(plugin / "rerank.py") + if str(plugin) not in sys.path: + sys.path.insert(0, str(plugin)) + import rerank as rerank_mod + + from search.rank_pool import run_ce_on_pool + + def predict(pairs): + scores = [] + for _, text in pairs: + if "gold" in text: + scores.append(2.0) + elif "partial" in text: + scores.append(1.0) + else: + scores.append(0.0) + return scores + + rerank_mod.set_predict(predict) + try: + rows = [ + { + "type": "doc", + "doc_id": "irr", + "text": "DOCID irr. Title: other", + }, + { + "type": "doc", + "doc_id": "gold", + "text": "DOCID gold. Title: gold kettle", + }, + { + "type": "doc", + "doc_id": "sub", + "text": "DOCID sub. Title: partial kettle", + }, + { + "type": "query", + "qid": "esci-1", + "query": "kettle", + "gold_doc_ids": ["gold", "sub"], + "gold_grades": {"gold": 1.0, "sub": 0.1}, + "candidate_doc_ids": ["irr", "gold", "sub"], + "candidate_grades": {"irr": 0.0, "gold": 1.0, "sub": 0.1}, + "slice": "esci-us", + }, + ] + result = run_ce_on_pool(rows, dataset_name="toy-pool.jsonl") + finally: + rerank_mod.set_predict(None) + self.assertEqual(result["status"], "ok") + self.assertTrue(result["rank_pool"]) + self.assertTrue(result["rank_pool_ce"]) + self.assertEqual(result["queries"][0]["hit_ids"][0], "gold") + self.assertGreater(result["metrics"]["ndcg@20"], 0.0) + + def test_ce_on_retrieved_reorders_hits_only(self): + import sys + + root = Path(__file__).resolve().parents[1] + plugin = root / "plugins" / "search-rerank" + self.require_local_artifact(plugin / "rerank.py") + if str(plugin) not in sys.path: + sys.path.insert(0, str(plugin)) + import rerank as rerank_mod + + from search.rerank_retrieved import run_ce_on_retrieved + + def predict(pairs): + scores = [] + for _, text in pairs: + if "gold" in text: + scores.append(2.0) + elif "partial" in text: + scores.append(1.0) + else: + scores.append(0.0) + return scores + + rerank_mod.set_predict(predict) + try: + rows = [ + { + "type": "doc", + "doc_id": "irr", + "text": "DOCID irr. Title: other", + }, + { + "type": "doc", + "doc_id": "gold", + "text": "DOCID gold. Title: gold kettle", + }, + { + "type": "doc", + "doc_id": "sub", + "text": "DOCID sub. Title: partial kettle", + }, + { + "type": "query", + "qid": "esci-1", + "query": "kettle", + "gold_doc_ids": ["gold", "sub"], + "gold_grades": {"gold": 1.0, "sub": 0.1}, + "slice": "esci-us", + }, + ] + eval_result = { + "k": 50, + "ks": [5, 10, 20, 50], + "rank_pool": False, + "fusion": "rrf", + "channels": ["passages"], + "n_docs": 3, + "queries": [ + { + "qid": "esci-1", + "query": "kettle", + "slice": "esci-us", + "gold_doc_ids": ["gold", "sub"], + "gold_grades": {"gold": 1.0, "sub": 0.1}, + "hit_ids": ["irr", "sub", "gold"], + "metrics": {}, + "retrieve_ms": 12.0, + "pool_coverage": None, + } + ], + } + result = run_ce_on_retrieved( + eval_result, + rows, + dataset_name="toy-retrieved.jsonl", + ) + finally: + rerank_mod.set_predict(None) + self.assertEqual(result["status"], "ok") + self.assertFalse(result["rank_pool_ce"]) + self.assertEqual(result["rerank"], "harness:cross-encoder") + self.assertEqual(result["queries"][0]["hit_ids"][0], "gold") + self.assertEqual(set(result["queries"][0]["hit_ids"]), {"irr", "sub", "gold"}) + self.assertGreater(result["metrics"]["ndcg@10"], 0.0) + + def test_electronics_marker_for_esci72_style_titles(self): + from search.miss import looks_like_electronics + + self.assertTrue(looks_like_electronics("Apple iPad Air 2, 16 GB")) + self.assertFalse(looks_like_electronics("Price Tags Without String")) + + def test_miss_strata_classifies_head_tail_and_total_miss(self): + from search.miss_strata import classify_eval, rewrite_query + + self.assertIsNone(rewrite_query("esci-72", "$100 things that are not electronics")) + self.assertIsNone(rewrite_query("esci-177", "p_num: integer not null i_num: integer")) + rewritten = rewrite_query( + "esci-113", + ""tortillas without interesterified soybean oil”", + ) + self.assertIsNotNone(rewritten) + self.assertNotIn(""", rewritten) + + eval_result = { + "k": 50, + "queries": [ + { + "qid": "q-head", + "query": "red mug", + "hit_ids": ["g1", "x"] + [f"n{i}" for i in range(48)], + "gold_doc_ids": ["g1", "g2"], + "gold_grades": {"g1": 1.0, "g2": 0.1}, + "metrics": {"recall@10": 0.5, "recall@50": 0.5}, + }, + { + "qid": "q-tail", + "query": "blue mug", + "hit_ids": ["n0"] * 10 + ["g2"] + [f"n{i}" for i in range(39)], + "gold_doc_ids": ["g2"], + "gold_grades": {"g2": 1.0}, + "metrics": {"recall@10": 0.0, "recall@50": 1.0}, + }, + { + "qid": "esci-113", + "query": ""tortillas without interesterified soybean oil”", + "hit_ids": [f"n{i}" for i in range(50)], + "gold_doc_ids": ["g3"], + "gold_grades": {"g3": 1.0}, + "metrics": {"recall@10": 0.0, "recall@50": 0.0}, + }, + ], + } + rows = [ + {"type": "doc", "doc_id": "g1", "text": "Title: red mug"}, + {"type": "doc", "doc_id": "g2", "text": "Title: blue mug"}, + {"type": "doc", "doc_id": "g3", "text": "Title: tortillas"}, + { + "type": "query", + "qid": "q-head", + "query": "red mug", + "gold_doc_ids": ["g1", "g2"], + "gold_grades": {"g1": 1.0, "g2": 0.1}, + }, + { + "type": "query", + "qid": "q-tail", + "query": "blue mug", + "gold_doc_ids": ["g2"], + "gold_grades": {"g2": 1.0}, + }, + { + "type": "query", + "qid": "esci-113", + "query": ""tortillas without interesterified soybean oil”", + "gold_doc_ids": ["g3"], + "gold_grades": {"g3": 1.0}, + }, + ] + taxonomy = classify_eval(eval_result, rows, k=50) + self.assertEqual(taxonomy["n_queries"], 3) + self.assertEqual(taxonomy["n_gold"], 4) + self.assertEqual(taxonomy["stratum_counts"]["head-ok"], 1) + self.assertEqual(taxonomy["stratum_counts"]["rank-too-low"], 1) + self.assertEqual(taxonomy["stratum_counts"]["total-miss"], 1) + self.assertTrue(taxonomy["run_query_side"]) + self.assertEqual(taxonomy["rewritable_qids"], ["esci-113"]) + + def test_spell_normalize_nfkc_and_accents(self): + from search.miss_strata import normalize_spelling, write_spell_jsonl + + self.assertEqual(normalize_spelling("niños!!!"), "ninos") + self.assertEqual(normalize_spelling(" sofa, azul. "), "sofa azul") + self.assertEqual(normalize_spelling("caf\u00e9"), "cafe") + with tempfile.TemporaryDirectory() as tmp: + dest = Path(tmp) / "spell.jsonl" + summary = write_spell_jsonl( + [ + {"type": "doc", "doc_id": "d1", "text": "sofa"}, + {"type": "query", "qid": "q1", "query": "sofá azul!!"}, + ], + dest, + ) + self.assertEqual(summary["n_queries_changed"], 1) + rows = dest.read_text(encoding="utf-8").strip().splitlines() + query = json.loads(rows[1]) + self.assertEqual(query["query"], "sofa azul") + self.assertEqual(query["query_original"], "sofá azul!!") + + def test_doc_meta_keys_copies_generic_catalog_fields(self): + from search.evaluate import doc_meta_keys + + self.assertEqual( + doc_meta_keys( + { + "doc_id": "it-bollitore", + "brand": "CasaLuce", + "color": "argento", + "locale": "it", + "title": "ignored", + } + ), + {"brand": "CasaLuce", "color": "argento", "locale": "it"}, + ) + self.assertIsNone(doc_meta_keys({"doc_id": "x", "text": "only text"})) + + def test_dense_holdout_excludes_eval_qids(self): + from search.finetune_esci_ce import held_out_query_ids + + holdout_path = self.require_local_artifact( + Path(__file__).resolve().parents[1] + / "benchmarks" + / "data" + / "search_esci_74.jsonl" + ) + holdout = held_out_query_ids(holdout_path) + self.assertIn("72", holdout) + self.assertIn("esci-72", holdout) + self.assertGreater(len(holdout), 74) + + def test_local_dense_scores_injected_ranks(self): + from search.local_dense import evaluate_dense + + queries = [ + { + "qid": "q1", + "query": "red mug", + "slice": "toy", + "gold_doc_ids": ["gold"], + "gold_grades": {"gold": 1.0, "sub": 0.1}, + } + ] + ranked = {"q1": ["gold", "sub", "irr"]} + metrics, per_query = evaluate_dense( + ranked, queries, ks=(5, 10, 20, 50), encode_ms=10.0 + ) + self.assertEqual(per_query[0]["hit_ids"][0], "gold") + self.assertEqual(metrics["recall@10"], 1.0) + self.assertGreater(metrics["ndcg@10"], 0.0) + + def test_exhaustive_ce_ranks_exact_above_irrelevant(self): + from search.rank_corpus import PROTOCOL, run_exhaustive_ce + + rows = [ + {"type": "doc", "doc_id": "irr", "text": "DOCID irr. Title: other"}, + {"type": "doc", "doc_id": "exact", "text": "DOCID exact. Title: gold kettle"}, + {"type": "doc", "doc_id": "maybe", "text": "DOCID maybe. Title: similar kettle"}, + { + "type": "query", + "qid": "esci-toy", + "query": "kettle", + "gold_doc_ids": ["exact"], + "gold_grades": {"exact": 1.0, "maybe": 0.1}, + "slice": "toy", + }, + ] + + def predict(pairs): + out = [] + for _, text in pairs: + if "gold kettle" in text: + out.append([0.95, 0.03, 0.01, 0.01]) + elif "similar" in text: + out.append([0.05, 0.80, 0.10, 0.05]) + else: + out.append([0.01, 0.02, 0.07, 0.90]) + return out + + result = run_exhaustive_ce( + rows, + predict=predict, + model_name="toy-4class", + dataset_name="toy-corpus.jsonl", + k=3, + ks=(5, 10, 20, 50), + ) + self.assertEqual(result["status"], "ok") + self.assertEqual(result["protocol"], PROTOCOL) + self.assertFalse(result["rank_pool"]) + self.assertEqual(result["queries"][0]["hit_ids"][0], "exact") + self.assertLess(result["queries"][0]["hit_ids"].index("exact"), result["queries"][0]["hit_ids"].index("irr")) + self.assertGreater(result["metrics"]["ndcg@10"], 0.0) + + def test_list_overlap_counts_sidecar_unique_golds(self): + from search.list_overlap import run_union, summarize_overlap, unique_golds + + gold = {"g1", "g2", "g3"} + extra = unique_golds( + ["g2", "noise", "g3"], + ["g1", "g2", "other"], + gold, + k=50, + ) + self.assertEqual(extra, ["g3"]) + passages = { + "run_id": "passages", + "queries": [ + { + "qid": "q1", + "hit_ids": ["g1", "g2", "p1"], + "gold_doc_ids": ["g1", "g2", "g3"], + "gold_grades": {"g1": 1.0, "g2": 1.0, "g3": 0.1}, + } + ], + } + bge = { + "run_id": "bge", + "queries": [{"qid": "q1", "hit_ids": ["g3", "g1", "b1"]}], + } + summary = summarize_overlap(passages, {"bge": bge}, k=50) + self.assertEqual(summary["runs"]["bge"]["unique_gold_hits"], 1) + rows = [ + {"type": "doc", "doc_id": "g1", "text": "g1"}, + {"type": "doc", "doc_id": "g2", "text": "g2"}, + {"type": "doc", "doc_id": "g3", "text": "g3"}, + { + "type": "query", + "qid": "q1", + "query": "mug", + "gold_doc_ids": ["g1", "g2", "g3"], + "gold_grades": {"g1": 1.0, "g2": 1.0, "g3": 0.1}, + }, + ] + union = run_union( + [passages, bge], + rows, + dataset_name="toy.jsonl", + k=50, + ks=(5, 10, 20, 50), + run_names=["passages", "bge"], + ) + self.assertEqual(union["channels"], ["harness-union"]) + self.assertIn("g3", union["queries"][0]["hit_ids"][:10]) + self.assertGreaterEqual(union["metrics"]["recall@10"], 1.0) + + def test_cascade_freezes_head_and_injects_tail_golds(self): + from search.list_overlap import CASCADE_PROTOCOL, cascade_frozen_head, run_cascade + + passages = [f"h{index}" for index in range(10)] + ["tail_gold", "irr"] + gold = {"h0", "tail_gold", "extra"} + ranked = cascade_frozen_head( + passages, + [["extra", "noise", "h0"]], + gold, + head_k=10, + k=50, + ) + self.assertEqual(ranked[:10], passages[:10]) + self.assertIn("extra", ranked[10:]) + self.assertNotIn("extra", ranked[:10]) + self.assertIn("tail_gold", ranked) + rows = [ + {"type": "doc", "doc_id": doc_id, "text": doc_id} + for doc_id in [*passages, "extra"] + ] + rows.append( + { + "type": "query", + "qid": "q1", + "query": "mug", + "gold_doc_ids": ["h0", "tail_gold", "extra"], + "gold_grades": {"h0": 1.0, "tail_gold": 1.0, "extra": 0.1}, + } + ) + result = run_cascade( + {"run_id": "passages", "queries": [{"qid": "q1", "hit_ids": passages}]}, + [{"run_id": "bge", "queries": [{"qid": "q1", "hit_ids": ["extra", "noise"]}]}], + rows, + dataset_name="toy.jsonl", + k=50, + head_k=10, + ks=(5, 10, 20, 50), + run_names=["passages", "bge"], + ) + self.assertEqual(result["protocol"], CASCADE_PROTOCOL) + self.assertEqual(result["channels"], ["harness-cascade"]) + self.assertEqual(result["queries"][0]["hit_ids"][:10], passages[:10]) + self.assertIn("extra", result["queries"][0]["hit_ids"][10:]) + self.assertEqual(result["metrics"]["recall@10"], 1.0 / 3.0) + + def test_frozen_head_merge_matches_harness_cascade(self): + from src.core.search.hybrid import frozen_head_merge + from search.list_overlap import cascade_frozen_head + + passages = [f"h{index}" for index in range(10)] + ["tail_gold", "irr"] + sidecars = [["extra", "noise", "h0"]] + gold = {"h0", "tail_gold", "extra"} + ranked = frozen_head_merge( + passages, + sidecars, + head_k=10, + k=50, + prefer_ids=gold, + ) + self.assertEqual( + ranked, + cascade_frozen_head(passages, sidecars, gold, head_k=10, k=50), + ) + live = frozen_head_merge(passages, sidecars, head_k=10, k=50) + self.assertEqual(live[:10], passages[:10]) + self.assertIn("extra", live[10:]) + self.assertIn("noise", live[10:]) + + def test_cascade_replay_esci74_stored_lists(self): + from search.dataset import load_records + from search.list_overlap import load_eval_run, run_cascade + + root = Path(__file__).resolve().parents[1] + runs = root / "benchmarks" / "runs" + passages_path = runs / "search-esci-74-passages-k50" / "eval.json" + bge_path = runs / "search-esci-74-bge-base-k50" / "eval.json" + colbert_path = runs / "search-esci-74-colbert-k50" / "eval.json" + dataset = root / "benchmarks" / "data" / "search_esci_74.jsonl" + if not all( + path.exists() + for path in (passages_path, bge_path, colbert_path, dataset) + ): + self.skipTest("stored US ESCI n=74 evals not present") + passages = load_eval_run(passages_path.parent) + bge = load_eval_run(bge_path.parent) + colbert = load_eval_run(colbert_path.parent) + rows = load_records(dataset) + result = run_cascade( + passages, + [bge, colbert], + rows, + dataset_name="search_esci_74.jsonl", + k=50, + head_k=10, + ks=(5, 10, 20, 50), + run_names=["passages", "bge", "colbert"], + ) + metrics = result["metrics"] + self.assertAlmostEqual(metrics["ndcg@10"], 0.500, places=3) + self.assertAlmostEqual(metrics["recall@10"], 0.379, places=3) + self.assertAlmostEqual(metrics["recall@50"], 0.889, places=3) + + def test_ltr_head_promotes_title_overlap_on_held_out_query(self): + from search.ltr_head import ( + example_from_eval_row, + features_for_doc, + fit_pairwise, + rerank_ids, + run_ltr_head, + ) + + feats = features_for_doc( + "navy velvet sofa", + "hit", + rrf_ids=["miss", "hit"], + bm25_ids=["hit"], + dense_ids=["miss"], + doc={"title": "Navy velvet sofa two seats", "brand": "AtelierNord"}, + ) + self.assertGreater(feats[3], 0.5) + self.assertEqual(feats[4], 0.0) + train_row = { + "qid": "q-train", + "query": "navy velvet sofa", + "hit_ids": ["irr", "gold"], + "bm25_ids": ["gold"], + "dense_ids": ["irr"], + "hits": [ + {"id": "irr", "doc_id": "irr"}, + {"id": "gold", "doc_id": "gold"}, + ], + "gold_doc_ids": ["gold"], + "gold_grades": {"gold": 1.0}, + } + test_row = { + "qid": "q-test", + "query": "navy velvet sofa", + "hit_ids": ["irr", "gold"], + "bm25_ids": ["gold"], + "dense_ids": ["irr"], + "hits": [ + {"id": "irr", "doc_id": "irr"}, + {"id": "gold", "doc_id": "gold"}, + ], + "gold_doc_ids": ["gold"], + "gold_grades": {"gold": 1.0}, + } + docs = { + "gold": {"doc_id": "gold", "title": "Navy velvet sofa"}, + "irr": {"doc_id": "irr", "title": "plastic kettle lid"}, + } + train_ex = example_from_eval_row(train_row, docs, k=10) + test_ex = example_from_eval_row(test_row, docs, k=10) + self.assertIsNotNone(train_ex) + self.assertIsNotNone(test_ex) + weights = fit_pairwise([train_ex], epochs=20, seed=0) + ranked = rerank_ids(test_ex, weights) + self.assertEqual(ranked[0], "gold") + result = run_ltr_head( + {"run_id": "toy", "queries": [train_row, test_row]}, + [ + {"type": "doc", "doc_id": "gold", "title": "Navy velvet sofa", "text": "Title: Navy velvet sofa"}, + {"type": "doc", "doc_id": "irr", "title": "plastic kettle lid", "text": "Title: plastic kettle lid"}, + { + "type": "query", + "qid": "q-train", + "query": "navy velvet sofa", + "gold_doc_ids": ["gold"], + "gold_grades": {"gold": 1.0}, + }, + { + "type": "query", + "qid": "q-test", + "query": "navy velvet sofa", + "gold_doc_ids": ["gold"], + "gold_grades": {"gold": 1.0}, + }, + ], + dataset_name="toy.jsonl", + k=10, + ks=(5, 10), + n_folds=2, + ) + self.assertEqual(result["protocol"], "ltr-head-cv") + self.assertGreaterEqual(result["metrics"]["recall@10"], 1.0) + + def test_ltr_pair_policy_skips_unlabeled_for_other_query_neg(self): + from search.ltr_head import ( + PAIR_OTHER_QUERY_NEG, + PAIR_UNLABELED_ZERO, + collect_pairs, + example_from_eval_row, + features_for_doc, + other_query_gold_ids, + ) + + row_a = { + "qid": "q-a", + "query": "navy sofa", + "hit_ids": ["gold-a", "gold-b", "unlabeled"], + "hits": [ + {"id": "gold-a", "doc_id": "gold-a"}, + {"id": "gold-b", "doc_id": "gold-b"}, + {"id": "unlabeled", "doc_id": "unlabeled"}, + ], + "gold_doc_ids": ["gold-a"], + "gold_grades": {"gold-a": 1.0}, + } + row_b = { + "qid": "q-b", + "query": "steel kettle", + "hit_ids": ["gold-b", "gold-a", "unlabeled"], + "hits": [ + {"id": "gold-b", "doc_id": "gold-b"}, + {"id": "gold-a", "doc_id": "gold-a"}, + {"id": "unlabeled", "doc_id": "unlabeled"}, + ], + "gold_doc_ids": ["gold-b"], + "gold_grades": {"gold-b": 1.0}, + } + docs = { + "gold-a": {"doc_id": "gold-a", "title": "Navy sofa"}, + "gold-b": {"doc_id": "gold-b", "title": "Steel kettle"}, + "unlabeled": {"doc_id": "unlabeled", "title": "random cable"}, + } + ex_a = example_from_eval_row(row_a, docs, k=10) + ex_b = example_from_eval_row(row_b, docs, k=10) + other = other_query_gold_ids([ex_a, ex_b], "q-a") + self.assertEqual(other, {"gold-b"}) + zero_pairs = collect_pairs(ex_a, pair_policy=PAIR_UNLABELED_ZERO) + neg_pairs = collect_pairs( + ex_a, pair_policy=PAIR_OTHER_QUERY_NEG, other_gold=other + ) + zero_docs = {(ex_a["ids"][hi], ex_a["ids"][lo]) for hi, lo in zero_pairs} + neg_docs = {(ex_a["ids"][hi], ex_a["ids"][lo]) for hi, lo in neg_pairs} + self.assertIn(("gold-a", "unlabeled"), zero_docs) + self.assertNotIn(("gold-a", "unlabeled"), neg_docs) + self.assertIn(("gold-a", "gold-b"), neg_docs) + feats = features_for_doc( + "navy sofa", + "gold-a", + rrf_ids=["gold-a"], + bm25_ids=["gold-a"], + dense_ids=["gold-a"], + doc={"title": "Navy sofa"}, + ce_gain=0.42, + ) + self.assertEqual(len(feats), 7) + self.assertAlmostEqual(float(feats[-1]), 0.42) + from search.ltr_head import rank_train_group + + grouped = rank_train_group( + ex_a, pair_policy=PAIR_OTHER_QUERY_NEG, other_gold=other + ) + self.assertIsNotNone(grouped) + _, labels = grouped + self.assertEqual(len(labels), 2) + + def test_ltr_lightgbm_promotes_high_ce_gain_on_held_out_query(self): + from search.ltr_head import ( + PAIR_OTHER_QUERY_NEG, + example_from_eval_row, + fit_lightgbm, + rerank_ids_model, + ) + + n_queries = 8 + docs: dict[str, dict[str, str]] = {} + rows: list[dict] = [] + for index in range(n_queries): + gold = f"gold-{index}" + other = f"gold-{(index + 1) % n_queries}" + miss = f"miss-{index}" + docs[gold] = {"doc_id": gold, "title": f"Navy sofa {index}"} + docs[miss] = {"doc_id": miss, "title": f"plastic lid {index}"} + rows.append( + { + "qid": f"q-{index}", + "query": "navy sofa", + "hit_ids": [miss, other, gold], + "bm25_ids": [gold], + "dense_ids": [miss], + "hits": [ + {"id": miss, "doc_id": miss}, + {"id": other, "doc_id": other}, + {"id": gold, "doc_id": gold}, + ], + "gold_doc_ids": [gold], + "gold_grades": {gold: 1.0}, + "ce": {gold: 0.95, other: 0.05, miss: 0.01}, + } + ) + examples = [ + example_from_eval_row( + row, docs, k=10, ce_scores=row["ce"] + ) + for row in rows + ] + self.assertTrue(all(item is not None for item in examples)) + model = fit_lightgbm( + examples[:-1], + pair_policy=PAIR_OTHER_QUERY_NEG, + min_data_in_leaf=1, + ) + ranked = rerank_ids_model(examples[-1], model) + self.assertEqual(ranked[0], "gold-7") + self.assertGreater(float(sum(model.feature_importances_)), 0.0) + + def test_ltr_apply_trains_on_other_run_not_eval_qid(self): + from search.ltr_head import PROTOCOL_APPLY, run_ltr_head + + train_docs = [ + {"type": "doc", "doc_id": "gold-t", "title": "Navy velvet sofa", "text": "Title: Navy velvet sofa"}, + {"type": "doc", "doc_id": "irr-t", "title": "plastic lid", "text": "Title: plastic lid"}, + ] + eval_docs = [ + {"type": "doc", "doc_id": "gold-e", "title": "Navy velvet sofa", "text": "Title: Navy velvet sofa"}, + {"type": "doc", "doc_id": "irr-e", "title": "plastic lid", "text": "Title: plastic lid"}, + ] + train_queries = [ + { + "type": "query", + "qid": "q-train", + "query": "navy velvet sofa", + "gold_doc_ids": ["gold-t"], + "gold_grades": {"gold-t": 1.0}, + } + ] + eval_queries = [ + { + "type": "query", + "qid": "q-eval", + "query": "navy velvet sofa", + "gold_doc_ids": ["gold-e"], + "gold_grades": {"gold-e": 1.0}, + } + ] + train_row = { + "qid": "q-train", + "query": "navy velvet sofa", + "hit_ids": ["irr-t", "gold-t"], + "bm25_ids": ["gold-t"], + "dense_ids": ["irr-t"], + "hits": [ + {"id": "irr-t", "doc_id": "irr-t"}, + {"id": "gold-t", "doc_id": "gold-t"}, + ], + "gold_doc_ids": ["gold-t"], + "gold_grades": {"gold-t": 1.0}, + } + eval_row = { + "qid": "q-eval", + "query": "navy velvet sofa", + "hit_ids": ["irr-e", "gold-e"], + "bm25_ids": ["gold-e"], + "dense_ids": ["irr-e"], + "hits": [ + {"id": "irr-e", "doc_id": "irr-e"}, + {"id": "gold-e", "doc_id": "gold-e"}, + ], + "gold_doc_ids": ["gold-e"], + "gold_grades": {"gold-e": 1.0}, + } + result = run_ltr_head( + {"run_id": "toy-eval", "queries": [eval_row]}, + eval_docs + eval_queries, + dataset_name="toy-eval.jsonl", + k=10, + ks=(5, 10), + n_folds=2, + train_eval_result={"run_id": "toy-train", "queries": [train_row]}, + train_rows=train_docs + train_queries, + train_source_run="toy-train", + ) + self.assertEqual(result["protocol"], PROTOCOL_APPLY) + self.assertEqual(result["ltr_train_run"], "toy-train") + self.assertEqual(result["ltr_n_train_queries"], 1) + self.assertEqual(result["queries"][0]["qid"], "q-eval") + self.assertEqual(result["queries"][0]["hit_ids"][0], "gold-e") + self.assertGreaterEqual(result["metrics"]["recall@10"], 1.0) + + def test_ltr_apply_lightgbm_trains_on_other_run_not_eval_qid(self): + from search.ltr_head import HEAD_LIGHTGBM, PROTOCOL_APPLY, run_ltr_head + + train_docs: list[dict] = [] + train_queries: list[dict] = [] + train_eval_rows: list[dict] = [] + for index in range(8): + gold = f"gold-t-{index}" + miss = f"miss-t-{index}" + train_docs.extend( + [ + { + "type": "doc", + "doc_id": gold, + "title": f"Navy velvet sofa {index}", + "text": f"Title: Navy velvet sofa {index}", + }, + { + "type": "doc", + "doc_id": miss, + "title": f"plastic lid {index}", + "text": f"Title: plastic lid {index}", + }, + ] + ) + qid = f"q-train-{index}" + train_queries.append( + { + "type": "query", + "qid": qid, + "query": "navy velvet sofa", + "gold_doc_ids": [gold], + "gold_grades": {gold: 1.0}, + } + ) + train_eval_rows.append( + { + "qid": qid, + "query": "navy velvet sofa", + "hit_ids": [miss, gold], + "bm25_ids": [gold], + "dense_ids": [miss], + "hits": [ + {"id": miss, "doc_id": miss}, + {"id": gold, "doc_id": gold}, + ], + "gold_doc_ids": [gold], + "gold_grades": {gold: 1.0}, + } + ) + eval_docs = [ + { + "type": "doc", + "doc_id": "gold-e", + "title": "Navy velvet sofa", + "text": "Title: Navy velvet sofa", + }, + { + "type": "doc", + "doc_id": "irr-e", + "title": "plastic lid", + "text": "Title: plastic lid", + }, + ] + eval_queries = [ + { + "type": "query", + "qid": "q-eval", + "query": "navy velvet sofa", + "gold_doc_ids": ["gold-e"], + "gold_grades": {"gold-e": 1.0}, + } + ] + eval_row = { + "qid": "q-eval", + "query": "navy velvet sofa", + "hit_ids": ["irr-e", "gold-e"], + "bm25_ids": ["gold-e"], + "dense_ids": ["irr-e"], + "hits": [ + {"id": "irr-e", "doc_id": "irr-e"}, + {"id": "gold-e", "doc_id": "gold-e"}, + ], + "gold_doc_ids": ["gold-e"], + "gold_grades": {"gold-e": 1.0}, + } + result = run_ltr_head( + {"run_id": "toy-eval", "queries": [eval_row]}, + eval_docs + eval_queries, + dataset_name="toy-eval.jsonl", + k=10, + ks=(5, 10), + n_folds=2, + ltr_head=HEAD_LIGHTGBM, + train_eval_result={"run_id": "toy-train", "queries": train_eval_rows}, + train_rows=train_docs + train_queries, + train_source_run="toy-train", + ) + self.assertEqual(result["protocol"], PROTOCOL_APPLY) + self.assertEqual(result["ltr_train_run"], "toy-train") + self.assertEqual(result["ltr_n_train_queries"], 8) + self.assertEqual(result["rerank"], "ltr-lightgbm") + self.assertEqual(result["queries"][0]["qid"], "q-eval") + self.assertNotEqual(result["queries"][0]["qid"], "q-train-0") + + def test_export_hybrid_lists_holdout_unlabeled_is_i(self): + from search.export_hybrid_lists import ( + PROTECTED_OUT_NAMES, + export_hybrid_lists, + rows_from_eval, + ) + + self.assertIn("esci_retrieved_lists.jsonl", PROTECTED_OUT_NAMES) + eval_result = { + "queries": [ + { + "qid": "esci-hold", + "query": "navy sofa", + "hit_ids": ["gold-h", "miss-h"], + "hits": [ + {"id": "gold-h", "doc_id": "gold-h"}, + {"id": "miss-h", "doc_id": "miss-h"}, + ], + "gold_grades": {"gold-h": 1.0}, + }, + { + "qid": "esci-ok", + "query": "navy sofa", + "hit_ids": ["gold-ok", "miss-ok"], + "hits": [ + {"id": "gold-ok", "doc_id": "gold-ok"}, + {"id": "miss-ok", "doc_id": "miss-ok"}, + ], + "gold_grades": {"gold-ok": 1.0}, + }, + ] + } + docs = [ + { + "type": "doc", + "doc_id": "gold-ok", + "title": "Navy sofa", + "text": "Title: Navy sofa", + }, + { + "type": "doc", + "doc_id": "miss-ok", + "title": "plastic lid", + "text": "Title: plastic lid", + }, + ] + queries = [ + { + "type": "query", + "qid": "esci-ok", + "query": "navy sofa", + "gold_doc_ids": ["gold-ok"], + "gold_grades": {"gold-ok": 1.0}, + }, + { + "type": "query", + "qid": "esci-hold", + "query": "navy sofa", + "gold_doc_ids": ["gold-h"], + "gold_grades": {"gold-h": 1.0}, + }, + ] + rows = rows_from_eval( + eval_result, + docs=docs, + queries=queries, + holdout={"esci-hold", "hold"}, + k=50, + ) + qids = {row["query_id"] for row in rows} + self.assertNotIn("esci-hold", qids) + self.assertIn("esci-ok", qids) + by_pid = {row["product_id"]: row["label"] for row in rows} + self.assertEqual(by_pid["gold-ok"], "E") + self.assertEqual(by_pid["miss-ok"], "I") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + dataset = root / "train.jsonl" + holdout_path = root / "holdout.jsonl" + out = root / "hybrid.jsonl" + dataset.write_text( + json.dumps(docs[0]) + + "\n" + + json.dumps(docs[1]) + + "\n" + + json.dumps(queries[0]) + + "\n", + encoding="utf-8", + ) + holdout_path.write_text( + json.dumps( + { + "type": "query", + "qid": "esci-hold", + "query": "navy sofa", + "gold_doc_ids": ["gold-h"], + } + ) + + "\n", + encoding="utf-8", + ) + meta = export_hybrid_lists( + eval_result=eval_result, + dataset_path=dataset, + holdout_path=holdout_path, + out_path=out, + k=50, + ) + self.assertEqual(meta["source"], "hybrid-k50") + self.assertEqual(meta["n_queries"], 1) + with self.assertRaises(ValueError): + export_hybrid_lists( + eval_result=eval_result, + dataset_path=dataset, + holdout_path=holdout_path, + out_path=root / "esci_retrieved_lists.jsonl", + k=50, + ) + + def test_retrieved_lists_unlabeled_is_irrelevant(self): + from search.finetune_esci_4class import held_out_query_ids + from search.mine_retrieved_lists import ( + is_held_out, + labeled_hits, + mine_from_groups, + select_groups, + ) + from search.pool_first_stage import Bm25Index, rank_docs, tokenize + + docs = [ + ("B001", tokenize("red ceramic mug")), + ("B002", tokenize("blue dinner plate")), + ("B003", tokenize("red mug gift set")), + ] + index = Bm25Index(docs) + qrels = {"B001": "E", "B003": "S"} + scores = index.scores(tokenize("red ceramic mug")) + ranked = rank_docs(scores, ["B001", "B002", "B003"]) + hits = labeled_hits(ranked, qrels, k=3) + by_id = dict(hits) + self.assertEqual(by_id["B001"], "E") + self.assertEqual(by_id["B003"], "S") + self.assertEqual(by_id["B002"], "I") + self.assertTrue(any(label == "I" for _, label in hits)) + + groups = { + "q1": { + "query": "red ceramic mug", + "qrels": {"B001": "E", "B003": "S"}, + } + } + passages = { + "B001": "Title: red ceramic mug", + "B002": "Title: blue dinner plate", + "B003": "Title: red mug gift set", + } + rows = mine_from_groups(groups, passages, k=3) + labels = {row["product_id"]: row["label"] for row in rows} + self.assertEqual(labels["B001"], "E") + self.assertEqual(labels["B002"], "I") + + holdout_path = self.require_local_artifact( + Path(__file__).resolve().parents[1] + / "benchmarks" + / "data" + / "search_esci_74.jsonl" + ) + holdout = held_out_query_ids(holdout_path) + self.assertTrue(is_held_out("72", holdout)) + selected = select_groups( + { + "72": {"query": "not electronics", "qrels": {"x": "E"}}, + "esci-113": {"query": "html", "qrels": {"y": "E"}}, + "900001": {"query": "ok", "qrels": {"z": "E"}}, + }, + holdout=holdout, + max_queries=10, + seed=1, + ) + self.assertNotIn("72", selected) + self.assertNotIn("esci-113", selected) + self.assertIn("900001", selected) + + def test_local_colbert_fake_encoder_ranks_match(self): + from search.local_colbert import PLUGIN_DIR, run_local_colbert + + import sys + + self.require_local_artifact(PLUGIN_DIR / "encode.py") + if str(PLUGIN_DIR) not in sys.path: + sys.path.insert(0, str(PLUGIN_DIR)) + import encode as colbert_encode + + def fake_tokens(text: str): + blob = (text or "").lower() + if "match" in blob or blob == "q": + return [[1.0, 0.0]] + return [[0.0, 1.0]] + + colbert_encode.set_encoder(fake_tokens) + try: + rows = [ + {"type": "doc", "doc_id": "hit", "text": "match"}, + {"type": "doc", "doc_id": "miss", "text": "other"}, + { + "type": "query", + "qid": "q1", + "query": "q", + "gold_doc_ids": ["hit"], + "gold_grades": {"hit": 1.0}, + }, + ] + result = run_local_colbert( + rows, + dataset_name="toy", + k=2, + ks=(5, 10, 20, 50), + brain_id="harness-local-colbert", + ) + self.assertEqual(result["channels"], ["harness-colbert"]) + self.assertEqual(result["queries"][0]["hit_ids"][0], "hit") + finally: + colbert_encode.set_encoder(None) + + def test_failed_report_skips_ledger(self): + from search.report import entry_from_report, update_reports_json + + failed = { + "status": "failed", + "run_id": "search-fail", + "brain_id": "searchbenchsmoke", + "dataset": "search_toy.jsonl", + "n_queries": 8, + "ndcg@10": 0.9, + } + self.assertIsNone(entry_from_report(failed)) + empty_ok = { + "status": "ok", + "run_id": "search-empty", + "n_queries": 0, + } + self.assertIsNone(entry_from_report(empty_ok)) + + ledger = { + "schema_version": 2, + "benchmarks": { + "locomo": {"name": "LoCoMo", "leaderboard": [{"run_id": "keep-me"}]}, + "recsys": {"name": "RecSys", "leaderboard": [{"run_id": "keep-recsys"}]}, + "search": {"name": "Search (hybrid BM25 + dense)", "leaderboard": []}, + }, + } + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "REPORTS.json" + path.write_text(json.dumps(ledger), encoding="utf-8") + with patch("search.report.REPORTS_PATH", path): + update_reports_json(failed) + after = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual( + after["benchmarks"]["locomo"]["leaderboard"][0]["run_id"], + "keep-me", + ) + self.assertEqual( + after["benchmarks"]["recsys"]["leaderboard"][0]["run_id"], + "keep-recsys", + ) + self.assertEqual(after["benchmarks"]["search"]["leaderboard"], []) + + ok = { + "status": "ok", + "run_id": "search-ok", + "brain_id": "searchbenchsmoke", + "dataset": "search_toy.jsonl", + "fusion": "rrf", + "n_queries": 8, + "ndcg@10": 0.5, + "recall@10": 0.6, + "mrr": 0.4, + "p50_retrieve_ms": 12.0, + "p95_retrieve_ms": 30.0, + "git_sha": "abc", + "recorded_at": "2026-08-18T00:00:00+00:00", + } + with patch("search.report.REPORTS_PATH", path): + update_reports_json(ok) + after_ok = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual( + after_ok["benchmarks"]["locomo"]["leaderboard"][0]["run_id"], + "keep-me", + ) + self.assertEqual(len(after_ok["benchmarks"]["search"]["leaderboard"]), 1) + self.assertEqual( + after_ok["benchmarks"]["search"]["leaderboard"][0]["run_id"], + "search-ok", + ) + + def test_cli_refuses_memory_brain(self): + from search.cli import main + + with self.assertRaises(SystemExit) as ctx: + main(["--brain", "locomoconv26", "dataset-stats"]) + self.assertIn("Refusing brain_id", str(ctx.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search_bm25.py b/tests/test_search_bm25.py new file mode 100644 index 0000000..65040a0 --- /dev/null +++ b/tests/test_search_bm25.py @@ -0,0 +1,256 @@ +import os +import unittest +from unittest.mock import MagicMock, patch + +ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", +} +for key, value in ENV_DEFAULTS.items(): + os.environ.setdefault(key, value) + +from src.core.search.bm25 import okapi_bm25_document +from src.constants.data import TextChunk +from src.lib.postgresql import data as pgdata + + +def _postgres_available() -> bool: + try: + import psycopg2 + + pg = pgdata.config.postgresql + pg.validate_credentials() + conn = psycopg2.connect( + host=pg.host, + port=pg.port, + user=pg.username, + password=pg.password, + dbname=pg.maintenance_database or "postgres", + connect_timeout=2, + ) + conn.close() + return True + except Exception: + return False + + +class OkapiBm25Tests(unittest.TestCase): + def test_term_overlap_ranks_above_non_overlap(self): + overlap = okapi_bm25_document( + {"license": 2.0}, + dl=8, + avgdl=8, + n_docs=2, + dfs={"license": 1}, + ) + non_overlap = okapi_bm25_document( + {}, + dl=8, + avgdl=8, + n_docs=2, + dfs={"license": 1}, + ) + self.assertGreater(overlap, non_overlap) + self.assertEqual(non_overlap, 0.0) + + def test_higher_tf_ranks_higher(self): + high = okapi_bm25_document( + {"license": 3.0}, + dl=10, + avgdl=10, + n_docs=4, + dfs={"license": 2}, + ) + low = okapi_bm25_document( + {"license": 1.0}, + dl=10, + avgdl=10, + n_docs=4, + dfs={"license": 2}, + ) + self.assertGreater(high, low) + + def test_search_ddl_is_gated_off_base_schema(self): + self.assertNotIn("search_tsv", pgdata._BRAIN_DDL) + self.assertIn("search_tsv", pgdata._SEARCH_DDL) + self.assertIn("gin", pgdata._SEARCH_DDL.lower()) + self.assertIn("to_tsvector", pgdata._SEARCH_DDL) + self.assertIn("to_tsvector('english'", pgdata._SEARCH_DDL) + self.assertNotIn("to_tsvector('italian'", pgdata._SEARCH_DDL) + self.assertNotIn("to_tsvector('spanish'", pgdata._SEARCH_DDL) + self.assertNotIn("search_tsv_alt", pgdata._SEARCH_DDL) + alt = pgdata._search_alt_ddl("italian") + self.assertIn("to_tsvector('italian'", alt) + self.assertIn("search_tsv_alt", alt) + with self.assertRaises(ValueError): + pgdata._search_alt_ddl("english") + + def test_english_bm25_sql_keeps_plainto_and(self): + sql = pgdata._bm25_sql("english", "search_tsv", "search_len") + self.assertIn("plainto_tsquery('english'", sql) + self.assertIn("search_tsv", sql) + self.assertNotIn("search_tsv_alt", sql) + self.assertNotIn("WHERE TRUE", sql) + + def test_alt_bm25_sql_matches_any_lexeme(self): + sql = pgdata._bm25_sql("italian", "search_tsv_alt", "search_len_alt") + self.assertNotIn("plainto_tsquery", sql) + self.assertIn("WHERE TRUE", sql) + self.assertIn("search_tsv_alt", sql) + self.assertIn("search_len_alt", sql) + and_sql = pgdata._bm25_sql( + "italian", "search_tsv_alt", "search_len_alt", query_match="and" + ) + self.assertIn("plainto_tsquery('italian'", and_sql) + + def test_searchbenchesci74_stays_english_and_path(self): + with ( + patch.object( + pgdata.config, "search_fts_brains", frozenset({"searchbenchitsmoke"}) + ), + patch.object(pgdata.config, "search_fts_regconfig", "italian"), + ): + self.assertIsNone( + pgdata.config.search_fts_regconfig_for_brain("searchbenchesci74") + ) + self.assertIsNone( + pgdata.config.search_fts_regconfig_for_brain("locomoconv26") + ) + + def test_node_search_schema_skipped_when_search_disabled(self): + from src.lib.postgresql.graph_store import PostgreSQLGraphStore + + store = PostgreSQLGraphStore.__new__(PostgreSQLGraphStore) + store._schema_ready = {"brain-a"} + store._search_ready_brains = set() + store._schema_lock = MagicMock() + with patch.object(pgdata.config, "search_enabled", False): + store._ensure_brain_schema("brain-a") + self.assertEqual(store._search_ready_brains, set()) + store._schema_lock.assert_not_called() + + def test_search_schema_skipped_when_search_disabled(self): + client = pgdata.PostgreSQLDataClient.__new__(pgdata.PostgreSQLDataClient) + client._initialized_brains = {"brain-a"} + client._search_ready_brains = set() + client._search_alt_brains = set() + client._lock = MagicMock() + with patch.object(pgdata.config, "search_enabled", False): + client._ensure_brain_schema("brain-a") + self.assertEqual(client._search_ready_brains, set()) + + @unittest.skipUnless(_postgres_available(), "postgres not available") + def test_postgres_bm25_ranks_term_overlap_first(self): + brain_id = "searchbench_bm25_unit" + with patch.object(pgdata.config, "search_enabled", True): + client = pgdata.PostgreSQLDataClient() + overlap = TextChunk( + id="overlap", + text="alice counseling license renewal board", + ) + other = TextChunk( + id="other", + text="weather forecast sunny tomorrow picnic", + ) + client.save_text_chunk(overlap, brain_id) + client.save_text_chunk(other, brain_id) + ranked = client.search_bm25("counseling license", brain_id, limit=10) + ids = [chunk.id for chunk, _score in ranked] + self.assertTrue(ids, "expected at least one BM25 hit") + self.assertEqual(ids[0], "overlap") + self.assertNotIn("other", ids) + + def test_fts_regconfig_only_allowlisted_searchbench(self): + with ( + patch.object( + pgdata.config, + "search_fts_brains", + frozenset({"locomoconv26", "searchbenchitsmoke", "demorecsys"}), + ), + patch.object(pgdata.config, "search_fts_regconfig", "italian"), + ): + self.assertEqual( + pgdata.config.search_fts_regconfig_for_brain("searchbenchitsmoke"), + "italian", + ) + self.assertIsNone( + pgdata.config.search_fts_regconfig_for_brain("searchbenchescies") + ) + self.assertIsNone(pgdata.config.search_fts_regconfig_for_brain("locomoconv26")) + self.assertIsNone(pgdata.config.search_fts_regconfig_for_brain("demorecsys")) + self.assertIsNone(pgdata.config.search_fts_regconfig_for_brain("beam1m1clean")) + + def test_alt_ddl_not_executed_for_memory_brain(self): + client = pgdata.PostgreSQLDataClient.__new__(pgdata.PostgreSQLDataClient) + client._initialized_brains = {"locomoconv26"} + client._search_ready_brains = {"locomoconv26"} + client._search_alt_brains = set() + client._lock = MagicMock() + with ( + patch.object(pgdata.config, "search_enabled", True), + patch.object( + pgdata.config, "search_fts_regconfig_for_brain", return_value=None + ), + patch.object(pgdata, "ensure_brain_database") as ensure_db, + ): + client._ensure_brain_schema("locomoconv26") + ensure_db.assert_not_called() + self.assertEqual(client._search_alt_brains, set()) + + def test_alt_ddl_executed_for_allowlisted_searchbench(self): + from contextlib import contextmanager + + client = pgdata.PostgreSQLDataClient.__new__(pgdata.PostgreSQLDataClient) + client._initialized_brains = {"searchbenchitsmoke"} + client._search_ready_brains = {"searchbenchitsmoke"} + client._search_alt_brains = set() + client._lock = MagicMock() + cur = MagicMock() + conn = MagicMock() + conn.cursor.return_value.__enter__.return_value = cur + conn.cursor.return_value.__exit__.return_value = False + + @contextmanager + def fake_borrow(_pool): + yield conn + + with ( + patch.object(pgdata.config, "search_enabled", True), + patch.object( + pgdata.config, "search_fts_regconfig_for_brain", return_value="italian" + ), + patch.object(pgdata, "ensure_brain_database"), + patch.object(pgdata, "get_brain_pool"), + patch.object(pgdata, "borrow", fake_borrow), + ): + client._ensure_brain_schema("searchbenchitsmoke") + executed = " ".join(str(call.args[0]) for call in cur.execute.call_args_list) + self.assertIn("search_tsv_alt", executed) + self.assertIn("to_tsvector('italian'", executed) + self.assertNotIn("to_tsvector('english'", executed) + self.assertIn("searchbenchitsmoke", client._search_alt_brains) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search_catalog.py b/tests/test_search_catalog.py new file mode 100644 index 0000000..62d22bd --- /dev/null +++ b/tests/test_search_catalog.py @@ -0,0 +1,687 @@ +import tempfile +import unittest +from pathlib import Path +import os + +ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", +} +for key, value in ENV_DEFAULTS.items(): + os.environ.setdefault(key, value) + + +class SearchCatalogTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + import sys + + root = Path(__file__).resolve().parents[1] + bench = root / "benchmarks" + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + if str(bench) not in sys.path: + sys.path.insert(0, str(bench)) + + def test_format_product_text_includes_marker(self): + from search.catalog import format_product_text + from search.dataset import doc_marker + + text = format_product_text( + "B07ABC", + title="Blue kettle", + description="1.5 liter stovetop kettle", + extras=(("Brand", "Acme"), ("Color", "blue")), + ) + self.assertIn(doc_marker("B07ABC"), text) + self.assertIn("Title: Blue kettle", text) + self.assertIn("Brand: Acme", text) + + def test_select_catalog_keeps_distractors_and_grades(self): + from search.catalog import ESCI_GAINS, _docs_and_queries, _select_catalog + + queries, needed = _select_catalog( + [ + { + "query_id": "10", + "query": "blue kettle", + "product_id": "p-exact", + "esci_label": "E", + }, + { + "query_id": "10", + "query": "blue kettle", + "product_id": "p-sub", + "esci_label": "S", + }, + { + "query_id": "10", + "query": "blue kettle", + "product_id": "p-irr", + "esci_label": "I", + }, + ], + max_queries=8, + max_docs=20, + candidates_per_query=40, + slice_name="esci-us", + dataset="esci", + gains=ESCI_GAINS, + label_key="esci_label", + query_text_key="query", + ) + self.assertEqual(needed, {"p-exact", "p-sub", "p-irr"}) + self.assertEqual(len(queries), 1) + self.assertEqual(queries[0]["gold_grades"]["p-exact"], 1.0) + self.assertEqual(queries[0]["gold_grades"]["p-sub"], 0.1) + self.assertNotIn("p-irr", queries[0]["gold_grades"]) + self.assertEqual(queries[0]["candidate_grades"]["p-irr"], 0.0) + self.assertEqual( + queries[0]["candidate_doc_ids"], + ["p-exact", "p-sub", "p-irr"], + ) + rows = _docs_and_queries( + queries, + products={ + "p-exact": {"product_title": "Blue kettle", "product_description": ""}, + "p-sub": {"product_title": "Teapot", "product_description": ""}, + "p-irr": {"product_title": "Lamp", "product_description": ""}, + }, + doc_ids=needed, + dataset="esci", + title_key="product_title", + description_key="product_description", + ) + docs = [row for row in rows if row["type"] == "doc"] + self.assertEqual({row["doc_id"] for row in docs}, needed) + self.assertEqual(docs[0]["title"], "Blue kettle") + + def test_select_catalog_skips_holdout_qids(self): + from search.catalog import ESCI_GAINS, _select_catalog + + queries, needed = _select_catalog( + [ + { + "query_id": "72", + "query": "held out kettle", + "product_id": "p-hold", + "esci_label": "E", + }, + { + "query_id": "99", + "query": "train sofa", + "product_id": "p-train", + "esci_label": "E", + }, + ], + max_queries=8, + max_docs=20, + candidates_per_query=40, + slice_name="esci-us", + dataset="esci", + gains=ESCI_GAINS, + label_key="esci_label", + query_text_key="query", + holdout_qids={"esci-72", "72"}, + ) + self.assertEqual([row["qid"] for row in queries], ["esci-99"]) + self.assertEqual(needed, {"p-train"}) + + def test_wands_gains_select_exact_and_partial_not_irrelevant(self): + from search.catalog import WANDS_GAINS, _select_catalog + + self.assertEqual( + WANDS_GAINS, + {"Exact": 1.0, "Partial": 0.5, "Irrelevant": 0.0}, + ) + queries, needed = _select_catalog( + [ + { + "query_id": "1", + "query": "velvet sofa", + "query_class": "Sofas", + "product_id": "p-exact", + "label": "Exact", + }, + { + "query_id": "1", + "query": "velvet sofa", + "query_class": "Sofas", + "product_id": "p-partial", + "label": "Partial", + }, + { + "query_id": "1", + "query": "velvet sofa", + "query_class": "Sofas", + "product_id": "p-irr", + "label": "Irrelevant", + }, + ], + max_queries=8, + max_docs=20, + candidates_per_query=40, + slice_name="wands", + dataset="wands", + gains=WANDS_GAINS, + label_key="label", + query_text_key="query", + query_slice_key="query_class", + ) + self.assertEqual(needed, {"p-exact", "p-partial", "p-irr"}) + self.assertEqual(len(queries), 1) + self.assertEqual(queries[0]["qid"], "wands-1") + self.assertEqual(queries[0]["gold_grades"]["p-exact"], 1.0) + self.assertEqual(queries[0]["gold_grades"]["p-partial"], 0.5) + self.assertNotIn("p-irr", queries[0]["gold_grades"]) + self.assertEqual(queries[0]["gold_doc_ids"], ["p-exact", "p-partial"]) + self.assertEqual(queries[0]["candidate_grades"]["p-irr"], 0.0) + + def test_frozen_wands_jsonl_blocks_overwrite(self): + from search.catalog import DATA_DIR, catalog_overwrite_blocked + + self.assertTrue( + catalog_overwrite_blocked(DATA_DIR / "search_esci_74.jsonl") + ) + wands = DATA_DIR / "search_wands.jsonl" + if not wands.exists(): + self.skipTest("optional frozen WANDS dataset is absent") + self.assertTrue(wands.exists() and wands.stat().st_size > 0) + self.assertTrue(catalog_overwrite_blocked(wands)) + self.assertFalse( + catalog_overwrite_blocked(DATA_DIR / "search_esci.jsonl") + ) + + def test_frozen_structured_brains_exclude_wandsgraph(self): + from search.config import FROZEN_STRUCTURED_BRAINS + + self.assertIn("searchbenchwands", FROZEN_STRUCTURED_BRAINS) + self.assertIn("searchbenchesci74", FROZEN_STRUCTURED_BRAINS) + self.assertNotIn("searchbenchwandsgraph", FROZEN_STRUCTURED_BRAINS) + self.assertNotIn("searchbenchjdslice", FROZEN_STRUCTURED_BRAINS) + + def test_evaluate_refuses_frozen_structured_ingest(self): + from search.evaluate import evaluate_search + + class _Client: + class _Settings: + brain_id = "searchbenchwands" + + settings = _Settings() + + with self.assertRaises(SystemExit): + evaluate_search(_Client(), [], ingest_graph=True) + with self.assertRaises(SystemExit): + evaluate_search( + _Client(), + [], + interactions=[{"user_id": "u", "item_id": "0"}], + ) + + def test_wands_feature_parse_and_catalog_triples(self): + from search.mapping import ( + docs_to_triples, + entity_uuid, + parse_feature_string, + ) + + parsed = parse_feature_string( + "|Color:Navy|Material:Velvet|InStock:yes|Finish:|" + ) + self.assertEqual(parsed, [("Color", "Navy"), ("Material", "Velvet")]) + docs = [ + { + "doc_id": f"d{i}", + "title": f"Item {i}", + "class": "sofas" if i == 1 else "", + "brand": "Acme" if i == 2 else "", + "features": "|Color:Navy|" if i == 3 else "", + "dataset": "wands", + } + for i in range(1, 6) + ] + triples = docs_to_triples(docs) + uuids = {row["subject"]["uuid"] for row in triples} + self.assertEqual(uuids, {entity_uuid(f"d{i}") for i in range(1, 6)}) + self.assertTrue(all(row.get("event") is None for row in triples)) + self.assertTrue(all(row["subj_event"]["name"] == "HAS" for row in triples)) + self.assertTrue(all("event_obj" not in row or row.get("event_obj") is None for row in triples)) + objects = {(row["object"]["type"], row["object"]["name"]) for row in triples} + self.assertIn(("CLASS", "sofas"), objects) + self.assertIn(("ATTR", "Acme"), objects) + self.assertIn(("ATTR", "Navy"), objects) + self.assertNotIn("demorecsys", str(triples)) + + def test_catalog_graph_sofas_modern_has_search_text(self): + from search.mapping import doc_to_triples, parse_feature_string, split_hierarchy + + parsed = parse_feature_string("dsprimaryproductstyle : modern|color : navy") + self.assertIn(("dsprimaryproductstyle", "modern"), parsed) + self.assertEqual(split_hierarchy("Furniture > Living Room > Sofas"), ["Furniture", "Living Room", "Sofas"]) + triples = doc_to_triples( + { + "doc_id": "sofa-1", + "title": "Velvet sofa", + "description": "A low modern sofa for small rooms", + "class": "sofas", + "hierarchy": "Furniture > Living Room > Sofas", + "features": "style:modern|color:navy", + "price": "899", + "dataset": "wands", + } + ) + objects = {(row["object"]["type"], row["object"]["name"]) for row in triples} + self.assertIn(("CLASS", "sofas"), objects) + self.assertIn(("ATTR", "modern"), objects) + self.assertIn(("TYPE", "Furniture"), objects) + subject = triples[0]["subject"] + self.assertIn("modern sofa", subject["properties"]["search_text"].lower()) + self.assertIn("sofas", subject["properties"]["search_text"].lower()) + self.assertIn("modern", subject["properties"]["search_text"].lower()) + self.assertIn("navy", subject["properties"]["search_text"].lower()) + self.assertEqual(subject["properties"]["price"], "899") + hub_texts = {row["object"]["properties"]["search_text"] for row in triples} + self.assertTrue(any("modern" in text.lower() for text in hub_texts)) + self.assertNotIn(("ATTR", "899"), objects) + self.assertTrue(all(row.get("event") is None for row in triples)) + self.assertTrue(all(row["subj_event"]["name"] == "HAS" for row in triples)) + self.assertTrue( + all("dsprimaryproductstyle" not in str(row.get("subj_event") or {}) for row in triples) + ) + + def test_entity_search_text_prefers_catalog_blob(self): + from search.mapping import doc_id_from_text, doc_to_triples, node_id_from_passage_text + + blob = ( + "DOCID wands-bed.\n" + "Title: solid wood platform bed\n" + "Class: Beds\n" + "Features: dsprimaryproductstyle : modern|woodspecies : rubberwood" + ) + triples = doc_to_triples( + { + "doc_id": "wands-bed", + "title": "solid wood platform bed", + "class": "Beds", + "features": "dsprimaryproductstyle : modern|woodspecies : rubberwood", + "text": blob, + } + ) + subject = triples[0]["subject"] + self.assertEqual(subject["properties"]["search_text"], blob) + self.assertIn("rubberwood", subject["properties"]["search_text"]) + self.assertIsNone(triples[0].get("event")) + self.assertEqual(triples[0]["subj_event"]["name"], "HAS") + self.assertNotIn("rubberwood", triples[0]["subj_event"]["name"]) + self.assertEqual(doc_id_from_text(blob), "wands-bed") + self.assertEqual(node_id_from_passage_text(blob), "wands-bed") + self.assertIsNone(node_id_from_passage_text("no marker here")) + + def test_catalog_has_is_direct_edge(self): + from search.mapping import doc_to_triples, is_static_has_triple + + triples = doc_to_triples( + { + "doc_id": "0", + "title": "solid wood platform bed", + "class": "Beds", + "features": "woodspecies : rubberwood", + } + ) + self.assertTrue(triples) + self.assertTrue(all(is_static_has_triple(row) for row in triples)) + self.assertTrue(all(row.get("event") is None for row in triples)) + kinds = {(row["object"]["type"], row["object"]["name"]) for row in triples} + self.assertIn(("CLASS", "Beds"), kinds) + self.assertIn(("ATTR", "rubberwood"), kinds) + + def test_entity_backfill_refuses_frozen_brains(self): + from search.backfill_entities import ( + apply_entity_text_backfill, + backfill_rows_from_docs, + refuse_entity_backfill, + ) + + self.assertEqual(refuse_entity_backfill("searchbenchwandsgraph"), "searchbenchwandsgraph") + with self.assertRaises(SystemExit): + refuse_entity_backfill("searchbenchwands") + with self.assertRaises(SystemExit): + refuse_entity_backfill("searchbenchesci74") + rows = backfill_rows_from_docs( + [ + { + "doc_id": "0", + "title": "solid wood platform bed", + "class": "Beds", + "features": "dsprimaryproductstyle : modern|woodspecies : rubberwood", + "text": "DOCID 0.\nTitle: solid wood platform bed\nFeatures: woodspecies : rubberwood", + } + ] + ) + self.assertEqual(rows[0]["uuid"], "0") + self.assertIn("rubberwood", rows[0]["search_text"]) + with self.assertRaises(SystemExit): + apply_entity_text_backfill( + brain_id="searchbenchwands", + rows=rows, + graph=None, + embeddings=None, + vector_store=None, + ) + + def test_apply_entity_backfill_updates_entity_not_event(self): + from types import SimpleNamespace + from unittest.mock import MagicMock + + from search.backfill_entities import apply_entity_text_backfill + from src.constants.embeddings import Vector + + node = SimpleNamespace( + uuid="0", + name="bed", + labels=["ENTITY"], + properties={"search_text": "bed", "v_id": "old-v"}, + ) + graph = MagicMock() + graph.get_by_uuid.return_value = node + embeddings = MagicMock() + embeddings.embed_text.return_value = Vector( + id="new", embeddings=[0.1], metadata={} + ) + store = MagicMock() + store.add_vectors.return_value = ["vid-1"] + summary = apply_entity_text_backfill( + brain_id="searchbenchwandsgraph", + rows=[{"uuid": "0", "name": "bed", "search_text": "rubberwood modern bed"}], + graph=graph, + embeddings=embeddings, + vector_store=store, + is_item=lambda uuid, labels: "ENTITY" in (labels or []), + ) + self.assertEqual(summary["updated"], 1) + self.assertEqual(summary["missing"], 0) + written = [ + call.kwargs.get("new_properties") or {} + for call in graph.update_node.call_args_list + ] + self.assertTrue(any("rubberwood" in str(props.get("search_text") or "") for props in written)) + event = SimpleNamespace( + uuid="evt-1", + name="HAS", + labels=["EVENT"], + properties={}, + ) + graph.get_by_uuid.return_value = event + skipped = apply_entity_text_backfill( + brain_id="searchbenchwandsgraph", + rows=[{"uuid": "evt-1", "name": "HAS", "search_text": "should not copy"}], + graph=graph, + embeddings=embeddings, + vector_store=store, + is_item=lambda uuid, labels: "ENTITY" in (labels or []), + ) + self.assertEqual(skipped["updated"], 0) + self.assertEqual(skipped["skipped"], 1) + + def test_wandsgraph_node_join_calls_neighbors(self): + from search.client import TimedResult + from search.evaluate import assert_wandsgraph_node_join + + class _Client: + class _Settings: + brain_id = "searchbenchwandsgraph" + + settings = _Settings() + + def get_neighbors(self, uuid, limit=5): + self.seen = uuid + return TimedResult(data={"count": 4}, latency_ms=1.0, status_code=200) + + client = _Client() + out = assert_wandsgraph_node_join( + client, + { + "queries": [ + { + "qid": "q1", + "hits": [ + { + "id": "chunk-1", + "doc_id": "0", + "node_id": "0", + "channel": "passages", + } + ], + } + ] + }, + ) + self.assertEqual(out["node_id"], "0") + self.assertEqual(out["status_code"], 200) + self.assertEqual(client.seen, "0") + skipped = assert_wandsgraph_node_join( + type("C", (), {"settings": type("S", (), {"brain_id": "searchbenchsmoke"})()})(), + {"queries": []}, + ) + self.assertTrue(skipped["skipped"]) + + def test_interaction_triples_have_happened_at(self): + from search.mapping import interactions_to_triples, load_interaction_rows + + rows = [ + { + "user_id": "u01", + "item_id": "sku-101", + "behavior": "view", + "timestamp": "2024-01-02T10:00:00Z", + "category": "phones", + "brand": "Acme", + }, + { + "user_id": "u01", + "item_id": "sku-102", + "behavior": "purchase", + "timestamp": "2024-01-05T12:00:00Z", + }, + { + "user_id": "u02", + "item_id": "sku-201", + "behavior": "view", + "timestamp": "2024-01-02T09:00:00Z", + }, + ] + triples = interactions_to_triples(rows) + events = [ + row["event"] + for row in triples + if row.get("event") and row["event"]["name"] != "HAS" + ] + self.assertGreaterEqual(len(events), 3) + self.assertTrue(all(event.get("happened_at") for event in events)) + self.assertTrue(all(event["type"] == "EVENT" for event in events)) + self.assertTrue(all(row["object"]["uuid"] in {"sku-101", "sku-102", "sku-201"} or row["object"]["type"] != "ENTITY" for row in triples)) + entity_ids = { + row["object"]["uuid"] + for row in triples + if row["object"]["type"] == "ENTITY" + } + self.assertEqual(entity_ids, {"sku-101", "sku-102", "sku-201"}) + self.assertNotIn("PRODUCT", str(triples)) + self.assertNotIn("Categorized", str(triples)) + self.assertNotIn("demorecsys", str(triples)) + root = Path(__file__).resolve().parents[1] + toy = root / "benchmarks" / "data" / "recsys_toy.jsonl" + if toy.exists(): + loaded = load_interaction_rows(toy)[:3] + self.assertEqual(len(loaded), 3) + self.assertEqual(loaded[0]["item_id"], "sku-101") + + def test_interaction_options_emit_prefers_not_catalog_has(self): + from search.mapping import hub_uuid, interaction_to_triples + + triples = interaction_to_triples( + { + "user_id": "u01", + "item_id": "sku-101", + "behavior": "view", + "timestamp": "2024-01-02T10:00:00Z", + "color": "navy", + "options": {"style": "70s", "color": "olive"}, + }, + seq=1, + ) + prefers = [ + row + for row in triples + if (row.get("subj_event") or {}).get("name") == "PREFERS" + ] + self.assertEqual(len(prefers), 2) + self.assertTrue(all(row.get("event") is None for row in prefers)) + self.assertTrue(all(row["subject"]["uuid"] == "user:u01" for row in prefers)) + style_hub = hub_uuid("attr", "70s") + olive_hub = hub_uuid("attr", "olive") + prefers_tips = {row["object"]["uuid"] for row in prefers} + self.assertEqual(prefers_tips, {style_hub, olive_hub}) + has_tips = { + row["object"]["uuid"] + for row in triples + if (row.get("subj_event") or {}).get("name") == "HAS" + } + self.assertIn(hub_uuid("attr", "navy"), has_tips) + self.assertNotIn(style_hub, has_tips) + self.assertNotIn(olive_hub, has_tips) + + def test_personalize_smoke_fixture_emits_prefers(self): + from search.mapping import hub_uuid, load_interaction_rows, interactions_to_triples + + root = Path(__file__).resolve().parents[1] + fixture = root / "benchmarks" / "data" / "search_personalize_smoke.jsonl" + if not fixture.exists(): + self.skipTest("optional personalization smoke fixture is absent") + rows = load_interaction_rows(fixture) + self.assertEqual(len(rows), 4) + triples = interactions_to_triples(rows) + prefers = [ + row + for row in triples + if (row.get("subj_event") or {}).get("name") == "PREFERS" + ] + self.assertGreaterEqual(len(prefers), 1) + self.assertIn(hub_uuid("attr", "70s"), {row["object"]["uuid"] for row in prefers}) + self.assertTrue(any(row["event"]["name"] == "Favorite" for row in triples if row.get("event"))) + + def test_cli_unknown_catalog(self): + from search.cli import main + + self.assertEqual(main(["download", "--name", "not-a-dataset"]), 1) + + def test_write_roundtrip(self): + from search.dataset import dataset_stats, load_records, write_records + + rows = [ + {"type": "doc", "doc_id": "a", "text": "DOCID a. Title: kettle"}, + { + "type": "query", + "qid": "q1", + "query": "kettle", + "gold_doc_ids": ["a"], + "gold_grades": {"a": 1.0}, + "slice": "esci-us", + }, + ] + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "search_esci.jsonl" + write_records(rows, path) + loaded = load_records(path) + stats = dataset_stats(loaded) + self.assertEqual(stats["n_docs"], 1) + self.assertEqual(stats["n_queries"], 1) + self.assertTrue(stats["graded"]) + + def test_esci_locale_paths_keep_us_default(self): + from search.catalog import ESCI_JSONL, catalog_jsonl_path, normalize_esci_locale + + self.assertEqual(catalog_jsonl_path("esci"), ESCI_JSONL) + self.assertEqual(catalog_jsonl_path("esci", locale="us"), ESCI_JSONL) + self.assertEqual(catalog_jsonl_path("esci", locale="US"), ESCI_JSONL) + es_path = catalog_jsonl_path("esci", locale="es") + jp_path = catalog_jsonl_path("esci", locale="jp") + self.assertEqual(es_path.name, "search_esci_es.jsonl") + self.assertEqual(jp_path.name, "search_esci_jp.jsonl") + self.assertNotEqual(es_path, ESCI_JSONL) + self.assertNotEqual(jp_path, ESCI_JSONL) + self.assertEqual(normalize_esci_locale("es"), "es") + + def test_esci_locale_rejects_italian(self): + from search.catalog import catalog_jsonl_path, normalize_esci_locale + from search.cli import main + + with self.assertRaises(ValueError) as ctx: + normalize_esci_locale("it") + self.assertIn("no Italian", str(ctx.exception)) + with self.assertRaises(ValueError): + catalog_jsonl_path("esci", locale="it") + self.assertEqual(main(["download", "--name", "esci", "--locale", "it"]), 1) + + def test_italian_smoke_is_not_esci(self): + from search.dataset import dataset_stats, load_records + + root = Path(__file__).resolve().parents[1] + path = root / "benchmarks" / "data" / "search_italian_smoke.jsonl" + if not path.exists(): + self.skipTest("optional Italian smoke fixture is absent") + rows = load_records(path) + stats = dataset_stats(rows) + self.assertGreaterEqual(stats["n_docs"], 3) + self.assertGreaterEqual(stats["n_queries"], 3) + self.assertEqual(stats["slices"], {"italian-smoke": stats["n_queries"]}) + for row in rows: + self.assertNotEqual(row.get("dataset"), "esci") + self.assertNotIn("esci-", str(row.get("qid") or "")) + self.assertNotIn("esci-", str(row.get("slice") or "")) + + def test_italian_inflect_is_not_esci(self): + from search.dataset import dataset_stats, load_records + + root = Path(__file__).resolve().parents[1] + path = root / "benchmarks" / "data" / "search_italian_smoke_inflect.jsonl" + if not path.exists(): + self.skipTest("optional Italian inflection fixture is absent") + rows = load_records(path) + stats = dataset_stats(rows) + self.assertGreaterEqual(stats["n_docs"], 3) + self.assertGreaterEqual(stats["n_queries"], 3) + self.assertEqual(stats["slices"], {"italian-smoke-inflect": stats["n_queries"]}) + queries = [row["query"] for row in rows if row.get("type") == "query"] + self.assertIn("bollitori acciaio", queries) + self.assertIn("divani velluto", queries) + self.assertIn("caffettiere alluminio", queries) + for row in rows: + self.assertNotEqual(row.get("dataset"), "esci") + self.assertNotIn("esci-", str(row.get("qid") or "")) + self.assertNotIn("esci-", str(row.get("slice") or "")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search_config.py b/tests/test_search_config.py new file mode 100644 index 0000000..872372f --- /dev/null +++ b/tests/test_search_config.py @@ -0,0 +1,145 @@ +import os +import unittest +from unittest.mock import patch + +ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", + "PIPELINE_MODE": "accurate", +} +for key, value in ENV_DEFAULTS.items(): + os.environ.setdefault(key, value) + +from src.config import ( + validate_context_passage_mode, + validate_search_config, + validate_search_fusion, +) + + +class SearchConfigValidationTests(unittest.TestCase): + def test_defaults_are_search_off_and_fused(self): + self.assertEqual(validate_search_fusion(None), "rrf") + self.assertEqual(validate_context_passage_mode(None), "hybrid") + validate_search_config( + enabled=False, + use_dense=False, + use_bm25=False, + data_db="mongo", + bm25_k1=1.2, + bm25_b=0.75, + ) + + def test_bm25_requires_postgres_data_db(self): + with self.assertRaises(ValueError) as ctx: + validate_search_config( + enabled=True, + use_dense=True, + use_bm25=True, + data_db="mongo", + bm25_k1=1.2, + bm25_b=0.75, + ) + self.assertIn("DATA_DB=postgresql", str(ctx.exception)) + + def test_dense_only_search_supports_non_postgres_data_db(self): + validate_search_config( + enabled=True, + use_dense=True, + use_bm25=False, + data_db="mongo", + bm25_k1=1.2, + bm25_b=0.75, + ) + + def test_enabled_requires_at_least_one_channel(self): + with self.assertRaises(ValueError) as ctx: + validate_search_config( + enabled=True, + use_dense=False, + use_bm25=False, + data_db="postgresql", + bm25_k1=1.2, + bm25_b=0.75, + ) + self.assertIn("SEARCH_USE_DENSE", str(ctx.exception)) + + def test_invalid_fusion_and_passage_mode(self): + with self.assertRaises(ValueError): + validate_search_fusion("weighted") + with self.assertRaises(ValueError): + validate_context_passage_mode("splade") + + def test_config_reads_search_defaults(self): + env = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", + "DATA_DB": "postgresql", + } + with patch.dict(os.environ, env, clear=False): + for key in ( + "SEARCH_ENABLED", + "SEARCH_USE_DENSE", + "SEARCH_USE_BM25", + "SEARCH_FUSION", + "SEARCH_LITERAL_FILL", + "CONTEXT_PASSAGE_MODE", + ): + os.environ.pop(key, None) + from src.config import Config + + cfg = Config() + self.assertFalse(cfg.search_enabled) + self.assertTrue(cfg.search_use_dense) + self.assertTrue(cfg.search_use_bm25) + self.assertEqual(cfg.search_fusion, "rrf") + self.assertEqual(cfg.search_bm25_k1, 1.2) + self.assertEqual(cfg.search_bm25_b, 0.75) + self.assertEqual(cfg.context_passage_mode, "hybrid") + self.assertFalse(cfg.search_literal_fill) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search_esci_4class.py b/tests/test_search_esci_4class.py new file mode 100644 index 0000000..ffdbf1f --- /dev/null +++ b/tests/test_search_esci_4class.py @@ -0,0 +1,156 @@ +import unittest +from pathlib import Path + + +class SearchEsci4ClassTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + import sys + + root = Path(__file__).resolve().parents[1] + bench = root / "benchmarks" + if str(bench) not in sys.path: + sys.path.insert(0, str(bench)) + + def test_weighted_scores_match_esci_gains(self): + from search.finetune_esci_4class import weighted_scores + + exact = weighted_scores([[1.0, 0.0, 0.0, 0.0]]) + sub = weighted_scores([[0.0, 1.0, 0.0, 0.0]]) + comp = weighted_scores([[0.0, 0.0, 1.0, 0.0]]) + irr = weighted_scores([[0.0, 0.0, 0.0, 1.0]]) + self.assertEqual(exact, [1.0]) + self.assertEqual(sub, [0.1]) + self.assertEqual(comp, [0.01]) + self.assertEqual(irr, [0.0]) + mixed = weighted_scores([[0.5, 0.5, 0.0, 0.0]]) + self.assertAlmostEqual(mixed[0], 0.55) + + def test_rank_doc_ids_orders_by_weighted_score(self): + from search.finetune_esci_4class import rank_doc_ids, weighted_scores + + doc_ids = ["irr", "exact", "sub"] + scores = weighted_scores( + [ + [0.05, 0.05, 0.1, 0.8], + [0.9, 0.05, 0.05, 0.0], + [0.1, 0.8, 0.05, 0.05], + ] + ) + self.assertEqual(rank_doc_ids(doc_ids, scores), ["exact", "sub", "irr"]) + + def test_class_weights_upweight_rare_labels(self): + from search.finetune_esci_4class import class_weights + + weights = class_weights([80, 10, 5, 5]) + self.assertGreater(weights[2], weights[0]) + self.assertGreater(weights[3], weights[0]) + + def test_held_out_query_ids_strip_esci_prefix(self): + import json + import tempfile + + from search.finetune_esci_4class import held_out_query_ids + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "slice.jsonl" + path.write_text( + json.dumps( + { + "type": "doc", + "doc_id": "p1", + "text": "DOCID p1.\nTitle: kettle", + } + ) + + "\n" + + json.dumps( + { + "type": "query", + "qid": "esci-99", + "query": "kettle", + "gold_doc_ids": ["p1"], + } + ) + + "\n", + encoding="utf-8", + ) + holdout = held_out_query_ids(path) + self.assertIn("esci-99", holdout) + self.assertIn("99", holdout) + + def test_run_4class_on_pool_keeps_irrelevant_and_ranks_exact_first(self): + from search.rank_pool_4class import run_4class_on_pool + + def predict(pairs): + scores = [] + for _, text in pairs: + if "gold kettle" in text: + scores.append([0.85, 0.1, 0.05, 0.0]) + elif "partial kettle" in text: + scores.append([0.1, 0.8, 0.05, 0.05]) + else: + scores.append([0.05, 0.05, 0.1, 0.8]) + return scores + + rows = [ + {"type": "doc", "doc_id": "irr", "text": "DOCID irr. Title: lamp"}, + { + "type": "doc", + "doc_id": "gold", + "text": "DOCID gold. Title: gold kettle", + }, + { + "type": "doc", + "doc_id": "sub", + "text": "DOCID sub. Title: partial kettle", + }, + { + "type": "query", + "qid": "esci-1", + "query": "kettle", + "gold_doc_ids": ["gold", "sub"], + "gold_grades": {"gold": 1.0, "sub": 0.1}, + "candidate_doc_ids": ["irr", "gold", "sub"], + "candidate_grades": {"irr": 0.0, "gold": 1.0, "sub": 0.1}, + "slice": "esci-us", + }, + ] + result = run_4class_on_pool( + rows, + dataset_name="toy.jsonl", + predict=predict, + model_name="toy-4class", + brain_id="searchbenchesci74", + ) + self.assertEqual(result["status"], "ok") + self.assertEqual(result["n_queries"], 1) + self.assertTrue(result["rank_pool"]) + ranked = result["queries"][0]["hit_ids"] + self.assertEqual(ranked, ["gold", "sub", "irr"]) + self.assertEqual(ranked[-1], "irr") + self.assertGreater(result["metrics"]["ndcg@20"], 0.9) + + def test_persist_tokenizer_max_length_writes_train_length(self): + import tempfile + from types import SimpleNamespace + + from search.finetune_esci_4class import persist_tokenizer_max_length + + class FakeTokenizer: + def __init__(self): + self.model_max_length = 10**18 + self.saved = None + + def save_pretrained(self, path): + self.saved = path + + tokenizer = FakeTokenizer() + model = SimpleNamespace(max_length=192, tokenizer=tokenizer) + with tempfile.TemporaryDirectory() as tmp: + persist_tokenizer_max_length(model, Path(tmp)) + self.assertEqual(tokenizer.model_max_length, 192) + self.assertEqual(tokenizer.saved, tmp) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search_graph.py b/tests/test_search_graph.py new file mode 100644 index 0000000..e1bf35d --- /dev/null +++ b/tests/test_search_graph.py @@ -0,0 +1,713 @@ +import os +import unittest +from unittest.mock import MagicMock, patch + +ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", +} +for key, value in ENV_DEFAULTS.items(): + os.environ.setdefault(key, value) + +from fastapi import HTTPException + +from src.constants.embeddings import Vector +from src.constants.kg import Node, SearchEntitiesResult +from src.core.search.graph_channels import ( + DEFAULT_COMMUNITY_LABELS, + collect_community_hits, + collect_entity_hits, + collect_event_hits, + expand_neighbor_hits, +) +from src.core.search.hybrid import fuse_passage_lists +from src.services.api.constants.requests import SearchHit, SearchRequestBody, SearchResponse + + +def _node( + uuid: str, + name: str, + labels: list[str], + happened_at: str | None = None, + description: str | None = None, + properties: dict | None = None, +) -> Node: + props = dict(properties or {}) + if happened_at: + props["happened_at"] = happened_at + return Node( + uuid=uuid, + name=name, + labels=labels, + happened_at=happened_at, + description=description, + properties=props, + ) + + +class FakeGraph: + def __init__(self, nodes: list[Node], neighbors: dict[str, list[Node]] | None = None): + self.nodes = {node.uuid: node for node in nodes} + self.neighbors = neighbors or {} + self.neighbor_calls = 0 + + def search_entities( + self, + brain_id: str = "default", + limit: int = 10, + skip: int = 0, + node_labels: list[str] | None = None, + query_text: str | None = None, + ) -> SearchEntitiesResult: + matched: list[Node] = [] + want = {str(item).upper() for item in (node_labels or [])} + needle = (query_text or "").lower() + for node in self.nodes.values(): + have = {str(item).upper() for item in node.labels} + if want and not (have & want): + continue + if needle and needle not in node.name.lower(): + continue + matched.append(node) + page = matched[skip : skip + limit] + return SearchEntitiesResult(results=page, total=len(matched)) + + def search_nodes_bm25( + self, + query_text: str, + brain_id: str = "default", + limit: int = 10, + node_labels: list[str] | None = None, + node_uuids: list[str] | None = None, + ): + needle = (query_text or "").lower() + want = {str(item).upper() for item in (node_labels or [])} + allowed = {str(item) for item in (node_uuids or [])} or None + scored: list[tuple[Node, float]] = [] + for node in self.nodes.values(): + if allowed is not None and node.uuid not in allowed: + continue + have = {str(item).upper() for item in node.labels} + if want and not (have & want): + continue + props = node.properties or {} + text = " ".join( + [ + node.name or "", + node.description or "", + str(props.get("search_text") or ""), + ] + ).lower() + score = 0.0 + if needle and needle in text: + score += 5.0 + for token in needle.split(): + if token and token in text: + score += 1.0 + if score: + scored.append((node, score)) + scored.sort(key=lambda item: (-item[1], item[0].uuid)) + return scored[:limit] + + def get_by_uuids(self, uuids: list[str], brain_id: str = "default") -> list[Node]: + return [self.nodes[item] for item in uuids if item in self.nodes] + + def get_by_uuid(self, uuid: str, brain_id: str = "default") -> Node: + return self.nodes[uuid] + + def get_neighbors(self, nodes, brain_id: str = "default", **kwargs): + self.neighbor_calls += 1 + keys = [] + for item in nodes: + keys.append(item if isinstance(item, str) else item.uuid) + out = {} + for key in keys: + out[key] = [(None, node) for node in self.neighbors.get(key, [])] + return out + + def get_event_centric_neighbors(self, nodes, brain_id: str = "default"): + return [] + + +class SearchGraphChannelTests(unittest.IsolatedAsyncioTestCase): + def test_contract_has_graph_fields_not_catalog_enums(self): + body = SearchRequestBody(query="sofa") + self.assertEqual(body.channels, ["passages"]) + self.assertEqual(body.expand, "none") + self.assertEqual(body.mode, "default") + self.assertIsNone(body.node_labels) + fields = set(SearchRequestBody.model_fields) + hit_fields = set(SearchHit.model_fields) + for banned in ( + "product_id", + "product_features", + "sku", + "category", + "brand", + "user_id", + ): + self.assertNotIn(banned, fields) + self.assertNotIn(banned, hit_fields) + self.assertIn("labels", hit_fields) + self.assertIn("extras", hit_fields) + self.assertIn("node_id", hit_fields) + self.assertIn("node_ids", set(SearchResponse.model_fields)) + self.assertIn("extras", fields) + self.assertEqual(list(DEFAULT_COMMUNITY_LABELS), ["TYPE", "CLASS", "TOPIC"]) + + async def test_unknown_channel_is_400(self): + from src.services.api.controllers import search as search_mod + + with patch.object(search_mod.config, "search_enabled", True): + with self.assertRaises(HTTPException) as ctx: + await search_mod.search( + SearchRequestBody(query="sofa", channels=["foo"]) + ) + self.assertEqual(ctx.exception.status_code, 400) + self.assertIn("Unknown search channel", str(ctx.exception.detail)) + + async def test_entities_empty_graph_is_200(self): + from src.services.api.controllers import search as search_mod + + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.1], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_nodes.return_value = [] + graph = FakeGraph([]) + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "graph_adapter", graph), + patch.object(search_mod, "data_adapter", MagicMock()), + ): + response = await search_mod.search( + SearchRequestBody(query="sofa", channels=["entities"], k=5) + ) + + self.assertEqual(response.hits, []) + + async def test_seeded_entity_name_in_top_k(self): + from src.services.api.controllers import search as search_mod + + graph = FakeGraph([_node("sku-1", "navy velvet sofa", ["ENTITY"])]) + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.2], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_nodes.return_value = [] + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "graph_adapter", graph), + patch.object(search_mod, "data_adapter", MagicMock()), + ): + response = await search_mod.search( + SearchRequestBody(query="velvet sofa", channels=["entities"], k=5) + ) + + self.assertEqual([hit.id for hit in response.hits], ["sku-1"]) + self.assertEqual(response.hits[0].channel, "entities") + self.assertIn("ENTITY", response.hits[0].labels) + self.assertEqual(response.hits[0].node_id, "sku-1") + self.assertEqual(response.node_ids, ["sku-1"]) + + async def test_node_labels_mismatch_is_empty_not_400(self): + from src.services.api.controllers import search as search_mod + + graph = FakeGraph([_node("sku-1", "navy velvet sofa", ["ENTITY"])]) + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.2], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_nodes.return_value = [] + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "graph_adapter", graph), + patch.object(search_mod, "data_adapter", MagicMock()), + ): + response = await search_mod.search( + SearchRequestBody( + query="velvet sofa", + channels=["entities"], + node_labels=["EVENT"], + k=5, + ) + ) + + self.assertEqual(response.hits, []) + + async def test_default_passages_does_not_touch_graph(self): + from src.services.api.controllers import search as search_mod + from src.constants.data import TextChunk + + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + (TextChunk(id="lex", text="license board"), 3.1) + ] + mock_graph = MagicMock() + mock_embeddings = MagicMock() + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "graph_adapter", mock_graph), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search(SearchRequestBody(query="license", k=5)) + + self.assertEqual([hit.id for hit in response.hits], ["lex"]) + self.assertEqual(response.hits[0].channel, "passages") + mock_graph.search_entities.assert_not_called() + mock_embeddings.embed_text.assert_not_called() + + async def test_events_empty_ok(self): + from src.services.api.controllers import search as search_mod + + graph = FakeGraph([_node("sku-1", "navy velvet sofa", ["ENTITY"])]) + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.2], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_nodes.return_value = [] + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "graph_adapter", graph), + patch.object(search_mod, "data_adapter", MagicMock()), + ): + response = await search_mod.search( + SearchRequestBody(query="view", channels=["events"], k=5) + ) + + self.assertEqual(response.hits, []) + + async def test_events_with_happened_at(self): + from src.services.api.controllers import search as search_mod + + graph = FakeGraph( + [ + _node( + "evt-1", + "View", + ["EVENT"], + happened_at="2024-01-05T12:00:00Z", + ) + ] + ) + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.2], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_nodes.return_value = [] + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "graph_adapter", graph), + patch.object(search_mod, "data_adapter", MagicMock()), + ): + response = await search_mod.search( + SearchRequestBody(query="View", channels=["events"], k=5) + ) + + self.assertEqual([hit.id for hit in response.hits], ["evt-1"]) + self.assertEqual(response.hits[0].channel, "events") + self.assertEqual(response.hits[0].extras["happened_at"], "2024-01-05T12:00:00Z") + + def test_community_hub_returns_members_with_fanout_cap(self): + hub = _node("class-sofas", "sofas", ["CLASS"]) + members = [ + _node("e1", "velvet sofa", ["ENTITY"]), + _node("e2", "leather sofa", ["ENTITY"]), + _node("e3", "loveseat", ["ENTITY"]), + ] + unrelated = _node("e4", "lamp", ["ENTITY"]) + graph = FakeGraph( + [hub, *members, unrelated], + neighbors={"class-sofas": members}, + ) + hits = collect_community_hits( + query="sofas", + brain_id="searchbenchsmoke", + k=10, + graph=graph, + fanout=2, + ) + ids = [hit.id for hit in hits] + self.assertTrue(set(ids).issubset({"e1", "e2", "e3"})) + self.assertNotIn("e4", ids) + self.assertNotIn("class-sofas", ids) + self.assertEqual(len(ids), 2) + self.assertTrue(all(hit.channel == "communities" for hit in hits)) + + def test_expand_neighbors_default_off(self): + seed = _node("sku-1", "velvet sofa", ["ENTITY"]) + neighbor = _node("class-sofas", "sofas", ["CLASS"]) + graph = FakeGraph([seed, neighbor], neighbors={"sku-1": [neighbor]}) + seeds = collect_entity_hits( + query="velvet sofa", + brain_id="searchbenchsmoke", + k=5, + graph=graph, + ) + self.assertEqual(graph.neighbor_calls, 0) + expanded = expand_neighbor_hits( + seeds, + brain_id="searchbenchsmoke", + k=5, + graph=graph, + fanout=10, + ) + self.assertEqual([hit.id for hit in expanded], ["class-sofas"]) + self.assertEqual(expanded[0].channel, "neighbors") + + async def test_expand_omitted_skips_hops(self): + from src.services.api.controllers import search as search_mod + + seed = _node("sku-1", "velvet sofa", ["ENTITY"]) + neighbor = _node("n1", "pillow", ["ENTITY"]) + graph = FakeGraph([seed, neighbor], neighbors={"sku-1": [neighbor]}) + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.2], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_nodes.return_value = [] + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "graph_adapter", graph), + patch.object(search_mod, "data_adapter", MagicMock()), + ): + response = await search_mod.search( + SearchRequestBody(query="velvet sofa", channels=["entities"], k=5) + ) + + self.assertEqual([hit.id for hit in response.hits], ["sku-1"]) + self.assertEqual(graph.neighbor_calls, 0) + + async def test_expand_neighbors_adds_hop(self): + from src.services.api.controllers import search as search_mod + + seed = _node("sku-1", "velvet sofa", ["ENTITY"]) + neighbor = _node("n1", "pillow", ["ENTITY"]) + graph = FakeGraph([seed, neighbor], neighbors={"sku-1": [neighbor]}) + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.2], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_nodes.return_value = [] + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "graph_adapter", graph), + patch.object(search_mod, "data_adapter", MagicMock()), + ): + response = await search_mod.search( + SearchRequestBody( + query="velvet sofa", + channels=["entities"], + expand="neighbors", + k=5, + ) + ) + + ids = [hit.id for hit in response.hits] + self.assertIn("sku-1", ids) + self.assertIn("n1", ids) + self.assertTrue(any(hit.channel == "neighbors" for hit in response.hits)) + + def test_fuse_mixed_chunk_and_node_ids(self): + fused = fuse_passage_lists( + ["chunk-1"], + [], + extra_id_lists=[["sku-1"]], + ) + ids = [item for item, _ in fused] + self.assertEqual(set(ids), {"chunk-1", "sku-1"}) + + async def test_fused_graph_lists_drop_hub_ids(self): + from src.constants.data import TextChunk + from src.services.api.controllers import search as search_mod + + hub = _node("hub:attr:red", "red", ["ATTR"]) + product = _node("sku-1", "red velvet sofa", ["ENTITY"]) + graph = FakeGraph([hub, product], neighbors={"hub:attr:red": [product]}) + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + (TextChunk(id="chunk-1", text="red velvet sofa"), 2.0) + ] + mock_embeddings = MagicMock() + mock_embeddings.embed_text.return_value = Vector( + id="q", embeddings=[0.2], metadata={} + ) + mock_vs = MagicMock() + mock_vs.search_nodes.return_value = [] + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "graph_adapter", graph), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody( + query="red sofa", + channels=["passages", "entities", "communities"], + k=10, + ) + ) + + ids = [hit.id for hit in response.hits] + self.assertNotIn("hub:attr:red", ids) + self.assertIn("sku-1", ids) + self.assertIn("chunk-1", ids) + self.assertEqual(response.channel_lists["bm25"], ["chunk-1"]) + self.assertIn("sku-1", response.channel_lists["entities"]) + self.assertNotIn("hub:attr:red", response.channel_lists["entities"]) + self.assertNotIn("hub:attr:red", response.channel_lists["communities"]) + + def test_entities_drop_attr_hubs(self): + hub = _node("hub:attr:red", "red", ["ATTR"]) + product = _node("sku-1", "red velvet sofa", ["ENTITY"]) + graph = FakeGraph([hub, product]) + hits = collect_entity_hits( + query="red sofa", + brain_id="searchbenchsmoke", + k=5, + graph=graph, + ) + ids = [hit.id for hit in hits] + self.assertEqual(ids, ["sku-1"]) + self.assertTrue(all(hit.channel == "entities" for hit in hits)) + + def test_community_skips_attr_member_hubs(self): + hub = _node("class-sofas", "sofas", ["CLASS"]) + product = _node("e1", "velvet sofa", ["ENTITY"]) + attr = _node("hub:attr:red", "red", ["ATTR"]) + graph = FakeGraph( + [hub, product, attr], + neighbors={"class-sofas": [product, attr]}, + ) + hits = collect_community_hits( + query="sofas", + brain_id="searchbenchsmoke", + k=10, + graph=graph, + fanout=5, + ) + ids = [hit.id for hit in hits] + self.assertEqual(ids, ["e1"]) + self.assertNotIn("hub:attr:red", ids) + + def test_community_attr_hub_returns_products(self): + hub = _node("hub:attr:red", "red", ["ATTR"]) + product = _node("e1", "red sofa", ["ENTITY"]) + graph = FakeGraph([hub, product], neighbors={"hub:attr:red": [product]}) + hits = collect_community_hits( + query="red sofa", + brain_id="searchbenchsmoke", + k=10, + graph=graph, + fanout=5, + ) + self.assertEqual([hit.id for hit in hits], ["e1"]) + self.assertEqual(hits[0].channel, "communities") + + def test_search_controller_does_not_import_retrieve_assembler(self): + from pathlib import Path + + root = Path(__file__).resolve().parents[1] + search_text = ( + root / "src" / "services" / "api" / "controllers" / "search.py" + ).read_text(encoding="utf-8") + retrieve_text = ( + root / "src" / "services" / "api" / "controllers" / "retrieve.py" + ).read_text(encoding="utf-8") + self.assertNotIn("controllers.retrieve", search_text) + self.assertNotIn("kg_topic_sessions", search_text) + self.assertNotIn("graph_channels", retrieve_text) + schema = ( + root / "src" / "services" / "api" / "constants" / "requests.py" + ).read_text(encoding="utf-8") + self.assertNotIn("product_id", schema.split("class SearchRequestBody")[1].split("class SearchHitScores")[0]) + + +class SearchGraphHelperTests(unittest.TestCase): + def test_entity_ann_and_name_match(self): + named = _node("sku-1", "navy velvet sofa", ["ENTITY"]) + vector_only = _node("sku-2", "other item", ["ENTITY"]) + graph = FakeGraph([named, vector_only]) + vs = MagicMock() + vs.search_nodes.return_value = [ + Vector(id="v2", metadata={"uuid": "sku-2", "name": "other item"}, distance=0.1) + ] + hits = collect_entity_hits( + query="velvet sofa", + brain_id="searchbenchsmoke", + k=5, + graph=graph, + vector_search=vs, + query_vector=[0.1, 0.2], + ) + ids = [hit.id for hit in hits] + self.assertIn("sku-1", ids) + self.assertIn("sku-2", ids) + + def test_events_rank_recent_higher(self): + older = _node("evt-old", "View", ["EVENT"], happened_at="2020-01-01T00:00:00Z") + newer = _node("evt-new", "View", ["EVENT"], happened_at="2026-08-01T00:00:00Z") + graph = FakeGraph([older, newer]) + hits = collect_event_hits( + query="View", + brain_id="searchbenchsmoke", + k=5, + graph=graph, + ) + self.assertEqual(hits[0].id, "evt-new") + self.assertGreater(hits[0].score, hits[1].score) + + def test_entity_bm25_matches_description_only(self): + named = _node("sku-1", "catalog item", ["ENTITY"]) + described = _node( + "sku-2", + "item two", + ["ENTITY"], + description="navy velvet ottoman for small rooms", + properties={"search_text": "item two navy velvet ottoman for small rooms"}, + ) + graph = FakeGraph([named, described]) + hits = collect_entity_hits( + query="ottoman", + brain_id="searchbenchsmoke", + k=5, + graph=graph, + ) + ids = [hit.id for hit in hits] + self.assertIn("sku-2", ids) + self.assertNotIn("sku-1", ids) + + def test_community_intersects_class_and_attr_hubs(self): + class_hub = _node("hub:class:sofas", "sofas", ["CLASS"]) + attr_hub = _node("hub:attr:modern", "modern", ["ATTR"]) + both = _node("e1", "velvet sofa", ["ENTITY"]) + class_only = _node("e2", "leather sofa", ["ENTITY"]) + graph = FakeGraph( + [class_hub, attr_hub, both, class_only], + neighbors={ + "hub:class:sofas": [both, class_only], + "hub:attr:modern": [both], + }, + ) + hits = collect_community_hits( + query="modern sofas", + brain_id="searchbenchsmoke", + k=10, + graph=graph, + fanout=5, + ) + ids = [hit.id for hit in hits] + self.assertEqual(ids, ["e1"]) + self.assertNotIn("e2", ids) + self.assertNotIn("hub:class:sofas", ids) + + def test_community_union_when_single_hub_kind(self): + class_hub = _node("hub:class:sofas", "sofas", ["CLASS"]) + a = _node("e1", "velvet sofa", ["ENTITY"]) + b = _node("e2", "leather sofa", ["ENTITY"]) + graph = FakeGraph( + [class_hub, a, b], + neighbors={"hub:class:sofas": [a, b]}, + ) + hits = collect_community_hits( + query="sofas", + brain_id="searchbenchsmoke", + k=10, + graph=graph, + fanout=5, + ) + self.assertEqual(set(hit.id for hit in hits), {"e1", "e2"}) + + def test_community_hybrid_ranks_by_node_bm25_not_degree(self): + class_hub = _node("hub:class:sofas", "sofas", ["CLASS"]) + attr_hub = _node("hub:attr:modern", "modern", ["ATTR"]) + weak = _node("e1", "sofa", ["ENTITY"], description="basic") + strong = _node( + "e2", + "plush seating", + ["ENTITY"], + description="modern sofas extra plush", + properties={"search_text": "plush seating modern sofas extra plush"}, + ) + graph = FakeGraph( + [class_hub, attr_hub, weak, strong], + neighbors={ + "hub:class:sofas": [weak, strong], + "hub:attr:modern": [weak, strong], + }, + ) + hits = collect_community_hits( + query="modern sofas", + brain_id="searchbenchsmoke", + k=10, + graph=graph, + fanout=5, + ) + self.assertEqual([hit.id for hit in hits], ["e2", "e1"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_search_jdsearch.py b/tests/test_search_jdsearch.py new file mode 100644 index 0000000..e2a415a --- /dev/null +++ b/tests/test_search_jdsearch.py @@ -0,0 +1,312 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock + +FIXTURE_DIR = ( + Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "jdsearch" +) + + +class SearchJdsearchTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + import sys + + root = Path(__file__).resolve().parents[1] + bench = root / "benchmarks" + if str(root) not in sys.path: + sys.path.insert(0, str(root)) + if str(bench) not in sys.path: + sys.path.insert(0, str(bench)) + + def _fixture_paths(self) -> tuple[Path, Path]: + product = FIXTURE_DIR / "product_meta_data.txt" + behavior = FIXTURE_DIR / "user_behavior_data.txt" + self.assertTrue(product.exists(), product) + self.assertTrue(behavior.exists(), behavior) + return product, behavior + + def test_term_and_list_separators(self): + from search.jdsearch import TERM_SEP, join_terms, split_list + + self.assertEqual(TERM_SEP, "\030") + self.assertEqual(join_terms(f"111{TERM_SEP}222"), "111 222") + self.assertEqual(split_list("100_200_999"), ["100", "200", "999"]) + self.assertEqual(split_list(""), []) + + def test_stats_histogram_and_truncated_count(self): + from search.jdsearch import PAPER_N_PRODUCTS, collect_jdsearch_stats + + product, behavior = self._fixture_paths() + stats = collect_jdsearch_stats( + product_path=product, + behavior_path=behavior, + extract=False, + ) + self.assertEqual(stats["n_behavior_rows"], 2) + self.assertEqual(stats["n_products_seen"], 2) + self.assertFalse(stats["truncated"]) + self.assertIn("1", stats["label_histogram"]) + self.assertEqual(stats["label_scheme"], "graded") + self.assertGreater(stats["history_type_counts"].get("CLICK", 0), 0) + self.assertGreater(stats["history_type_counts"].get("FLW", 0), 0) + self.assertIn("999", {"999"}) + self.assertGreater(stats["n_wids_missing_meta"], 0) + + truncated = collect_jdsearch_stats( + product_path=product, + behavior_path=behavior, + extract=False, + max_behavior_rows=1, + max_product_rows=1, + ) + self.assertTrue(truncated["truncated"]) + self.assertEqual(truncated["n_behavior_rows"], 1) + self.assertEqual(truncated["n_products_seen"], 1) + self.assertNotEqual(truncated["n_products_seen"], PAPER_N_PRODUCTS) + self.assertLess(truncated["n_products_seen"], 12_000_000) + + def test_prepare_rows_caps_target_and_gold(self): + from search.catalog import FROZEN_JSONL_IF_EXISTS + from search.dataset import dataset_stats, load_records, split_corpus + from search.jdsearch import ( + jdsearch_interactions_path, + prepare_jdsearch_bundle, + prepare_jdsearch_rows, + ) + + product, behavior = self._fixture_paths() + wands = ( + Path(__file__).resolve().parents[1] + / "benchmarks" + / "data" + / "search_wands.jsonl" + ) + wands_mtime = wands.stat().st_mtime if wands.exists() else None + rows, interactions, stats = prepare_jdsearch_rows( + max_queries=80, + max_docs=2000, + product_path=product, + behavior_path=behavior, + extract=False, + ) + docs, queries = split_corpus(rows) + self.assertLessEqual(len(queries), 80) + self.assertLessEqual(len(docs), 2000) + self.assertEqual(stats["label_scheme"], "graded") + self.assertTrue(queries) + for query in queries: + self.assertTrue(str(query.get("target") or "").startswith("jd-u")) + self.assertTrue(query.get("gold_doc_ids")) + self.assertTrue(query.get("gold_grades")) + self.assertTrue(query.get("candidate_doc_ids")) + self.assertTrue(any(row.get("behavior") == "follow" for row in interactions)) + self.assertTrue(any(row.get("behavior") == "click" for row in interactions)) + qids = {query["qid"] for query in queries} + self.assertTrue(qids) + for query in queries: + self.assertNotIn("999", query["gold_doc_ids"]) + self.assertIn("search_jdsearch.jsonl", FROZEN_JSONL_IF_EXISTS) + with tempfile.TemporaryDirectory() as tmp: + dest = Path(tmp) / "search_jdsearch.jsonl" + written = prepare_jdsearch_bundle( + dest, + max_queries=80, + max_docs=2000, + product_path=product, + behavior_path=behavior, + extract=False, + ) + self.assertEqual(written, dest) + loaded = load_records(dest) + stats = dataset_stats(loaded) + self.assertLessEqual(stats["n_queries"], 80) + self.assertLessEqual(stats["n_docs"], 2000) + _, loaded_queries = split_corpus(loaded) + self.assertTrue(all(q.get("target") and q.get("gold_doc_ids") for q in loaded_queries)) + inter_path = jdsearch_interactions_path(dest) + self.assertTrue(inter_path.exists()) + self.assertFalse((Path(tmp) / "search_wands.jsonl").exists()) + self.assertFalse((Path(tmp) / "search_esci_74.jsonl").exists()) + if wands_mtime is not None: + self.assertEqual(wands.stat().st_mtime, wands_mtime) + + def test_prepare_drops_all_missing_gold(self): + from search.jdsearch import prepare_jdsearch_rows + + product, behavior = self._fixture_paths() + with tempfile.TemporaryDirectory() as tmp: + extra_behavior = Path(tmp) / "user_behavior_data.txt" + extra_behavior.write_text( + behavior.read_text(encoding="utf-8") + + "onlymissing\t999\t1.0\t-1\t\t\t0\n", + encoding="utf-8", + ) + rows, _interactions, _stats = prepare_jdsearch_rows( + product_path=product, + behavior_path=extra_behavior, + extract=False, + ) + queries = [row for row in rows if row.get("type") == "query"] + self.assertTrue(all("999" not in (q.get("gold_doc_ids") or []) for q in queries)) + self.assertTrue(all(q.get("gold_doc_ids") for q in queries)) + + def test_follow_event_not_has(self): + import importlib.util + + from search.mapping import HAS_EVENT_NAME, interaction_to_triples + + triples = interaction_to_triples( + [ + { + "user_id": "jd-u0", + "item_id": "100", + "behavior": "follow", + "timestamp": "2022-10-16T00:00:00Z", + } + ] + ) + events = [ + row["event"]["name"] + for row in triples + if isinstance(row.get("event"), dict) and row["event"].get("name") + ] + self.assertIn("Follow", events) + self.assertNotIn(HAS_EVENT_NAME, events) + mapping_path = ( + Path(__file__).resolve().parents[1] + / "plugins" + / "features-rec" + / "models" + / "mapping.py" + ) + if not mapping_path.is_file(): + self.skipTest("optional features-rec plugin is not installed") + spec = importlib.util.spec_from_file_location( + "features_rec_mapping_jdsearch", mapping_path + ) + self.assertIsNotNone(spec) + self.assertIsNotNone(spec.loader) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + self.assertEqual(mod.normalize_behavior("FLW"), ("Follow", "TARGETED")) + self.assertEqual(mod.behavior_weight("FLW"), 0.5) + self.assertEqual(mod.behavior_weight("follow"), 0.5) + + def test_evaluate_sends_target_only_when_personalized(self): + from search.client import TimedResult + from search.evaluate import evaluate_search + from src.services.api.constants.requests import SearchRequestBody + + fields = set(SearchRequestBody.model_fields) + self.assertIn("target", fields) + self.assertNotIn("user_id", fields) + + class _Client: + class _Settings: + brain_id = "searchbenchjdslice" + + settings = _Settings() + + def __init__(self): + self.calls = [] + + def search(self, query, **kwargs): + self.calls.append({"query": query, **kwargs}) + return TimedResult(data={"hits": []}, latency_ms=1.0) + + def list_text_chunks(self, **kwargs): + skip = int(kwargs.get("skip") or 0) + if skip: + return TimedResult(data={"data": [], "total": 1}, latency_ms=1.0) + return TimedResult( + data={ + "data": [{"id": "c1", "text": "DOCID 100. 111 222"}], + "total": 1, + }, + latency_ms=1.0, + ) + + rows = [ + { + "type": "doc", + "doc_id": "100", + "text": "DOCID 100. 111 222", + }, + { + "type": "query", + "qid": "jdsearch-0", + "query": "111 222", + "target": "jd-u0", + "gold_doc_ids": ["100"], + "gold_grades": {"100": 1.0}, + "candidate_doc_ids": ["100"], + }, + ] + client = _Client() + evaluate_search(client, rows, skip_ingest=True, personalize=True) + self.assertEqual(client.calls[0].get("target"), "jd-u0") + client.calls.clear() + evaluate_search(client, rows, skip_ingest=True, personalize=False) + self.assertFalse(client.calls[0].get("target")) + client.calls.clear() + evaluate_search( + client, + [{**rows[1], "target": ""}, rows[0]] + if False + else [ + rows[0], + {**rows[1], "target": ""}, + ], + skip_ingest=True, + personalize=True, + ) + self.assertFalse(client.calls[0].get("target")) + + def test_client_search_body_omits_target_by_default(self): + from search.client import BrainAPIClient, TimedResult + from search.config import Settings + + captured = {} + + class _Stub(BrainAPIClient): + def __post_init__(self): + self._client = MagicMock() + + def _request(self, method, path, *, json=None, params=None): + captured["json"] = json + return TimedResult(data={"hits": []}, latency_ms=1.0) + + settings = MagicMock(spec=Settings) + settings.brain_id = "searchbenchjdslice" + settings.brainapi_url = "http://127.0.0.1" + settings.brainpat_token = "token" + settings.require_brainapi = lambda: None + client = _Stub(settings) + client.search("111 222") + self.assertNotIn("target", captured["json"]) + client.search("111 222", target="jd-u0") + self.assertEqual(captured["json"]["target"], "jd-u0") + self.assertNotIn("user_id", captured["json"]) + + def test_bm25_tokens_present_in_doc_text(self): + from search.jdsearch import prepare_jdsearch_rows + + product, behavior = self._fixture_paths() + rows, _interactions, _stats = prepare_jdsearch_rows( + product_path=product, + behavior_path=behavior, + extract=False, + ) + docs = [row for row in rows if row.get("type") == "doc"] + queries = [row for row in rows if row.get("type") == "query"] + self.assertTrue(docs) + blob = " ".join(str(doc.get("text") or "") for doc in docs) + self.assertIn("DOCID 100.", blob) + nonempty = 0 + for query in queries: + tokens = str(query.get("query") or "").split() + if any(token and token in blob for token in tokens): + nonempty += 1 + self.assertGreater(nonempty, 0) diff --git a/tests/test_search_personalize.py b/tests/test_search_personalize.py new file mode 100644 index 0000000..e0ebc7b --- /dev/null +++ b/tests/test_search_personalize.py @@ -0,0 +1,324 @@ +import os +import unittest +from unittest.mock import MagicMock, patch + +ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", +} +for key, value in ENV_DEFAULTS.items(): + os.environ.setdefault(key, value) + +from src.constants.data import TextChunk +from src.constants.kg import Node, Predicate +from src.core.search.personalize import ( + blend_ranked, + personalize_ranked_ids, + query_personalize_lambda, + score_nodes_for_user, + user_pref_weights, +) +from src.services.api.constants.requests import SearchRequestBody + + +def _node(uuid: str, name: str, labels: list[str], happened_at: str | None = None) -> Node: + return Node( + uuid=uuid, + name=name, + labels=labels, + happened_at=happened_at, + properties={}, + ) + + +def _pred(name: str, amount: float | None = None, weight: float | None = None) -> Predicate: + props = {} + if weight is not None: + props["weight"] = weight + return Predicate( + uuid=f"rel:{name}:{id(name)}", + name=name, + description="", + amount=amount, + properties=props, + ) + + +class FakeGraph: + def __init__(self): + self.nodes: dict[str, Node] = {} + self.edges: dict[str, list[tuple[Predicate, Node]]] = {} + + def add(self, node: Node) -> Node: + self.nodes[node.uuid] = node + self.edges.setdefault(node.uuid, []) + return node + + def link(self, tail: str, pred: Predicate, tip: str) -> None: + self.edges.setdefault(tail, []).append((pred, self.nodes[tip])) + + def get_by_uuids(self, uuids, brain_id="default"): + return [self.nodes[uid] for uid in uuids if uid in self.nodes] + + def get_by_uuid(self, uuid, brain_id="default"): + return self.nodes.get(uuid) + + def get_neighbors(self, nodes, brain_id="default", of_types=None, **kwargs): + out = {} + for item in nodes: + uid = item if isinstance(item, str) else item.uuid + neigh = list(self.edges.get(uid, [])) + if of_types: + wanted = {label.upper() for label in of_types} + neigh = [ + (pred, node) + for pred, node in neigh + if wanted & {label.upper() for label in (node.labels or [])} + ] + out[uid] = neigh + return out + + +def _style_graph() -> tuple[FakeGraph, Node]: + graph = FakeGraph() + user = graph.add(_node("user:u01", "u01", ["USER"])) + recent = graph.add( + _node("evt-recent", "View", ["EVENT"], happened_at="08/20/2026") + ) + old = graph.add(_node("evt-old", "View", ["EVENT"], happened_at="08/21/2025")) + seventies = graph.add(_node("70s-item", "lamp 70s", ["ENTITY"])) + modern = graph.add(_node("modern-item", "lamp modern", ["ENTITY"])) + attr_70s = graph.add(_node("hub:attr:70s", "70s", ["ATTR", "STYLE"])) + attr_modern = graph.add(_node("hub:attr:modern", "modern", ["ATTR", "STYLE"])) + graph.link("user:u01", _pred("MADE"), "evt-recent") + graph.link("evt-recent", _pred("TARGETED"), "70s-item") + graph.link("70s-item", _pred("HAS"), "hub:attr:70s") + graph.link("user:u01", _pred("MADE"), "evt-old") + graph.link("evt-old", _pred("TARGETED"), "modern-item") + graph.link("modern-item", _pred("HAS"), "hub:attr:modern") + graph.link( + "user:u01", + _pred("PREFERS", amount=0.4, weight=0.4), + "hub:attr:70s", + ) + return graph, user + + +class QueryLambdaTests(unittest.TestCase): + def test_generic_one_token(self): + self.assertAlmostEqual(query_personalize_lambda("lamp"), 0.85) + + def test_two_and_three_tokens(self): + self.assertAlmostEqual(query_personalize_lambda("oak table"), 0.5) + self.assertAlmostEqual(query_personalize_lambda("oak dining table"), 0.25) + + def test_four_plus_tokens(self): + self.assertAlmostEqual(query_personalize_lambda("oak dining table set"), 0.1) + + def test_digits_are_specific(self): + self.assertEqual(query_personalize_lambda("oak dining table 180cm"), 0.0) + self.assertEqual(query_personalize_lambda("sku 7468"), 0.0) + + +class BlendTests(unittest.TestCase): + def test_zero_lambda_preserves_order(self): + ids = ["a", "b", "c"] + ordered, _ = blend_ranked(ids, {"a": 3, "b": 2, "c": 1}, {"c": 9}, 0.0) + self.assertEqual(ordered, ids) + + def test_all_zero_prefs_preserve_order(self): + ids = ["a", "b", "c"] + ordered, _ = blend_ranked(ids, {"a": 3, "b": 2, "c": 1}, {}, 0.9) + self.assertEqual(ordered, ids) + + def test_high_lambda_promotes_overlap_without_dropping(self): + ids = ["a", "b", "c"] + ordered, _ = blend_ranked( + ids, {"a": 3, "b": 2, "c": 1}, {"a": 0, "b": 0, "c": 10}, 1.0 + ) + self.assertEqual(ordered[0], "c") + self.assertEqual(set(ordered), set(ids)) + + +class PrefVectorTests(unittest.TestCase): + def test_recent_70s_outranks_old_modern_and_combines_prefers(self): + graph, user = _style_graph() + with patch("src.core.search.personalize.graph_adapter", graph): + weights = user_pref_weights(user, "brain") + self.assertGreater(weights["hub:attr:70s"], weights["hub:attr:modern"]) + self.assertGreater(weights["hub:attr:70s"], 0.4) + + def test_missing_user_is_empty(self): + graph = FakeGraph() + with patch("src.core.search.personalize.graph_adapter", graph): + self.assertEqual(user_pref_weights(None, "brain"), {}) + self.assertEqual( + user_pref_weights(_node("missing", "x", ["USER"]), "brain"), + {}, + ) + + def test_score_nodes_overlap_only(self): + graph, user = _style_graph() + with patch("src.core.search.personalize.graph_adapter", graph): + prefs = user_pref_weights(user, "brain") + scores = score_nodes_for_user( + ["70s-item", "modern-item", "ghost"], prefs, "brain" + ) + self.assertGreater(scores["70s-item"], scores["modern-item"]) + self.assertEqual(scores["ghost"], 0.0) + + +class SearchPersonalizeTests(unittest.IsolatedAsyncioTestCase): + async def _search(self, body: SearchRequestBody, chunks: list): + from src.services.api.controllers import search as search_mod + + mock_data = MagicMock() + mock_data.search_bm25.return_value = chunks + mock_data.get_text_chunks_by_ids.return_value = ( + [item[0] for item in chunks], + None, + ) + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod.config, "search_literal_fill", False), + patch.object(search_mod, "embeddings_adapter", MagicMock()), + patch.object(search_mod, "vector_search", MagicMock()), + patch.object(search_mod, "data_adapter", mock_data), + ): + return await search_mod.search(body) + + def test_schema_allows_target_not_user_id(self): + body = SearchRequestBody(query="lamp", target="u01") + self.assertEqual(body.target, "u01") + self.assertEqual(body.channels, ["passages"]) + fields = set(SearchRequestBody.model_fields) + self.assertIn("target", fields) + for banned in ("product_id", "sku", "brand", "user_id"): + self.assertNotIn(banned, fields) + + async def test_omitted_target_keeps_retrieve_order(self): + chunks = [ + (TextChunk(id="first", text="DOCID 70s-item. lamp"), 9.0), + (TextChunk(id="second", text="DOCID modern-item. lamp"), 8.0), + (TextChunk(id="third", text="no join"), 7.0), + ] + response = await self._search(SearchRequestBody(query="lamp", k=3), chunks) + self.assertEqual([hit.id for hit in response.hits], ["first", "second", "third"]) + self.assertTrue(all(hit.scores.personalize is None for hit in response.hits)) + + async def test_target_promotes_70s_on_generic_query(self): + graph, _ = _style_graph() + chunks = [ + (TextChunk(id="first", text="DOCID modern-item. lamp"), 9.0), + (TextChunk(id="second", text="DOCID 70s-item. lamp"), 8.0), + (TextChunk(id="third", text="no join lamp"), 7.0), + ] + with patch("src.core.search.personalize.graph_adapter", graph): + response = await self._search( + SearchRequestBody(query="lamp", k=3, target="u01"), + chunks, + ) + ids = [hit.id for hit in response.hits] + self.assertEqual(ids[0], "second") + self.assertIn("third", ids) + self.assertEqual(response.hits[0].node_id, "70s-item") + self.assertIsNotNone(response.hits[0].scores.personalize) + + async def test_specific_query_does_not_reorder(self): + graph, _ = _style_graph() + chunks = [ + (TextChunk(id="first", text="DOCID modern-item. oak"), 9.0), + (TextChunk(id="second", text="DOCID 70s-item. oak"), 8.0), + ] + with patch("src.core.search.personalize.graph_adapter", graph): + response = await self._search( + SearchRequestBody( + query="oak dining table 180cm", k=2, target="u01" + ), + chunks, + ) + self.assertEqual([hit.id for hit in response.hits], ["first", "second"]) + + async def test_extras_and_survives_personalize(self): + graph, _ = _style_graph() + chunks = [ + ( + TextChunk( + id="first", + text="DOCID modern-item. lamp", + metadata={"color": "navy"}, + ), + 9.0, + ), + ( + TextChunk( + id="second", + text="DOCID 70s-item. lamp", + metadata={"color": "olive"}, + ), + 8.0, + ), + ( + TextChunk( + id="third", + text="DOCID other-item. lamp", + metadata={"color": "navy"}, + ), + 7.0, + ), + ] + with patch("src.core.search.personalize.graph_adapter", graph): + response = await self._search( + SearchRequestBody( + query="lamp", + k=3, + target="u01", + extras={"color": "navy"}, + ), + chunks, + ) + ids = [hit.id for hit in response.hits] + self.assertNotIn("second", ids) + self.assertEqual(set(ids), {"first", "third"}) + self.assertTrue( + all((hit.extras or {}).get("color") == "navy" for hit in response.hits) + ) + self.assertTrue( + all((hit.extras or {}).get("style") != "70s" for hit in response.hits) + ) + + def test_personalize_never_filters_ids(self): + graph, _ = _style_graph() + with patch("src.core.search.personalize.graph_adapter", graph): + ordered, _ = personalize_ranked_ids( + query="lamp", + ranked_ids=["a", "b"], + retrieve_scores={"a": 2, "b": 1}, + node_id_by_hit={"a": "modern-item", "b": "70s-item"}, + target="u01", + brain_id="brain", + ) + self.assertEqual(set(ordered), {"a", "b"}) diff --git a/tests/test_search_plugins.py b/tests/test_search_plugins.py new file mode 100644 index 0000000..eaa9933 --- /dev/null +++ b/tests/test_search_plugins.py @@ -0,0 +1,455 @@ +import os +import sys +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +ENV_DEFAULTS = { + "BRAINPAT_TOKEN": "test-token", + "MODELS_MODE": "local", + "EMBEDDINGS_LOCAL_MODEL": "local-model", + "EMBEDDINGS_SMALL_MODEL": "small-model", + "EMBEDDING_NODES_DIMENSION": "3", + "EMBEDDING_TRIPLETS_DIMENSION": "3", + "EMBEDDING_OBSERVATIONS_DIMENSION": "3", + "EMBEDDING_DATA_DIMENSION": "3", + "EMBEDDING_RELATIONSHIPS_DIMENSION": "3", + "REDIS_HOST": "localhost", + "REDIS_PORT": "6379", + "NEO4J_HOST": "localhost", + "NEO4J_PORT": "7687", + "NEO4J_USERNAME": "neo4j", + "NEO4J_PASSWORD": "password", + "MILVUS_HOST": "localhost", + "MILVUS_PORT": "19530", + "MONGO_CONNECTION_STRING": "mongodb://localhost:27017", + "CELERY_WORKER_CONCURRENCY": "1", + "OLLAMA_HOST": "localhost", + "OLLAMA_PORT": "11434", + "OLLAMA_LLM_SMALL_MODEL": "small", + "OLLAMA_LLM_LARGE_MODEL": "large", +} +for key, value in ENV_DEFAULTS.items(): + os.environ.setdefault(key, value) + +from fastapi import HTTPException + +from src.constants.data import TextChunk +from src.core.search.hooks import ( + register_search_reranker, + register_search_retriever, + reset_search_plugins, +) +from src.services.api.constants.requests import SearchRequestBody + +ROOT = Path(__file__).resolve().parents[1] + + +class SearchPluginHookTests(unittest.IsolatedAsyncioTestCase): + def setUp(self): + reset_search_plugins() + + def tearDown(self): + reset_search_plugins() + + async def test_unknown_rerank_plugin_is_400(self): + from src.services.api.controllers import search as search_mod + + with patch.object(search_mod.config, "search_enabled", True): + with self.assertRaises(HTTPException) as ctx: + await search_mod.search( + SearchRequestBody( + query="license", + rerank="plugin:missing-ce", + ) + ) + self.assertEqual(ctx.exception.status_code, 400) + self.assertIn("Unknown search rerank plugin", str(ctx.exception.detail)) + + async def test_unknown_retriever_plugin_is_400(self): + from src.services.api.controllers import search as search_mod + + with patch.object(search_mod.config, "search_enabled", True): + with self.assertRaises(HTTPException) as ctx: + await search_mod.search( + SearchRequestBody( + query="license", + channels=["plugin:splade"], + ) + ) + self.assertEqual(ctx.exception.status_code, 400) + self.assertIn("Unknown search retriever plugin", str(ctx.exception.detail)) + + async def test_linear_rerank_is_400_not_silent(self): + from src.services.api.controllers import search as search_mod + + with patch.object(search_mod.config, "search_enabled", True): + with self.assertRaises(HTTPException) as ctx: + await search_mod.search( + SearchRequestBody(query="license", rerank="linear") + ) + self.assertEqual(ctx.exception.status_code, 400) + + async def test_cross_encoder_reranks_top_hits(self): + from src.services.api.controllers import search as search_mod + + def reverse_rerank(query, candidates, k): + ordered = list(reversed(candidates)) + for index, item in enumerate(ordered): + item["score"] = 10 - index + return ordered[:k] + + register_search_reranker("cross-encoder", reverse_rerank) + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + (TextChunk(id="first", text="alpha"), 9.0), + (TextChunk(id="second", text="beta"), 8.0), + (TextChunk(id="third", text="gamma"), 7.0), + ] + mock_embeddings = MagicMock() + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody( + query="license", + k=3, + rerank="plugin:cross-encoder", + ) + ) + + self.assertEqual([hit.id for hit in response.hits], ["third", "second", "first"]) + self.assertEqual(response.hits[0].scores.rerank, 10) + mock_embeddings.embed_text.assert_not_called() + + def test_catalog_retrieve_and_rerank_caps(self): + from src.core.search.hooks import ( + CATALOG_RERANK_MAX_K, + RERANK_MAX_K, + rerank_max_k_for_mode, + retrieve_k_for_mode, + ) + + self.assertEqual(RERANK_MAX_K, 10) + self.assertEqual(CATALOG_RERANK_MAX_K, 50) + self.assertEqual(retrieve_k_for_mode("default", 10), 10) + self.assertEqual(retrieve_k_for_mode("catalog", 10), 50) + self.assertEqual(retrieve_k_for_mode("catalog", 50), 50) + self.assertEqual(retrieve_k_for_mode("catalog", 200), 200) + self.assertEqual(rerank_max_k_for_mode("default"), 10) + self.assertEqual(rerank_max_k_for_mode("catalog"), 50) + + async def test_default_reranks_at_most_ten(self): + from src.core.search.hooks import RERANK_MAX_K + from src.services.api.controllers import search as search_mod + + seen: list[tuple[int, int]] = [] + + def capture_rerank(query, candidates, k): + seen.append((len(candidates), k)) + return list(candidates)[:k] + + register_search_reranker("cross-encoder", capture_rerank) + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + (TextChunk(id=f"h{index}", text=f"t{index}"), float(60 - index)) + for index in range(50) + ] + mock_embeddings = MagicMock() + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody( + query="license", + k=50, + rerank="plugin:cross-encoder", + ) + ) + + self.assertEqual(RERANK_MAX_K, 10) + self.assertEqual(seen, [(10, 10)]) + self.assertEqual(len(response.hits), 50) + mock_data.search_bm25.assert_called_with("license", "default", limit=50) + + async def test_catalog_mode_reranks_fifty_when_k_fifty(self): + from src.core.search.hooks import CATALOG_RERANK_MAX_K + from src.services.api.controllers import search as search_mod + + seen: list[tuple[int, int]] = [] + + def capture_rerank(query, candidates, k): + seen.append((len(candidates), k)) + ordered = list(reversed(candidates)) + return ordered[:k] + + register_search_reranker("cross-encoder", capture_rerank) + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + (TextChunk(id=f"h{index}", text=f"t{index}"), float(60 - index)) + for index in range(50) + ] + mock_embeddings = MagicMock() + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody( + query="license", + k=50, + mode="catalog", + rerank="plugin:cross-encoder", + ) + ) + + self.assertEqual(CATALOG_RERANK_MAX_K, 50) + self.assertEqual(seen, [(50, 50)]) + self.assertEqual(len(response.hits), 50) + self.assertEqual(response.hits[0].id, "h49") + mock_data.search_bm25.assert_called_with("license", "default", limit=50) + + async def test_catalog_mode_retrieves_fifty_when_k_ten(self): + from src.services.api.controllers import search as search_mod + + seen: list[tuple[int, int]] = [] + + def capture_rerank(query, candidates, k): + seen.append((len(candidates), k)) + return list(candidates)[:k] + + register_search_reranker("cross-encoder", capture_rerank) + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + (TextChunk(id=f"h{index}", text=f"t{index}"), float(60 - index)) + for index in range(50) + ] + mock_embeddings = MagicMock() + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody( + query="license", + k=10, + mode="catalog", + rerank="plugin:cross-encoder", + ) + ) + + self.assertEqual(seen, [(50, 50)]) + self.assertEqual(len(response.hits), 10) + mock_data.search_bm25.assert_called_with("license", "default", limit=50) + + async def test_splade_channel_fuses_without_silent_bm25_fallback(self): + from src.services.api.controllers import search as search_mod + + def fake_splade(query, brain_id, k): + return ["splade-hit"], {"splade-hit": 4.2}, {"splade-hit": "sparse hit"} + + register_search_retriever("splade", fake_splade) + mock_data = MagicMock() + mock_embeddings = MagicMock() + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody( + query="license", + k=5, + channels=["plugin:splade"], + ) + ) + + self.assertEqual([hit.id for hit in response.hits], ["splade-hit"]) + self.assertEqual(response.hits[0].scores.plugin["splade"], 4.2) + mock_data.search_bm25.assert_not_called() + + async def test_plugin_lists_fill_frozen_tail_not_head(self): + from src.services.api.controllers import search as search_mod + + def fake_splade(query, brain_id, k): + return ( + ["plugin-extra"], + {"plugin-extra": 9.9}, + {"plugin-extra": "sparse extra"}, + ) + + register_search_retriever("splade", fake_splade) + mock_data = MagicMock() + mock_data.search_bm25.return_value = [ + (TextChunk(id=f"h{index}", text=f"t{index}"), float(20 - index)) + for index in range(12) + ] + mock_embeddings = MagicMock() + mock_vs = MagicMock() + + with ( + patch.object(search_mod.config, "search_enabled", True), + patch.object(search_mod.config, "search_use_dense", False), + patch.object(search_mod.config, "search_use_bm25", True), + patch.object(search_mod.config, "search_literal_fill", False), + patch.object(search_mod, "embeddings_adapter", mock_embeddings), + patch.object(search_mod, "vector_search", mock_vs), + patch.object(search_mod, "data_adapter", mock_data), + ): + response = await search_mod.search( + SearchRequestBody( + query="license", + k=15, + channels=["passages", "plugin:splade"], + ) + ) + + ids = [hit.id for hit in response.hits] + self.assertEqual(ids[:10], [f"h{index}" for index in range(10)]) + self.assertIn("plugin-extra", ids[10:]) + self.assertNotIn("plugin-extra", ids[:10]) + + def test_context_controller_does_not_import_search_hooks(self): + retrieve = ROOT / "src" / "services" / "api" / "controllers" / "retrieve.py" + text = retrieve.read_text(encoding="utf-8") + self.assertNotIn("src.core.search.hooks", text) + self.assertNotIn("src.core.search.graph_channels", text) + self.assertNotIn("register_search_reranker", text) + self.assertNotIn("plugin:cross-encoder", text) + + +class SearchPluginPackageTests(unittest.TestCase): + def setUp(self): + reset_search_plugins() + self._extra_paths: list[str] = [] + self._extra_modules: list[str] = [] + + def tearDown(self): + reset_search_plugins() + for name in self._extra_modules: + sys.modules.pop(name, None) + for path in self._extra_paths: + if path in sys.path: + sys.path.remove(path) + + def _load_plugin_dir(self, name: str): + plugin_path = ROOT / "plugins" / name + if not (plugin_path / "main.py").is_file(): + self.skipTest(f"optional search plugin is not installed: {name}") + plugin_dir = str(plugin_path) + sys.path.insert(0, plugin_dir) + self._extra_paths.append(plugin_dir) + return plugin_dir + + def test_cross_encoder_plugin_reranks_with_injected_predict(self): + self._load_plugin_dir("search-rerank") + import rerank as ce + + self._extra_modules.extend(["rerank", "main", "routes"]) + ce.set_predict(lambda pairs: [float(len(text)) for _, text in pairs]) + ranked = ce.rerank( + "q", + [ + {"id": "short", "text": "ab", "score": 1}, + {"id": "long", "text": "abcdefghij", "score": 1}, + ], + 10, + ) + self.assertEqual([row["id"] for row in ranked], ["long", "short"]) + from main import register as register_ce + + class Ctx: + _app = None + + def register_search_reranker(self, name, fn): + register_search_reranker(name, fn) + + def register_search_retriever(self, name, fn): + register_search_retriever(name, fn) + + register_ce(Ctx()) + from src.core.search.hooks import get_search_reranker + + self.assertIsNotNone(get_search_reranker("cross-encoder")) + + def test_splade_index_and_retrieve_with_fake_encoder(self): + self._load_plugin_dir("search-splade") + import encode as splade_encode + import index as splade_index + + self._extra_modules.extend(["encode", "index", "routes", "main"]) + splade_encode.set_encoder( + lambda text: {token: 1.0 for token in text.split() if token} + ) + splade_index.index_chunks( + "searchbenchsmoke", + [ + {"id": "a", "text": "alice license portland"}, + {"id": "b", "text": "piano thursday"}, + ], + ) + ids, scores, texts = splade_index.retrieve( + "alice license", "searchbenchsmoke", 2 + ) + self.assertEqual(ids[0], "a") + self.assertGreater(scores["a"], scores.get("b", 0)) + self.assertIn("alice", texts["a"]) + + def test_colbert_maxsim_retrieve_with_fake_encoder(self): + self._load_plugin_dir("search-colbert") + import encode as colbert_encode + import index as colbert_index + + self._extra_modules.extend(["encode", "index", "routes", "main"]) + + def fake_tokens(text: str): + table = { + "q": [[1.0, 0.0]], + "match": [[1.0, 0.0]], + "other": [[0.0, 1.0]], + } + return table.get(text, [[0.0, 0.0]]) + + colbert_encode.set_encoder(fake_tokens) + colbert_index.index_chunks( + "searchbenchsmoke", + [{"id": "hit", "text": "match"}, {"id": "miss", "text": "other"}], + ) + ids, scores, _ = colbert_index.retrieve("q", "searchbenchsmoke", 2) + self.assertEqual(ids[0], "hit") + self.assertGreater(scores["hit"], scores["miss"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_structured_triple_persistence.py b/tests/test_structured_triple_persistence.py index 16ba756..d4ca4c3 100644 --- a/tests/test_structured_triple_persistence.py +++ b/tests/test_structured_triple_persistence.py @@ -85,6 +85,91 @@ def test_full_triple_produces_subject_event_and_event_object_edges(self): self.assertIn(("alice", "person"), entities) self.assertIn(("purchase", "event", "01/01/2024"), entities) + def test_direct_has_is_one_edge_not_event(self): + from src.core.agents.architect_agent import ingestion_triples_to_relationships + from src.services.api.constants.requests import IngestionTripleSet + + triple = IngestionTripleSet( + subject={"name": "bed", "type": "ENTITY", "uuid": "0"}, + subj_event={"name": "HAS", "uuid": "rel-has-1"}, + object={"name": "rubberwood", "type": "ATTR", "uuid": "hub-attr-rubberwood"}, + ) + relationships, entities = ingestion_triples_to_relationships([triple], []) + self.assertEqual(len(relationships), 1) + self.assertEqual(relationships[0].tail.uuid, "0") + self.assertEqual(relationships[0].name, "HAS") + self.assertEqual(relationships[0].tip.uuid, "hub-attr-rubberwood") + self.assertEqual(relationships[0].tip.type, "ATTR") + self.assertNotIn(("has", "event", ""), entities) + self.assertNotIn(("has", "event"), entities) + + def test_has_wrapper_without_happened_at_collapses(self): + from src.core.agents.architect_agent import ingestion_triples_to_relationships + from src.services.api.constants.requests import IngestionTripleSet + + triple = IngestionTripleSet( + subject={"name": "bed", "type": "ENTITY", "uuid": "0"}, + subj_event={"name": "HAS", "uuid": "rel-1"}, + event={"name": "HAS", "type": "EVENT", "uuid": "evt-has"}, + event_obj={"name": "HAS", "uuid": "rel-2"}, + object={"name": "modern", "type": "ATTR", "uuid": "hub-modern"}, + ) + relationships, entities = ingestion_triples_to_relationships([triple], []) + self.assertEqual(len(relationships), 1) + self.assertEqual(relationships[0].tail.uuid, "0") + self.assertEqual(relationships[0].tip.uuid, "hub-modern") + self.assertEqual(relationships[0].name, "HAS") + self.assertTrue( + all(getattr(rel.tail, "type", "").upper() != "EVENT" for rel in relationships) + ) + self.assertTrue( + all(getattr(rel.tip, "type", "").upper() != "EVENT" for rel in relationships) + ) + + def test_has_event_with_happened_at_stays_two_edges(self): + from src.core.agents.architect_agent import ingestion_triples_to_relationships + from src.services.api.constants.requests import IngestionTripleSet + + triple = IngestionTripleSet( + subject={"name": "u1", "type": "USER", "uuid": "user-1"}, + subj_event={"name": "HAS", "uuid": "rel-1"}, + event={ + "name": "HAS", + "type": "EVENT", + "uuid": "evt-dated", + "happened_at": "01/02/2024", + }, + event_obj={"name": "HAS", "uuid": "rel-2"}, + object={"name": "sku-1", "type": "ENTITY", "uuid": "sku-1"}, + ) + relationships, _ = ingestion_triples_to_relationships([triple], []) + self.assertEqual(len(relationships), 2) + self.assertEqual(relationships[0].tip.uuid, "evt-dated") + self.assertEqual(relationships[1].tail.uuid, "evt-dated") + + def test_direct_prefers_is_one_edge(self): + from src.core.agents.architect_agent import ingestion_triples_to_relationships + from src.services.api.constants.requests import IngestionTripleSet + + triple = IngestionTripleSet( + subject={"name": "u01", "type": "USER", "uuid": "user:u01"}, + subj_event={ + "name": "PREFERS", + "uuid": "rel-pref-1", + "amount": 0.2, + "properties": {"facet": "style", "value": "70s"}, + }, + object={"name": "70s", "type": "ATTR", "uuid": "hub:attr:70s"}, + ) + relationships, entities = ingestion_triples_to_relationships([triple], []) + self.assertEqual(len(relationships), 1) + self.assertEqual(relationships[0].tail.uuid, "user:u01") + self.assertEqual(relationships[0].name, "PREFERS") + self.assertEqual(relationships[0].tip.uuid, "hub:attr:70s") + self.assertEqual(relationships[0].amount, 0.2) + self.assertNotIn(("prefers", "event", ""), entities) + self.assertNotIn(("prefers", "event"), entities) + class StructuredTriplePersistenceTests(unittest.TestCase): def test_triple_only_persists_without_llm(self): diff --git a/tui/README.md b/tui/README.md index acb748c..22c283f 100644 --- a/tui/README.md +++ b/tui/README.md @@ -94,9 +94,10 @@ The interactive flow asks (any matching CLI flag skips that prompt): 2. Otherwise: Vector DB → Data DB → Graph DB → Models mode 3. For `remote` models mode: pick provider, then provide GCP credentials, project, and model names 4. For `local` models mode: probe Ollama, wait for it to start, then verify pulled models -5. Connection details for only the services you actually selected -6. `BRAINPAT_TOKEN` (generate or paste) -7. Optionally start the docker compose containers now +5. Search: enable `/retrieve/search`? If yes, retrieval = both fused (default) / dense only / BM25 only. Requires `DATA_DB=postgresql`. Search p50 SLO is < 200 ms excluding embed RTT. +6. Connection details for only the services you actually selected +7. `BRAINPAT_TOKEN` (generate or paste) +8. Optionally start the docker compose containers now If Python (>=3.11) or Docker is missing, the TUI walks you through installing them with platform-aware commands and retries detection automatically. diff --git a/tui/src/commands/config.ts b/tui/src/commands/config.ts index bb1c151..9feb22e 100644 --- a/tui/src/commands/config.ts +++ b/tui/src/commands/config.ts @@ -11,6 +11,7 @@ import { } from "../flows/pipeline.js"; import { askConnections } from "../flows/connections.js"; import { askAuth } from "../flows/auth.js"; +import { askSearch } from "../flows/search.js"; import { askServicesRuntime } from "../flows/services-runtime.js"; import { runSetupWizard, @@ -37,6 +38,7 @@ type ConfigMenuSection = | "runtime" | "models" | "pipeline" + | "search" | "connections" | "auth" | "plugins" @@ -66,6 +68,15 @@ function draftSummary(draft: ConfigDraft): string { ? `Pipeline OCR: ${draft.pipeline.ocrMode}` : "Pipeline OCR: not configured", ); + if (draft.search) { + lines.push( + draft.search.enabled + ? `Search: enabled (${draft.search.retrieval})` + : "Search: disabled", + ); + } else { + lines.push("Search: not configured (default off)"); + } lines.push( draft.connections ? "Connections: configured" : "Connections: not configured", ); @@ -100,6 +111,7 @@ async function configMainMenu(): Promise { { value: "runtime", label: "Services runtime" }, { value: "models", label: "Models (LLM & embeddings)" }, { value: "pipeline", label: "Pipeline (OCR)" }, + { value: "search", label: "Search (BM25 + dense)" }, { value: "connections", label: "Connection details" }, { value: "auth", label: "Authentication token" }, { value: "plugins", label: "Plugins (install/uninstall)" }, @@ -190,6 +202,20 @@ async function configurePipeline(draft: ConfigDraft): Promise { p.log.success("Pipeline updated."); } +async function configureSearch(draft: ConfigDraft): Promise { + const search = await askSearch({ + allowBack: true, + backHint: MENU_BACK_HINT, + initial: draft.search, + dataDb: draft.dbs?.dataDb, + }); + if (isPromptBack(search)) { + return; + } + draft.search = search; + p.log.success("Search updated."); +} + async function configureConnections(draft: ConfigDraft): Promise { if (!draft.dbs) { p.log.warn("Configure databases first."); @@ -393,6 +419,9 @@ export async function runConfig(): Promise { case "pipeline": await configurePipeline(draft); break; + case "search": + await configureSearch(draft); + break; case "connections": await configureConnections(draft); break; diff --git a/tui/src/constants.ts b/tui/src/constants.ts index f0313d5..e3ebcb6 100644 --- a/tui/src/constants.ts +++ b/tui/src/constants.ts @@ -236,6 +236,14 @@ export const ENV_KEYS = { embeddingDataDimension: "EMBEDDING_DATA_DIMENSION", embeddingRelationshipsDimension: "EMBEDDING_RELATIONSHIPS_DIMENSION", enabledPlugins: "ENABLED_PLUGINS", + searchEnabled: "SEARCH_ENABLED", + searchUseDense: "SEARCH_USE_DENSE", + searchUseBm25: "SEARCH_USE_BM25", + searchFusion: "SEARCH_FUSION", + searchFusionAlpha: "SEARCH_FUSION_ALPHA", + searchBm25K1: "SEARCH_BM25_K1", + searchBm25B: "SEARCH_BM25_B", + contextPassageMode: "CONTEXT_PASSAGE_MODE", } as const; export const SERVICE_COMPOSE_FILES = { diff --git a/tui/src/flows/search.ts b/tui/src/flows/search.ts new file mode 100644 index 0000000..ba6b453 --- /dev/null +++ b/tui/src/flows/search.ts @@ -0,0 +1,117 @@ +import * as p from "@clack/prompts"; +import pc from "picocolors"; +import { isPromptBack, pickOne, type PromptBack } from "../lib/prompts.js"; +import type { SearchChoices, SearchRetrieval } from "../types.js"; + +const DEFAULT_SEARCH: SearchChoices = { + enabled: false, + retrieval: "hybrid", +}; + +export function defaultSearchChoices(): SearchChoices { + return { ...DEFAULT_SEARCH }; +} + +export async function askSearch(options?: { + allowBack?: false; + initial?: SearchChoices; + dataDb?: string; +}): Promise; +export async function askSearch(options: { + allowBack: true; + backHint?: string; + initial?: SearchChoices; + dataDb?: string; +}): Promise; +export async function askSearch(options?: { + allowBack?: boolean; + backHint?: string; + initial?: SearchChoices; + dataDb?: string; +}): Promise { + p.log.step("Search"); + if (options?.dataDb && options.dataDb !== "postgresql") { + p.log.info( + pc.dim( + "Hybrid search needs DATA_DB=postgresql. Leaving SEARCH_ENABLED=false.", + ), + ); + return { enabled: false, retrieval: "hybrid" }; + } + + const enabledChoice = options?.allowBack + ? await pickOne<"yes" | "no">({ + message: "Enable hybrid search API (/retrieve/search)?", + options: [ + { + value: "no", + label: "No (default)", + hint: "memory path unchanged; search routes return 404", + }, + { + value: "yes", + label: "Yes", + hint: "BM25 + dense ANN; fused by default; p50 < 200 ms ex-embed", + }, + ], + initialValue: options.initial?.enabled ? "yes" : "no", + allowBack: true, + backHint: options.backHint, + }) + : await pickOne<"yes" | "no">({ + message: "Enable hybrid search API (/retrieve/search)?", + options: [ + { + value: "no", + label: "No (default)", + hint: "memory path unchanged; search routes return 404", + }, + { + value: "yes", + label: "Yes", + hint: "BM25 + dense ANN; fused by default; p50 < 200 ms ex-embed", + }, + ], + initialValue: options?.initial?.enabled ? "yes" : "no", + }); + if (isPromptBack(enabledChoice)) { + return enabledChoice; + } + if (enabledChoice === "no") { + return { enabled: false, retrieval: "hybrid" }; + } + + const retrieval = options?.allowBack + ? await pickOne({ + message: "Search retrieval", + options: [ + { + value: "hybrid", + label: "Both fused (default)", + hint: "parallel BM25 ∪ dense, RRF", + }, + { value: "dense", label: "Dense only" }, + { value: "bm25", label: "BM25 only" }, + ], + initialValue: options.initial?.retrieval ?? "hybrid", + allowBack: true, + backHint: options.backHint, + }) + : await pickOne({ + message: "Search retrieval", + options: [ + { + value: "hybrid", + label: "Both fused (default)", + hint: "parallel BM25 ∪ dense, RRF", + }, + { value: "dense", label: "Dense only" }, + { value: "bm25", label: "BM25 only" }, + ], + initialValue: options?.initial?.retrieval ?? "hybrid", + }); + if (isPromptBack(retrieval)) { + return retrieval; + } + return { enabled: true, retrieval }; +} diff --git a/tui/src/flows/setup-wizard.ts b/tui/src/flows/setup-wizard.ts index e54993d..cd5d63f 100644 --- a/tui/src/flows/setup-wizard.ts +++ b/tui/src/flows/setup-wizard.ts @@ -3,6 +3,7 @@ import { askUseDefaults, DEFAULT_DBS } from "./defaults.js"; import { askDatabases } from "./services.js"; import { askModels } from "./models.js"; import { askPipeline } from "./pipeline.js"; +import { askSearch, defaultSearchChoices } from "./search.js"; import { askConnections } from "./connections.js"; import { askAuth } from "./auth.js"; import { askServicesRuntime } from "./services-runtime.js"; @@ -20,6 +21,7 @@ import type { ModelsChoices, PluginChoice, PipelineChoices, + SearchChoices, ServicesRuntime, } from "../types.js"; @@ -28,6 +30,7 @@ export interface SetupDraft { servicesRuntime?: ServicesRuntime; models?: ModelsChoices; pipeline?: PipelineChoices; + search?: SearchChoices; connections?: Connections; auth?: AuthChoices; plugins?: PluginChoice[]; @@ -47,6 +50,7 @@ export function toInitChoices(draft: SetupDraft): InitChoices { servicesRuntime: draft.servicesRuntime!, models: draft.models!, pipeline: draft.pipeline!, + search: draft.search ?? defaultSearchChoices(), connections: draft.connections!, auth: draft.auth!, plugins: draft.plugins ?? [], @@ -243,19 +247,35 @@ export async function runSetupWizard( } if (step === 5) { + const search = await askSearch({ + allowBack: true, + backHint: stepBackHint, + initial: draft.search, + dataDb: draft.dbs?.dataDb, + }); + if (isPromptBack(search)) { + step = draft.usedDefaults ? 3 : 4; + continue; + } + draft.search = search; + step = 6; + continue; + } + + if (step === 6) { if (!draft.dbs) { step = draft.usedDefaults ? 0 : 1; continue; } draft.connections = await askConnections(draft.dbs, flags); - step = 6; + step = 7; continue; } - if (step === 6) { + if (step === 7) { if (flags?.brainpatToken) { draft.auth = { brainpatToken: flags.brainpatToken }; - step = 7; + step = 8; continue; } const auth = await askAuth({ @@ -264,15 +284,15 @@ export async function runSetupWizard( prechosenToken: flags?.brainpatToken, }); if (isPromptBack(auth)) { - step = 5; + step = 6; continue; } draft.auth = auth; - step = 7; + step = 8; continue; } - if (step === 7) { + if (step === 8) { if (flags?.noPlugins || flags?.plugins !== undefined) { draft.plugins = flags.noPlugins ? [] : (flags.plugins ?? []); return true; @@ -285,7 +305,7 @@ export async function runSetupWizard( skip: flags?.noPlugins, }); if (isPromptBack(plugins)) { - step = 6; + step = 7; continue; } draft.plugins = plugins; diff --git a/tui/src/lib/write-env.ts b/tui/src/lib/write-env.ts index b49fb9b..216137e 100644 --- a/tui/src/lib/write-env.ts +++ b/tui/src/lib/write-env.ts @@ -65,6 +65,8 @@ export async function writeEnvFromChoices(choices: InitChoices): Promise { values[ENV_KEYS.runGraphConsolidator] = "true"; values[ENV_KEYS.celeryWorkerConcurrency] = 4; + applySearchValues(values, choices); + values[ENV_KEYS.brainpatToken] = choices.auth.brainpatToken; values[ENV_KEYS.enabledPlugins] = choices.plugins.map((plugin) => plugin.name).join(","); values["ENV"] = "development"; @@ -85,6 +87,25 @@ export async function writeEnvFromChoices(choices: InitChoices): Promise { await saveEnv(env); } +function applySearchValues( + values: Record, + choices: InitChoices, +): void { + const search = choices.search ?? { enabled: false, retrieval: "hybrid" }; + const enabled = search.enabled; + const retrieval = search.retrieval ?? "hybrid"; + values[ENV_KEYS.searchEnabled] = enabled ? "true" : "false"; + values[ENV_KEYS.searchUseDense] = + !enabled || retrieval === "hybrid" || retrieval === "dense" ? "true" : "false"; + values[ENV_KEYS.searchUseBm25] = + !enabled || retrieval === "hybrid" || retrieval === "bm25" ? "true" : "false"; + values[ENV_KEYS.searchFusion] = "rrf"; + values[ENV_KEYS.searchFusionAlpha] = "0.5"; + values[ENV_KEYS.searchBm25K1] = "1.2"; + values[ENV_KEYS.searchBm25B] = "0.75"; + values[ENV_KEYS.contextPassageMode] = enabled ? "hybrid" : "hybrid"; +} + function applyPostgresValues( values: Record, choices: InitChoices, diff --git a/tui/src/types.ts b/tui/src/types.ts index 987a334..9d7dabc 100644 --- a/tui/src/types.ts +++ b/tui/src/types.ts @@ -12,6 +12,7 @@ export type ModelProvider = | "amazon_bedrock"; export type OcrMode = "docparser" | "docling"; export type PipelineMode = "accurate" | "lightweight"; +export type SearchRetrieval = "hybrid" | "dense" | "bm25"; export type ServicesRuntime = "docker" | "manual"; export type PluginSource = "local" | "registry"; @@ -156,6 +157,11 @@ export interface PipelineChoices { ocrMode: OcrMode; } +export interface SearchChoices { + enabled: boolean; + retrieval: SearchRetrieval; +} + export interface PluginChoice { name: string; source: PluginSource; @@ -167,6 +173,7 @@ export interface InitChoices { dbs: DbChoices; models: ModelsChoices; pipeline: PipelineChoices; + search: SearchChoices; connections: Connections; auth: AuthChoices; plugins: PluginChoice[]; diff --git a/uv.lock b/uv.lock index d053e27..5c93f5e 100644 --- a/uv.lock +++ b/uv.lock @@ -18,7 +18,7 @@ resolution-markers = [ [[package]] name = "accelerate" -version = "1.13.0" +version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, @@ -29,9 +29,9 @@ dependencies = [ { name = "safetensors" }, { name = "torch" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ca/14/787e5498cd062640f0f3d92ef4ae4063174f76f9afd29d13fc52a319daae/accelerate-1.13.0.tar.gz", hash = "sha256:d631b4e0f5b3de4aff2d7e9e6857d164810dfc3237d54d017f075122d057b236", size = 402835, upload-time = "2026-03-04T19:34:12.359Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/75/94cd5d389649578aca399e5aa822637eec18319a1dadc400ffe2f9a7493f/accelerate-1.14.0.tar.gz", hash = "sha256:41b9c4377a54e0b460a959b0defa1b736e4ca0a2373252d9a539964c2afe3c8d", size = 412167, upload-time = "2026-06-11T13:45:52.326Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/46/02ac5e262d4af18054b3e922b2baedbb2a03289ee792162de60a865defc5/accelerate-1.13.0-py3-none-any.whl", hash = "sha256:cf1a3efb96c18f7b152eb0fa7490f3710b19c3f395699358f08decca2b8b62e0", size = 383744, upload-time = "2026-03-04T19:34:10.313Z" }, + { url = "https://files.pythonhosted.org/packages/a8/db/253133d7e7cb40d3af384bb2f5c0b4a2b7fdcffbc95c688cc67a20a3c103/accelerate-1.14.0-py3-none-any.whl", hash = "sha256:e94390c2863b873be18f623f9df48a0d8fe5eff13ea7f1a00092b0a7904888c6", size = 389246, upload-time = "2026-06-11T13:45:50.477Z" }, ] [[package]] @@ -272,7 +272,7 @@ wheels = [ [[package]] name = "brainapi2" -version = "2.14.3.dev0" +version = "2.17.0rc1" source = { editable = "." } dependencies = [ { name = "anthropic" }, @@ -285,8 +285,7 @@ dependencies = [ { name = "langchain" }, { name = "langchain-aws" }, { name = "langchain-core" }, - { name = "langchain-google-vertexai", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, - { name = "langchain-google-vertexai", version = "3.2.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "langchain-google-vertexai" }, { name = "langchain-openai" }, { name = "langid" }, { name = "levenshtein" }, @@ -296,6 +295,7 @@ dependencies = [ { name = "numpy" }, { name = "openai" }, { name = "pgvector" }, + { name = "pillow" }, { name = "psycopg2-binary" }, { name = "pylint" }, { name = "pymilvus" }, @@ -303,11 +303,11 @@ dependencies = [ { name = "python-dotenv" }, { name = "python-multipart" }, { name = "redis" }, - { name = "sentence-transformers" }, { name = "spacy" }, { name = "spacy-langdetect" }, { name = "sympy" }, { name = "tenacity" }, + { name = "tiktoken" }, { name = "typer" }, { name = "uvicorn" }, { name = "watchfiles" }, @@ -318,6 +318,9 @@ docling-ocr = [ { name = "accelerate" }, { name = "docling" }, ] +local-ml = [ + { name = "sentence-transformers" }, +] postgresql-backend = [ { name = "networkx" }, { name = "pgvector" }, @@ -326,23 +329,23 @@ postgresql-backend = [ [package.metadata] requires-dist = [ - { name = "accelerate", marker = "extra == 'docling-ocr'", specifier = ">=1.0.0,<2.0.0" }, + { name = "accelerate", marker = "extra == 'docling-ocr'", specifier = ">=1.14.0,<2.0.0" }, { name = "anthropic", specifier = ">=0.57.1,<0.58.0" }, { name = "boto3", specifier = ">=1.40.19,<2.0.0" }, { name = "celery", specifier = ">=5.5.3,<6.0.0" }, { name = "cpp-chunker", specifier = ">=0.1.4,<0.2.0" }, - { name = "docling", marker = "extra == 'docling-ocr'", specifier = ">=2.0.0,<3.0.0" }, - { name = "fastapi", specifier = ">=0.119.0,<0.120.0" }, + { name = "docling", marker = "extra == 'docling-ocr'", specifier = ">=2.121.0,<3.0.0" }, + { name = "fastapi", specifier = ">=0.141.1,<0.142.0" }, { name = "google-genai", specifier = ">=1.46.0,<2.0.0" }, { name = "kombu", extras = ["rabbitmq"], specifier = ">=5.3.0,<6.0.0" }, - { name = "langchain", specifier = ">=1.0.0,<2.0.0" }, + { name = "langchain", specifier = ">=1.3.16,<2.0.0" }, { name = "langchain-aws", specifier = ">=1.0.0,<2.0.0" }, - { name = "langchain-core", specifier = ">=1.0.0,<2.0.0" }, + { name = "langchain-core", specifier = ">=1.6.0,<2.0.0" }, { name = "langchain-google-vertexai", specifier = ">=3.0.1,<4.0.0" }, - { name = "langchain-openai", specifier = ">=1.0.0,<2.0.0" }, + { name = "langchain-openai", specifier = ">=1.6.0,<2.0.0" }, { name = "langid", specifier = ">=1.1.6,<2.0.0" }, { name = "levenshtein", specifier = ">=0.27.1,<0.28.0" }, - { name = "mcp", specifier = ">=1.26.0,<2.0.0" }, + { name = "mcp", specifier = ">=1.28.1,<2.0.0" }, { name = "neo4j", specifier = ">=6.0.2,<7.0.0" }, { name = "networkx", specifier = ">=3.0,<4.0.0" }, { name = "networkx", marker = "extra == 'postgresql-backend'", specifier = ">=3.0,<4.0.0" }, @@ -350,24 +353,26 @@ requires-dist = [ { name = "openai", specifier = ">=2.5.0,<3.0.0" }, { name = "pgvector", specifier = ">=0.3.0,<1.0.0" }, { name = "pgvector", marker = "extra == 'postgresql-backend'", specifier = ">=0.3.0,<1.0.0" }, + { name = "pillow", specifier = ">=12.3.0,<13.0.0" }, { name = "psycopg2-binary", specifier = ">=2.9.6,<3.0.0" }, { name = "psycopg2-binary", marker = "extra == 'postgresql-backend'", specifier = ">=2.9.6,<3.0.0" }, { name = "pylint", specifier = ">=4.0.1,<5.0.0" }, { name = "pymilvus", specifier = ">=2.6.2,<3.0.0" }, { name = "pymongo", specifier = ">=4.15.3,<5.0.0" }, - { name = "python-dotenv", specifier = ">=1.1.1,<2.0.0" }, - { name = "python-multipart", specifier = ">=0.0.20,<0.0.21" }, + { name = "python-dotenv", specifier = ">=1.2.3,<2.0.0" }, + { name = "python-multipart", specifier = ">=0.0.32,<0.1.0" }, { name = "redis", specifier = ">=6.4.0,<7.0.0" }, - { name = "sentence-transformers", specifier = ">=2.2.0,<3.0.0" }, + { name = "sentence-transformers", marker = "extra == 'local-ml'", specifier = ">=6.0.0,<7.0.0" }, { name = "spacy", specifier = ">=3.0.0,<4.0.0" }, { name = "spacy-langdetect", specifier = ">=0.1.2,<0.2.0" }, { name = "sympy", specifier = ">=1.12.1,<1.14.0" }, { name = "tenacity", specifier = ">=8.2.3,<10.0.0" }, - { name = "typer", specifier = ">=0.16.0,<0.17.0" }, - { name = "uvicorn", specifier = ">=0.38.0,<0.39.0" }, + { name = "tiktoken", specifier = ">=0.12.0,<1.0.0" }, + { name = "typer", specifier = ">=0.26.0,<0.27.0" }, + { name = "uvicorn", specifier = ">=0.52.4,<0.53.0" }, { name = "watchfiles", specifier = ">=1.2.0,<2.0.0" }, ] -provides-extras = ["docling-ocr", "postgresql-backend"] +provides-extras = ["docling-ocr", "local-ml", "postgresql-backend"] [[package]] name = "cachetools" @@ -561,14 +566,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.1" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, ] [[package]] @@ -665,61 +670,58 @@ wheels = [ [[package]] name = "cryptography" -version = "46.0.5" +version = "50.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/60/04/ee2a9e8542e4fa2773b81771ff8349ff19cdd56b7258a0cc442639052edb/cryptography-46.0.5.tar.gz", hash = "sha256:abace499247268e3757271b2f1e244b36b06f8515cf27c4d49468fc9eb16e93d", size = 750064, upload-time = "2026-02-10T19:18:38.255Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/81/b0bb27f2ba931a65409c6b8a8b358a7f03c0e46eceacddff55f7c84b1f3b/cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad", size = 7176289, upload-time = "2026-02-10T19:17:08.274Z" }, - { url = "https://files.pythonhosted.org/packages/ff/9e/6b4397a3e3d15123de3b1806ef342522393d50736c13b20ec4c9ea6693a6/cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b", size = 4275637, upload-time = "2026-02-10T19:17:10.53Z" }, - { url = "https://files.pythonhosted.org/packages/63/e7/471ab61099a3920b0c77852ea3f0ea611c9702f651600397ac567848b897/cryptography-46.0.5-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d7e3d356b8cd4ea5aff04f129d5f66ebdc7b6f8eae802b93739ed520c47c79b", size = 4424742, upload-time = "2026-02-10T19:17:12.388Z" }, - { url = "https://files.pythonhosted.org/packages/37/53/a18500f270342d66bf7e4d9f091114e31e5ee9e7375a5aba2e85a91e0044/cryptography-46.0.5-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:50bfb6925eff619c9c023b967d5b77a54e04256c4281b0e21336a130cd7fc263", size = 4277528, upload-time = "2026-02-10T19:17:13.853Z" }, - { url = "https://files.pythonhosted.org/packages/22/29/c2e812ebc38c57b40e7c583895e73c8c5adb4d1e4a0cc4c5a4fdab2b1acc/cryptography-46.0.5-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:803812e111e75d1aa73690d2facc295eaefd4439be1023fefc4995eaea2af90d", size = 4947993, upload-time = "2026-02-10T19:17:15.618Z" }, - { url = "https://files.pythonhosted.org/packages/6b/e7/237155ae19a9023de7e30ec64e5d99a9431a567407ac21170a046d22a5a3/cryptography-46.0.5-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ee190460e2fbe447175cda91b88b84ae8322a104fc27766ad09428754a618ed", size = 4456855, upload-time = "2026-02-10T19:17:17.221Z" }, - { url = "https://files.pythonhosted.org/packages/2d/87/fc628a7ad85b81206738abbd213b07702bcbdada1dd43f72236ef3cffbb5/cryptography-46.0.5-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:f145bba11b878005c496e93e257c1e88f154d278d2638e6450d17e0f31e558d2", size = 3984635, upload-time = "2026-02-10T19:17:18.792Z" }, - { url = "https://files.pythonhosted.org/packages/84/29/65b55622bde135aedf4565dc509d99b560ee4095e56989e815f8fd2aa910/cryptography-46.0.5-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:e9251e3be159d1020c4030bd2e5f84d6a43fe54b6c19c12f51cde9542a2817b2", size = 4277038, upload-time = "2026-02-10T19:17:20.256Z" }, - { url = "https://files.pythonhosted.org/packages/bc/36/45e76c68d7311432741faf1fbf7fac8a196a0a735ca21f504c75d37e2558/cryptography-46.0.5-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:47fb8a66058b80e509c47118ef8a75d14c455e81ac369050f20ba0d23e77fee0", size = 4912181, upload-time = "2026-02-10T19:17:21.825Z" }, - { url = "https://files.pythonhosted.org/packages/6d/1a/c1ba8fead184d6e3d5afcf03d569acac5ad063f3ac9fb7258af158f7e378/cryptography-46.0.5-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:4c3341037c136030cb46e4b1e17b7418ea4cbd9dd207e4a6f3b2b24e0d4ac731", size = 4456482, upload-time = "2026-02-10T19:17:25.133Z" }, - { url = "https://files.pythonhosted.org/packages/f9/e5/3fb22e37f66827ced3b902cf895e6a6bc1d095b5b26be26bd13c441fdf19/cryptography-46.0.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:890bcb4abd5a2d3f852196437129eb3667d62630333aacc13dfd470fad3aaa82", size = 4405497, upload-time = "2026-02-10T19:17:26.66Z" }, - { url = "https://files.pythonhosted.org/packages/1a/df/9d58bb32b1121a8a2f27383fabae4d63080c7ca60b9b5c88be742be04ee7/cryptography-46.0.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:80a8d7bfdf38f87ca30a5391c0c9ce4ed2926918e017c29ddf643d0ed2778ea1", size = 4667819, upload-time = "2026-02-10T19:17:28.569Z" }, - { url = "https://files.pythonhosted.org/packages/ea/ed/325d2a490c5e94038cdb0117da9397ece1f11201f425c4e9c57fe5b9f08b/cryptography-46.0.5-cp311-abi3-win32.whl", hash = "sha256:60ee7e19e95104d4c03871d7d7dfb3d22ef8a9b9c6778c94e1c8fcc8365afd48", size = 3028230, upload-time = "2026-02-10T19:17:30.518Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5a/ac0f49e48063ab4255d9e3b79f5def51697fce1a95ea1370f03dc9db76f6/cryptography-46.0.5-cp311-abi3-win_amd64.whl", hash = "sha256:38946c54b16c885c72c4f59846be9743d699eee2b69b6988e0a00a01f46a61a4", size = 3480909, upload-time = "2026-02-10T19:17:32.083Z" }, - { url = "https://files.pythonhosted.org/packages/00/13/3d278bfa7a15a96b9dc22db5a12ad1e48a9eb3d40e1827ef66a5df75d0d0/cryptography-46.0.5-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:94a76daa32eb78d61339aff7952ea819b1734b46f73646a07decb40e5b3448e2", size = 7119287, upload-time = "2026-02-10T19:17:33.801Z" }, - { url = "https://files.pythonhosted.org/packages/67/c8/581a6702e14f0898a0848105cbefd20c058099e2c2d22ef4e476dfec75d7/cryptography-46.0.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5be7bf2fb40769e05739dd0046e7b26f9d4670badc7b032d6ce4db64dddc0678", size = 4265728, upload-time = "2026-02-10T19:17:35.569Z" }, - { url = "https://files.pythonhosted.org/packages/dd/4a/ba1a65ce8fc65435e5a849558379896c957870dd64fecea97b1ad5f46a37/cryptography-46.0.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe346b143ff9685e40192a4960938545c699054ba11d4f9029f94751e3f71d87", size = 4408287, upload-time = "2026-02-10T19:17:36.938Z" }, - { url = "https://files.pythonhosted.org/packages/f8/67/8ffdbf7b65ed1ac224d1c2df3943553766914a8ca718747ee3871da6107e/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c69fd885df7d089548a42d5ec05be26050ebcd2283d89b3d30676eb32ff87dee", size = 4270291, upload-time = "2026-02-10T19:17:38.748Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e5/f52377ee93bc2f2bba55a41a886fd208c15276ffbd2569f2ddc89d50e2c5/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:8293f3dea7fc929ef7240796ba231413afa7b68ce38fd21da2995549f5961981", size = 4927539, upload-time = "2026-02-10T19:17:40.241Z" }, - { url = "https://files.pythonhosted.org/packages/3b/02/cfe39181b02419bbbbcf3abdd16c1c5c8541f03ca8bda240debc467d5a12/cryptography-46.0.5-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:1abfdb89b41c3be0365328a410baa9df3ff8a9110fb75e7b52e66803ddabc9a9", size = 4442199, upload-time = "2026-02-10T19:17:41.789Z" }, - { url = "https://files.pythonhosted.org/packages/c0/96/2fcaeb4873e536cf71421a388a6c11b5bc846e986b2b069c79363dc1648e/cryptography-46.0.5-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d66e421495fdb797610a08f43b05269e0a5ea7f5e652a89bfd5a7d3c1dee3648", size = 3960131, upload-time = "2026-02-10T19:17:43.379Z" }, - { url = "https://files.pythonhosted.org/packages/d8/d2/b27631f401ddd644e94c5cf33c9a4069f72011821cf3dc7309546b0642a0/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4e817a8920bfbcff8940ecfd60f23d01836408242b30f1a708d93198393a80b4", size = 4270072, upload-time = "2026-02-10T19:17:45.481Z" }, - { url = "https://files.pythonhosted.org/packages/f4/a7/60d32b0370dae0b4ebe55ffa10e8599a2a59935b5ece1b9f06edb73abdeb/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:68f68d13f2e1cb95163fa3b4db4bf9a159a418f5f6e7242564fc75fcae667fd0", size = 4892170, upload-time = "2026-02-10T19:17:46.997Z" }, - { url = "https://files.pythonhosted.org/packages/d2/b9/cf73ddf8ef1164330eb0b199a589103c363afa0cf794218c24d524a58eab/cryptography-46.0.5-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:a3d1fae9863299076f05cb8a778c467578262fae09f9dc0ee9b12eb4268ce663", size = 4441741, upload-time = "2026-02-10T19:17:48.661Z" }, - { url = "https://files.pythonhosted.org/packages/5f/eb/eee00b28c84c726fe8fa0158c65afe312d9c3b78d9d01daf700f1f6e37ff/cryptography-46.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c4143987a42a2397f2fc3b4d7e3a7d313fbe684f67ff443999e803dd75a76826", size = 4396728, upload-time = "2026-02-10T19:17:50.058Z" }, - { url = "https://files.pythonhosted.org/packages/65/f4/6bc1a9ed5aef7145045114b75b77c2a8261b4d38717bd8dea111a63c3442/cryptography-46.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7d731d4b107030987fd61a7f8ab512b25b53cef8f233a97379ede116f30eb67d", size = 4652001, upload-time = "2026-02-10T19:17:51.54Z" }, - { url = "https://files.pythonhosted.org/packages/86/ef/5d00ef966ddd71ac2e6951d278884a84a40ffbd88948ef0e294b214ae9e4/cryptography-46.0.5-cp314-cp314t-win32.whl", hash = "sha256:c3bcce8521d785d510b2aad26ae2c966092b7daa8f45dd8f44734a104dc0bc1a", size = 3003637, upload-time = "2026-02-10T19:17:52.997Z" }, - { url = "https://files.pythonhosted.org/packages/b7/57/f3f4160123da6d098db78350fdfd9705057aad21de7388eacb2401dceab9/cryptography-46.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:4d8ae8659ab18c65ced284993c2265910f6c9e650189d4e3f68445ef82a810e4", size = 3469487, upload-time = "2026-02-10T19:17:54.549Z" }, - { url = "https://files.pythonhosted.org/packages/e2/fa/a66aa722105ad6a458bebd64086ca2b72cdd361fed31763d20390f6f1389/cryptography-46.0.5-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:4108d4c09fbbf2789d0c926eb4152ae1760d5a2d97612b92d508d96c861e4d31", size = 7170514, upload-time = "2026-02-10T19:17:56.267Z" }, - { url = "https://files.pythonhosted.org/packages/0f/04/c85bdeab78c8bc77b701bf0d9bdcf514c044e18a46dcff330df5448631b0/cryptography-46.0.5-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1f30a86d2757199cb2d56e48cce14deddf1f9c95f1ef1b64ee91ea43fe2e18", size = 4275349, upload-time = "2026-02-10T19:17:58.419Z" }, - { url = "https://files.pythonhosted.org/packages/5c/32/9b87132a2f91ee7f5223b091dc963055503e9b442c98fc0b8a5ca765fab0/cryptography-46.0.5-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:039917b0dc418bb9f6edce8a906572d69e74bd330b0b3fea4f79dab7f8ddd235", size = 4420667, upload-time = "2026-02-10T19:18:00.619Z" }, - { url = "https://files.pythonhosted.org/packages/a1/a6/a7cb7010bec4b7c5692ca6f024150371b295ee1c108bdc1c400e4c44562b/cryptography-46.0.5-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ba2a27ff02f48193fc4daeadf8ad2590516fa3d0adeeb34336b96f7fa64c1e3a", size = 4276980, upload-time = "2026-02-10T19:18:02.379Z" }, - { url = "https://files.pythonhosted.org/packages/8e/7c/c4f45e0eeff9b91e3f12dbd0e165fcf2a38847288fcfd889deea99fb7b6d/cryptography-46.0.5-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:61aa400dce22cb001a98014f647dc21cda08f7915ceb95df0c9eaf84b4b6af76", size = 4939143, upload-time = "2026-02-10T19:18:03.964Z" }, - { url = "https://files.pythonhosted.org/packages/37/19/e1b8f964a834eddb44fa1b9a9976f4e414cbb7aa62809b6760c8803d22d1/cryptography-46.0.5-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3ce58ba46e1bc2aac4f7d9290223cead56743fa6ab94a5d53292ffaac6a91614", size = 4453674, upload-time = "2026-02-10T19:18:05.588Z" }, - { url = "https://files.pythonhosted.org/packages/db/ed/db15d3956f65264ca204625597c410d420e26530c4e2943e05a0d2f24d51/cryptography-46.0.5-cp38-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:420d0e909050490d04359e7fdb5ed7e667ca5c3c402b809ae2563d7e66a92229", size = 3978801, upload-time = "2026-02-10T19:18:07.167Z" }, - { url = "https://files.pythonhosted.org/packages/41/e2/df40a31d82df0a70a0daf69791f91dbb70e47644c58581d654879b382d11/cryptography-46.0.5-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:582f5fcd2afa31622f317f80426a027f30dc792e9c80ffee87b993200ea115f1", size = 4276755, upload-time = "2026-02-10T19:18:09.813Z" }, - { url = "https://files.pythonhosted.org/packages/33/45/726809d1176959f4a896b86907b98ff4391a8aa29c0aaaf9450a8a10630e/cryptography-46.0.5-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:bfd56bb4b37ed4f330b82402f6f435845a5f5648edf1ad497da51a8452d5d62d", size = 4901539, upload-time = "2026-02-10T19:18:11.263Z" }, - { url = "https://files.pythonhosted.org/packages/99/0f/a3076874e9c88ecb2ecc31382f6e7c21b428ede6f55aafa1aa272613e3cd/cryptography-46.0.5-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:a3d507bb6a513ca96ba84443226af944b0f7f47dcc9a399d110cd6146481d24c", size = 4452794, upload-time = "2026-02-10T19:18:12.914Z" }, - { url = "https://files.pythonhosted.org/packages/02/ef/ffeb542d3683d24194a38f66ca17c0a4b8bf10631feef44a7ef64e631b1a/cryptography-46.0.5-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:9f16fbdf4da055efb21c22d81b89f155f02ba420558db21288b3d0035bafd5f4", size = 4404160, upload-time = "2026-02-10T19:18:14.375Z" }, - { url = "https://files.pythonhosted.org/packages/96/93/682d2b43c1d5f1406ed048f377c0fc9fc8f7b0447a478d5c65ab3d3a66eb/cryptography-46.0.5-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ced80795227d70549a411a4ab66e8ce307899fad2220ce5ab2f296e687eacde9", size = 4667123, upload-time = "2026-02-10T19:18:15.886Z" }, - { url = "https://files.pythonhosted.org/packages/45/2d/9c5f2926cb5300a8eefc3f4f0b3f3df39db7f7ce40c8365444c49363cbda/cryptography-46.0.5-cp38-abi3-win32.whl", hash = "sha256:02f547fce831f5096c9a567fd41bc12ca8f11df260959ecc7c3202555cc47a72", size = 3010220, upload-time = "2026-02-10T19:18:17.361Z" }, - { url = "https://files.pythonhosted.org/packages/48/ef/0c2f4a8e31018a986949d34a01115dd057bf536905dca38897bacd21fac3/cryptography-46.0.5-cp38-abi3-win_amd64.whl", hash = "sha256:556e106ee01aa13484ce9b0239bca667be5004efb0aabbed28d353df86445595", size = 3467050, upload-time = "2026-02-10T19:18:18.899Z" }, - { url = "https://files.pythonhosted.org/packages/eb/dd/2d9fdb07cebdf3d51179730afb7d5e576153c6744c3ff8fded23030c204e/cryptography-46.0.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:3b4995dc971c9fb83c25aa44cf45f02ba86f71ee600d81091c2f0cbae116b06c", size = 3476964, upload-time = "2026-02-10T19:18:20.687Z" }, - { url = "https://files.pythonhosted.org/packages/e9/6f/6cc6cc9955caa6eaf83660b0da2b077c7fe8ff9950a3c5e45d605038d439/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:bc84e875994c3b445871ea7181d424588171efec3e185dced958dad9e001950a", size = 4218321, upload-time = "2026-02-10T19:18:22.349Z" }, - { url = "https://files.pythonhosted.org/packages/3e/5d/c4da701939eeee699566a6c1367427ab91a8b7088cc2328c09dbee940415/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:2ae6971afd6246710480e3f15824ed3029a60fc16991db250034efd0b9fb4356", size = 4381786, upload-time = "2026-02-10T19:18:24.529Z" }, - { url = "https://files.pythonhosted.org/packages/ac/97/a538654732974a94ff96c1db621fa464f455c02d4bb7d2652f4edc21d600/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d861ee9e76ace6cf36a6a89b959ec08e7bc2493ee39d07ffe5acb23ef46d27da", size = 4217990, upload-time = "2026-02-10T19:18:25.957Z" }, - { url = "https://files.pythonhosted.org/packages/ae/11/7e500d2dd3ba891197b9efd2da5454b74336d64a7cc419aa7327ab74e5f6/cryptography-46.0.5-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:2b7a67c9cd56372f3249b39699f2ad479f6991e62ea15800973b956f4b73e257", size = 4381252, upload-time = "2026-02-10T19:18:27.496Z" }, - { url = "https://files.pythonhosted.org/packages/bc/58/6b3d24e6b9bc474a2dcdee65dfd1f008867015408a271562e4b690561a4d/cryptography-46.0.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8456928655f856c6e1533ff59d5be76578a7157224dbd9ce6872f25055ab9ab7", size = 3407605, upload-time = "2026-02-10T19:18:29.233Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/5c/59086b4aac5e879d38ddbcf74e4be7ade89cebc3eb199a55da998c3bb46a/cryptography-50.0.0-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:031e2d5dd4bb9caa3ca9c82e5a197fd8ae680232cee62603d1a813f3f07e3d03", size = 4001252, upload-time = "2026-07-31T14:23:33.331Z" }, + { url = "https://files.pythonhosted.org/packages/57/ef/8f2df13c7216bcad3e1c74e07f6e193d93e998e114f524a53877c9af27ad/cryptography-50.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fd9192b7b70c573d7f214eb1ae35e00d359f6f5e4b27c7e21e30de1fc6204645", size = 4719554, upload-time = "2026-07-31T14:23:35.611Z" }, + { url = "https://files.pythonhosted.org/packages/d9/41/029086c34d91052fc3b88bcc8056f709a7c915c7a23b235a54eb800b1c97/cryptography-50.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:06a32a980526a6ab9a4b9bf8f7385800791e2bb960903cb6b530e4817509a3b7", size = 4702130, upload-time = "2026-07-31T14:23:37.635Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ff/b6ce0954962e7f7b969f850a883744197bb3910bdfd7b6da162eab7d9f68/cryptography-50.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a1b30560f2acc95aa8b2e06e716a13dbfc97314747b80d9707e307f77b40d6b3", size = 4725244, upload-time = "2026-07-31T14:23:39.471Z" }, + { url = "https://files.pythonhosted.org/packages/06/1e/63a1027cb7fec360a182208e1b7767d5aa1fe57be3d6aa856e69a321edc0/cryptography-50.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:8d89f3976b10b4ce31118de72329025f70d2c6ead14a8217c5514dd2c6d5a78f", size = 5342265, upload-time = "2026-07-31T14:23:41.286Z" }, + { url = "https://files.pythonhosted.org/packages/6b/72/a1116d683a6d7ece94590013882515de087edf9ef0e6292aae615a44df73/cryptography-50.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b42a28c1844fd9de8f3f7d540e36b66f3a9c83fceac7170ebc7a6a19edd9dcae", size = 4734609, upload-time = "2026-07-31T14:23:43.139Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/36a9c479bbe49acea2636c7fd3360d20f7b7e079c300352011c44850b181/cryptography-50.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:900131fafd8aead39ac7dd3a7e833be754c17a95cfd91221636949fe4eb0aa8a", size = 4356517, upload-time = "2026-07-31T14:23:44.939Z" }, + { url = "https://files.pythonhosted.org/packages/32/98/8a151d64367204cbc63ec65d37502f1d9c53cf4bfc6ec3c532614dbec60d/cryptography-50.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:07949c449a1abcf60d1ee6e88956d89404c7df3c8258f46589e912988e551987", size = 4724529, upload-time = "2026-07-31T14:23:46.93Z" }, + { url = "https://files.pythonhosted.org/packages/22/f6/ec13b470172126464a86bf54d2294a46d29837fc51ba3e45d4047946fb5e/cryptography-50.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f89831ef99dd7dd169ab06d63a831adb9e20a87aac6d380266bbda5823349169", size = 5299852, upload-time = "2026-07-31T14:23:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/da/3a/f05e32c99d440c9bb891ea0e36c9091891e36be5a9a87ab2ee6ea20729f6/cryptography-50.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:82148ec5bddac30b51a5b3c1945075f896fa022cb93f8e4a01e9f6ee95292c5f", size = 4734462, upload-time = "2026-07-31T14:23:50.861Z" }, + { url = "https://files.pythonhosted.org/packages/ca/dc/bd72b26be8953f80625f63151efd38eee71c76ca6cf591c08ff34615a79e/cryptography-50.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1489e263a8048bb8b6a8bac662eb2d402ea5d2b7b4699b72f385f1e2772db105", size = 4852708, upload-time = "2026-07-31T14:23:52.715Z" }, + { url = "https://files.pythonhosted.org/packages/27/20/c930314a2ab476d15dec966ec87e2e9637bb02b06106b12c0396c57bb603/cryptography-50.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:7cec5b856506da6defb290f30c9ee687d5f5e8cb0bd3f6459dde43b0b4fa40ef", size = 5004179, upload-time = "2026-07-31T14:23:54.887Z" }, + { url = "https://files.pythonhosted.org/packages/32/2e/c9db68a0c4bfa28e310707527c0ee3a2bd254104d2e02e68f368e197aa4c/cryptography-50.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:bd1c592e4d5974f0d08d4888e432157adba757c66da0246918e43677fafa2d30", size = 3840395, upload-time = "2026-07-31T14:23:56.677Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fb/951032a3bf22a5697c83183fb6294a4843772947a70e616c57b3ff5f522e/cryptography-50.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:49e7d93abdbd2990caced757e5fade25302f719c3c8fb6e6fff2dde98999fc41", size = 3989258, upload-time = "2026-07-31T14:23:58.881Z" }, + { url = "https://files.pythonhosted.org/packages/d4/67/91eb047e69c5e845f2f14b8a2e4a1aab0f283cb885531e9e22c8adb176bc/cryptography-50.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:19736989797678c6af1e55cd49055cdbcb55d8f6b5583ac5335f933aba9101dc", size = 4700648, upload-time = "2026-07-31T14:24:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/30/82/85f0f7425c856b9f96459411eb12e74ef72df9caf6f8f15bf23a33ff131f/cryptography-50.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80b63928fa35083b33966ce1efb70e5b9607181e49dcd1c22c8c005e319f667f", size = 4682442, upload-time = "2026-07-31T14:24:02.538Z" }, + { url = "https://files.pythonhosted.org/packages/1a/28/b555a365adff1cca2fbe7b9e487d68a40de6bc67ff2cb587473eb43de0e7/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d58c3db7cd6eed54e6c06744db55456b65ebd7492ddeae9c1e93cfca7aa857d3", size = 4707596, upload-time = "2026-07-31T14:24:04.394Z" }, + { url = "https://files.pythonhosted.org/packages/72/d8/f52538140cc719df62a01cf87d1c7142318d235817109d6f4054d7c352d6/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:df2a58a472f332225671c35b0a830208b86d004f82baa8530fa3782c85646533", size = 5314552, upload-time = "2026-07-31T14:24:06.31Z" }, + { url = "https://files.pythonhosted.org/packages/38/14/6120e5bd7c5aa022ad15424ba4d5c5269d0d9448ed4d55e492ea91e3c1c4/cryptography-50.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:11b74db56cdbe3cdee6e3f6982ecb70334fa10dce99ed58bf7894aaaa3b2a037", size = 4717113, upload-time = "2026-07-31T14:24:08.349Z" }, + { url = "https://files.pythonhosted.org/packages/fa/71/190bf38c3ee2e0f8efc9860ae100c9df4169742eef274b91e7aa1cb133b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f59e38625469987d7ef6d495323c55e7db6c212eaf6112267e0d3b565a2e9c9f", size = 4338580, upload-time = "2026-07-31T14:24:10.227Z" }, + { url = "https://files.pythonhosted.org/packages/3a/63/504ccfbbe61fd8aa983f7f146399cdf034c72c2fc55f5b2dfdcdcdb20c99/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ecfed7367f965a0328cfbdd70da860f15441f002f613185668c6e6ebf5a0ac11", size = 4707038, upload-time = "2026-07-31T14:24:12.169Z" }, + { url = "https://files.pythonhosted.org/packages/01/77/2cf79bbfc4d12ca106437a6e170d6aaa01a373e93093118aaaef0e801bd4/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:9aa87839c383bdbab6ef865787a1fb877af8dd03464c4400322726feaaadfc6d", size = 5273110, upload-time = "2026-07-31T14:24:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/e5/45/8aae2972c520145377ea3559a605a899bebe227bf070b33cdb445929a9b9/cryptography-50.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:6ba6a53445bd3cfa809ef3ef5f1589aa6ba08784a1d962bf47d0940e871dab1c", size = 4716439, upload-time = "2026-07-31T14:24:16.415Z" }, + { url = "https://files.pythonhosted.org/packages/7b/20/4fe50b619a48c2525cc46e2dbc1ac490708d704be5d467bdaac6dc955682/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3f5735ffe4996d28b809371756219f5354864902a3b9e7c0b9ee87041209fc9c", size = 4837383, upload-time = "2026-07-31T14:24:18.553Z" }, + { url = "https://files.pythonhosted.org/packages/92/91/3a31366e183343d3703f8995c095f5734676bd6938118047e50fcf279eb4/cryptography-50.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1b4a266766514614f8aa60416e71f2fc6e575d36e7bdc90f644fadb2f4b75b95", size = 4985772, upload-time = "2026-07-31T14:24:20.385Z" }, + { url = "https://files.pythonhosted.org/packages/74/9a/02ffe35b2853d121689871eb5dce862092562b3a1ed5cc98f1aaed441506/cryptography-50.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:12b9c6996425c76ea6c457ace4f3073e715b8c545add07cd1a8f3a4f90691269", size = 3816291, upload-time = "2026-07-31T14:24:22.125Z" }, + { url = "https://files.pythonhosted.org/packages/03/37/73d005be173aff344af30e9fd2a576575cb2391a7101d9cd3842e1fa8cce/cryptography-50.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ccdc4a71a4dabae05de219404f9f4abc38e3b58422177ff93d0da05967dafa07", size = 4036009, upload-time = "2026-07-31T14:24:24.122Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c6/7a6202a534e32103a285b7834a120869557fe198d51d7cfe59754c8bda9c/cryptography-50.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:910e1d2668e7de9648f2bcee30e180db2a6b15c30f887d7c4c93ddf96e3992e3", size = 4745252, upload-time = "2026-07-31T14:24:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/85/4f/0fa8c2f4428198f15d9ff8d63400e27afbf94ce833f6108da1eb3753f945/cryptography-50.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a91296cb61e8df6f86d0c19cc4068228da256bf59bf86049fbd821084565327f", size = 4728939, upload-time = "2026-07-31T14:24:27.994Z" }, + { url = "https://files.pythonhosted.org/packages/d1/63/54dd723490ba2dc09b299682c10b38db38f159728bcaae8c591b8af2f22d/cryptography-50.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e722f16708d854fe924790e051061f6704a472c3bac347b6fd88033ea8dd0dc5", size = 4748483, upload-time = "2026-07-31T14:24:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/1d/dd/7c77d26285cc7f6991efce64a0f5b4f9383bfa5dd8c5033003eaf7db4cdb/cryptography-50.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:d764dcf130c428ef66786f866dd750f53182bc608813489915e9fc106bb0c82f", size = 5367599, upload-time = "2026-07-31T14:24:32.457Z" }, + { url = "https://files.pythonhosted.org/packages/46/c9/f60aed34c013f317f92817b6c171c2d22a78270fa41109bd4b08af26b194/cryptography-50.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:105110f43a471dbd0060b9c9516cb8a6a79233631a04cc2ba16f28323ac6e025", size = 4762647, upload-time = "2026-07-31T14:24:34.599Z" }, + { url = "https://files.pythonhosted.org/packages/be/f3/f9a0173b139372c3a48ed98154b45cc6b9de17c789d5ab552e621c293609/cryptography-50.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:828743d939e9629bc267b8e2d08d8bb67cd4319c771a33d4b18b22dd8fb7440a", size = 4385197, upload-time = "2026-07-31T14:24:36.647Z" }, + { url = "https://files.pythonhosted.org/packages/d8/36/83bb81f6e569bc38e1e4a7bc80f29b46bb9601920bc455fc8e888f5d5742/cryptography-50.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a8183b489dc1f7f80f135780fadc1108f14b31b8a40411c7a5b17425f65f28b", size = 4748095, upload-time = "2026-07-31T14:24:39.493Z" }, + { url = "https://files.pythonhosted.org/packages/6b/16/d3008eff98c764979865834c3d386d4fd041b5f52e7f34fc29ac1a5eb515/cryptography-50.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:6e7d61120573a7f2cd94cc095f9e81f6967c61ccdf194285aa143ecec8e0b708", size = 5325948, upload-time = "2026-07-31T14:24:41.556Z" }, + { url = "https://files.pythonhosted.org/packages/9c/f8/d97f9603efda3888187bfdb893f26c41be4735c10631d05d284ee6b047c4/cryptography-50.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:37fdb0d0111f1e2ff07139dfb79f1b49531f8e213c46f1163dd7642979b58c47", size = 4762400, upload-time = "2026-07-31T14:24:43.636Z" }, + { url = "https://files.pythonhosted.org/packages/64/a2/4615c8f7d81a00b1d6e6afe19f694e1543582349fb5f4076f6cb5dc36485/cryptography-50.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87f62a3d3b9888ed0fdde100ec06aa61ca9cd44bad9057d1dff9a516b5f5bb9", size = 4878208, upload-time = "2026-07-31T14:24:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/d2/1a/efcfb02f91407149a0dacffffab791f7e19bf6385f63b3666dc8b5e5c9c8/cryptography-50.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:65c2c3add92b45fd0709db8594536aea39c2a67af0e27ffcf049c498501140b7", size = 5037050, upload-time = "2026-07-31T14:24:47.697Z" }, + { url = "https://files.pythonhosted.org/packages/57/30/4a22984d4f1bdfb8c054f07a92bc176b97a3134cc1d6c4b3bffb1f3688b4/cryptography-50.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:d24fead1d4d076e1bfb006dcec392074a3cd8d7b4fc8a595aa64073b2b7a96ba", size = 3874135, upload-time = "2026-07-31T14:24:50.085Z" }, + { url = "https://files.pythonhosted.org/packages/9d/3e/e54cde8c01631a5a8226ccd617eab9e57fd5cfdad90f1a9e6bb570794631/cryptography-50.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5e34edd123674534acd70147f0ca331eaa2c74e6325fb2028c886aa26ba0b68c", size = 3963170, upload-time = "2026-07-31T14:24:51.968Z" }, + { url = "https://files.pythonhosted.org/packages/01/b6/0b9e125e90f3d2dcf599a218a899cda7326a3158cfa258723f0b398b08f6/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:8eb5e1172eb569ea8a872796576e6a67c276351728b6455d5beb01242b027c6a", size = 4692441, upload-time = "2026-07-31T14:24:53.743Z" }, + { url = "https://files.pythonhosted.org/packages/53/c9/a5151588710785a96d7bc4de27d4cd62f263bbbcb203cfe29df537eb6505/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:910d11e1a385c654bf738bf3e6b8e6ed5de0f5610fcae2be9e5b398d8081d20e", size = 4699810, upload-time = "2026-07-31T14:24:55.746Z" }, + { url = "https://files.pythonhosted.org/packages/c7/1a/15b92b25eb6ce3089cd49377ae990a0f3ad485a510f968aed1f19dbdcdf2/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:62598a8a57f815db4c6259a4e97d857dab56697e7de8e8ab02352ab74da1995d", size = 4691924, upload-time = "2026-07-31T14:24:58.082Z" }, + { url = "https://files.pythonhosted.org/packages/62/15/219075012ab13e8905f3cd572204f4acb4b111df787104346b9bc0cea789/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:07479a1cb08219ab719147e742e76090c9c773321959bb94946fffdd397a6437", size = 4699593, upload-time = "2026-07-31T14:24:59.951Z" }, + { url = "https://files.pythonhosted.org/packages/8e/b5/c2c5fce26f0ee40d21bafe7f191d29a34b35a65ac4fe8a1191d1983612e9/cryptography-50.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c99c003e088647b8a5b7c145d6f78c335f6348332b62e142d411c4b63d1460b9", size = 3813796, upload-time = "2026-07-31T14:25:02.298Z" }, ] [[package]] @@ -839,65 +841,52 @@ wheels = [ ] [[package]] -name = "docling" -version = "2.77.0" +name = "doclang" +version = "0.7.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "accelerate" }, - { name = "beautifulsoup4" }, - { name = "certifi" }, - { name = "defusedxml" }, - { name = "docling-core", extra = ["chunking"] }, - { name = "docling-ibm-models" }, - { name = "docling-parse" }, - { name = "filetype" }, - { name = "huggingface-hub" }, { name = "lxml" }, - { name = "marko" }, - { name = "ocrmac", marker = "sys_platform == 'darwin'" }, - { name = "openpyxl" }, - { name = "pandas" }, - { name = "pillow" }, - { name = "pluggy" }, - { name = "polyfactory" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pylatexenc" }, - { name = "pypdfium2" }, - { name = "python-docx" }, - { name = "python-pptx" }, - { name = "rapidocr" }, - { name = "requests" }, - { name = "rtree" }, - { name = "scipy" }, - { name = "tqdm" }, { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5e/2c/5fd1ff2666c3e7a75f9bfa058056be10c3495d57c16a9a4fd60b91b47ba8/docling-2.77.0.tar.gz", hash = "sha256:d410e9a4f457951f9da43a2ed2e6103747d60494f5e75ac64c54038d57ef903d", size = 372156, upload-time = "2026-03-06T13:46:44.234Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/3a/005e4856ad8e9b9879414a4df4dbc56dc3663b96f9d8c920ef210e8931cf/doclang-0.7.3.tar.gz", hash = "sha256:ca50615357e46ebf9597bb9065b9112367103ec24bd539f8ae12649224cf50b0", size = 31569, upload-time = "2026-07-15T08:11:02.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/81/334ccc0f0cd7c3d75996b6b596e7f4c62c4c46a0ca042003315c28170159/doclang-0.7.3-py3-none-any.whl", hash = "sha256:9440c4ca9f7e061a7b8d33bdf15b1029be69a4c13cd8952dd6ce541884e4c685", size = 32267, upload-time = "2026-07-15T08:11:01.977Z" }, +] + +[[package]] +name = "docling" +version = "2.121.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docling-slim", extra = ["standard"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/8d/97e3763f0cc13448521a928298d609b22a77b1eef11c698341bf19ddd41d/docling-2.121.0.tar.gz", hash = "sha256:c8970cbd3907892db218c7f342ff62db5b393ca3cf568db50c8f2384e248a22d", size = 9013, upload-time = "2026-08-20T09:07:44.568Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/08/0330d839d618b79188f8087551662dfbb8efd6fbfd5d33a2511138d98df0/docling-2.77.0-py3-none-any.whl", hash = "sha256:e6f7df5fbd878a47e9c6ce4d1746aada8c4d674c5342345c8992bd19bd07a9ac", size = 400712, upload-time = "2026-03-06T13:46:42.377Z" }, + { url = "https://files.pythonhosted.org/packages/d3/c8/190eaffb6113dc3f7fdc4282cd67efdd51bba3bacb2e5b31bc5d77a78df6/docling-2.121.0-py3-none-any.whl", hash = "sha256:d1577ac7660ac6cb3ac7e213f1541f8bed1990ff8da4274d94b5bfc91e0fec55", size = 5177, upload-time = "2026-08-20T09:07:43.424Z" }, ] [[package]] name = "docling-core" -version = "2.69.0" +version = "2.92.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "defusedxml" }, + { name = "doclang" }, { name = "jsonref" }, { name = "jsonschema" }, { name = "latex2mathml" }, { name = "pandas" }, { name = "pillow" }, { name = "pydantic" }, + { name = "pydantic-settings" }, { name = "pyyaml" }, { name = "tabulate" }, { name = "typer" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/28/58/00e46ffb2c184fdb9a90ae1d93e522f70f4679cf9ac8312688a2c3f67c59/docling_core-2.69.0.tar.gz", hash = "sha256:9760e13063de5f373dbc3417de3ef9a3709c6eb18c0e60ada3c1f96c8b37c14e", size = 267556, upload-time = "2026-03-09T04:32:12.542Z" } +sdist = { url = "https://files.pythonhosted.org/packages/47/9c/8ab960c09e501b57dda1b823dff3264bed70ab70ef60328767be38ca30f5/docling_core-2.92.0.tar.gz", hash = "sha256:33fd25e38c199336447a21925374400aca13a6b9316a032f111972c2dc0f085c", size = 360897, upload-time = "2026-08-19T10:55:21.431Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/7b/a49ff5ae79f15ed9f56c8dffdbf5d7cfc5ca8a60f237e0e63fabed0582dd/docling_core-2.69.0-py3-none-any.whl", hash = "sha256:4c3817ce00c7cb5622e59ddb4e5268a58083ae67f6e1436b36a84b6b4bbf6899", size = 247127, upload-time = "2026-03-09T04:32:11.022Z" }, + { url = "https://files.pythonhosted.org/packages/bb/d1/7f6d9e737ea5c41c0fd4d2819732fc43416c0a60c5870342381db149d368/docling_core-2.92.0-py3-none-any.whl", hash = "sha256:726d89c23197e53be2f7192bb4c00108ff545830cf9ab7b981fb014fa92b5c02", size = 295507, upload-time = "2026-08-19T10:55:19.992Z" }, ] [package.optional-dependencies] @@ -913,7 +902,7 @@ chunking = [ [[package]] name = "docling-ibm-models" -version = "3.12.0" +version = "3.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "accelerate" }, @@ -930,40 +919,94 @@ dependencies = [ { name = "tqdm" }, { name = "transformers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/95/57/b4cee1ea5a7d34a8a96787aa2dc371e4dd17a1a3bd6131cf3ced9024f1be/docling_ibm_models-3.12.0.tar.gz", hash = "sha256:85c2b6c9dbb7fbb8eaf0f2a462b5984626457a6dc33148643491270c27767b46", size = 98458, upload-time = "2026-03-09T12:27:35.744Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/b5/f95bd7df8acc3b792fd11c6d00c8a9ea3966b10edd734fb194d07be29606/docling_ibm_models-3.14.0.tar.gz", hash = "sha256:def964e3d524f66c7321ef9d48d4021278f14319f01d3f78058cd2324f641e22", size = 100765, upload-time = "2026-08-11T06:58:25.953Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/ed/820fdcea9aa329119855a658366fb13098b375a2b024358b1d7836f47419/docling_ibm_models-3.12.0-py3-none-any.whl", hash = "sha256:008fe1f5571db413782efe510c1d6327ea9df20b5255d416d0f4b56cfd090238", size = 93800, upload-time = "2026-03-09T12:27:34.477Z" }, + { url = "https://files.pythonhosted.org/packages/4e/96/c168e6d31397203eeed572c3b2cd1bb1efff7f9b0753cb53855fd505894a/docling_ibm_models-3.14.0-py3-none-any.whl", hash = "sha256:795d39cd0f7b1e14a702e681b0ef0f9bd31deaedddb4e2686ad577296ecb8fc9", size = 94362, upload-time = "2026-08-11T06:58:24.513Z" }, ] [[package]] name = "docling-parse" -version = "5.5.0" +version = "7.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "docling-core" }, { name = "pillow" }, { name = "pydantic" }, { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "tabulate" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e6/94/68453bf4136e82f7c94168f0332466822cdb5f226c8a0e1335de21c595ed/docling_parse-5.5.0.tar.gz", hash = "sha256:0914c7174f8fe497d406f4814a70cdfccb4e09d8b2ba90a6e92d02704f5a4a65", size = 57526362, upload-time = "2026-03-04T10:27:44.689Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/e4/1454330adcc7f850dc639a0fe7d50c69becaf1cb4d3a6cf484671cecd2de/docling_parse-7.15.0.tar.gz", hash = "sha256:fcb1b4e9a0e447f29e8248e46eea502ab4c7a75fe733a2ed911635a9b74b5092", size = 6911331, upload-time = "2026-08-20T08:31:13.52Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/f3/387ea2d5c006036f5d9f8dd510aa1b9a968f932ee676475e6c702e92a3f5/docling_parse-5.5.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:980591e6c71db27e2cdcc3b7f999b77401793b897b3d2f87b83c9c3913b77098", size = 7794931, upload-time = "2026-03-04T10:27:17.398Z" }, - { url = "https://files.pythonhosted.org/packages/22/16/6c70f5c4f9c2bf80f77378e780a16d887ba7827ad376c295b9ce90f38e80/docling_parse-5.5.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccbb063c665016e3bd231720db18c13eba98c2bf78a745c9276413e3d7ee46ea", size = 8196345, upload-time = "2026-03-04T10:27:18.694Z" }, - { url = "https://files.pythonhosted.org/packages/8e/3e/62f767db4684d761e1ad4890759c25d202012d36ce113dd6b89765e3c448/docling_parse-5.5.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70545978146d6a4d763f524d09d9f1ae6f58819a71d8aa186d7aa1d6a20d4a92", size = 8298038, upload-time = "2026-03-04T10:27:19.934Z" }, - { url = "https://files.pythonhosted.org/packages/21/76/734246f62997d4da08ef1af569b26bbaef6f09d97f82a871052720080c70/docling_parse-5.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:13eac1ef3cd99b5d2659cf01799d11b9cd8e2fb6bf90392ac081a1bf54b5fdcd", size = 9202075, upload-time = "2026-03-04T10:27:21.324Z" }, - { url = "https://files.pythonhosted.org/packages/6a/f5/7fbebe472f2964be18527e6728a3d8e618b6272b9555a08f38f9e7912515/docling_parse-5.5.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:75521736460264dc77ee54b2f5bb96fa2607983ba97c99a06e3f4a4d534dbf53", size = 7795798, upload-time = "2026-03-04T10:27:23.044Z" }, - { url = "https://files.pythonhosted.org/packages/78/43/c4443f5f88e892a6337165c541cc348e3c4ecf29fa04781e144cde56064b/docling_parse-5.5.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a464610e170c50ba25c41f053ba5141fa4fa338fcabec1c857416cfcf23dfa8", size = 8196508, upload-time = "2026-03-04T10:27:24.564Z" }, - { url = "https://files.pythonhosted.org/packages/15/12/0d71fa0ce26b7f77ebd86de1108a4954a08be4969ff48fd027ed90a90594/docling_parse-5.5.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:574b0d3d2beaeaa6e4aeafc63563204ae784758179c14980fa63e26a4b315a39", size = 8295306, upload-time = "2026-03-04T10:27:25.898Z" }, - { url = "https://files.pythonhosted.org/packages/b3/12/a0f6d8ad390d27d300ea04834ccd1c776e3713534c78c0cecc08a5af0414/docling_parse-5.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:be0ee5c30455a9865e1fa6b5d9ff0a2fdc42094ac12c04902296429e3f4d8c39", size = 9203995, upload-time = "2026-03-04T10:27:28.045Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5d/e09e9d8a994cea16cec13276a9d5a67703fd27f1c67fcea04bb175ea8acf/docling_parse-5.5.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:a6916033bab153741f97fe96020f8db9b3567d5d5db71d470ac9a7184889ca59", size = 7795798, upload-time = "2026-03-04T10:27:29.875Z" }, - { url = "https://files.pythonhosted.org/packages/4f/27/2b7e67cd1fe918fe91b62afea242d9fac8d49da50f90271962b40823edaa/docling_parse-5.5.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88f9dde1903652bd6b153fdeaa1f9043f3cc6762386b2b69258ad2414019db1d", size = 8196635, upload-time = "2026-03-04T10:27:32.272Z" }, - { url = "https://files.pythonhosted.org/packages/a3/29/352858ee62c09d27aad6312f383d15608526b63a0c7f10fe769ffc7e8735/docling_parse-5.5.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b0e78c3b32078b2ffa8ea08dbada5e3708c70f302a1b9d1468626d2682b93901", size = 8295906, upload-time = "2026-03-04T10:27:33.831Z" }, - { url = "https://files.pythonhosted.org/packages/51/74/34fbe48a058fb4fef8e3ded0392b8ba64741a32561e5f23b677cb29b39a2/docling_parse-5.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:db271c7b2323b09b8296eb9c321e016262241bf62f3586a9d3b12f53b97d5a29", size = 9203923, upload-time = "2026-03-04T10:27:35.779Z" }, - { url = "https://files.pythonhosted.org/packages/bd/b0/c2ca426423abc6bf92cafa02c0d1b16c03223d9c8e90c46bf12b247890ab/docling_parse-5.5.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:fbee5f433bc26bd643affb391c8eea2ffb59207526caded09327f1ac25bc5164", size = 7796472, upload-time = "2026-03-04T10:27:37.329Z" }, - { url = "https://files.pythonhosted.org/packages/49/93/8fe80eaf72b514b6ba0e2a07f57a006572608bda05ac18279482cc547958/docling_parse-5.5.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f29ee02a63fac5a5611f318954d4ae383758cd22c851b5298a926320e368a4e", size = 8196803, upload-time = "2026-03-04T10:27:38.989Z" }, - { url = "https://files.pythonhosted.org/packages/14/58/af4769eb8c716e232f5fddc369c1ac821d8f9030f8c9f85d22ef104930c2/docling_parse-5.5.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:75753287a042f47119fbb131effc17668bd54fce71b940dac41d7ed1a37a1a0d", size = 8296215, upload-time = "2026-03-04T10:27:40.646Z" }, - { url = "https://files.pythonhosted.org/packages/92/c8/8c04fe1582fae1e65e1ada2dd651a2bd229caa031ffbcdc0f4719392f50e/docling_parse-5.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:ce24075b85557ae5943d99793d007379c0ef61f2529cf942334fa1306c58022c", size = 9564482, upload-time = "2026-03-04T10:27:42.254Z" }, + { url = "https://files.pythonhosted.org/packages/ef/cf/687829d3d52bfa63d9ea8dc928c158a55a8a35eb9a67498be89675e23bd4/docling_parse-7.15.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:b6453b243d998a73a2fbd1245d202dd315fb1f915e1ccfafdce98d391cade8ae", size = 9742716, upload-time = "2026-08-20T08:30:30.609Z" }, + { url = "https://files.pythonhosted.org/packages/a1/38/75dae655a1af5bf90fd1b5a5d3424dfd92b4b2613b89931acd827f069c75/docling_parse-7.15.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a00c19112264b4e6b0d7d5baa9ccd74997b6252924e512f1c88cf30de5bea1f5", size = 10263321, upload-time = "2026-08-20T08:30:33.148Z" }, + { url = "https://files.pythonhosted.org/packages/a9/20/e940091a488a9fccac7502389f2662e588364cfab736a1507c63d9149821/docling_parse-7.15.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4cb6494f8ca158c774cd7b68d96b11054ec8a92f64fb3e8cc355a8cc858997fc", size = 10659211, upload-time = "2026-08-20T08:30:34.903Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1f/237eb95370bbac40effeab82435237ba3cbd02e89e7d989f7b7d99a67f40/docling_parse-7.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:bbb301b0e7a283745fa54fa20c4279a95a0500a2228362cff5a4a6f82e740fd5", size = 11718460, upload-time = "2026-08-20T08:30:36.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/06/42755b3035f13d0dc317cf352d8e98b03df5ee414e75ad469c9eb3b7a1d1/docling_parse-7.15.0-cp311-cp311-win_arm64.whl", hash = "sha256:b59940048cf0216e09a9de9c5c1bd86d061f89e7fe0ca49f6388061626d79043", size = 9031611, upload-time = "2026-08-20T08:30:39.15Z" }, + { url = "https://files.pythonhosted.org/packages/4f/20/e9ecf45568c7d1e8c0e775190bf88bc65b7fa03adb223cd2976e4989b66a/docling_parse-7.15.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a425599d40aa8b72a443c5fc9e22c120163d398a341af988c953e0e70f1ad0f3", size = 9744461, upload-time = "2026-08-20T08:30:41.97Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fb/09afa8b1813ed0f3b691d59b80b335180e0a69bc8f2906b9aac9737c695d/docling_parse-7.15.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4ea8d79e870319a583a68b3e88e95d048dfa496d8b6e5928f5333fe26cc597e3", size = 10266993, upload-time = "2026-08-20T08:30:44.143Z" }, + { url = "https://files.pythonhosted.org/packages/02/34/3d290b2f5f0576645c9d056f087a77f4e45c6662eb6ce77834c85f00affe/docling_parse-7.15.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca982b7df91f05b0bdcb28f437b2637623baccd205b941297f43b9e93aeb1f14", size = 10664829, upload-time = "2026-08-20T08:30:46.172Z" }, + { url = "https://files.pythonhosted.org/packages/08/84/4733989d2f1687c7dabb08cff13f53005878d1a6638142898f324bc233f4/docling_parse-7.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:6143e824b00dbaf14bd3692792bd0b41348f582fa7ed221ff462079935c127f6", size = 11724625, upload-time = "2026-08-20T08:30:48.371Z" }, + { url = "https://files.pythonhosted.org/packages/47/a3/1a327900335a30b86e2f1330764e340d98c693cc26ec64a1159efcc1ab5f/docling_parse-7.15.0-cp312-cp312-win_arm64.whl", hash = "sha256:f528916ec21c1bec5916db2e9c6ecac4c472fedd6aae4a9cf308f59cf8fcbd7b", size = 9032075, upload-time = "2026-08-20T08:30:50.803Z" }, + { url = "https://files.pythonhosted.org/packages/23/79/a23aa5ea3da51cf2fd320c9fd46e1c031f8b96c5762221315b2b92257a95/docling_parse-7.15.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:01d853d3b7f9c231351f9d15c1cb0746ab56c3d8329a850a1b706ef279af09ed", size = 9744478, upload-time = "2026-08-20T08:30:53.208Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e3/dca5e16ace7c0bcbf6f92fd3318068be2a8d2b0bffef90d6fab5c0e09130/docling_parse-7.15.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa573d66c428ceed2fb3ae968823af79eb529abffaf95d60a7ed74cde9f9cc5b", size = 10266974, upload-time = "2026-08-20T08:30:55.226Z" }, + { url = "https://files.pythonhosted.org/packages/11/9f/c65a613bcd5a5fc6c024b1d0a9bb8a00a20111750c60e6b34e708c8dc683/docling_parse-7.15.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3523b631ca10180fe414c2d9cd11579171289936353fa5af3dd29caba541458d", size = 10664851, upload-time = "2026-08-20T08:30:57.243Z" }, + { url = "https://files.pythonhosted.org/packages/3d/07/cdfa96bafea1fed0a078902c38ff677656ed67bb11fe7c6140304931cad5/docling_parse-7.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:6ff600dfeb2c2d9e0e66a00efeb15d59c8d543b38c6f1bcfcb1f86156216e086", size = 11724552, upload-time = "2026-08-20T08:30:59.406Z" }, + { url = "https://files.pythonhosted.org/packages/14/ce/62e5b4a833bf640153dafe3a838e36f12fa045fb77a511c356a927ad8399/docling_parse-7.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:e4bc3bf49bfd56447f35d1fe2e86897893da3b6c154f0c44643ff311e12ff8b5", size = 9032111, upload-time = "2026-08-20T08:31:01.299Z" }, + { url = "https://files.pythonhosted.org/packages/35/b4/fb0f1543245692d261f6d8da7128c60cbdb7b771f24454f834eda797b797/docling_parse-7.15.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:59b0dfa1d1ac97bc2d9d7ec457299e97afcf408bf4126e0cdc790818b8781510", size = 9744706, upload-time = "2026-08-20T08:31:03.491Z" }, + { url = "https://files.pythonhosted.org/packages/8c/4f/bf23632d5111f5c5b6fe27ce17c6d25e5db0413bcdd47ebe4f7e8346dd6f/docling_parse-7.15.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88dd0b133febd1ea21cffd43f3aafeb4161b33b74f0c59ae7c7420438d1abd4a", size = 10267571, upload-time = "2026-08-20T08:31:05.541Z" }, + { url = "https://files.pythonhosted.org/packages/69/13/ef9fe96d023d08603d9d280fa5fd2912ca1ba2190c43e3a80070a858697d/docling_parse-7.15.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5fc8ddd9cb3e864f4894d89271fc239277bdf5efc6c8deeecf1560cb163b3c4c", size = 10665251, upload-time = "2026-08-20T08:31:07.589Z" }, + { url = "https://files.pythonhosted.org/packages/e5/5e/e4bcc86900f1ce665b813f1797b96aea0d7aca00585eb9b1547fcbb5aa56/docling_parse-7.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:9c8d5aa23a90b5d082ed21b22a3384e13ceac4b45ce3d52230f5e05893f5cc6c", size = 12159194, upload-time = "2026-08-20T08:31:09.466Z" }, + { url = "https://files.pythonhosted.org/packages/e3/78/739780b74c5e5ee47e314f9340994f8883cb6173f99ca91e327479af46b5/docling_parse-7.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:b844748fdc27a48737b6d2f0ee0a07cb25cbd340ad8535ac2ae9048f9e214b18", size = 9398542, upload-time = "2026-08-20T08:31:11.511Z" }, +] + +[[package]] +name = "docling-slim" +version = "2.121.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "docling-core" }, + { name = "filetype" }, + { name = "pluggy" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "requests" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/29/a9f372eaf99d81d49b29de33565982e0ac7444e6ec246852601694487d01/docling_slim-2.121.0.tar.gz", hash = "sha256:f3069ded34f62f4e6a4d2ffda78cba0baad2ced4944a4b2d70cb646670ef327b", size = 603636, upload-time = "2026-08-20T09:06:21.054Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/b4/e59422978e7e6a738e9a62399bfa3585b2b0eb4c331592d5e69e100d373c/docling_slim-2.121.0-py3-none-any.whl", hash = "sha256:c97fc9cd889b61a66b9cf5d2d87c6b6935f29fe7aacfd8fd39802be7425e4996", size = 746537, upload-time = "2026-08-20T09:06:19.003Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "accelerate" }, + { name = "beautifulsoup4" }, + { name = "defusedxml" }, + { name = "docling-core", extra = ["chunking"] }, + { name = "docling-ibm-models" }, + { name = "docling-parse" }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "mail-parser" }, + { name = "marko" }, + { name = "numpy" }, + { name = "openpyxl" }, + { name = "pillow" }, + { name = "polyfactory" }, + { name = "pylatexenc" }, + { name = "pypdfium2" }, + { name = "python-docx" }, + { name = "python-dotenv" }, + { name = "python-oxmsg" }, + { name = "python-pptx" }, + { name = "rapidocr" }, + { name = "rich" }, + { name = "rtree" }, + { name = "scipy" }, + { name = "torch" }, + { name = "torchvision" }, + { name = "typer" }, + { name = "websockets" }, ] [[package]] @@ -998,16 +1041,18 @@ wheels = [ [[package]] name = "fastapi" -version = "0.119.1" +version = "0.141.1" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "annotated-doc" }, { name = "pydantic" }, { name = "starlette" }, { name = "typing-extensions" }, + { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/f4/152127681182e6413e7a89684c434e19e7414ed7ac0c632999c3c6980640/fastapi-0.119.1.tar.gz", hash = "sha256:a5e3426edce3fe221af4e1992c6d79011b247e3b03cc57999d697fe76cbf8ae0", size = 338616, upload-time = "2025-10-20T11:30:27.734Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/26/e6d959b4ac959fdb3e9c4154656fc160794db6af8e64673d52759456bf07/fastapi-0.119.1-py3-none-any.whl", hash = "sha256:0b8c2a2cce853216e150e9bd4faaed88227f8eb37de21cb200771f491586a27f", size = 108123, upload-time = "2025-10-20T11:30:26.185Z" }, + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, ] [[package]] @@ -1080,15 +1125,17 @@ requests = [ [[package]] name = "google-cloud-aiplatform" -version = "1.140.0" +version = "1.165.1" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "certifi" }, { name = "docstring-parser" }, { name = "google-api-core", extra = ["grpc"] }, { name = "google-auth" }, { name = "google-cloud-bigquery" }, { name = "google-cloud-resource-manager" }, - { name = "google-cloud-storage" }, + { name = "google-cloud-storage", version = "3.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "google-cloud-storage", version = "3.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "google-genai" }, { name = "packaging" }, { name = "proto-plus" }, @@ -1096,9 +1143,9 @@ dependencies = [ { name = "pydantic" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1b/14/1c223faf986afffdd61c994a10c30a04985ed5ba072201058af2c6e1e572/google_cloud_aiplatform-1.140.0.tar.gz", hash = "sha256:ea7eb1870b4cf600f8c2472102e21c3a1bcaf723d6e49f00ed51bc6b88d54fff", size = 10146640, upload-time = "2026-03-04T00:56:38.95Z" } +sdist = { url = "https://files.pythonhosted.org/packages/13/19/45df8264bf80dcc133c777822390c503aa75b878a57e9286d6f520b08c87/google_cloud_aiplatform-1.165.1.tar.gz", hash = "sha256:bd62ba7590255cacd66f9d0439eb731060af460d35bb48dbbeeefce9dfc0a359", size = 11325730, upload-time = "2026-08-19T21:12:30.858Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/5c/bb64aee2da24895d57611eed00fac54739bfa34f98ab344020a6605875bf/google_cloud_aiplatform-1.140.0-py2.py3-none-any.whl", hash = "sha256:e94493a2682b9d17efa7146a53bb3665bf1595c3394fd3d0f45d18f71623fddc", size = 8355660, upload-time = "2026-03-04T00:56:34.441Z" }, + { url = "https://files.pythonhosted.org/packages/b4/66/08c325e817a2713f23a028badc72220fc4b80521e999211ad9d935176446/google_cloud_aiplatform-1.165.1-py2.py3-none-any.whl", hash = "sha256:93874bd7993d1d901291595df693a8b9d4d7c89b93d4f41f8664d73f088dec89", size = 9456088, upload-time = "2026-08-19T21:12:26.018Z" }, ] [[package]] @@ -1153,29 +1200,62 @@ wheels = [ name = "google-cloud-storage" version = "3.9.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version < '3.12' and sys_platform == 'darwin'", + "python_full_version < '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux')", +] dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-cloud-core" }, - { name = "google-crc32c" }, - { name = "google-resumable-media" }, - { name = "requests" }, + { name = "google-api-core", marker = "python_full_version < '3.13'" }, + { name = "google-auth", marker = "python_full_version < '3.13'" }, + { name = "google-cloud-core", marker = "python_full_version < '3.13'" }, + { name = "google-crc32c", marker = "python_full_version < '3.13'" }, + { name = "google-resumable-media", marker = "python_full_version < '3.13'" }, + { name = "requests", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f7/b1/4f0798e88285b50dfc60ed3a7de071def538b358db2da468c2e0deecbb40/google_cloud_storage-3.9.0.tar.gz", hash = "sha256:f2d8ca7db2f652be757e92573b2196e10fbc09649b5c016f8b422ad593c641cc", size = 17298544, upload-time = "2026-02-02T13:36:34.119Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/46/0b/816a6ae3c9fd096937d2e5f9670558908811d57d59ddf69dd4b83b326fd1/google_cloud_storage-3.9.0-py3-none-any.whl", hash = "sha256:2dce75a9e8b3387078cbbdad44757d410ecdb916101f8ba308abf202b6968066", size = 321324, upload-time = "2026-02-02T13:36:32.271Z" }, ] +[[package]] +name = "google-cloud-storage" +version = "3.13.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'darwin'", + "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux')", + "python_full_version == '3.13.*' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux')", +] +dependencies = [ + { name = "google-api-core", marker = "python_full_version >= '3.13'" }, + { name = "google-auth", marker = "python_full_version >= '3.13'" }, + { name = "google-cloud-core", marker = "python_full_version >= '3.13'" }, + { name = "google-crc32c", marker = "python_full_version >= '3.13'" }, + { name = "google-resumable-media", marker = "python_full_version >= '3.13'" }, + { name = "requests", marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/7e/73bb7512df1d1aad6ce3f9aed847cd40e0cd400ba4a85d86ab8eb412e9cc/google_cloud_storage-3.13.1.tar.gz", hash = "sha256:a80bf8cac2794808aa61c50c5f769ecbbe2d10331bacd0d69d30e59b14b346b2", size = 17341051, upload-time = "2026-08-06T06:24:42.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/6f/d69f0e185e08ddb58c323a0a935af2b492907b5de362bc08933b0a3b5644/google_cloud_storage-3.13.1-py3-none-any.whl", hash = "sha256:98208de6c21e85cecd3eb44551894efff33d98365500e178867d4305854a770a", size = 341486, upload-time = "2026-08-06T06:23:36.548Z" }, +] + [[package]] name = "google-cloud-vectorsearch" version = "0.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "google-api-core", extra = ["grpc"], marker = "python_full_version < '3.13'" }, - { name = "google-auth", marker = "python_full_version < '3.13'" }, - { name = "grpcio", marker = "python_full_version < '3.13'" }, - { name = "proto-plus", marker = "python_full_version < '3.13'" }, - { name = "protobuf", marker = "python_full_version < '3.13'" }, + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "grpcio" }, + { name = "proto-plus" }, + { name = "protobuf" }, ] sdist = { url = "https://files.pythonhosted.org/packages/78/fe/d5d0548a4e6773ac3c2964d1f4e1cfde4f26971d73e0c7510b77fce17ada/google_cloud_vectorsearch-0.6.0.tar.gz", hash = "sha256:b6a62f9f0aa34713982ff98650c21f052a56e8d36c663b737ad37947f0eab968", size = 405492, upload-time = "2026-03-06T21:52:53.77Z" } wheels = [ @@ -1352,34 +1432,26 @@ wheels = [ [[package]] name = "hf-xet" -version = "1.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8b/cb/9bb543bd987ffa1ee48202cc96a756951b734b79a542335c566148ade36c/hf_xet-1.3.2.tar.gz", hash = "sha256:e130ee08984783d12717444e538587fa2119385e5bd8fc2bb9f930419b73a7af", size = 643646, upload-time = "2026-02-27T17:26:08.051Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/49/75/462285971954269432aad2e7938c5c7ff9ec7d60129cec542ab37121e3d6/hf_xet-1.3.2-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:335a8f36c55fd35a92d0062f4e9201b4015057e62747b7e7001ffb203c0ee1d2", size = 3761019, upload-time = "2026-02-27T17:25:49.441Z" }, - { url = "https://files.pythonhosted.org/packages/35/56/987b0537ddaf88e17192ea09afa8eca853e55f39a4721578be436f8409df/hf_xet-1.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:c1ae4d3a716afc774e66922f3cac8206bfa707db13f6a7e62dfff74bfc95c9a8", size = 3521565, upload-time = "2026-02-27T17:25:47.469Z" }, - { url = "https://files.pythonhosted.org/packages/a8/5c/7e4a33a3d689f77761156cc34558047569e54af92e4d15a8f493229f6767/hf_xet-1.3.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d6dbdf231efac0b9b39adcf12a07f0c030498f9212a18e8c50224d0e84ab803d", size = 4176494, upload-time = "2026-02-27T17:25:40.247Z" }, - { url = "https://files.pythonhosted.org/packages/6b/b3/71e856bf9d9a69b3931837e8bf22e095775f268c8edcd4a9e8c355f92484/hf_xet-1.3.2-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c1980abfb68ecf6c1c7983379ed7b1e2b49a1aaf1a5aca9acc7d48e5e2e0a961", size = 3955601, upload-time = "2026-02-27T17:25:38.376Z" }, - { url = "https://files.pythonhosted.org/packages/63/d7/aecf97b3f0a981600a67ff4db15e2d433389d698a284bb0ea5d8fcdd6f7f/hf_xet-1.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1c88fbd90ad0d27c46b77a445f0a436ebaa94e14965c581123b68b1c52f5fd30", size = 4154770, upload-time = "2026-02-27T17:25:56.756Z" }, - { url = "https://files.pythonhosted.org/packages/e2/e1/3af961f71a40e09bf5ee909842127b6b00f5ab4ee3817599dc0771b79893/hf_xet-1.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:35b855024ca37f2dd113ac1c08993e997fbe167b9d61f9ef66d3d4f84015e508", size = 4394161, upload-time = "2026-02-27T17:25:58.111Z" }, - { url = "https://files.pythonhosted.org/packages/a1/c3/859509bade9178e21b8b1db867b8e10e9f817ab9ac1de77cb9f461ced765/hf_xet-1.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:31612ba0629046e425ba50375685a2586e11fb9144270ebabd75878c3eaf6378", size = 3637377, upload-time = "2026-02-27T17:26:10.611Z" }, - { url = "https://files.pythonhosted.org/packages/05/7f/724cfbef4da92d577b71f68bf832961c8919f36c60d28d289a9fc9d024d4/hf_xet-1.3.2-cp313-cp313t-win_arm64.whl", hash = "sha256:433c77c9f4e132b562f37d66c9b22c05b5479f243a1f06a120c1c06ce8b1502a", size = 3497875, upload-time = "2026-02-27T17:26:09.034Z" }, - { url = "https://files.pythonhosted.org/packages/ba/75/9d54c1ae1d05fb704f977eca1671747babf1957f19f38ae75c5933bc2dc1/hf_xet-1.3.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c34e2c7aefad15792d57067c1c89b2b02c1bbaeabd7f8456ae3d07b4bbaf4094", size = 3761076, upload-time = "2026-02-27T17:25:55.42Z" }, - { url = "https://files.pythonhosted.org/packages/f2/8a/08a24b6c6f52b5d26848c16e4b6d790bb810d1bf62c3505bed179f7032d3/hf_xet-1.3.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4bc995d6c41992831f762096020dc14a65fdf3963f86ffed580b596d04de32e3", size = 3521745, upload-time = "2026-02-27T17:25:54.217Z" }, - { url = "https://files.pythonhosted.org/packages/b5/db/a75cf400dd8a1a8acf226a12955ff6ee999f272dfc0505bafd8079a61267/hf_xet-1.3.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:959083c89dee30f7d6f890b36cdadda823386c4de63b1a30384a75bfd2ae995d", size = 4176301, upload-time = "2026-02-27T17:25:46.044Z" }, - { url = "https://files.pythonhosted.org/packages/01/40/6c4c798ffdd83e740dd3925c4e47793b07442a9efa3bc3866ba141a82365/hf_xet-1.3.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cfa760888633b08c01b398d212ce7e8c0d7adac6c86e4b20dfb2397d8acd78ee", size = 3955437, upload-time = "2026-02-27T17:25:44.703Z" }, - { url = "https://files.pythonhosted.org/packages/0c/09/9a3aa7c5f07d3e5cc57bb750d12a124ffa72c273a87164bd848f9ac5cc14/hf_xet-1.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3155a02e083aa21fd733a7485c7c36025e49d5975c8d6bda0453d224dd0b0ac4", size = 4154535, upload-time = "2026-02-27T17:26:05.207Z" }, - { url = "https://files.pythonhosted.org/packages/ae/e0/831f7fa6d90cb47a230bc23284b502c700e1483bbe459437b3844cdc0776/hf_xet-1.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:91b1dc03c31cbf733d35dc03df7c5353686233d86af045e716f1e0ea4a2673cf", size = 4393891, upload-time = "2026-02-27T17:26:06.607Z" }, - { url = "https://files.pythonhosted.org/packages/ab/96/6ed472fdce7f8b70f5da6e3f05be76816a610063003bfd6d9cea0bbb58a3/hf_xet-1.3.2-cp314-cp314t-win_amd64.whl", hash = "sha256:211f30098512d95e85ad03ae63bd7dd2c4df476558a5095d09f9e38e78cbf674", size = 3637583, upload-time = "2026-02-27T17:26:17.349Z" }, - { url = "https://files.pythonhosted.org/packages/8b/e8/a069edc4570b3f8e123c0b80fadc94530f3d7b01394e1fc1bb223339366c/hf_xet-1.3.2-cp314-cp314t-win_arm64.whl", hash = "sha256:4a6817c41de7c48ed9270da0b02849347e089c5ece9a0e72ae4f4b3a57617f82", size = 3497977, upload-time = "2026-02-27T17:26:14.966Z" }, - { url = "https://files.pythonhosted.org/packages/d8/28/dbb024e2e3907f6f3052847ca7d1a2f7a3972fafcd53ff79018977fcb3e4/hf_xet-1.3.2-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f93b7595f1d8fefddfede775c18b5c9256757824f7f6832930b49858483cd56f", size = 3763961, upload-time = "2026-02-27T17:25:52.537Z" }, - { url = "https://files.pythonhosted.org/packages/e4/71/b99aed3823c9d1795e4865cf437d651097356a3f38c7d5877e4ac544b8e4/hf_xet-1.3.2-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:a85d3d43743174393afe27835bde0cd146e652b5fcfdbcd624602daef2ef3259", size = 3526171, upload-time = "2026-02-27T17:25:50.968Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ca/907890ce6ef5598b5920514f255ed0a65f558f820515b18db75a51b2f878/hf_xet-1.3.2-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7c2a054a97c44e136b1f7f5a78f12b3efffdf2eed3abc6746fc5ea4b39511633", size = 4180750, upload-time = "2026-02-27T17:25:43.125Z" }, - { url = "https://files.pythonhosted.org/packages/8c/ad/bc7f41f87173d51d0bce497b171c4ee0cbde1eed2d7b4216db5d0ada9f50/hf_xet-1.3.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:06b724a361f670ae557836e57801b82c75b534812e351a87a2c739f77d1e0635", size = 3961035, upload-time = "2026-02-27T17:25:41.837Z" }, - { url = "https://files.pythonhosted.org/packages/73/38/600f4dda40c4a33133404d9fe644f1d35ff2d9babb4d0435c646c63dd107/hf_xet-1.3.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:305f5489d7241a47e0458ef49334be02411d1d0f480846363c1c8084ed9916f7", size = 4161378, upload-time = "2026-02-27T17:26:00.365Z" }, - { url = "https://files.pythonhosted.org/packages/00/b3/7bc1ff91d1ac18420b7ad1e169b618b27c00001b96310a89f8a9294fe509/hf_xet-1.3.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:06cdbde243c85f39a63b28e9034321399c507bcd5e7befdd17ed2ccc06dfe14e", size = 4398020, upload-time = "2026-02-27T17:26:03.977Z" }, - { url = "https://files.pythonhosted.org/packages/2b/0b/99bfd948a3ed3620ab709276df3ad3710dcea61976918cce8706502927af/hf_xet-1.3.2-cp37-abi3-win_amd64.whl", hash = "sha256:9298b47cce6037b7045ae41482e703c471ce36b52e73e49f71226d2e8e5685a1", size = 3641624, upload-time = "2026-02-27T17:26:13.542Z" }, - { url = "https://files.pythonhosted.org/packages/cc/02/9a6e4ca1f3f73a164c0cd48e41b3cc56585dcc37e809250de443d673266f/hf_xet-1.3.2-cp37-abi3-win_arm64.whl", hash = "sha256:83d8ec273136171431833a6957e8f3af496bee227a0fe47c7b8b39c106d1749a", size = 3503976, upload-time = "2026-02-27T17:26:12.123Z" }, +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] [[package]] @@ -1421,30 +1493,31 @@ wheels = [ [[package]] name = "huggingface-hub" -version = "0.36.2" +version = "1.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "click" }, { name = "filelock" }, { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, { name = "packaging" }, { name = "pyyaml" }, - { name = "requests" }, { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7c/b7/8cb61d2eece5fb05a83271da168186721c450eb74e3c31f7ef3169fa475b/huggingface_hub-0.36.2.tar.gz", hash = "sha256:1934304d2fb224f8afa3b87007d58501acfda9215b334eed53072dd5e815ff7a", size = 649782, upload-time = "2026-02-06T09:24:13.098Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/ae/222a91937ebee7f62c0ca8f5ee0afd97577caf24c0abb927d1f5c7e9f6d2/huggingface_hub-1.28.0.tar.gz", hash = "sha256:46a2e950c09234de54093d587d1675382f0d08dbd600d9fb599b5932f5b2c6cb", size = 959609, upload-time = "2026-08-18T12:27:15.101Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/af/48ac8483240de756d2438c380746e7130d1c6f75802ef22f3c6d49982787/huggingface_hub-0.36.2-py3-none-any.whl", hash = "sha256:48f0c8eac16145dfce371e9d2d7772854a4f591bcb56c9cf548accf531d54270", size = 566395, upload-time = "2026-02-06T09:24:11.133Z" }, + { url = "https://files.pythonhosted.org/packages/51/0e/eafef18f1a75e125e68395db21131db0cf868a128ecd2fce69b4df6c584b/huggingface_hub-1.28.0-py3-none-any.whl", hash = "sha256:58a8bacb03072edfc38067065e9dc24bbb34805410fcd36a1632de0b329660bb", size = 793202, upload-time = "2026-08-18T12:27:12.719Z" }, ] [[package]] name = "idna" -version = "3.11" +version = "3.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, ] [[package]] @@ -1666,16 +1739,16 @@ wheels = [ [[package]] name = "langchain" -version = "1.2.10" +version = "1.3.16" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "langgraph" }, { name = "pydantic" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/16/22/a4d4ac98fc2e393537130bbfba0d71a8113e6f884d96f935923e247397fe/langchain-1.2.10.tar.gz", hash = "sha256:bdcd7218d9c79a413cf15e106e4eb94408ac0963df9333ccd095b9ed43bf3be7", size = 570071, upload-time = "2026-02-10T14:56:49.74Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/d5/cda22848e95e17addb8f365a210f78d4474d2e78895a4cc79ddf29ee186c/langchain-1.3.16.tar.gz", hash = "sha256:781fd3bca96813e9d0d74c58a323d0ac05a0c0d5cb2c56ac34185f4f3ec0c4bb", size = 666372, upload-time = "2026-08-20T14:35:14.682Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/06/c3394327f815fade875724c0f6cff529777c96a1e17fea066deb997f8cf5/langchain-1.2.10-py3-none-any.whl", hash = "sha256:e07a377204451fffaed88276b8193e894893b1003e25c5bca6539288ccca3698", size = 111738, upload-time = "2026-02-10T14:56:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/c3/92/854f52036abe590c70ae14e398d00a1553cedadcf93e59de6b7b4026b2bb/langchain-1.3.16-py3-none-any.whl", hash = "sha256:67dcb28d69f96018f3062da2075f99b25ea048af7f5baae570afe1d91e033743", size = 147826, upload-time = "2026-08-20T14:35:13.109Z" }, ] [[package]] @@ -1695,9 +1768,10 @@ wheels = [ [[package]] name = "langchain-core" -version = "1.4.0" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "httpx" }, { name = "jsonpatch" }, { name = "langchain-protocol" }, { name = "langsmith" }, @@ -1708,117 +1782,59 @@ dependencies = [ { name = "typing-extensions" }, { name = "uuid-utils" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/59/de/679a53472c25860837e32c0442c962fa86e95317a36460e2c9d5c91b17c2/langchain_core-1.4.0.tar.gz", hash = "sha256:1dc341eed802ed9c117c0df3923c991e5e9e226571e5725c194eeb5bd93d1a7f", size = 920260, upload-time = "2026-05-11T18:42:35.919Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/1a/86c38c27b81913a1c6c12448cab55defb5a1097c7dc9a4cea83f55477a2d/langchain_core-1.4.0-py3-none-any.whl", hash = "sha256:23cbbdb46e38ddd1dd5247e6167e96013eae74bea4c5949c550809970a9e565c", size = 548120, upload-time = "2026-05-11T18:42:33.992Z" }, -] - -[[package]] -name = "langchain-google-vertexai" -version = "3.2.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.14' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version >= '3.14' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'linux')", - "python_full_version == '3.13.*' and sys_platform == 'darwin'", - "python_full_version == '3.13.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.13.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.13.*' and sys_platform != 'darwin' and sys_platform != 'linux')", -] -dependencies = [ - { name = "bottleneck", marker = "python_full_version >= '3.13'" }, - { name = "google-cloud-aiplatform", marker = "python_full_version >= '3.13'" }, - { name = "google-cloud-storage", marker = "python_full_version >= '3.13'" }, - { name = "httpx", marker = "python_full_version >= '3.13'" }, - { name = "httpx-sse", marker = "python_full_version >= '3.13'" }, - { name = "langchain-core", marker = "python_full_version >= '3.13'" }, - { name = "numexpr", marker = "python_full_version >= '3.13'" }, - { name = "pyarrow", marker = "python_full_version >= '3.13'" }, - { name = "pydantic", marker = "python_full_version >= '3.13'" }, - { name = "validators", marker = "python_full_version >= '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/5f/55a5b568104c32e265970d4217083d76252c2140f532f382bb42f35886a8/langchain_google_vertexai-3.2.1.tar.gz", hash = "sha256:8913e8aa7ca300eb7d9b8681ba2487dad787debe2511a903a249dc03709720d2", size = 360287, upload-time = "2026-01-05T21:47:58.287Z" } +sdist = { url = "https://files.pythonhosted.org/packages/97/88/ebc98df187c525d729725ab39759337c56d5d2803423632376aa35bde899/langchain_core-1.6.0.tar.gz", hash = "sha256:dc72e36678ed26683ec0ad8829b44011fba461d10f9b31dbd9110215e5ec2333", size = 992493, upload-time = "2026-08-19T15:55:40.642Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/60/90/e2b1493df6ad06a7c1194f6d8238ddbd7fedc0cfe1e32d1c3980903c9d05/langchain_google_vertexai-3.2.1-py3-none-any.whl", hash = "sha256:57a25680290060c896fb740bdaafa987e0518b599f793b3dfb2d07a7aec97bd8", size = 103650, upload-time = "2026-01-05T21:47:57.136Z" }, + { url = "https://files.pythonhosted.org/packages/84/4c/508a90b9d2e3bd7738fd93cb2ac2178ce734662c75e69b3b81c445f1b360/langchain_core-1.6.0-py3-none-any.whl", hash = "sha256:d8bb924cd413955d9d3192ccced140407427c3ff51e50ffb941222648da92cb2", size = 570006, upload-time = "2026-08-19T15:55:38.935Z" }, ] [[package]] name = "langchain-google-vertexai" -version = "3.2.2" +version = "3.2.4" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.12.*' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version == '3.12.*' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version == '3.12.*' and sys_platform != 'darwin' and sys_platform != 'linux')", - "python_full_version < '3.12' and sys_platform == 'darwin'", - "python_full_version < '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", - "(python_full_version < '3.12' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.12' and sys_platform != 'darwin' and sys_platform != 'linux')", -] dependencies = [ - { name = "bottleneck", marker = "python_full_version < '3.13'" }, - { name = "google-cloud-aiplatform", marker = "python_full_version < '3.13'" }, - { name = "google-cloud-storage", marker = "python_full_version < '3.13'" }, - { name = "google-cloud-vectorsearch", marker = "python_full_version < '3.13'" }, - { name = "httpx", marker = "python_full_version < '3.13'" }, - { name = "httpx-sse", marker = "python_full_version < '3.13'" }, - { name = "langchain-core", marker = "python_full_version < '3.13'" }, - { name = "langchain-tests", marker = "python_full_version < '3.13'" }, - { name = "numexpr", marker = "python_full_version < '3.13'" }, - { name = "pyarrow", marker = "python_full_version < '3.13'" }, - { name = "pydantic", marker = "python_full_version < '3.13'" }, - { name = "validators", marker = "python_full_version < '3.13'" }, + { name = "bottleneck" }, + { name = "google-cloud-aiplatform" }, + { name = "google-cloud-storage", version = "3.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "google-cloud-storage", version = "3.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "google-cloud-vectorsearch" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "langchain-core" }, + { name = "numexpr" }, + { name = "pyarrow" }, + { name = "pydantic" }, + { name = "validators" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/81/36/6924d8321f661733c15685738d525de30b4dee5f0b288545f0a525ddf4e6/langchain_google_vertexai-3.2.2.tar.gz", hash = "sha256:089200b44e0002ef0a571243bf19cd6897446e8b5d17b7c3a8e6579820cda3a7", size = 375968, upload-time = "2026-01-30T18:29:13.229Z" } +sdist = { url = "https://files.pythonhosted.org/packages/39/1c/4efdcfe094ef6711d9c130428c4c42af61a49a1580fa718dd1e1f6bd5fc3/langchain_google_vertexai-3.2.4.tar.gz", hash = "sha256:f0c16b4b639037f60357dbf6d4aa3b0f301f23be6b33b0688c75902235cb9434", size = 351222, upload-time = "2026-06-09T14:57:58.316Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/d9/3adf09ff844a6d5c9dad9fe9ad6a032b706a1184eae71caa4c89bb470cbd/langchain_google_vertexai-3.2.2-py3-none-any.whl", hash = "sha256:aee8ea79f5aa19da74058e3905c78e0d3b882d5bc9eaf8549d92c13a5c458fda", size = 113381, upload-time = "2026-01-30T18:29:12.13Z" }, + { url = "https://files.pythonhosted.org/packages/04/a2/af563ff45208d22abc28d0a0e44a7fb8aceaadac201afa6a11c77bfb6338/langchain_google_vertexai-3.2.4-py3-none-any.whl", hash = "sha256:65b5615e596fdabc2e149f0160fded88bebef2bbc1ea70095ff81714f7570183", size = 118884, upload-time = "2026-06-09T14:57:57.062Z" }, ] [[package]] name = "langchain-openai" -version = "1.1.10" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "certifi" }, { name = "langchain-core" }, { name = "openai" }, { name = "tiktoken" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d6/0f/01147f842499338ae3b0dd0a351fb83006d9ed623cf3a999bd68ba5bbe2d/langchain_openai-1.1.10.tar.gz", hash = "sha256:ca6fae7cf19425acc81814efed59c7d205ec9a1f284fd1d08aae9bda85d6501b", size = 1059755, upload-time = "2026-02-17T18:03:44.506Z" } +sdist = { url = "https://files.pythonhosted.org/packages/58/aa/5b4dbe8f90c44729fb7800bfde113ce1bc7c9687115d2e0d20fb377671e9/langchain_openai-1.6.0.tar.gz", hash = "sha256:7c545cc1f2bee048a7e219f375b78cd820b32c876ccf825320e25e1e94132577", size = 3290712, upload-time = "2026-08-19T21:44:45.334Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/72/17/3785cbcdc81c451179247e4176d2697879cb4f45ab2c59d949ca574e072d/langchain_openai-1.1.10-py3-none-any.whl", hash = "sha256:d91b2c09e9fbc70f7af45345d3aa477744962d41c73a029beb46b4f83b824827", size = 87205, upload-time = "2026-02-17T18:03:43.502Z" }, + { url = "https://files.pythonhosted.org/packages/2b/fd/e22f2ffa2e2cf6043eb56f6a6bc5e18ac24da9b48b92cd458ba54af24b9e/langchain_openai-1.6.0-py3-none-any.whl", hash = "sha256:648112bbd135aa51d60d4aa2fd4ad353628a8d92a2f8e779244a0d9869a16f18", size = 125078, upload-time = "2026-08-19T21:44:44.242Z" }, ] [[package]] name = "langchain-protocol" -version = "0.0.16" +version = "0.0.18" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/36/e7/8300ba22d968653051fd06e3117d783872dddf3dcebdd6b1d386836eb43c/langchain_protocol-0.0.16.tar.gz", hash = "sha256:806c7cdd951b1c4f692fa40fce60821ff0f221d4360e27673ddf2c2b99c2b7ff", size = 5969, upload-time = "2026-05-28T23:05:11.121Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/9c/06dfcc88d02a6364e8d864c421ddd3736305cb0a6c853f75c302c80fe17c/langchain_protocol-0.0.16-py3-none-any.whl", hash = "sha256:3658c142c5d0fb3a023a4be442ce4c15c6d626aab6135eb79a76dc64ad19c3c3", size = 7037, upload-time = "2026-05-28T23:05:10.163Z" }, -] - -[[package]] -name = "langchain-tests" -version = "1.1.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx", marker = "python_full_version < '3.13'" }, - { name = "langchain-core", marker = "python_full_version < '3.13'" }, - { name = "numpy", marker = "python_full_version < '3.13'" }, - { name = "pytest", marker = "python_full_version < '3.13'" }, - { name = "pytest-asyncio", marker = "python_full_version < '3.13'" }, - { name = "pytest-benchmark", marker = "python_full_version < '3.13'" }, - { name = "pytest-codspeed", marker = "python_full_version < '3.13'" }, - { name = "pytest-recording", marker = "python_full_version < '3.13'" }, - { name = "pytest-socket", marker = "python_full_version < '3.13'" }, - { name = "syrupy", marker = "python_full_version < '3.13'" }, - { name = "vcrpy", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/21/94/e626a40c14a5bc7b60563446a23330a06654d0b1e3804109ed792ca0c638/langchain_tests-1.1.5.tar.gz", hash = "sha256:add75c24ea4aacb5f4efa02670fcceee5d5276848586f724e90135da6d3e070e", size = 154114, upload-time = "2026-02-18T16:08:31.745Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/59/b5959aea96faa9146e2e49a7a22882b3528c62efafe9a6a95beab30c2305/langchain_protocol-0.0.18.tar.gz", hash = "sha256:ec3e11782f1ed0c9db38e5a9ed01b0e7a0d3fba406faa8aef6594b73c56a63e6", size = 6150, upload-time = "2026-06-18T17:08:26.959Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/2c/c7641310171ed49d1e603c56ce38a9f3e157c23257b226f69ce5cb6a7428/langchain_tests-1.1.5-py3-none-any.whl", hash = "sha256:535429fb31d17a6cf8d7f61a3f81013a650f090c0f57b4daf1a03be7771d61ff", size = 55731, upload-time = "2026-02-18T16:08:30.811Z" }, + { url = "https://files.pythonhosted.org/packages/99/2e/d82db9eec13ad0f72e7aaad5c4bc730ab111934fdc83c85523206eb9b0a0/langchain_protocol-0.0.18-py3-none-any.whl", hash = "sha256:70b53a86fbf9cedc863555effe44da192ab02d556ddbf2cf95b8873adcf41b5a", size = 7221, upload-time = "2026-06-18T17:08:25.996Z" }, ] [[package]] @@ -1832,7 +1848,7 @@ sdist = { url = "https://files.pythonhosted.org/packages/59/59/4bc44158a767a6d66 [[package]] name = "langgraph" -version = "1.0.10" +version = "1.2.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, @@ -1842,48 +1858,51 @@ dependencies = [ { name = "pydantic" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/55/92/14df6fefba28c10caf1cb05aa5b8c7bf005838fe32a86d903b6c7cc4018d/langgraph-1.0.10.tar.gz", hash = "sha256:73bd10ee14a8020f31ef07e9cd4c1a70c35cc07b9c2b9cd637509a10d9d51e29", size = 511644, upload-time = "2026-02-27T21:04:38.743Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/0d/c8e7ee98896659e1b6555db0ab115a9ca899844744645d5d894032bab1d7/langgraph-1.2.11.tar.gz", hash = "sha256:9ecfe11e50d338b34b15cf4d8a442642de103e8ae6971320efba84e4542eb363", size = 725753, upload-time = "2026-08-11T14:00:36.945Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/60/260e0c04620a37ba8916b712766c341cc5fc685dabc6948c899494bbc2ae/langgraph-1.0.10-py3-none-any.whl", hash = "sha256:7c298bef4f6ea292fcf9824d6088fe41a6727e2904ad6066f240c4095af12247", size = 160920, upload-time = "2026-02-27T21:04:35.932Z" }, + { url = "https://files.pythonhosted.org/packages/0a/7f/c5c30e4be99ff821029c7ac872a480676bb179c9f3df85ea3f38d13f86d4/langgraph-1.2.11-py3-none-any.whl", hash = "sha256:8bab70de7b2d00b5300fb289bcf38d8b241400f3184c1e95e8ce706fb0e8686b", size = 248854, upload-time = "2026-08-11T14:00:35.494Z" }, ] [[package]] name = "langgraph-checkpoint" -version = "4.0.1" +version = "4.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "ormsgpack" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/44/a8df45d1e8b4637e29789fa8bae1db022c953cc7ac80093cfc52e923547e/langgraph_checkpoint-4.0.1.tar.gz", hash = "sha256:b433123735df11ade28829e40ce25b9be614930cd50245ff2af60629234befd9", size = 158135, upload-time = "2026-02-27T21:06:16.092Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/e1/089c4c9e0a2fec7f883f82ae8e6a727138d50074cfeb6644bc2d13b1019b/langgraph_checkpoint-4.2.0.tar.gz", hash = "sha256:51a593b6bee684b0818e5d6e58e28ab340c6db7794575056ce7bd1b746a84ed7", size = 180239, upload-time = "2026-08-07T20:05:03.756Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/65/4c/09a4a0c42f5d2fc38d6c4d67884788eff7fd2cfdf367fdf7033de908b4c0/langgraph_checkpoint-4.0.1-py3-none-any.whl", hash = "sha256:e3adcd7a0e0166f3b48b8cf508ce0ea366e7420b5a73aa81289888727769b034", size = 50453, upload-time = "2026-02-27T21:06:14.293Z" }, + { url = "https://files.pythonhosted.org/packages/05/71/3b475f09bd57d3a5649792c66353312b4432afd843f301739dfcebd157f0/langgraph_checkpoint-4.2.0-py3-none-any.whl", hash = "sha256:0547fd228935a0b758865de3a3d6d7a2537c308895d0f9ab092ce9151b5da942", size = 56833, upload-time = "2026-08-07T20:05:02.655Z" }, ] [[package]] name = "langgraph-prebuilt" -version = "1.0.8" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "langchain-core" }, { name = "langgraph-checkpoint" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0d/06/dd61a5c2dce009d1b03b1d56f2a85b3127659fdddf5b3be5d8f1d60820fb/langgraph_prebuilt-1.0.8.tar.gz", hash = "sha256:0cd3cf5473ced8a6cd687cc5294e08d3de57529d8dd14fdc6ae4899549efcf69", size = 164442, upload-time = "2026-02-19T18:14:39.083Z" } +sdist = { url = "https://files.pythonhosted.org/packages/29/66/ed9b93f56bc17ef22d551892f0ac2b225a97fe0fcf23a511b857f70d590b/langgraph_prebuilt-1.1.0.tar.gz", hash = "sha256:3c579cf6eed2d17f9c157c2d0fcaddcd8688524e7022d3b22b37a3bf4589d528", size = 178833, upload-time = "2026-05-12T03:37:49.332Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/41/ec966424ad3f2ed3996d24079d3342c8cd6c0bd0653c12b2a917a685ec6c/langgraph_prebuilt-1.0.8-py3-none-any.whl", hash = "sha256:d16a731e591ba4470f3e313a319c7eee7dbc40895bcf15c821f985a3522a7ce0", size = 35648, upload-time = "2026-02-19T18:14:37.611Z" }, + { url = "https://files.pythonhosted.org/packages/e9/43/3fe1a700b8490ed02679cdbbc8c915eb23a092faf496c9c1118abcd10be3/langgraph_prebuilt-1.1.0-py3-none-any.whl", hash = "sha256:51e311747d755b751d5c6b39b0c1446124d3a7643d2515017e6714b323508fc9", size = 41043, upload-time = "2026-05-12T03:37:48.007Z" }, ] [[package]] name = "langgraph-sdk" -version = "0.3.9" +version = "0.4.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, + { name = "langchain-core" }, + { name = "langchain-protocol" }, { name = "orjson" }, + { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3a/bd/ca8ae5c6a34be6d4f7aa86016e010ff96b3a939456041565797952e3014d/langgraph_sdk-0.3.9.tar.gz", hash = "sha256:8be8958529b3f6d493ec248fdb46e539362efda75784654a42a7091d22504e0e", size = 184287, upload-time = "2026-02-24T18:39:03.276Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/0e/3c0347fd517390e807d9772d2037f9cc65c0227af27f07052ba9c8ebdfa2/langgraph_sdk-0.4.3.tar.gz", hash = "sha256:f101cc043ddd7400ceaef66d934c42fa2f0e7d4ea5029754d286260d39038251", size = 344145, upload-time = "2026-08-19T18:05:21.838Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/4c/7a7510260fbda788efd13bf4650d3e7d80988118441ac811ec78e0aa03ac/langgraph_sdk-0.3.9-py3-none-any.whl", hash = "sha256:94654294250c920789b6ed0d8a70c0117fed5736b61efc24ff647157359453c5", size = 90511, upload-time = "2026-02-24T18:39:02.012Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a0/f3bc44b46ef730588f66fdf9e9ddc54189db78482a9e914ae6a554a2b2e0/langgraph_sdk-0.4.3-py3-none-any.whl", hash = "sha256:1b7920b39b6dc439843d122a06f04a0cb8b65c02fd086ddf68e18596a230a0e7", size = 161794, upload-time = "2026-08-19T18:05:20.495Z" }, ] [[package]] @@ -1897,22 +1916,27 @@ sdist = { url = "https://files.pythonhosted.org/packages/ea/4c/0fb7d900d3b0b9c87 [[package]] name = "langsmith" -version = "0.7.15" +version = "0.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "anyio" }, + { name = "distro" }, { name = "httpx" }, { name = "orjson", marker = "platform_python_implementation != 'PyPy'" }, { name = "packaging" }, { name = "pydantic" }, { name = "requests" }, { name = "requests-toolbelt" }, + { name = "sniffio" }, + { name = "typing-extensions" }, { name = "uuid-utils" }, + { name = "websockets" }, { name = "xxhash" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/09/e1/39dbcd7b066ba976e7bf3e6d4a9996f2a332eef87f8355ee22404b951a5c/langsmith-0.7.15.tar.gz", hash = "sha256:4d7848f236f1374606e51d7ae1aff240d4e57049d9d58130714059cd70d488c7", size = 1114101, upload-time = "2026-03-09T18:24:02.772Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/57/7b6c11080c9e082ebf1456a2e2372fae8f23a85a5ae2869bdd5ab9a6507d/langsmith-0.11.1.tar.gz", hash = "sha256:47998977366acb3ba3093881fd465cbf11a5f8c2f4e87e40a17dda203f6dedf4", size = 4814724, upload-time = "2026-08-19T15:47:44.116Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/02/e7/1087537f189882846b3b8c18004ac3b583693abc04e4517fad05b18e3aad/langsmith-0.7.15-py3-none-any.whl", hash = "sha256:12d5eff6072002051c9dbfa04e7d06d56b9caad4e813d75827afd3cc44e379fa", size = 347360, upload-time = "2026-03-09T18:24:01.337Z" }, + { url = "https://files.pythonhosted.org/packages/f1/85/0ad6df25588122760b2b40ec182eafc55532c9e66015c83e16675bd34a29/langsmith-0.11.1-py3-none-any.whl", hash = "sha256:cfc3437a9cf27440cd0095c24df945edbceb6df10b579bc8b3980b4ad367835f", size = 744589, upload-time = "2026-08-19T15:47:42.043Z" }, ] [[package]] @@ -2108,6 +2132,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/77/d7f491cbc05303ac6801651aabeb262d43f319288c1ea96c66b1d2692ff3/lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e", size = 3518768, upload-time = "2025-09-22T04:04:57.097Z" }, ] +[[package]] +name = "mail-parser" +version = "4.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/0f/b6fc717f7a7d9188e451e9d30e98b3f61866313958f866f9da7ff5ed3c84/mail_parser-4.6.4.tar.gz", hash = "sha256:1929109b7934ee061c1533a897b18c57fd0adb3548b79b5e28d901819bade872", size = 2898988, upload-time = "2026-08-21T21:47:13.58Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/f9/b35a02c12346111e997ebfaa46ebeb759828373a3eb9f0e82c1e826b3530/mail_parser-4.6.4-py3-none-any.whl", hash = "sha256:bc6e437b3afe38091893e7b6ea49c7f2188ad7616abdd4fa3059e85f1b69efce", size = 50073, upload-time = "2026-08-21T21:47:12.07Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -2214,7 +2247,7 @@ wheels = [ [[package]] name = "mcp" -version = "1.26.0" +version = "1.29.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2232,9 +2265,9 @@ dependencies = [ { name = "typing-inspection" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fc/6d/62e76bbb8144d6ed86e202b5edd8a4cb631e7c8130f3f4893c3f90262b10/mcp-1.26.0.tar.gz", hash = "sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66", size = 608005, upload-time = "2026-01-24T19:40:32.468Z" } +sdist = { url = "https://files.pythonhosted.org/packages/30/d3/f9acc21dfc886e4f78e2add1a47db46ce16884346afde53f8a064c02c891/mcp-1.29.0.tar.gz", hash = "sha256:52d01f334de1868cc3bb2d6604931126a67631f99a6c5d3b82ba47290315ec36", size = 643148, upload-time = "2026-07-28T13:41:41.939Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/d9/eaa1f80170d2b7c5ba23f3b59f766f3a0bb41155fbc32a69adfa1adaaef9/mcp-1.26.0-py3-none-any.whl", hash = "sha256:904a21c33c25aa98ddbeb47273033c435e595bbacfdb177f4bd87f6dceebe1ca", size = 233615, upload-time = "2026-01-24T19:40:30.652Z" }, + { url = "https://files.pythonhosted.org/packages/01/c8/248b201f6d753d69fd5d6506011abbb35a946d9142b2ae311a948fd0be3d/mcp-1.29.0-py3-none-any.whl", hash = "sha256:f5a075bb611f23d6f4d080c6a1699fa62772eebc562ba9e66b306ddde1c755f7", size = 223436, upload-time = "2026-07-28T13:41:40.337Z" }, ] [[package]] @@ -2589,17 +2622,12 @@ wheels = [ ] [[package]] -name = "ocrmac" -version = "1.0.1" +name = "olefile" +version = "0.47" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click", marker = "sys_platform == 'darwin'" }, - { name = "pillow", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-vision", marker = "sys_platform == 'darwin'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5e/07/3e15ab404f75875c5e48c47163300eb90b7409044d8711fc3aaf52503f2e/ocrmac-1.0.1.tar.gz", hash = "sha256:507fe5e4cbd67b2d03f6729a52bbc11f9d0b58241134eb958a5daafd4b9d93d9", size = 1454317, upload-time = "2026-01-08T16:44:26.412Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/1b/077b508e3e500e1629d366249c3ccb32f95e50258b231705c09e3c7a4366/olefile-0.47.zip", hash = "sha256:599383381a0bf3dfbd932ca0ca6515acd174ed48870cbf7fee123d698c192c1c", size = 112240, upload-time = "2023-12-01T16:22:53.025Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/37/15/7cc16507a2aca927abe395f1c545f17ae76b1f8ed44f43ebe4e8670ee203/ocrmac-1.0.1-py3-none-any.whl", hash = "sha256:1cef25426f7ae6bbd57fe3dc5553b25461ae8ad0d2b428a9bbadbf5907349024", size = 9955, upload-time = "2026-01-08T16:44:25.555Z" }, + { url = "https://files.pythonhosted.org/packages/17/d3/b64c356a907242d719fc668b71befd73324e47ab46c8ebbbede252c154b2/olefile-0.47-py2.py3-none-any.whl", hash = "sha256:543c7da2a7adadf21214938bb79c83ea12b473a4b6ee4ad4bf854e7715e13d1f", size = 114565, upload-time = "2023-12-01T16:22:51.518Z" }, ] [[package]] @@ -2617,7 +2645,7 @@ wheels = [ [[package]] name = "openai" -version = "2.26.0" +version = "2.54.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2629,9 +2657,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d7/91/2a06c4e9597c338cac1e5e5a8dd6f29e1836fc229c4c523529dca387fda8/openai-2.26.0.tar.gz", hash = "sha256:b41f37c140ae0034a6e92b0c509376d907f3a66109935fba2c1b471a7c05a8fb", size = 666702, upload-time = "2026-03-05T23:17:35.874Z" } +sdist = { url = "https://files.pythonhosted.org/packages/50/9a/8c75e8c8a5b407a0586faeb2afac91674ff955c191ecc1d6d3b6669f6788/openai-2.54.0.tar.gz", hash = "sha256:e3e6f8bc1ba30ddf381ace1a14340eed381cb984a1a59bd0f34b5be3b5d49cfa", size = 1100285, upload-time = "2026-08-11T18:46:59.035Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/2e/3f73e8ca53718952222cacd0cf7eecc9db439d020f0c1fe7ae717e4e199a/openai-2.26.0-py3-none-any.whl", hash = "sha256:6151bf8f83802f036117f06cc8a57b3a4da60da9926826cc96747888b57f394f", size = 1136409, upload-time = "2026-03-05T23:17:34.072Z" }, + { url = "https://files.pythonhosted.org/packages/64/a8/bb76c7356de8ad57f59d5ff993d434df0607f07f08bcc9c9a5c275e399c0/openai-2.54.0-py3-none-any.whl", hash = "sha256:89089789197ccdb87f173a03145ed1598d00795220c93e96cf712b1cbf5e5f2b", size = 1660351, upload-time = "2026-08-11T18:46:56.684Z" }, ] [[package]] @@ -2856,89 +2884,87 @@ wheels = [ [[package]] name = "pillow" -version = "12.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1f/42/5c74462b4fd957fcd7b13b04fb3205ff8349236ea74c7c375766d6c82288/pillow-12.1.1.tar.gz", hash = "sha256:9ad8fa5937ab05218e2b6a4cff30295ad35afd2f83ac592e68c0d871bb0fdbc4", size = 46980264, upload-time = "2026-02-11T04:23:07.146Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/46/5da1ec4a5171ee7bf1a0efa064aba70ba3d6e0788ce3f5acd1375d23c8c0/pillow-12.1.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:e879bb6cd5c73848ef3b2b48b8af9ff08c5b71ecda8048b7dd22d8a33f60be32", size = 5304084, upload-time = "2026-02-11T04:20:27.501Z" }, - { url = "https://files.pythonhosted.org/packages/78/93/a29e9bc02d1cf557a834da780ceccd54e02421627200696fcf805ebdc3fb/pillow-12.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:365b10bb9417dd4498c0e3b128018c4a624dc11c7b97d8cc54effe3b096f4c38", size = 4657866, upload-time = "2026-02-11T04:20:29.827Z" }, - { url = "https://files.pythonhosted.org/packages/13/84/583a4558d492a179d31e4aae32eadce94b9acf49c0337c4ce0b70e0a01f2/pillow-12.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d4ce8e329c93845720cd2014659ca67eac35f6433fd3050393d85f3ecef0dad5", size = 6232148, upload-time = "2026-02-11T04:20:31.329Z" }, - { url = "https://files.pythonhosted.org/packages/d5/e2/53c43334bbbb2d3b938978532fbda8e62bb6e0b23a26ce8592f36bcc4987/pillow-12.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc354a04072b765eccf2204f588a7a532c9511e8b9c7f900e1b64e3e33487090", size = 8038007, upload-time = "2026-02-11T04:20:34.225Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a6/3d0e79c8a9d58150dd98e199d7c1c56861027f3829a3a60b3c2784190180/pillow-12.1.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e7976bf1910a8116b523b9f9f58bf410f3e8aa330cd9a2bb2953f9266ab49af", size = 6345418, upload-time = "2026-02-11T04:20:35.858Z" }, - { url = "https://files.pythonhosted.org/packages/a2/c8/46dfeac5825e600579157eea177be43e2f7ff4a99da9d0d0a49533509ac5/pillow-12.1.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:597bd9c8419bc7c6af5604e55847789b69123bbe25d65cc6ad3012b4f3c98d8b", size = 7034590, upload-time = "2026-02-11T04:20:37.91Z" }, - { url = "https://files.pythonhosted.org/packages/af/bf/e6f65d3db8a8bbfeaf9e13cc0417813f6319863a73de934f14b2229ada18/pillow-12.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2c1fc0f2ca5f96a3c8407e41cca26a16e46b21060fe6d5b099d2cb01412222f5", size = 6458655, upload-time = "2026-02-11T04:20:39.496Z" }, - { url = "https://files.pythonhosted.org/packages/f9/c2/66091f3f34a25894ca129362e510b956ef26f8fb67a0e6417bc5744e56f1/pillow-12.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:578510d88c6229d735855e1f278aa305270438d36a05031dfaae5067cc8eb04d", size = 7159286, upload-time = "2026-02-11T04:20:41.139Z" }, - { url = "https://files.pythonhosted.org/packages/7b/5a/24bc8eb526a22f957d0cec6243146744966d40857e3d8deb68f7902ca6c1/pillow-12.1.1-cp311-cp311-win32.whl", hash = "sha256:7311c0a0dcadb89b36b7025dfd8326ecfa36964e29913074d47382706e516a7c", size = 6328663, upload-time = "2026-02-11T04:20:43.184Z" }, - { url = "https://files.pythonhosted.org/packages/31/03/bef822e4f2d8f9d7448c133d0a18185d3cce3e70472774fffefe8b0ed562/pillow-12.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:fbfa2a7c10cc2623f412753cddf391c7f971c52ca40a3f65dc5039b2939e8563", size = 7031448, upload-time = "2026-02-11T04:20:44.696Z" }, - { url = "https://files.pythonhosted.org/packages/49/70/f76296f53610bd17b2e7d31728b8b7825e3ac3b5b3688b51f52eab7c0818/pillow-12.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:b81b5e3511211631b3f672a595e3221252c90af017e399056d0faabb9538aa80", size = 2453651, upload-time = "2026-02-11T04:20:46.243Z" }, - { url = "https://files.pythonhosted.org/packages/07/d3/8df65da0d4df36b094351dce696f2989bec731d4f10e743b1c5f4da4d3bf/pillow-12.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab323b787d6e18b3d91a72fc99b1a2c28651e4358749842b8f8dfacd28ef2052", size = 5262803, upload-time = "2026-02-11T04:20:47.653Z" }, - { url = "https://files.pythonhosted.org/packages/d6/71/5026395b290ff404b836e636f51d7297e6c83beceaa87c592718747e670f/pillow-12.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:adebb5bee0f0af4909c30db0d890c773d1a92ffe83da908e2e9e720f8edf3984", size = 4657601, upload-time = "2026-02-11T04:20:49.328Z" }, - { url = "https://files.pythonhosted.org/packages/b1/2e/1001613d941c67442f745aff0f7cc66dd8df9a9c084eb497e6a543ee6f7e/pillow-12.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb66b7cc26f50977108790e2456b7921e773f23db5630261102233eb355a3b79", size = 6234995, upload-time = "2026-02-11T04:20:51.032Z" }, - { url = "https://files.pythonhosted.org/packages/07/26/246ab11455b2549b9233dbd44d358d033a2f780fa9007b61a913c5b2d24e/pillow-12.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aee2810642b2898bb187ced9b349e95d2a7272930796e022efaf12e99dccd293", size = 8045012, upload-time = "2026-02-11T04:20:52.882Z" }, - { url = "https://files.pythonhosted.org/packages/b2/8b/07587069c27be7535ac1fe33874e32de118fbd34e2a73b7f83436a88368c/pillow-12.1.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a0b1cd6232e2b618adcc54d9882e4e662a089d5768cd188f7c245b4c8c44a397", size = 6349638, upload-time = "2026-02-11T04:20:54.444Z" }, - { url = "https://files.pythonhosted.org/packages/ff/79/6df7b2ee763d619cda2fb4fea498e5f79d984dae304d45a8999b80d6cf5c/pillow-12.1.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7aac39bcf8d4770d089588a2e1dd111cbaa42df5a94be3114222057d68336bd0", size = 7041540, upload-time = "2026-02-11T04:20:55.97Z" }, - { url = "https://files.pythonhosted.org/packages/2c/5e/2ba19e7e7236d7529f4d873bdaf317a318896bac289abebd4bb00ef247f0/pillow-12.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ab174cd7d29a62dd139c44bf74b698039328f45cb03b4596c43473a46656b2f3", size = 6462613, upload-time = "2026-02-11T04:20:57.542Z" }, - { url = "https://files.pythonhosted.org/packages/03/03/31216ec124bb5c3dacd74ce8efff4cc7f52643653bad4825f8f08c697743/pillow-12.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:339ffdcb7cbeaa08221cd401d517d4b1fe7a9ed5d400e4a8039719238620ca35", size = 7166745, upload-time = "2026-02-11T04:20:59.196Z" }, - { url = "https://files.pythonhosted.org/packages/1f/e7/7c4552d80052337eb28653b617eafdef39adfb137c49dd7e831b8dc13bc5/pillow-12.1.1-cp312-cp312-win32.whl", hash = "sha256:5d1f9575a12bed9e9eedd9a4972834b08c97a352bd17955ccdebfeca5913fa0a", size = 6328823, upload-time = "2026-02-11T04:21:01.385Z" }, - { url = "https://files.pythonhosted.org/packages/3d/17/688626d192d7261bbbf98846fc98995726bddc2c945344b65bec3a29d731/pillow-12.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:21329ec8c96c6e979cd0dfd29406c40c1d52521a90544463057d2aaa937d66a6", size = 7033367, upload-time = "2026-02-11T04:21:03.536Z" }, - { url = "https://files.pythonhosted.org/packages/ed/fe/a0ef1f73f939b0eca03ee2c108d0043a87468664770612602c63266a43c4/pillow-12.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:af9a332e572978f0218686636610555ae3defd1633597be015ed50289a03c523", size = 2453811, upload-time = "2026-02-11T04:21:05.116Z" }, - { url = "https://files.pythonhosted.org/packages/d5/11/6db24d4bd7685583caeae54b7009584e38da3c3d4488ed4cd25b439de486/pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d242e8ac078781f1de88bf823d70c1a9b3c7950a44cdf4b7c012e22ccbcd8e4e", size = 4062689, upload-time = "2026-02-11T04:21:06.804Z" }, - { url = "https://files.pythonhosted.org/packages/33/c0/ce6d3b1fe190f0021203e0d9b5b99e57843e345f15f9ef22fcd43842fd21/pillow-12.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:02f84dfad02693676692746df05b89cf25597560db2857363a208e393429f5e9", size = 4138535, upload-time = "2026-02-11T04:21:08.452Z" }, - { url = "https://files.pythonhosted.org/packages/a0/c6/d5eb6a4fb32a3f9c21a8c7613ec706534ea1cf9f4b3663e99f0d83f6fca8/pillow-12.1.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:e65498daf4b583091ccbb2556c7000abf0f3349fcd57ef7adc9a84a394ed29f6", size = 3601364, upload-time = "2026-02-11T04:21:10.194Z" }, - { url = "https://files.pythonhosted.org/packages/14/a1/16c4b823838ba4c9c52c0e6bbda903a3fe5a1bdbf1b8eb4fff7156f3e318/pillow-12.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c6db3b84c87d48d0088943bf33440e0c42370b99b1c2a7989216f7b42eede60", size = 5262561, upload-time = "2026-02-11T04:21:11.742Z" }, - { url = "https://files.pythonhosted.org/packages/bb/ad/ad9dc98ff24f485008aa5cdedaf1a219876f6f6c42a4626c08bc4e80b120/pillow-12.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8b7e5304e34942bf62e15184219a7b5ad4ff7f3bb5cca4d984f37df1a0e1aee2", size = 4657460, upload-time = "2026-02-11T04:21:13.786Z" }, - { url = "https://files.pythonhosted.org/packages/9e/1b/f1a4ea9a895b5732152789326202a82464d5254759fbacae4deea3069334/pillow-12.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:18e5bddd742a44b7e6b1e773ab5db102bd7a94c32555ba656e76d319d19c3850", size = 6232698, upload-time = "2026-02-11T04:21:15.949Z" }, - { url = "https://files.pythonhosted.org/packages/95/f4/86f51b8745070daf21fd2e5b1fe0eb35d4db9ca26e6d58366562fb56a743/pillow-12.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fc44ef1f3de4f45b50ccf9136999d71abb99dca7706bc75d222ed350b9fd2289", size = 8041706, upload-time = "2026-02-11T04:21:17.723Z" }, - { url = "https://files.pythonhosted.org/packages/29/9b/d6ecd956bb1266dd1045e995cce9b8d77759e740953a1c9aad9502a0461e/pillow-12.1.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a8eb7ed8d4198bccbd07058416eeec51686b498e784eda166395a23eb99138e", size = 6346621, upload-time = "2026-02-11T04:21:19.547Z" }, - { url = "https://files.pythonhosted.org/packages/71/24/538bff45bde96535d7d998c6fed1a751c75ac7c53c37c90dc2601b243893/pillow-12.1.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47b94983da0c642de92ced1702c5b6c292a84bd3a8e1d1702ff923f183594717", size = 7038069, upload-time = "2026-02-11T04:21:21.378Z" }, - { url = "https://files.pythonhosted.org/packages/94/0e/58cb1a6bc48f746bc4cb3adb8cabff73e2742c92b3bf7a220b7cf69b9177/pillow-12.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:518a48c2aab7ce596d3bf79d0e275661b846e86e4d0e7dec34712c30fe07f02a", size = 6460040, upload-time = "2026-02-11T04:21:23.148Z" }, - { url = "https://files.pythonhosted.org/packages/6c/57/9045cb3ff11eeb6c1adce3b2d60d7d299d7b273a2e6c8381a524abfdc474/pillow-12.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a550ae29b95c6dc13cf69e2c9dc5747f814c54eeb2e32d683e5e93af56caa029", size = 7164523, upload-time = "2026-02-11T04:21:25.01Z" }, - { url = "https://files.pythonhosted.org/packages/73/f2/9be9cb99f2175f0d4dbadd6616ce1bf068ee54a28277ea1bf1fbf729c250/pillow-12.1.1-cp313-cp313-win32.whl", hash = "sha256:a003d7422449f6d1e3a34e3dd4110c22148336918ddbfc6a32581cd54b2e0b2b", size = 6332552, upload-time = "2026-02-11T04:21:27.238Z" }, - { url = "https://files.pythonhosted.org/packages/3f/eb/b0834ad8b583d7d9d42b80becff092082a1c3c156bb582590fcc973f1c7c/pillow-12.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:344cf1e3dab3be4b1fa08e449323d98a2a3f819ad20f4b22e77a0ede31f0faa1", size = 7040108, upload-time = "2026-02-11T04:21:29.462Z" }, - { url = "https://files.pythonhosted.org/packages/d5/7d/fc09634e2aabdd0feabaff4a32f4a7d97789223e7c2042fd805ea4b4d2c2/pillow-12.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:5c0dd1636633e7e6a0afe7bf6a51a14992b7f8e60de5789018ebbdfae55b040a", size = 2453712, upload-time = "2026-02-11T04:21:31.072Z" }, - { url = "https://files.pythonhosted.org/packages/19/2a/b9d62794fc8a0dd14c1943df68347badbd5511103e0d04c035ffe5cf2255/pillow-12.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0330d233c1a0ead844fc097a7d16c0abff4c12e856c0b325f231820fee1f39da", size = 5264880, upload-time = "2026-02-11T04:21:32.865Z" }, - { url = "https://files.pythonhosted.org/packages/26/9d/e03d857d1347fa5ed9247e123fcd2a97b6220e15e9cb73ca0a8d91702c6e/pillow-12.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5dae5f21afb91322f2ff791895ddd8889e5e947ff59f71b46041c8ce6db790bc", size = 4660616, upload-time = "2026-02-11T04:21:34.97Z" }, - { url = "https://files.pythonhosted.org/packages/f7/ec/8a6d22afd02570d30954e043f09c32772bfe143ba9285e2fdb11284952cd/pillow-12.1.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e0c664be47252947d870ac0d327fea7e63985a08794758aa8af5b6cb6ec0c9c", size = 6269008, upload-time = "2026-02-11T04:21:36.623Z" }, - { url = "https://files.pythonhosted.org/packages/3d/1d/6d875422c9f28a4a361f495a5f68d9de4a66941dc2c619103ca335fa6446/pillow-12.1.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:691ab2ac363b8217f7d31b3497108fb1f50faab2f75dfb03284ec2f217e87bf8", size = 8073226, upload-time = "2026-02-11T04:21:38.585Z" }, - { url = "https://files.pythonhosted.org/packages/a1/cd/134b0b6ee5eda6dc09e25e24b40fdafe11a520bc725c1d0bbaa5e00bf95b/pillow-12.1.1-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9e8064fb1cc019296958595f6db671fba95209e3ceb0c4734c9baf97de04b20", size = 6380136, upload-time = "2026-02-11T04:21:40.562Z" }, - { url = "https://files.pythonhosted.org/packages/7a/a9/7628f013f18f001c1b98d8fffe3452f306a70dc6aba7d931019e0492f45e/pillow-12.1.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:472a8d7ded663e6162dafdf20015c486a7009483ca671cece7a9279b512fcb13", size = 7067129, upload-time = "2026-02-11T04:21:42.521Z" }, - { url = "https://files.pythonhosted.org/packages/1e/f8/66ab30a2193b277785601e82ee2d49f68ea575d9637e5e234faaa98efa4c/pillow-12.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:89b54027a766529136a06cfebeecb3a04900397a3590fd252160b888479517bf", size = 6491807, upload-time = "2026-02-11T04:21:44.22Z" }, - { url = "https://files.pythonhosted.org/packages/da/0b/a877a6627dc8318fdb84e357c5e1a758c0941ab1ddffdafd231983788579/pillow-12.1.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:86172b0831b82ce4f7877f280055892b31179e1576aa00d0df3bb1bbf8c3e524", size = 7190954, upload-time = "2026-02-11T04:21:46.114Z" }, - { url = "https://files.pythonhosted.org/packages/83/43/6f732ff85743cf746b1361b91665d9f5155e1483817f693f8d57ea93147f/pillow-12.1.1-cp313-cp313t-win32.whl", hash = "sha256:44ce27545b6efcf0fdbdceb31c9a5bdea9333e664cda58a7e674bb74608b3986", size = 6336441, upload-time = "2026-02-11T04:21:48.22Z" }, - { url = "https://files.pythonhosted.org/packages/3b/44/e865ef3986611bb75bfabdf94a590016ea327833f434558801122979cd0e/pillow-12.1.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a285e3eb7a5a45a2ff504e31f4a8d1b12ef62e84e5411c6804a42197c1cf586c", size = 7045383, upload-time = "2026-02-11T04:21:50.015Z" }, - { url = "https://files.pythonhosted.org/packages/a8/c6/f4fb24268d0c6908b9f04143697ea18b0379490cb74ba9e8d41b898bd005/pillow-12.1.1-cp313-cp313t-win_arm64.whl", hash = "sha256:cc7d296b5ea4d29e6570dabeaed58d31c3fea35a633a69679fb03d7664f43fb3", size = 2456104, upload-time = "2026-02-11T04:21:51.633Z" }, - { url = "https://files.pythonhosted.org/packages/03/d0/bebb3ffbf31c5a8e97241476c4cf8b9828954693ce6744b4a2326af3e16b/pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:417423db963cb4be8bac3fc1204fe61610f6abeed1580a7a2cbb2fbda20f12af", size = 4062652, upload-time = "2026-02-11T04:21:53.19Z" }, - { url = "https://files.pythonhosted.org/packages/2d/c0/0e16fb0addda4851445c28f8350d8c512f09de27bbb0d6d0bbf8b6709605/pillow-12.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:b957b71c6b2387610f556a7eb0828afbe40b4a98036fc0d2acfa5a44a0c2036f", size = 4138823, upload-time = "2026-02-11T04:22:03.088Z" }, - { url = "https://files.pythonhosted.org/packages/6b/fb/6170ec655d6f6bb6630a013dd7cf7bc218423d7b5fa9071bf63dc32175ae/pillow-12.1.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:097690ba1f2efdeb165a20469d59d8bb03c55fb6621eb2041a060ae8ea3e9642", size = 3601143, upload-time = "2026-02-11T04:22:04.909Z" }, - { url = "https://files.pythonhosted.org/packages/59/04/dc5c3f297510ba9a6837cbb318b87dd2b8f73eb41a43cc63767f65cb599c/pillow-12.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2815a87ab27848db0321fb78c7f0b2c8649dee134b7f2b80c6a45c6831d75ccd", size = 5266254, upload-time = "2026-02-11T04:22:07.656Z" }, - { url = "https://files.pythonhosted.org/packages/05/30/5db1236b0d6313f03ebf97f5e17cda9ca060f524b2fcc875149a8360b21c/pillow-12.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f7ed2c6543bad5a7d5530eb9e78c53132f93dfa44a28492db88b41cdab885202", size = 4657499, upload-time = "2026-02-11T04:22:09.613Z" }, - { url = "https://files.pythonhosted.org/packages/6f/18/008d2ca0eb612e81968e8be0bbae5051efba24d52debf930126d7eaacbba/pillow-12.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:652a2c9ccfb556235b2b501a3a7cf3742148cd22e04b5625c5fe057ea3e3191f", size = 6232137, upload-time = "2026-02-11T04:22:11.434Z" }, - { url = "https://files.pythonhosted.org/packages/70/f1/f14d5b8eeb4b2cd62b9f9f847eb6605f103df89ef619ac68f92f748614ea/pillow-12.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d6e4571eedf43af33d0fc233a382a76e849badbccdf1ac438841308652a08e1f", size = 8042721, upload-time = "2026-02-11T04:22:13.321Z" }, - { url = "https://files.pythonhosted.org/packages/5a/d6/17824509146e4babbdabf04d8171491fa9d776f7061ff6e727522df9bd03/pillow-12.1.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b574c51cf7d5d62e9be37ba446224b59a2da26dc4c1bb2ecbe936a4fb1a7cb7f", size = 6347798, upload-time = "2026-02-11T04:22:15.449Z" }, - { url = "https://files.pythonhosted.org/packages/d1/ee/c85a38a9ab92037a75615aba572c85ea51e605265036e00c5b67dfafbfe2/pillow-12.1.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a37691702ed687799de29a518d63d4682d9016932db66d4e90c345831b02fb4e", size = 7039315, upload-time = "2026-02-11T04:22:17.24Z" }, - { url = "https://files.pythonhosted.org/packages/ec/f3/bc8ccc6e08a148290d7523bde4d9a0d6c981db34631390dc6e6ec34cacf6/pillow-12.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f95c00d5d6700b2b890479664a06e754974848afaae5e21beb4d83c106923fd0", size = 6462360, upload-time = "2026-02-11T04:22:19.111Z" }, - { url = "https://files.pythonhosted.org/packages/f6/ab/69a42656adb1d0665ab051eec58a41f169ad295cf81ad45406963105408f/pillow-12.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:559b38da23606e68681337ad74622c4dbba02254fc9cb4488a305dd5975c7eeb", size = 7165438, upload-time = "2026-02-11T04:22:21.041Z" }, - { url = "https://files.pythonhosted.org/packages/02/46/81f7aa8941873f0f01d4b55cc543b0a3d03ec2ee30d617a0448bf6bd6dec/pillow-12.1.1-cp314-cp314-win32.whl", hash = "sha256:03edcc34d688572014ff223c125a3f77fb08091e4607e7745002fc214070b35f", size = 6431503, upload-time = "2026-02-11T04:22:22.833Z" }, - { url = "https://files.pythonhosted.org/packages/40/72/4c245f7d1044b67affc7f134a09ea619d4895333d35322b775b928180044/pillow-12.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:50480dcd74fa63b8e78235957d302d98d98d82ccbfac4c7e12108ba9ecbdba15", size = 7176748, upload-time = "2026-02-11T04:22:24.64Z" }, - { url = "https://files.pythonhosted.org/packages/e4/ad/8a87bdbe038c5c698736e3348af5c2194ffb872ea52f11894c95f9305435/pillow-12.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:5cb1785d97b0c3d1d1a16bc1d710c4a0049daefc4935f3a8f31f827f4d3d2e7f", size = 2544314, upload-time = "2026-02-11T04:22:26.685Z" }, - { url = "https://files.pythonhosted.org/packages/6c/9d/efd18493f9de13b87ede7c47e69184b9e859e4427225ea962e32e56a49bc/pillow-12.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1f90cff8aa76835cba5769f0b3121a22bd4eb9e6884cfe338216e557a9a548b8", size = 5268612, upload-time = "2026-02-11T04:22:29.884Z" }, - { url = "https://files.pythonhosted.org/packages/f8/f1/4f42eb2b388eb2ffc660dcb7f7b556c1015c53ebd5f7f754965ef997585b/pillow-12.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1f1be78ce9466a7ee64bfda57bdba0f7cc499d9794d518b854816c41bf0aa4e9", size = 4660567, upload-time = "2026-02-11T04:22:31.799Z" }, - { url = "https://files.pythonhosted.org/packages/01/54/df6ef130fa43e4b82e32624a7b821a2be1c5653a5fdad8469687a7db4e00/pillow-12.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:42fc1f4677106188ad9a55562bbade416f8b55456f522430fadab3cef7cd4e60", size = 6269951, upload-time = "2026-02-11T04:22:33.921Z" }, - { url = "https://files.pythonhosted.org/packages/a9/48/618752d06cc44bb4aae8ce0cd4e6426871929ed7b46215638088270d9b34/pillow-12.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:98edb152429ab62a1818039744d8fbb3ccab98a7c29fc3d5fcef158f3f1f68b7", size = 8074769, upload-time = "2026-02-11T04:22:35.877Z" }, - { url = "https://files.pythonhosted.org/packages/c3/bd/f1d71eb39a72fa088d938655afba3e00b38018d052752f435838961127d8/pillow-12.1.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d470ab1178551dd17fdba0fef463359c41aaa613cdcd7ff8373f54be629f9f8f", size = 6381358, upload-time = "2026-02-11T04:22:37.698Z" }, - { url = "https://files.pythonhosted.org/packages/64/ef/c784e20b96674ed36a5af839305f55616f8b4f8aa8eeccf8531a6e312243/pillow-12.1.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6408a7b064595afcab0a49393a413732a35788f2a5092fdc6266952ed67de586", size = 7068558, upload-time = "2026-02-11T04:22:39.597Z" }, - { url = "https://files.pythonhosted.org/packages/73/cb/8059688b74422ae61278202c4e1ad992e8a2e7375227be0a21c6b87ca8d5/pillow-12.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5d8c41325b382c07799a3682c1c258469ea2ff97103c53717b7893862d0c98ce", size = 6493028, upload-time = "2026-02-11T04:22:42.73Z" }, - { url = "https://files.pythonhosted.org/packages/c6/da/e3c008ed7d2dd1f905b15949325934510b9d1931e5df999bb15972756818/pillow-12.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c7697918b5be27424e9ce568193efd13d925c4481dd364e43f5dff72d33e10f8", size = 7191940, upload-time = "2026-02-11T04:22:44.543Z" }, - { url = "https://files.pythonhosted.org/packages/01/4a/9202e8d11714c1fc5951f2e1ef362f2d7fbc595e1f6717971d5dd750e969/pillow-12.1.1-cp314-cp314t-win32.whl", hash = "sha256:d2912fd8114fc5545aa3a4b5576512f64c55a03f3ebcca4c10194d593d43ea36", size = 6438736, upload-time = "2026-02-11T04:22:46.347Z" }, - { url = "https://files.pythonhosted.org/packages/f3/ca/cbce2327eb9885476b3957b2e82eb12c866a8b16ad77392864ad601022ce/pillow-12.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4ceb838d4bd9dab43e06c363cab2eebf63846d6a4aeaea283bbdfd8f1a8ed58b", size = 7182894, upload-time = "2026-02-11T04:22:48.114Z" }, - { url = "https://files.pythonhosted.org/packages/ec/d2/de599c95ba0a973b94410477f8bf0b6f0b5e67360eb89bcb1ad365258beb/pillow-12.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7b03048319bfc6170e93bd60728a1af51d3dd7704935feb228c4d4faab35d334", size = 2546446, upload-time = "2026-02-11T04:22:50.342Z" }, - { url = "https://files.pythonhosted.org/packages/56/11/5d43209aa4cb58e0cc80127956ff1796a68b928e6324bbf06ef4db34367b/pillow-12.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:600fd103672b925fe62ed08e0d874ea34d692474df6f4bf7ebe148b30f89f39f", size = 5228606, upload-time = "2026-02-11T04:22:52.106Z" }, - { url = "https://files.pythonhosted.org/packages/5f/d5/3b005b4e4fda6698b371fa6c21b097d4707585d7db99e98d9b0b87ac612a/pillow-12.1.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:665e1b916b043cef294bc54d47bf02d87e13f769bc4bc5fa225a24b3a6c5aca9", size = 4622321, upload-time = "2026-02-11T04:22:53.827Z" }, - { url = "https://files.pythonhosted.org/packages/df/36/ed3ea2d594356fd8037e5a01f6156c74bc8d92dbb0fa60746cc96cabb6e8/pillow-12.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:495c302af3aad1ca67420ddd5c7bd480c8867ad173528767d906428057a11f0e", size = 5247579, upload-time = "2026-02-11T04:22:56.094Z" }, - { url = "https://files.pythonhosted.org/packages/54/9a/9cc3e029683cf6d20ae5085da0dafc63148e3252c2f13328e553aaa13cfb/pillow-12.1.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8fd420ef0c52c88b5a035a0886f367748c72147b2b8f384c9d12656678dfdfa9", size = 6989094, upload-time = "2026-02-11T04:22:58.288Z" }, - { url = "https://files.pythonhosted.org/packages/00/98/fc53ab36da80b88df0967896b6c4b4cd948a0dc5aa40a754266aa3ae48b3/pillow-12.1.1-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f975aa7ef9684ce7e2c18a3aa8f8e2106ce1e46b94ab713d156b2898811651d3", size = 5313850, upload-time = "2026-02-11T04:23:00.554Z" }, - { url = "https://files.pythonhosted.org/packages/30/02/00fa585abfd9fe9d73e5f6e554dc36cc2b842898cbfc46d70353dae227f8/pillow-12.1.1-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8089c852a56c2966cf18835db62d9b34fef7ba74c726ad943928d494fa7f4735", size = 5963343, upload-time = "2026-02-11T04:23:02.934Z" }, - { url = "https://files.pythonhosted.org/packages/f2/26/c56ce33ca856e358d27fda9676c055395abddb82c35ac0f593877ed4562e/pillow-12.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:cb9bb857b2d057c6dfc72ac5f3b44836924ba15721882ef103cecb40d002d80e", size = 7029880, upload-time = "2026-02-11T04:23:04.783Z" }, +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/c8/0a78b0e02d7ac54bc03e5321c9220da52f0c2ea83b21f7c40e7f3169c502/pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756", size = 5392415, upload-time = "2026-07-01T11:53:47.162Z" }, + { url = "https://files.pythonhosted.org/packages/b2/5b/a02d30018abd97ced9f5a6c63d28597694a00d066516b9c1c6de45859fc9/pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6", size = 4785266, upload-time = "2026-07-01T11:53:49.079Z" }, + { url = "https://files.pythonhosted.org/packages/c8/98/766667a4be768150a202836acd9fad19c06824ca86c4286d3cf6b274964e/pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd", size = 6263814, upload-time = "2026-07-01T11:53:51.32Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2d/ede717bc1144f63886c21fd349bb95860b0d1a21149ff16f2bb362b612b6/pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd", size = 6934408, upload-time = "2026-07-01T11:53:53.487Z" }, + { url = "https://files.pythonhosted.org/packages/a3/48/9c58b685e69d49c31af6c8eb9012055fab7e665785165c84796e2c73ce72/pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c", size = 6337160, upload-time = "2026-07-01T11:53:55.457Z" }, + { url = "https://files.pythonhosted.org/packages/ff/fa/dc2a5c0ba6df93f67c31d34b808b7ce440b40cdbf96f0b81cde1d1e6fa93/pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5", size = 7045172, upload-time = "2026-07-01T11:53:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/86/a5/444817a4d4c4c2417df00513086ca196f388d8f9ef40c2e4ccd1ad1af54b/pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b", size = 6472232, upload-time = "2026-07-01T11:53:59.767Z" }, + { url = "https://files.pythonhosted.org/packages/63/c6/4bad1b18d132a50b27e1365e1ab163616f7a5bb56d330f66f9d1d9d4f9d4/pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a", size = 7233653, upload-time = "2026-07-01T11:54:02.066Z" }, + { url = "https://files.pythonhosted.org/packages/fd/16/00f91ab7760dc842f5aad55217e80fc4a7067a0604535249bc8a2d6d9870/pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26", size = 2568195, upload-time = "2026-07-01T11:54:04.622Z" }, + { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, + { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, + { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, + { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, + { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, + { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, + { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, + { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, + { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, + { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, + { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" }, + { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" }, + { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, + { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, + { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, + { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" }, + { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, + { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, + { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" }, + { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" }, + { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, + { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, + { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, + { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" }, + { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" }, + { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" }, + { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, + { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, + { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, + { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, + { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, + { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, + { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, + { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, + { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, + { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, + { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, + { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, + { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, + { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, + { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, + { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, + { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, + { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, + { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, + { url = "https://files.pythonhosted.org/packages/75/18/2e8b40223153ccbc60df07f9e8928dc0c76202aa4e55ae9f53962b6510d6/pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468", size = 5302510, upload-time = "2026-07-01T11:56:25.736Z" }, + { url = "https://files.pythonhosted.org/packages/46/3e/51fabf59d5ab801ceab709453d3ab6b180083496579549de4c45ced6528a/pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94", size = 4736058, upload-time = "2026-07-01T11:56:28.041Z" }, + { url = "https://files.pythonhosted.org/packages/bf/20/22fe9384b7949e25fb1293bcfc84fb82590ff4ea6b37c95b24d26d793d86/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e", size = 5237776, upload-time = "2026-07-01T11:56:30.263Z" }, + { url = "https://files.pythonhosted.org/packages/08/14/f6ba68107680ffa74b39985f3f30884e41318fbc4250caa423c79b4788bb/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3", size = 5860358, upload-time = "2026-07-01T11:56:32.68Z" }, + { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, ] [[package]] @@ -3050,17 +3076,17 @@ wheels = [ [[package]] name = "protobuf" -version = "6.33.5" +version = "6.33.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/25/7c72c307aafc96fa87062aa6291d9f7c94836e43214d43722e86037aac02/protobuf-6.33.5.tar.gz", hash = "sha256:6ddcac2a081f8b7b9642c09406bc6a4290128fce5f471cddd165960bb9119e5c", size = 444465, upload-time = "2026-01-29T21:51:33.494Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b1/79/af92d0a8369732b027e6d6084251dd8e782c685c72da161bd4a2e00fbabb/protobuf-6.33.5-cp310-abi3-win32.whl", hash = "sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b", size = 425769, upload-time = "2026-01-29T21:51:21.751Z" }, - { url = "https://files.pythonhosted.org/packages/55/75/bb9bc917d10e9ee13dee8607eb9ab963b7cf8be607c46e7862c748aa2af7/protobuf-6.33.5-cp310-abi3-win_amd64.whl", hash = "sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c", size = 437118, upload-time = "2026-01-29T21:51:24.022Z" }, - { url = "https://files.pythonhosted.org/packages/a2/6b/e48dfc1191bc5b52950246275bf4089773e91cb5ba3592621723cdddca62/protobuf-6.33.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:a5cb85982d95d906df1e2210e58f8e4f1e3cdc088e52c921a041f9c9a0386de5", size = 427766, upload-time = "2026-01-29T21:51:25.413Z" }, - { url = "https://files.pythonhosted.org/packages/4e/b1/c79468184310de09d75095ed1314b839eb2f72df71097db9d1404a1b2717/protobuf-6.33.5-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:9b71e0281f36f179d00cbcb119cb19dec4d14a81393e5ea220f64b286173e190", size = 324638, upload-time = "2026-01-29T21:51:26.423Z" }, - { url = "https://files.pythonhosted.org/packages/c5/f5/65d838092fd01c44d16037953fd4c2cc851e783de9b8f02b27ec4ffd906f/protobuf-6.33.5-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:8afa18e1d6d20af15b417e728e9f60f3aa108ee76f23c3b2c07a2c3b546d3afd", size = 339411, upload-time = "2026-01-29T21:51:27.446Z" }, - { url = "https://files.pythonhosted.org/packages/9b/53/a9443aa3ca9ba8724fdfa02dd1887c1bcd8e89556b715cfbacca6b63dbec/protobuf-6.33.5-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:cbf16ba3350fb7b889fca858fb215967792dc125b35c7976ca4818bee3521cf0", size = 323465, upload-time = "2026-01-29T21:51:28.925Z" }, - { url = "https://files.pythonhosted.org/packages/57/bf/2086963c69bdac3d7cff1cc7ff79b8ce5ea0bec6797a017e1be338a46248/protobuf-6.33.5-py3-none-any.whl", hash = "sha256:69915a973dd0f60f31a08b8318b73eab2bd6a392c79184b3612226b0a3f8ec02", size = 170687, upload-time = "2026-01-29T21:51:32.557Z" }, + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] [[package]] @@ -3143,72 +3169,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/be/b732c8418ffa5bcfda002890f5dc4c869fc17db66ff11f53b17cfe44afc0/psycopg2_binary-2.9.12-cp314-cp314-win_amd64.whl", hash = "sha256:f12ae41fcafadb39b2785e64a40f9db05d6de2ac114077457e0e7c597f3af980", size = 2848762, upload-time = "2026-04-20T23:35:46.421Z" }, ] -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - [[package]] name = "pyarrow" -version = "22.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/30/53/04a7fdc63e6056116c9ddc8b43bc28c12cdd181b85cbeadb79278475f3ae/pyarrow-22.0.0.tar.gz", hash = "sha256:3d600dc583260d845c7d8a6db540339dd883081925da2bd1c5cb808f720b3cd9", size = 1151151, upload-time = "2025-10-24T12:30:00.762Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/b7/18f611a8cdc43417f9394a3ccd3eace2f32183c08b9eddc3d17681819f37/pyarrow-22.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:3e294c5eadfb93d78b0763e859a0c16d4051fc1c5231ae8956d61cb0b5666f5a", size = 34272022, upload-time = "2025-10-24T10:04:28.973Z" }, - { url = "https://files.pythonhosted.org/packages/26/5c/f259e2526c67eb4b9e511741b19870a02363a47a35edbebc55c3178db22d/pyarrow-22.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:69763ab2445f632d90b504a815a2a033f74332997052b721002298ed6de40f2e", size = 35995834, upload-time = "2025-10-24T10:04:35.467Z" }, - { url = "https://files.pythonhosted.org/packages/50/8d/281f0f9b9376d4b7f146913b26fac0aa2829cd1ee7e997f53a27411bbb92/pyarrow-22.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:b41f37cabfe2463232684de44bad753d6be08a7a072f6a83447eeaf0e4d2a215", size = 45030348, upload-time = "2025-10-24T10:04:43.366Z" }, - { url = "https://files.pythonhosted.org/packages/f5/e5/53c0a1c428f0976bf22f513d79c73000926cb00b9c138d8e02daf2102e18/pyarrow-22.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:35ad0f0378c9359b3f297299c3309778bb03b8612f987399a0333a560b43862d", size = 47699480, upload-time = "2025-10-24T10:04:51.486Z" }, - { url = "https://files.pythonhosted.org/packages/95/e1/9dbe4c465c3365959d183e6345d0a8d1dc5b02ca3f8db4760b3bc834cf25/pyarrow-22.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8382ad21458075c2e66a82a29d650f963ce51c7708c7c0ff313a8c206c4fd5e8", size = 48011148, upload-time = "2025-10-24T10:04:59.585Z" }, - { url = "https://files.pythonhosted.org/packages/c5/b4/7caf5d21930061444c3cf4fa7535c82faf5263e22ce43af7c2759ceb5b8b/pyarrow-22.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1a812a5b727bc09c3d7ea072c4eebf657c2f7066155506ba31ebf4792f88f016", size = 50276964, upload-time = "2025-10-24T10:05:08.175Z" }, - { url = "https://files.pythonhosted.org/packages/ae/f3/cec89bd99fa3abf826f14d4e53d3d11340ce6f6af4d14bdcd54cd83b6576/pyarrow-22.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:ec5d40dd494882704fb876c16fa7261a69791e784ae34e6b5992e977bd2e238c", size = 28106517, upload-time = "2025-10-24T10:05:14.314Z" }, - { url = "https://files.pythonhosted.org/packages/af/63/ba23862d69652f85b615ca14ad14f3bcfc5bf1b99ef3f0cd04ff93fdad5a/pyarrow-22.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:bea79263d55c24a32b0d79c00a1c58bb2ee5f0757ed95656b01c0fb310c5af3d", size = 34211578, upload-time = "2025-10-24T10:05:21.583Z" }, - { url = "https://files.pythonhosted.org/packages/b1/d0/f9ad86fe809efd2bcc8be32032fa72e8b0d112b01ae56a053006376c5930/pyarrow-22.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:12fe549c9b10ac98c91cf791d2945e878875d95508e1a5d14091a7aaa66d9cf8", size = 35989906, upload-time = "2025-10-24T10:05:29.485Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a8/f910afcb14630e64d673f15904ec27dd31f1e009b77033c365c84e8c1e1d/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:334f900ff08ce0423407af97e6c26ad5d4e3b0763645559ece6fbf3747d6a8f5", size = 45021677, upload-time = "2025-10-24T10:05:38.274Z" }, - { url = "https://files.pythonhosted.org/packages/13/95/aec81f781c75cd10554dc17a25849c720d54feafb6f7847690478dcf5ef8/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c6c791b09c57ed76a18b03f2631753a4960eefbbca80f846da8baefc6491fcfe", size = 47726315, upload-time = "2025-10-24T10:05:47.314Z" }, - { url = "https://files.pythonhosted.org/packages/bb/d4/74ac9f7a54cfde12ee42734ea25d5a3c9a45db78f9def949307a92720d37/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c3200cb41cdbc65156e5f8c908d739b0dfed57e890329413da2748d1a2cd1a4e", size = 47990906, upload-time = "2025-10-24T10:05:58.254Z" }, - { url = "https://files.pythonhosted.org/packages/2e/71/fedf2499bf7a95062eafc989ace56572f3343432570e1c54e6599d5b88da/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ac93252226cf288753d8b46280f4edf3433bf9508b6977f8dd8526b521a1bbb9", size = 50306783, upload-time = "2025-10-24T10:06:08.08Z" }, - { url = "https://files.pythonhosted.org/packages/68/ed/b202abd5a5b78f519722f3d29063dda03c114711093c1995a33b8e2e0f4b/pyarrow-22.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:44729980b6c50a5f2bfcc2668d36c569ce17f8b17bccaf470c4313dcbbf13c9d", size = 27972883, upload-time = "2025-10-24T10:06:14.204Z" }, - { url = "https://files.pythonhosted.org/packages/a6/d6/d0fac16a2963002fc22c8fa75180a838737203d558f0ed3b564c4a54eef5/pyarrow-22.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e6e95176209257803a8b3d0394f21604e796dadb643d2f7ca21b66c9c0b30c9a", size = 34204629, upload-time = "2025-10-24T10:06:20.274Z" }, - { url = "https://files.pythonhosted.org/packages/c6/9c/1d6357347fbae062ad3f17082f9ebc29cc733321e892c0d2085f42a2212b/pyarrow-22.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:001ea83a58024818826a9e3f89bf9310a114f7e26dfe404a4c32686f97bd7901", size = 35985783, upload-time = "2025-10-24T10:06:27.301Z" }, - { url = "https://files.pythonhosted.org/packages/ff/c0/782344c2ce58afbea010150df07e3a2f5fdad299cd631697ae7bd3bac6e3/pyarrow-22.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ce20fe000754f477c8a9125543f1936ea5b8867c5406757c224d745ed033e691", size = 45020999, upload-time = "2025-10-24T10:06:35.387Z" }, - { url = "https://files.pythonhosted.org/packages/1b/8b/5362443737a5307a7b67c1017c42cd104213189b4970bf607e05faf9c525/pyarrow-22.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e0a15757fccb38c410947df156f9749ae4a3c89b2393741a50521f39a8cf202a", size = 47724601, upload-time = "2025-10-24T10:06:43.551Z" }, - { url = "https://files.pythonhosted.org/packages/69/4d/76e567a4fc2e190ee6072967cb4672b7d9249ac59ae65af2d7e3047afa3b/pyarrow-22.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cedb9dd9358e4ea1d9bce3665ce0797f6adf97ff142c8e25b46ba9cdd508e9b6", size = 48001050, upload-time = "2025-10-24T10:06:52.284Z" }, - { url = "https://files.pythonhosted.org/packages/01/5e/5653f0535d2a1aef8223cee9d92944cb6bccfee5cf1cd3f462d7cb022790/pyarrow-22.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:252be4a05f9d9185bb8c18e83764ebcfea7185076c07a7a662253af3a8c07941", size = 50307877, upload-time = "2025-10-24T10:07:02.405Z" }, - { url = "https://files.pythonhosted.org/packages/2d/f8/1d0bd75bf9328a3b826e24a16e5517cd7f9fbf8d34a3184a4566ef5a7f29/pyarrow-22.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:a4893d31e5ef780b6edcaf63122df0f8d321088bb0dee4c8c06eccb1ca28d145", size = 27977099, upload-time = "2025-10-24T10:08:07.259Z" }, - { url = "https://files.pythonhosted.org/packages/90/81/db56870c997805bf2b0f6eeeb2d68458bf4654652dccdcf1bf7a42d80903/pyarrow-22.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:f7fe3dbe871294ba70d789be16b6e7e52b418311e166e0e3cba9522f0f437fb1", size = 34336685, upload-time = "2025-10-24T10:07:11.47Z" }, - { url = "https://files.pythonhosted.org/packages/1c/98/0727947f199aba8a120f47dfc229eeb05df15bcd7a6f1b669e9f882afc58/pyarrow-22.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:ba95112d15fd4f1105fb2402c4eab9068f0554435e9b7085924bcfaac2cc306f", size = 36032158, upload-time = "2025-10-24T10:07:18.626Z" }, - { url = "https://files.pythonhosted.org/packages/96/b4/9babdef9c01720a0785945c7cf550e4acd0ebcd7bdd2e6f0aa7981fa85e2/pyarrow-22.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c064e28361c05d72eed8e744c9605cbd6d2bb7481a511c74071fd9b24bc65d7d", size = 44892060, upload-time = "2025-10-24T10:07:26.002Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ca/2f8804edd6279f78a37062d813de3f16f29183874447ef6d1aadbb4efa0f/pyarrow-22.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:6f9762274496c244d951c819348afbcf212714902742225f649cf02823a6a10f", size = 47504395, upload-time = "2025-10-24T10:07:34.09Z" }, - { url = "https://files.pythonhosted.org/packages/b9/f0/77aa5198fd3943682b2e4faaf179a674f0edea0d55d326d83cb2277d9363/pyarrow-22.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a9d9ffdc2ab696f6b15b4d1f7cec6658e1d788124418cb30030afbae31c64746", size = 48066216, upload-time = "2025-10-24T10:07:43.528Z" }, - { url = "https://files.pythonhosted.org/packages/79/87/a1937b6e78b2aff18b706d738c9e46ade5bfcf11b294e39c87706a0089ac/pyarrow-22.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ec1a15968a9d80da01e1d30349b2b0d7cc91e96588ee324ce1b5228175043e95", size = 50288552, upload-time = "2025-10-24T10:07:53.519Z" }, - { url = "https://files.pythonhosted.org/packages/60/ae/b5a5811e11f25788ccfdaa8f26b6791c9807119dffcf80514505527c384c/pyarrow-22.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:bba208d9c7decf9961998edf5c65e3ea4355d5818dd6cd0f6809bec1afb951cc", size = 28262504, upload-time = "2025-10-24T10:08:00.932Z" }, - { url = "https://files.pythonhosted.org/packages/bd/b0/0fa4d28a8edb42b0a7144edd20befd04173ac79819547216f8a9f36f9e50/pyarrow-22.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:9bddc2cade6561f6820d4cd73f99a0243532ad506bc510a75a5a65a522b2d74d", size = 34224062, upload-time = "2025-10-24T10:08:14.101Z" }, - { url = "https://files.pythonhosted.org/packages/0f/a8/7a719076b3c1be0acef56a07220c586f25cd24de0e3f3102b438d18ae5df/pyarrow-22.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e70ff90c64419709d38c8932ea9fe1cc98415c4f87ea8da81719e43f02534bc9", size = 35990057, upload-time = "2025-10-24T10:08:21.842Z" }, - { url = "https://files.pythonhosted.org/packages/89/3c/359ed54c93b47fb6fe30ed16cdf50e3f0e8b9ccfb11b86218c3619ae50a8/pyarrow-22.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:92843c305330aa94a36e706c16209cd4df274693e777ca47112617db7d0ef3d7", size = 45068002, upload-time = "2025-10-24T10:08:29.034Z" }, - { url = "https://files.pythonhosted.org/packages/55/fc/4945896cc8638536ee787a3bd6ce7cec8ec9acf452d78ec39ab328efa0a1/pyarrow-22.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:6dda1ddac033d27421c20d7a7943eec60be44e0db4e079f33cc5af3b8280ccde", size = 47737765, upload-time = "2025-10-24T10:08:38.559Z" }, - { url = "https://files.pythonhosted.org/packages/cd/5e/7cb7edeb2abfaa1f79b5d5eb89432356155c8426f75d3753cbcb9592c0fd/pyarrow-22.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:84378110dd9a6c06323b41b56e129c504d157d1a983ce8f5443761eb5256bafc", size = 48048139, upload-time = "2025-10-24T10:08:46.784Z" }, - { url = "https://files.pythonhosted.org/packages/88/c6/546baa7c48185f5e9d6e59277c4b19f30f48c94d9dd938c2a80d4d6b067c/pyarrow-22.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:854794239111d2b88b40b6ef92aa478024d1e5074f364033e73e21e3f76b25e0", size = 50314244, upload-time = "2025-10-24T10:08:55.771Z" }, - { url = "https://files.pythonhosted.org/packages/3c/79/755ff2d145aafec8d347bf18f95e4e81c00127f06d080135dfc86aea417c/pyarrow-22.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:b883fe6fd85adad7932b3271c38ac289c65b7337c2c132e9569f9d3940620730", size = 28757501, upload-time = "2025-10-24T10:09:59.891Z" }, - { url = "https://files.pythonhosted.org/packages/0e/d2/237d75ac28ced3147912954e3c1a174df43a95f4f88e467809118a8165e0/pyarrow-22.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7a820d8ae11facf32585507c11f04e3f38343c1e784c9b5a8b1da5c930547fe2", size = 34355506, upload-time = "2025-10-24T10:09:02.953Z" }, - { url = "https://files.pythonhosted.org/packages/1e/2c/733dfffe6d3069740f98e57ff81007809067d68626c5faef293434d11bd6/pyarrow-22.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:c6ec3675d98915bf1ec8b3c7986422682f7232ea76cad276f4c8abd5b7319b70", size = 36047312, upload-time = "2025-10-24T10:09:10.334Z" }, - { url = "https://files.pythonhosted.org/packages/7c/2b/29d6e3782dc1f299727462c1543af357a0f2c1d3c160ce199950d9ca51eb/pyarrow-22.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:3e739edd001b04f654b166204fc7a9de896cf6007eaff33409ee9e50ceaff754", size = 45081609, upload-time = "2025-10-24T10:09:18.61Z" }, - { url = "https://files.pythonhosted.org/packages/8d/42/aa9355ecc05997915af1b7b947a7f66c02dcaa927f3203b87871c114ba10/pyarrow-22.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:7388ac685cab5b279a41dfe0a6ccd99e4dbf322edfb63e02fc0443bf24134e91", size = 47703663, upload-time = "2025-10-24T10:09:27.369Z" }, - { url = "https://files.pythonhosted.org/packages/ee/62/45abedde480168e83a1de005b7b7043fd553321c1e8c5a9a114425f64842/pyarrow-22.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f633074f36dbc33d5c05b5dc75371e5660f1dbf9c8b1d95669def05e5425989c", size = 48066543, upload-time = "2025-10-24T10:09:34.908Z" }, - { url = "https://files.pythonhosted.org/packages/84/e9/7878940a5b072e4f3bf998770acafeae13b267f9893af5f6d4ab3904b67e/pyarrow-22.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4c19236ae2402a8663a2c8f21f1870a03cc57f0bef7e4b6eb3238cc82944de80", size = 50288838, upload-time = "2025-10-24T10:09:44.394Z" }, - { url = "https://files.pythonhosted.org/packages/7b/03/f335d6c52b4a4761bcc83499789a1e2e16d9d201a58c327a9b5cc9a41bd9/pyarrow-22.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0c34fe18094686194f204a3b1787a27456897d8a2d62caf84b61e8dfbc0252ae", size = 29185594, upload-time = "2025-10-24T10:09:53.111Z" }, +version = "23.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/88/22/134986a4cc224d593c1afde5494d18ff629393d74cc2eddb176669f234a4/pyarrow-23.0.1.tar.gz", hash = "sha256:b8c5873e33440b2bc2f4a79d2b47017a89c5a24116c055625e6f2ee50523f019", size = 1167336, upload-time = "2026-02-16T10:14:12.39Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/41/8e6b6ef7e225d4ceead8459427a52afdc23379768f54dd3566014d7618c1/pyarrow-23.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:6f0147ee9e0386f519c952cc670eb4a8b05caa594eeffe01af0e25f699e4e9bb", size = 34302230, upload-time = "2026-02-16T10:09:03.859Z" }, + { url = "https://files.pythonhosted.org/packages/bf/4a/1472c00392f521fea03ae93408bf445cc7bfa1ab81683faf9bc188e36629/pyarrow-23.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:0ae6e17c828455b6265d590100c295193f93cc5675eb0af59e49dbd00d2de350", size = 35850050, upload-time = "2026-02-16T10:09:11.877Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b2/bd1f2f05ded56af7f54d702c8364c9c43cd6abb91b0e9933f3d77b4f4132/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fed7020203e9ef273360b9e45be52a2a47d3103caf156a30ace5247ffb51bdbd", size = 44491918, upload-time = "2026-02-16T10:09:18.144Z" }, + { url = "https://files.pythonhosted.org/packages/0b/62/96459ef5b67957eac38a90f541d1c28833d1b367f014a482cb63f3b7cd2d/pyarrow-23.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26d50dee49d741ac0e82185033488d28d35be4d763ae6f321f97d1140eb7a0e9", size = 47562811, upload-time = "2026-02-16T10:09:25.792Z" }, + { url = "https://files.pythonhosted.org/packages/7d/94/1170e235add1f5f45a954e26cd0e906e7e74e23392dcb560de471f7366ec/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c30143b17161310f151f4a2bcfe41b5ff744238c1039338779424e38579d701", size = 48183766, upload-time = "2026-02-16T10:09:34.645Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2d/39a42af4570377b99774cdb47f63ee6c7da7616bd55b3d5001aa18edfe4f/pyarrow-23.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db2190fa79c80a23fdd29fef4b8992893f024ae7c17d2f5f4db7171fa30c2c78", size = 50607669, upload-time = "2026-02-16T10:09:44.153Z" }, + { url = "https://files.pythonhosted.org/packages/00/ca/db94101c187f3df742133ac837e93b1f269ebdac49427f8310ee40b6a58f/pyarrow-23.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:f00f993a8179e0e1c9713bcc0baf6d6c01326a406a9c23495ec1ba9c9ebf2919", size = 27527698, upload-time = "2026-02-16T10:09:50.263Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4b/4166bb5abbfe6f750fc60ad337c43ecf61340fa52ab386da6e8dbf9e63c4/pyarrow-23.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:f4b0dbfa124c0bb161f8b5ebb40f1a680b70279aa0c9901d44a2b5a20806039f", size = 34214575, upload-time = "2026-02-16T10:09:56.225Z" }, + { url = "https://files.pythonhosted.org/packages/e1/da/3f941e3734ac8088ea588b53e860baeddac8323ea40ce22e3d0baa865cc9/pyarrow-23.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:7707d2b6673f7de054e2e83d59f9e805939038eebe1763fe811ee8fa5c0cd1a7", size = 35832540, upload-time = "2026-02-16T10:10:03.428Z" }, + { url = "https://files.pythonhosted.org/packages/88/7c/3d841c366620e906d54430817531b877ba646310296df42ef697308c2705/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:86ff03fb9f1a320266e0de855dee4b17da6794c595d207f89bba40d16b5c78b9", size = 44470940, upload-time = "2026-02-16T10:10:10.704Z" }, + { url = "https://files.pythonhosted.org/packages/2c/a5/da83046273d990f256cb79796a190bbf7ec999269705ddc609403f8c6b06/pyarrow-23.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:813d99f31275919c383aab17f0f455a04f5a429c261cc411b1e9a8f5e4aaaa05", size = 47586063, upload-time = "2026-02-16T10:10:17.95Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/b7d2ebcff47a514f47f9da1e74b7949138c58cfeb108cdd4ee62f43f0cf3/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bf5842f960cddd2ef757d486041d57c96483efc295a8c4a0e20e704cbbf39c67", size = 48173045, upload-time = "2026-02-16T10:10:25.363Z" }, + { url = "https://files.pythonhosted.org/packages/43/b2/b40961262213beaba6acfc88698eb773dfce32ecdf34d19291db94c2bd73/pyarrow-23.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564baf97c858ecc03ec01a41062e8f4698abc3e6e2acd79c01c2e97880a19730", size = 50621741, upload-time = "2026-02-16T10:10:33.477Z" }, + { url = "https://files.pythonhosted.org/packages/f6/70/1fdda42d65b28b078e93d75d371b2185a61da89dda4def8ba6ba41ebdeb4/pyarrow-23.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:07deae7783782ac7250989a7b2ecde9b3c343a643f82e8a4df03d93b633006f0", size = 27620678, upload-time = "2026-02-16T10:10:39.31Z" }, + { url = "https://files.pythonhosted.org/packages/47/10/2cbe4c6f0fb83d2de37249567373d64327a5e4d8db72f486db42875b08f6/pyarrow-23.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:6b8fda694640b00e8af3c824f99f789e836720aa8c9379fb435d4c4953a756b8", size = 34210066, upload-time = "2026-02-16T10:10:45.487Z" }, + { url = "https://files.pythonhosted.org/packages/cb/4f/679fa7e84dadbaca7a65f7cdba8d6c83febbd93ca12fa4adf40ba3b6362b/pyarrow-23.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:8ff51b1addc469b9444b7c6f3548e19dc931b172ab234e995a60aea9f6e6025f", size = 35825526, upload-time = "2026-02-16T10:10:52.266Z" }, + { url = "https://files.pythonhosted.org/packages/f9/63/d2747d930882c9d661e9398eefc54f15696547b8983aaaf11d4a2e8b5426/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:71c5be5cbf1e1cb6169d2a0980850bccb558ddc9b747b6206435313c47c37677", size = 44473279, upload-time = "2026-02-16T10:11:01.557Z" }, + { url = "https://files.pythonhosted.org/packages/b3/93/10a48b5e238de6d562a411af6467e71e7aedbc9b87f8d3a35f1560ae30fb/pyarrow-23.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9b6f4f17b43bc39d56fec96e53fe89d94bac3eb134137964371b45352d40d0c2", size = 47585798, upload-time = "2026-02-16T10:11:09.401Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/476943001c54ef078dbf9542280e22741219a184a0632862bca4feccd666/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fc13fc6c403d1337acab46a2c4346ca6c9dec5780c3c697cf8abfd5e19b6b37", size = 48179446, upload-time = "2026-02-16T10:11:17.781Z" }, + { url = "https://files.pythonhosted.org/packages/4b/b6/5dd0c47b335fcd8edba9bfab78ad961bd0fd55ebe53468cc393f45e0be60/pyarrow-23.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c16ed4f53247fa3ffb12a14d236de4213a4415d127fe9cebed33d51671113e2", size = 50623972, upload-time = "2026-02-16T10:11:26.185Z" }, + { url = "https://files.pythonhosted.org/packages/d5/09/a532297c9591a727d67760e2e756b83905dd89adb365a7f6e9c72578bcc1/pyarrow-23.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:cecfb12ef629cf6be0b1887f9f86463b0dd3dc3195ae6224e74006be4736035a", size = 27540749, upload-time = "2026-02-16T10:12:23.297Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8e/38749c4b1303e6ae76b3c80618f84861ae0c55dd3c2273842ea6f8258233/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:29f7f7419a0e30264ea261fdc0e5fe63ce5a6095003db2945d7cd78df391a7e1", size = 34471544, upload-time = "2026-02-16T10:11:32.535Z" }, + { url = "https://files.pythonhosted.org/packages/a3/73/f237b2bc8c669212f842bcfd842b04fc8d936bfc9d471630569132dc920d/pyarrow-23.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:33d648dc25b51fd8055c19e4261e813dfc4d2427f068bcecc8b53d01b81b0500", size = 35949911, upload-time = "2026-02-16T10:11:39.813Z" }, + { url = "https://files.pythonhosted.org/packages/0c/86/b912195eee0903b5611bf596833def7d146ab2d301afeb4b722c57ffc966/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:cd395abf8f91c673dd3589cadc8cc1ee4e8674fa61b2e923c8dd215d9c7d1f41", size = 44520337, upload-time = "2026-02-16T10:11:47.764Z" }, + { url = "https://files.pythonhosted.org/packages/69/c2/f2a717fb824f62d0be952ea724b4f6f9372a17eed6f704b5c9526f12f2f1/pyarrow-23.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:00be9576d970c31defb5c32eb72ef585bf600ef6d0a82d5eccaae96639cf9d07", size = 47548944, upload-time = "2026-02-16T10:11:56.607Z" }, + { url = "https://files.pythonhosted.org/packages/84/a7/90007d476b9f0dc308e3bc57b832d004f848fd6c0da601375d20d92d1519/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c2139549494445609f35a5cda4eb94e2c9e4d704ce60a095b342f82460c73a83", size = 48236269, upload-time = "2026-02-16T10:12:04.47Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3f/b16fab3e77709856eb6ac328ce35f57a6d4a18462c7ca5186ef31b45e0e0/pyarrow-23.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7044b442f184d84e2351e5084600f0d7343d6117aabcbc1ac78eb1ae11eb4125", size = 50604794, upload-time = "2026-02-16T10:12:11.797Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a1/22df0620a9fac31d68397a75465c344e83c3dfe521f7612aea33e27ab6c0/pyarrow-23.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:a35581e856a2fafa12f3f54fce4331862b1cfb0bef5758347a858a4aa9d6bae8", size = 27660642, upload-time = "2026-02-16T10:12:17.746Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1b/6da9a89583ce7b23ac611f183ae4843cd3a6cf54f079549b0e8c14031e73/pyarrow-23.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:5df1161da23636a70838099d4aaa65142777185cc0cdba4037a18cee7d8db9ca", size = 34238755, upload-time = "2026-02-16T10:12:32.819Z" }, + { url = "https://files.pythonhosted.org/packages/ae/b5/d58a241fbe324dbaeb8df07be6af8752c846192d78d2272e551098f74e88/pyarrow-23.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:fa8e51cb04b9f8c9c5ace6bab63af9a1f88d35c0d6cbf53e8c17c098552285e1", size = 35847826, upload-time = "2026-02-16T10:12:38.949Z" }, + { url = "https://files.pythonhosted.org/packages/54/a5/8cbc83f04aba433ca7b331b38f39e000efd9f0c7ce47128670e737542996/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b95a3994f015be13c63148fef8832e8a23938128c185ee951c98908a696e0eb", size = 44536859, upload-time = "2026-02-16T10:12:45.467Z" }, + { url = "https://files.pythonhosted.org/packages/36/2e/c0f017c405fcdc252dbccafbe05e36b0d0eb1ea9a958f081e01c6972927f/pyarrow-23.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4982d71350b1a6e5cfe1af742c53dfb759b11ce14141870d05d9e540d13bc5d1", size = 47614443, upload-time = "2026-02-16T10:12:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/af/6b/2314a78057912f5627afa13ba43809d9d653e6630859618b0fd81a4e0759/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c250248f1fe266db627921c89b47b7c06fee0489ad95b04d50353537d74d6886", size = 48232991, upload-time = "2026-02-16T10:13:04.729Z" }, + { url = "https://files.pythonhosted.org/packages/40/f2/1bcb1d3be3460832ef3370d621142216e15a2c7c62602a4ea19ec240dd64/pyarrow-23.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f4763b83c11c16e5f4c15601ba6dfa849e20723b46aa2617cb4bffe8768479f", size = 50645077, upload-time = "2026-02-16T10:13:14.147Z" }, + { url = "https://files.pythonhosted.org/packages/eb/3f/b1da7b61cd66566a4d4c8383d376c606d1c34a906c3f1cb35c479f59d1aa/pyarrow-23.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:3a4c85ef66c134161987c17b147d6bffdca4566f9a4c1d81a0a01cdf08414ea5", size = 28234271, upload-time = "2026-02-16T10:14:09.397Z" }, + { url = "https://files.pythonhosted.org/packages/b5/78/07f67434e910a0f7323269be7bfbf58699bd0c1d080b18a1ab49ba943fe8/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:17cd28e906c18af486a499422740298c52d7c6795344ea5002a7720b4eadf16d", size = 34488692, upload-time = "2026-02-16T10:13:21.541Z" }, + { url = "https://files.pythonhosted.org/packages/50/76/34cf7ae93ece1f740a04910d9f7e80ba166b9b4ab9596a953e9e62b90fe1/pyarrow-23.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:76e823d0e86b4fb5e1cf4a58d293036e678b5a4b03539be933d3b31f9406859f", size = 35964383, upload-time = "2026-02-16T10:13:28.63Z" }, + { url = "https://files.pythonhosted.org/packages/46/90/459b827238936d4244214be7c684e1b366a63f8c78c380807ae25ed92199/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a62e1899e3078bf65943078b3ad2a6ddcacf2373bc06379aac61b1e548a75814", size = 44538119, upload-time = "2026-02-16T10:13:35.506Z" }, + { url = "https://files.pythonhosted.org/packages/28/a1/93a71ae5881e99d1f9de1d4554a87be37da11cd6b152239fb5bd924fdc64/pyarrow-23.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:df088e8f640c9fae3b1f495b3c64755c4e719091caf250f3a74d095ddf3c836d", size = 47571199, upload-time = "2026-02-16T10:13:42.504Z" }, + { url = "https://files.pythonhosted.org/packages/88/a3/d2c462d4ef313521eaf2eff04d204ac60775263f1fb08c374b543f79f610/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:46718a220d64677c93bc243af1d44b55998255427588e400677d7192671845c7", size = 48259435, upload-time = "2026-02-16T10:13:49.226Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f1/11a544b8c3d38a759eb3fbb022039117fd633e9a7b19e4841cc3da091915/pyarrow-23.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a09f3876e87f48bc2f13583ab551f0379e5dfb83210391e68ace404181a20690", size = 50629149, upload-time = "2026-02-16T10:13:57.238Z" }, + { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" }, ] [[package]] name = "pyasn1" -version = "0.6.2" +version = "0.6.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/b6/6e630dff89739fcd427e3f72b3d905ce0acb85a45d4ec3e2678718a3487f/pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b", size = 146586, upload-time = "2026-01-16T18:04:18.534Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/44/b5/a96872e5184f354da9c84ae119971a0a4c221fe9b27a4d94bd43f2596727/pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf", size = 83371, upload-time = "2026-01-16T18:04:17.174Z" }, + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, ] [[package]] @@ -3392,34 +3409,34 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.13.1" +version = "2.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/52/6d/fffca34caecc4a3f97bda81b2098da5e8ab7efc9a66e819074a11955d87e/pydantic_settings-2.13.1.tar.gz", hash = "sha256:b4c11847b15237fb0171e1462bf540e294affb9b86db4d9aa5c01730bdbe4025", size = 223826, upload-time = "2026-02-19T13:45:08.055Z" } +sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/00/4b/ccc026168948fec4f7555b9164c724cf4125eac006e176541483d2c959be/pydantic_settings-2.13.1-py3-none-any.whl", hash = "sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237", size = 58929, upload-time = "2026-02-19T13:45:06.034Z" }, + { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, ] [[package]] name = "pygments" -version = "2.19.2" +version = "2.21.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] [[package]] name = "pyjwt" -version = "2.11.0" +version = "2.13.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5a/b46fa56bf322901eee5b0454a34343cdbdae202cd421775a8ee4e42fd519/pyjwt-2.11.0.tar.gz", hash = "sha256:35f95c1f0fbe5d5ba6e43f00271c275f7a1a4db1dab27bf708073b75318ea623", size = 98019, upload-time = "2026-01-30T19:59:55.694Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/01/c26ce75ba460d5cd503da9e13b21a33804d38c2165dec7b716d06b13010c/pyjwt-2.11.0-py3-none-any.whl", hash = "sha256:94a6bde30eb5c8e04fee991062b534071fd1439ef58d2adc9ccb823e7bcd0469", size = 28224, upload-time = "2026-01-30T19:59:54.539Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, ] [package.optional-dependencies] @@ -3530,93 +3547,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/cd/ddc794cdc8500f6f28c119c624252fb6dfb19481c6d7ed150f13cf468a6d/pymongo-4.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6b2a20edb5452ac8daa395890eeb076c570790dfce6b7a44d788af74c2f8cf96", size = 1047725, upload-time = "2026-01-07T18:05:28.47Z" }, ] -[[package]] -name = "pyobjc-core" -version = "12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b8/b6/d5612eb40be4fd5ef88c259339e6313f46ba67577a95d86c3470b951fce0/pyobjc_core-12.1.tar.gz", hash = "sha256:2bb3903f5387f72422145e1466b3ac3f7f0ef2e9960afa9bcd8961c5cbf8bd21", size = 1000532, upload-time = "2025-11-14T10:08:28.292Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/95/df/d2b290708e9da86d6e7a9a2a2022b91915cf2e712a5a82e306cb6ee99792/pyobjc_core-12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c918ebca280925e7fcb14c5c43ce12dcb9574a33cccb889be7c8c17f3bcce8b6", size = 671263, upload-time = "2025-11-14T09:31:35.231Z" }, - { url = "https://files.pythonhosted.org/packages/64/5a/6b15e499de73050f4a2c88fff664ae154307d25dc04da8fb38998a428358/pyobjc_core-12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:818bcc6723561f207e5b5453efe9703f34bc8781d11ce9b8be286bb415eb4962", size = 678335, upload-time = "2025-11-14T09:32:20.107Z" }, - { url = "https://files.pythonhosted.org/packages/f4/d2/29e5e536adc07bc3d33dd09f3f7cf844bf7b4981820dc2a91dd810f3c782/pyobjc_core-12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:01c0cf500596f03e21c23aef9b5f326b9fb1f8f118cf0d8b66749b6cf4cbb37a", size = 677370, upload-time = "2025-11-14T09:33:05.273Z" }, - { url = "https://files.pythonhosted.org/packages/1b/f0/4b4ed8924cd04e425f2a07269943018d43949afad1c348c3ed4d9d032787/pyobjc_core-12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:177aaca84bb369a483e4961186704f64b2697708046745f8167e818d968c88fc", size = 719586, upload-time = "2025-11-14T09:33:53.302Z" }, - { url = "https://files.pythonhosted.org/packages/25/98/9f4ed07162de69603144ff480be35cd021808faa7f730d082b92f7ebf2b5/pyobjc_core-12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:844515f5d86395b979d02152576e7dee9cc679acc0b32dc626ef5bda315eaa43", size = 670164, upload-time = "2025-11-14T09:34:37.458Z" }, - { url = "https://files.pythonhosted.org/packages/62/50/dc076965c96c7f0de25c0a32b7f8aa98133ed244deaeeacfc758783f1f30/pyobjc_core-12.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:453b191df1a4b80e756445b935491b974714456ae2cbae816840bd96f86db882", size = 712204, upload-time = "2025-11-14T09:35:24.148Z" }, -] - -[[package]] -name = "pyobjc-framework-cocoa" -version = "12.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/02/a3/16ca9a15e77c061a9250afbae2eae26f2e1579eb8ca9462ae2d2c71e1169/pyobjc_framework_cocoa-12.1.tar.gz", hash = "sha256:5556c87db95711b985d5efdaaf01c917ddd41d148b1e52a0c66b1a2e2c5c1640", size = 2772191, upload-time = "2025-11-14T10:13:02.069Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/07/5760735c0fffc65107e648eaf7e0991f46da442ac4493501be5380e6d9d4/pyobjc_framework_cocoa-12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f52228bcf38da64b77328787967d464e28b981492b33a7675585141e1b0a01e6", size = 383812, upload-time = "2025-11-14T09:40:53.169Z" }, - { url = "https://files.pythonhosted.org/packages/95/bf/ee4f27ec3920d5c6fc63c63e797c5b2cc4e20fe439217085d01ea5b63856/pyobjc_framework_cocoa-12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:547c182837214b7ec4796dac5aee3aa25abc665757b75d7f44f83c994bcb0858", size = 384590, upload-time = "2025-11-14T09:41:17.336Z" }, - { url = "https://files.pythonhosted.org/packages/ad/31/0c2e734165abb46215797bd830c4bdcb780b699854b15f2b6240515edcc6/pyobjc_framework_cocoa-12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5a3dcd491cacc2f5a197142b3c556d8aafa3963011110102a093349017705118", size = 384689, upload-time = "2025-11-14T09:41:41.478Z" }, - { url = "https://files.pythonhosted.org/packages/23/3b/b9f61be7b9f9b4e0a6db18b3c35c4c4d589f2d04e963e2174d38c6555a92/pyobjc_framework_cocoa-12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:914b74328c22d8ca261d78c23ef2befc29776e0b85555973927b338c5734ca44", size = 388843, upload-time = "2025-11-14T09:42:05.719Z" }, - { url = "https://files.pythonhosted.org/packages/59/bb/f777cc9e775fc7dae77b569254570fe46eb842516b3e4fe383ab49eab598/pyobjc_framework_cocoa-12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:03342a60fc0015bcdf9b93ac0b4f457d3938e9ef761b28df9564c91a14f0129a", size = 384932, upload-time = "2025-11-14T09:42:29.771Z" }, - { url = "https://files.pythonhosted.org/packages/58/27/b457b7b37089cad692c8aada90119162dfb4c4a16f513b79a8b2b022b33b/pyobjc_framework_cocoa-12.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6ba1dc1bfa4da42d04e93d2363491275fb2e2be5c20790e561c8a9e09b8cf2cc", size = 388970, upload-time = "2025-11-14T09:42:53.964Z" }, -] - -[[package]] -name = "pyobjc-framework-coreml" -version = "12.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/2d/baa9ea02cbb1c200683cb7273b69b4bee5070e86f2060b77e6a27c2a9d7e/pyobjc_framework_coreml-12.1.tar.gz", hash = "sha256:0d1a4216891a18775c9e0170d908714c18e4f53f9dc79fb0f5263b2aa81609ba", size = 40465, upload-time = "2025-11-14T10:14:02.265Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/0f/f55369da4a33cfe1db38a3512aac4487602783d3a1d572d2c8c4ccce6abc/pyobjc_framework_coreml-12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:16dafcfb123f022e62f47a590a7eccf7d0cb5957a77fd5f062b5ee751cb5a423", size = 11331, upload-time = "2025-11-14T09:45:50.445Z" }, - { url = "https://files.pythonhosted.org/packages/bb/39/4defef0deb25c5d7e3b7826d301e71ac5b54ef901b7dac4db1adc00f172d/pyobjc_framework_coreml-12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:10dc8e8db53d7631ebc712cad146e3a9a9a443f4e1a037e844149a24c3c42669", size = 11356, upload-time = "2025-11-14T09:45:52.271Z" }, - { url = "https://files.pythonhosted.org/packages/ae/3f/3749964aa3583f8c30d9996f0d15541120b78d307bb3070f5e47154ef38d/pyobjc_framework_coreml-12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:48fa3bb4a03fa23e0e36c93936dca2969598e4102f4b441e1663f535fc99cd31", size = 11371, upload-time = "2025-11-14T09:45:54.105Z" }, - { url = "https://files.pythonhosted.org/packages/9c/c8/cf20ea91ae33f05f3b92dec648c6f44a65f86d1a64c1d6375c95b85ccb7c/pyobjc_framework_coreml-12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:71de5b37e6a017e3ed16645c5d6533138f24708da5b56c35c818ae49d0253ee1", size = 11600, upload-time = "2025-11-14T09:45:55.976Z" }, - { url = "https://files.pythonhosted.org/packages/bc/5c/510ae8e3663238d32e653ed6a09ac65611dd045a7241f12633c1ab48bb9b/pyobjc_framework_coreml-12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:a04a96e512ecf6999aa9e1f60ad5635cb9d1cd839be470341d8d1541797baef6", size = 11418, upload-time = "2025-11-14T09:45:57.75Z" }, - { url = "https://files.pythonhosted.org/packages/d3/1a/b7367819381b07c440fa5797d2b0487e31f09aa72079a693ceab6875fa0a/pyobjc_framework_coreml-12.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:7762b3dd2de01565b7cf3049ce1e4c27341ba179d97016b0b7607448e1c39865", size = 11593, upload-time = "2025-11-14T09:45:59.623Z" }, -] - -[[package]] -name = "pyobjc-framework-quartz" -version = "12.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/94/18/cc59f3d4355c9456fc945eae7fe8797003c4da99212dd531ad1b0de8a0c6/pyobjc_framework_quartz-12.1.tar.gz", hash = "sha256:27f782f3513ac88ec9b6c82d9767eef95a5cf4175ce88a1e5a65875fee799608", size = 3159099, upload-time = "2025-11-14T10:21:24.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ef/dcd22b743e38b3c430fce4788176c2c5afa8bfb01085b8143b02d1e75201/pyobjc_framework_quartz-12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:19f99ac49a0b15dd892e155644fe80242d741411a9ed9c119b18b7466048625a", size = 217795, upload-time = "2025-11-14T09:59:46.922Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9b/780f057e5962f690f23fdff1083a4cfda5a96d5b4d3bb49505cac4f624f2/pyobjc_framework_quartz-12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7730cdce46c7e985535b5a42c31381af4aa6556e5642dc55b5e6597595e57a16", size = 218798, upload-time = "2025-11-14T10:00:01.236Z" }, - { url = "https://files.pythonhosted.org/packages/ba/2d/e8f495328101898c16c32ac10e7b14b08ff2c443a756a76fd1271915f097/pyobjc_framework_quartz-12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:629b7971b1b43a11617f1460cd218bd308dfea247cd4ee3842eb40ca6f588860", size = 219206, upload-time = "2025-11-14T10:00:15.623Z" }, - { url = "https://files.pythonhosted.org/packages/67/43/b1f0ad3b842ab150a7e6b7d97f6257eab6af241b4c7d14cb8e7fde9214b8/pyobjc_framework_quartz-12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:53b84e880c358ba1ddcd7e8d5ea0407d760eca58b96f0d344829162cda5f37b3", size = 224317, upload-time = "2025-11-14T10:00:30.703Z" }, - { url = "https://files.pythonhosted.org/packages/4a/00/96249c5c7e5aaca5f688ca18b8d8ad05cd7886ebd639b3c71a6a4cadbe75/pyobjc_framework_quartz-12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:42d306b07f05ae7d155984503e0fb1b701fecd31dcc5c79fe8ab9790ff7e0de0", size = 219558, upload-time = "2025-11-14T10:00:45.476Z" }, - { url = "https://files.pythonhosted.org/packages/4d/a6/708a55f3ff7a18c403b30a29a11dccfed0410485a7548c60a4b6d4cc0676/pyobjc_framework_quartz-12.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0cc08fddb339b2760df60dea1057453557588908e42bdc62184b6396ce2d6e9a", size = 224580, upload-time = "2025-11-14T10:01:00.091Z" }, -] - -[[package]] -name = "pyobjc-framework-vision" -version = "12.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyobjc-core", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-coreml", marker = "sys_platform == 'darwin'" }, - { name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c2/5a/08bb3e278f870443d226c141af14205ff41c0274da1e053b72b11dfc9fb2/pyobjc_framework_vision-12.1.tar.gz", hash = "sha256:a30959100e85dcede3a786c544e621ad6eb65ff6abf85721f805822b8c5fe9b0", size = 59538, upload-time = "2025-11-14T10:23:21.979Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/37/e30cf4eef2b4c7e20ccadc1249117c77305fbc38b2e5904eb42e3753f63c/pyobjc_framework_vision-12.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1edbf2fc18ce3b31108f845901a88f2236783ae6bf0bc68438d7ece572dc2a29", size = 21432, upload-time = "2025-11-14T10:06:42.373Z" }, - { url = "https://files.pythonhosted.org/packages/3a/5a/23502935b3fc877d7573e743fc3e6c28748f33a45c43851d503bde52cde7/pyobjc_framework_vision-12.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:6b3211d84f3a12aad0cde752cfd43a80d0218960ac9e6b46b141c730e7d655bd", size = 16625, upload-time = "2025-11-14T10:06:44.422Z" }, - { url = "https://files.pythonhosted.org/packages/f5/e4/e87361a31b82b22f8c0a59652d6e17625870dd002e8da75cb2343a84f2f9/pyobjc_framework_vision-12.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7273e2508db4c2e88523b4b7ff38ac54808756e7ba01d78e6c08ea68f32577d2", size = 16640, upload-time = "2025-11-14T10:06:46.653Z" }, - { url = "https://files.pythonhosted.org/packages/b1/dd/def55d8a80b0817f486f2712fc6243482c3264d373dc5ff75037b3aeb7ea/pyobjc_framework_vision-12.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:04296f0848cc8cdead66c76df6063720885cbdf24fdfd1900749a6e2297313db", size = 16782, upload-time = "2025-11-14T10:06:48.816Z" }, - { url = "https://files.pythonhosted.org/packages/a7/a4/ee1ef14d6e1df6617e64dbaaa0ecf8ecb9e0af1425613fa633f6a94049c1/pyobjc_framework_vision-12.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:631add775ed1dafb221a6116137cdcd78432addc16200ca434571c2a039c0e03", size = 16614, upload-time = "2025-11-14T10:06:50.852Z" }, - { url = "https://files.pythonhosted.org/packages/af/53/187743d9244becd4499a77f8ee699ae286e2f6ade7c0c7ad2975ae60f187/pyobjc_framework_vision-12.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fe41a1a70cc91068aee7b5293fa09dc66d1c666a8da79fdf948900988b439df6", size = 16771, upload-time = "2025-11-14T10:06:53.04Z" }, -] - [[package]] name = "pypdfium2" version = "5.6.0" @@ -3662,81 +3592,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, ] -[[package]] -name = "pytest-asyncio" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest", marker = "python_full_version < '3.13'" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, -] - -[[package]] -name = "pytest-benchmark" -version = "5.2.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "py-cpuinfo", marker = "python_full_version < '3.13'" }, - { name = "pytest", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/24/34/9f732b76456d64faffbef6232f1f9dbec7a7c4999ff46282fa418bd1af66/pytest_benchmark-5.2.3.tar.gz", hash = "sha256:deb7317998a23c650fd4ff76e1230066a76cb45dcece0aca5607143c619e7779", size = 341340, upload-time = "2025-11-09T18:48:43.215Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/33/29/e756e715a48959f1c0045342088d7ca9762a2f509b945f362a316e9412b7/pytest_benchmark-5.2.3-py3-none-any.whl", hash = "sha256:bc839726ad20e99aaa0d11a127445457b4219bdb9e80a1afc4b51da7f96b0803", size = 45255, upload-time = "2025-11-09T18:48:39.765Z" }, -] - -[[package]] -name = "pytest-codspeed" -version = "4.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "python_full_version < '3.13'" }, - { name = "pytest", marker = "python_full_version < '3.13'" }, - { name = "rich", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/ab/eca41967d11c95392829a8b4bfa9220a51cffc4a33ec4653358000356918/pytest_codspeed-4.3.0.tar.gz", hash = "sha256:5230d9d65f39063a313ed1820df775166227ec5c20a1122968f85653d5efee48", size = 124745, upload-time = "2026-02-09T15:23:34.745Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/15/ec0ac1f022173b3134c9638f2a35f21fbb3142c75da066d9e49e5a8bb4bd/pytest_codspeed-4.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbeff1eb2f2e36df088658b556fa993e6937bf64ffb07406de4db16fd2b26874", size = 347076, upload-time = "2026-02-09T15:23:19.989Z" }, - { url = "https://files.pythonhosted.org/packages/a5/e8/1fe375794ad02b7835f378a7bcfa8fbac9acadefe600a782a7c4a7064db7/pytest_codspeed-4.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:878aad5e4bb7b401ad8d82f3af5186030cd2bd0d0446782e10dabb9db8827466", size = 342215, upload-time = "2026-02-09T15:23:20.954Z" }, - { url = "https://files.pythonhosted.org/packages/09/58/50df94e9a78e1c77818a492c90557eeb1309af025120c9a21e6375950c52/pytest_codspeed-4.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527a3a02eaa3e4d4583adc4ba2327eef79628f3e1c682a4b959439551a72588e", size = 347395, upload-time = "2026-02-09T15:23:21.986Z" }, - { url = "https://files.pythonhosted.org/packages/e4/56/7dfbd3eefd112a14e6fb65f9ff31dacf2e9c381cb94b27332b81d2b13f8d/pytest_codspeed-4.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9858c2a6e1f391d5696757e7b6e9484749a7376c46f8b4dd9aebf093479a9667", size = 342625, upload-time = "2026-02-09T15:23:23.035Z" }, - { url = "https://files.pythonhosted.org/packages/7f/53/7255f6a25bc56ff1745b254b21545dfe0be2268f5b91ce78f7e8a908f0ad/pytest_codspeed-4.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34f2fd8497456eefbd325673f677ea80d93bb1bc08a578c1fa43a09cec3d1879", size = 347325, upload-time = "2026-02-09T15:23:23.998Z" }, - { url = "https://files.pythonhosted.org/packages/2e/f8/82ae570d8b9ad30f33c9d4002a7a1b2740de0e090540c69a28e4f711ebe2/pytest_codspeed-4.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df6a36a2a9da1406bc50428437f657f0bd8c842ae54bee5fb3ad30e01d50c0f5", size = 342558, upload-time = "2026-02-09T15:23:25.656Z" }, - { url = "https://files.pythonhosted.org/packages/b3/e1/55cfe9474f91d174c7a4b04d257b5fc6d4d06f3d3680f2da672ee59ccc10/pytest_codspeed-4.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bec30f4fc9c4973143cd80f0d33fa780e9fa3e01e4dbe8cedf229e72f1212c62", size = 347383, upload-time = "2026-02-09T15:23:26.68Z" }, - { url = "https://files.pythonhosted.org/packages/7f/3b/8fd781d959bbe789b3de8ce4c50d5706a684a0df377147dfb27b200c20c1/pytest_codspeed-4.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e6584e641cadf27d894ae90b87c50377232a97cbfd76ee0c7ecd0c056fa3f7f4", size = 342481, upload-time = "2026-02-09T15:23:27.686Z" }, - { url = "https://files.pythonhosted.org/packages/bb/0c/368045133c6effa2c665b1634b7b8a9c88b307f877fa31f1f8df47885b51/pytest_codspeed-4.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df0d1f6ea594f29b745c634d66d5f5f1caa1c3abd2af82fea49d656038e8fc77", size = 353680, upload-time = "2026-02-09T15:23:28.726Z" }, - { url = "https://files.pythonhosted.org/packages/59/21/e543abcd72244294e25ae88ec3a9311ade24d6913f8c8f42569d671700bc/pytest_codspeed-4.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2f5bb6d8898bea7db45e3c8b916ee48e36905b929477bb511b79c5a3ccacda4", size = 347888, upload-time = "2026-02-09T15:23:30.443Z" }, - { url = "https://files.pythonhosted.org/packages/55/d9/b8a53c20cf5b41042c205bb9d36d37da00418d30fd1a94bf9eb147820720/pytest_codspeed-4.3.0-py3-none-any.whl", hash = "sha256:05baff2a61dc9f3e92b92b9c2ab5fb45d9b802438f5373073f5766a91319ed7a", size = 125224, upload-time = "2026-02-09T15:23:33.774Z" }, -] - -[[package]] -name = "pytest-recording" -version = "0.13.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest", marker = "python_full_version < '3.13'" }, - { name = "vcrpy", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/32/9c/f4027c5f1693847b06d11caf4b4f6bb09f22c1581ada4663877ec166b8c6/pytest_recording-0.13.4.tar.gz", hash = "sha256:568d64b2a85992eec4ae0a419c855d5fd96782c5fb016784d86f18053792768c", size = 26576, upload-time = "2025-05-08T10:41:11.231Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/42/c2/ce34735972cc42d912173e79f200fe66530225190c06655c5632a9d88f1e/pytest_recording-0.13.4-py3-none-any.whl", hash = "sha256:ad49a434b51b1c4f78e85b1e6b74fdcc2a0a581ca16e52c798c6ace971f7f439", size = 13723, upload-time = "2025-05-08T10:41:09.684Z" }, -] - -[[package]] -name = "pytest-socket" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/05/ff/90c7e1e746baf3d62ce864c479fd53410b534818b9437413903596f81580/pytest_socket-0.7.0.tar.gz", hash = "sha256:71ab048cbbcb085c15a4423b73b619a8b35d6a307f46f78ea46be51b1b7e11b3", size = 12389, upload-time = "2024-01-28T20:17:23.177Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/19/58/5d14cb5cb59409e491ebe816c47bf81423cd03098ea92281336320ae5681/pytest_socket-0.7.0-py3-none-any.whl", hash = "sha256:7e0f4642177d55d317bbd58fc68c6bd9048d6eadb2d46a89307fa9221336ce45", size = 6754, upload-time = "2024-01-28T20:17:22.105Z" }, -] - [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -3764,20 +3619,34 @@ wheels = [ [[package]] name = "python-dotenv" -version = "1.2.2" +version = "1.2.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/53/ed9d74092561d4b01a2ef1349d52cdbc135e526c245f366b089cfca6de49/python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35", size = 58945, upload-time = "2026-08-16T16:54:54.067Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, + { url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" }, ] [[package]] name = "python-multipart" -version = "0.0.20" +version = "0.0.32" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/87/f44d7c9f274c7ee665a29b885ec97089ec5dc034c7f3fafa03da9e39a09e/python_multipart-0.0.20.tar.gz", hash = "sha256:8dd0cab45b8e23064ae09147625994d090fa46f5b0d1e13af944c331a7fa9d13", size = 37158, upload-time = "2024-12-16T19:45:46.972Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/45/58/38b5afbc1a800eeea951b9285d3912613f2603bdf897a4ab0f4bd7f405fc/python_multipart-0.0.20-py3-none-any.whl", hash = "sha256:8a62d3a8335e06589fe01f2a3e178cdcc632f3fbe0d492ad9ee0ec35aab1f104", size = 24546, upload-time = "2024-12-16T19:45:44.423Z" }, + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, +] + +[[package]] +name = "python-oxmsg" +version = "0.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "olefile" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/4e/869f34faedbc968796d2c7e9837dede079c9cb9750917356b1f1eda926e9/python_oxmsg-0.0.2.tar.gz", hash = "sha256:a6aff4deb1b5975d44d49dab1d9384089ffeec819e19c6940bc7ffbc84775fad", size = 34713, upload-time = "2025-02-03T17:13:47.415Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/67/f56c69a98c7eb244025845506387d0f961681657c9fcd8b2d2edd148f9d2/python_oxmsg-0.0.2-py3-none-any.whl", hash = "sha256:22be29b14c46016bcd05e34abddfd8e05ee82082f53b82753d115da3fc7d0355", size = 31455, upload-time = "2025-02-03T17:13:46.061Z" }, ] [[package]] @@ -3959,7 +3828,7 @@ wheels = [ [[package]] name = "rapidocr" -version = "3.7.0" +version = "3.9.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorlog" }, @@ -3975,7 +3844,7 @@ dependencies = [ { name = "tqdm" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/b8/011338eec8aea40cf9b82da7481f3e65e100537cff4c866b3c1b1e719b97/rapidocr-3.7.0-py3-none-any.whl", hash = "sha256:ace47f037956fa3780875f8556a0f27ab20d91962d36a9a2816aa367bb48718f", size = 15080131, upload-time = "2026-03-04T15:38:20.339Z" }, + { url = "https://files.pythonhosted.org/packages/55/ed/0ee9b9281986974be9d2406ae0134c8d7c91d2fc613f16ffda9701eeda6f/rapidocr-3.9.2-py3-none-any.whl", hash = "sha256:04d6b8d151f823d930bd91910555f57bea897c0c44fa6794267b94cf9c1ef9a0", size = 27275208, upload-time = "2026-07-21T10:59:01.599Z" }, ] [[package]] @@ -4110,7 +3979,7 @@ wheels = [ [[package]] name = "requests" -version = "2.32.5" +version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -4118,9 +3987,9 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] @@ -4461,30 +4330,31 @@ wheels = [ [[package]] name = "sentence-transformers" -version = "2.7.0" +version = "6.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, { name = "numpy" }, - { name = "pillow" }, { name = "scikit-learn" }, { name = "scipy" }, + { name = "tokenizers" }, { name = "torch" }, { name = "tqdm" }, { name = "transformers" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/65/8d/8d6147fdef0ed7aeff3dab487bd17619b512afab845eb295faa08b20a5d0/sentence_transformers-2.7.0.tar.gz", hash = "sha256:2f7df99d1c021dded471ed2d079e9d1e4fc8e30ecb06f957be060511b36f24ea", size = 128393, upload-time = "2024-04-17T13:15:44.223Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/43/6b53e6a2098440ce21478742facbc058f1a66ba2cb80b24bdc64942e1e2c/sentence_transformers-6.0.0.tar.gz", hash = "sha256:9e8c2c24f3b1c7473cd5f519a3d3cff60daaeb95533b82d045ffb43ee5f2dac4", size = 575048, upload-time = "2026-08-18T13:33:49.919Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/76/2c/bd95032aeb087b0706596af0a4518c4bfe0439a1bb149048ece18b617766/sentence_transformers-2.7.0-py3-none-any.whl", hash = "sha256:6a7276b05a95931581bbfa4ba49d780b2cf6904fa4a171ec7fd66c343f761c98", size = 171480, upload-time = "2024-04-17T13:15:42.048Z" }, + { url = "https://files.pythonhosted.org/packages/04/fe/9d19b01fe87945f9455c617bf5d33dfbf29fe06ab6580bc0bea06080c788/sentence_transformers-6.0.0-py3-none-any.whl", hash = "sha256:b974ac67523ea2a955afa87b1024129305472bd884367dcc969261cb086790e9", size = 739640, upload-time = "2026-08-18T13:33:48.428Z" }, ] [[package]] name = "setuptools" -version = "82.0.1" +version = "84.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, + { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, ] [[package]] @@ -4744,15 +4614,15 @@ wheels = [ [[package]] name = "starlette" -version = "0.48.0" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a7/a5/d6f429d43394057b67a6b5bbe6eae2f77a6bf7459d961fdb224bf206eee6/starlette-0.48.0.tar.gz", hash = "sha256:7e8cee469a8ab2352911528110ce9088fdc6a37d9876926e73da7ce4aa4c7a46", size = 2652949, upload-time = "2025-09-13T08:41:05.699Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/b4/205b0d5241d934e8add0c38aa924c4f9fb7330834ff11e5444db964ec3f9/starlette-1.6.0.tar.gz", hash = "sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b", size = 2716969, upload-time = "2026-08-08T18:27:57.512Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/72/2db2f49247d0a18b4f1bb9a5a39a0162869acf235f3a96418363947b3d46/starlette-0.48.0-py3-none-any.whl", hash = "sha256:0764ca97b097582558ecb498132ed0c7d942f233f365b86ba37770e026510659", size = 73736, upload-time = "2025-09-13T08:41:03.869Z" }, + { url = "https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl", hash = "sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c", size = 75969, upload-time = "2026-08-08T18:27:56.196Z" }, ] [[package]] @@ -4767,18 +4637,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/ff/c87e0622b1dadea79d2fb0b25ade9ed98954c9033722eb707053d310d4f3/sympy-1.13.3-py3-none-any.whl", hash = "sha256:54612cf55a62755ee71824ce692986f23c88ffa77207b30c1368eda4a7060f73", size = 6189483, upload-time = "2024-09-18T21:54:23.097Z" }, ] -[[package]] -name = "syrupy" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2e/b0/24bca682d6a6337854be37f242d116cceeda9942571d5804c44bc1bdd427/syrupy-5.1.0.tar.gz", hash = "sha256:df543c7aa50d3cf1246e83d58fe490afe5f7dab7b41e74ecc0d8d23ae19bd4b8", size = 50495, upload-time = "2026-01-25T14:53:06.2Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/70/cf880c3b95a6034ef673e74b369941b42315c01f1554a5637a4f8b911009/syrupy-5.1.0-py3-none-any.whl", hash = "sha256:95162d2b05e61ed3e13f117b88dfab7c58bd6f90e66ebbf918e8a77114ad51c5", size = 51658, upload-time = "2026-01-25T14:53:05.105Z" }, -] - [[package]] name = "tabulate" version = "0.10.0" @@ -5065,23 +4923,22 @@ wheels = [ [[package]] name = "transformers" -version = "4.57.6" +version = "5.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock" }, { name = "huggingface-hub" }, { name = "numpy" }, { name = "packaging" }, { name = "pyyaml" }, { name = "regex" }, - { name = "requests" }, { name = "safetensors" }, { name = "tokenizers" }, { name = "tqdm" }, + { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c4/35/67252acc1b929dc88b6602e8c4a982e64f31e733b804c14bc24b47da35e6/transformers-4.57.6.tar.gz", hash = "sha256:55e44126ece9dc0a291521b7e5492b572e6ef2766338a610b9ab5afbb70689d3", size = 10134912, upload-time = "2026-01-16T10:38:39.284Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/e6/4134ea2fbea322cddc7ffc94a0d8ee47fe32ce8e876b320cd37d88edfc4d/transformers-5.8.1.tar.gz", hash = "sha256:4dd5b6de4105725104d84fd6abd74b305f4debfc251b38c648ee5dd087cf543b", size = 8532019, upload-time = "2026-05-13T03:21:57.234Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/03/b8/e484ef633af3887baeeb4b6ad12743363af7cce68ae51e938e00aaa0529d/transformers-4.57.6-py3-none-any.whl", hash = "sha256:4c9e9de11333ddfe5114bc872c9f370509198acf0b87a832a0ab9458e2bd0550", size = 11993498, upload-time = "2026-01-16T10:38:31.289Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b1/8be7e7ef0b5200491312201918b6125ef9c9df9dd0f0240ccef9ac824e6b/transformers-5.8.1-py3-none-any.whl", hash = "sha256:5340fb95962162cdfdae5cc91d7f8fedd92ed75216c1154c5e1f590fcf56dd0e", size = 10632882, upload-time = "2026-05-13T03:21:52.876Z" }, ] [[package]] @@ -5197,17 +5054,17 @@ wheels = [ [[package]] name = "typer" -version = "0.16.1" +version = "0.26.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "rich" }, { name = "shellingham" }, - { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/43/78/d90f616bf5f88f8710ad067c1f8705bf7618059836ca084e5bb2a0855d75/typer-0.16.1.tar.gz", hash = "sha256:d358c65a464a7a90f338e3bb7ff0c74ac081449e53884b12ba658cbd72990614", size = 102836, upload-time = "2025-08-18T19:18:22.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/f7/68adc395201b20b872d68e975386832e8005ffeacedd43a1d837a32815be/typer-0.26.8.tar.gz", hash = "sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e", size = 202097, upload-time = "2026-06-26T09:22:45.705Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/76/06dbe78f39b2203d2a47d5facc5df5102d0561e2807396471b5f7c5a30a1/typer-0.16.1-py3-none-any.whl", hash = "sha256:90ee01cb02d9b8395ae21ee3368421faf21fa138cb2a541ed369c08cec5237c9", size = 46397, upload-time = "2025-08-18T19:18:21.663Z" }, + { url = "https://files.pythonhosted.org/packages/80/87/b9fd69c92c6102a066e1b86a35243f53e70bd4c709f2a26d9f4fee4f4dc0/typer-0.26.8-py3-none-any.whl", hash = "sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c", size = 122564, upload-time = "2026-06-26T09:22:44.72Z" }, ] [[package]] @@ -5267,11 +5124,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.6.3" +version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] [[package]] @@ -5305,15 +5162,15 @@ wheels = [ [[package]] name = "uvicorn" -version = "0.38.0" +version = "0.52.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cb/ce/f06b84e2697fef4688ca63bdb2fdf113ca0a3be33f94488f2cadb690b0cf/uvicorn-0.38.0.tar.gz", hash = "sha256:fd97093bdd120a2609fc0d3afe931d4d4ad688b6e75f0f929fde1bc36fe0e91d", size = 80605, upload-time = "2025-10-18T13:46:44.63Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/0f/3f86e61397dd33bf2ccf28188c40db6a740658aeebbbf6e7dbc101a1f487/uvicorn-0.52.4.tar.gz", hash = "sha256:73acfee47a0b133c5de13d219492d62d8a31e935f4fe6e41a232451a15379f86", size = 100627, upload-time = "2026-08-19T06:27:41.821Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/d9/d88e73ca598f4f6ff671fb5fde8a32925c2e08a637303a1d12883c7305fa/uvicorn-0.38.0-py3-none-any.whl", hash = "sha256:48c0afd214ceb59340075b4a052ea1ee91c16fbc2a9b1469cca0e54566977b02", size = 68109, upload-time = "2025-10-18T13:46:42.958Z" }, + { url = "https://files.pythonhosted.org/packages/f1/79/4a20b54ab0491485ccd8c077db2d39187c7f12b3e15485d38a7be37c81b4/uvicorn-0.52.4-py3-none-any.whl", hash = "sha256:f86e41a149d7d05a9969337e3946a9c171c06a5d42680896daaba624aeac8da1", size = 79871, upload-time = "2026-08-19T06:27:40.36Z" }, ] [[package]] @@ -5325,19 +5182,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/6e/3e955517e22cbdd565f2f8b2e73d52528b14b8bcfdb04f62466b071de847/validators-0.35.0-py3-none-any.whl", hash = "sha256:e8c947097eae7892cb3d26868d637f79f47b4a0554bc6b80065dfe5aac3705dd", size = 44712, upload-time = "2025-05-01T05:42:04.203Z" }, ] -[[package]] -name = "vcrpy" -version = "8.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml", marker = "python_full_version < '3.13'" }, - { name = "wrapt", marker = "python_full_version < '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/07/bcfd5ebd7cb308026ab78a353e091bd699593358be49197d39d004e5ad83/vcrpy-8.1.1.tar.gz", hash = "sha256:58e3053e33b423f3594031cb758c3f4d1df931307f1e67928e30cf352df7709f", size = 85770, upload-time = "2026-01-04T19:22:03.886Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/d7/f79b05a5d728f8786876a7d75dfb0c5cae27e428081b2d60152fb52f155f/vcrpy-8.1.1-py3-none-any.whl", hash = "sha256:2d16f31ad56493efb6165182dd99767207031b0da3f68b18f975545ede8ac4b9", size = 42445, upload-time = "2026-01-04T19:22:02.532Z" }, -] - [[package]] name = "vine" version = "5.1.0"