From c88a9b01efded896aeb68757455acb165e238289 Mon Sep 17 00:00:00 2001 From: Julian Risch Date: Mon, 15 Jun 2026 16:00:27 +0200 Subject: [PATCH 1/2] chore: exclude haystack-core-integrations packages from exclude-newer cutoff Add all packages maintained in deepset-ai/haystack-core-integrations to the [exclude-newer-package] table in uv.toml so that the global "P3D" (3-day) exclude-newer cutoff does not apply to our own first-party integration packages. This lets tutorials install freshly released integration versions without waiting for the supply-chain safety window. Co-Authored-By: Claude Opus 4.8 (1M context) --- uv.toml | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/uv.toml b/uv.toml index 822d47ce..b26e525c 100644 --- a/uv.toml +++ b/uv.toml @@ -2,7 +2,87 @@ # attacks via compromised dependencies. exclude-newer = "P3D" -# first-party dependencies can be excluded from the global cutoff by adding entries below +# first-party dependencies can be excluded from the global cutoff by adding entries below. +# This includes haystack-ai and all packages maintained in +# https://github.com/deepset-ai/haystack-core-integrations [exclude-newer-package] haystack-ai = false +aimlapi-haystack = false +alloydb-haystack = false +amazon-bedrock-haystack = false +amazon-sagemaker-haystack = false +amazon-textract-haystack = false +anthropic-haystack = false +arangodb-haystack = false +arcadedb-haystack = false +astra-haystack = false +azure-ai-search-haystack = false +azure-doc-intelligence-haystack = false +brave-haystack = false +chonkie-haystack = false +chroma-haystack = false +cognee-haystack = false +cohere-haystack = false +cometapi-haystack = false +deepeval-haystack = false +docling-haystack = false +docling-serve-haystack = false +dspy-haystack = false +e2b-haystack = false +elasticsearch-haystack = false +faiss-haystack = false +falkordb-haystack = false +fastembed-haystack = false +firecrawl-haystack = false +funasr-haystack = false +github-haystack = false +google-ai-haystack = false +google-genai-haystack = false +google-vertex-haystack = false +hanlp-haystack = false +huggingface-api-haystack = false +jina-haystack = false +kreuzberg-haystack = false +langfuse-haystack = false +lara-haystack = false +libreoffice-haystack = false +litellm-haystack = false +llama-cpp-haystack = false +llama-stack-haystack = false +markitdown-haystack = false +mcp-haystack = false +mem0-haystack = false +meta-llama-haystack = false +mistral-haystack = false +mongodb-atlas-haystack = false +nvidia-haystack = false +ollama-haystack = false +openrouter-haystack = false +opensearch-haystack = false +optimum-haystack = false +oracle-haystack = false +paddleocr-haystack = false +perplexity-haystack = false +pgvector-haystack = false +pinecone-haystack = false +presidio-haystack = false +pyversity-haystack = false +qdrant-haystack = false +ragas-haystack = false +searchapi-haystack = false +serperdev-haystack = false +snowflake-haystack = false +spacy-haystack = false +sqlalchemy-haystack = false +stackit-haystack = false +supabase-haystack = false +tavily-haystack = false +togetherai-haystack = false transformers-haystack = false +unstructured-fileconverter-haystack = false +valkey-haystack = false +vespa-haystack = false +vllm-haystack = false +watsonx-haystack = false +weave-haystack = false +weaviate-haystack = false From 2577de1b8c21e5a1d4194aa0472d62b39e0749a1 Mon Sep 17 00:00:00 2001 From: Julian Risch Date: Mon, 15 Jun 2026 16:36:22 +0200 Subject: [PATCH 2/2] ci: auto-sync exclude-newer-package table with haystack-core-integrations Add scripts/generate_uv_excludes.py, which regenerates the [exclude-newer-package] table in uv.toml from the package names declared in a checkout of deepset-ai/haystack-core-integrations, and a scheduled "Sync uv excludes" workflow that runs the script weekly and opens a PR when the set of integration packages changes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/sync_uv_excludes.yml | 57 ++++++++++++ scripts/generate_uv_excludes.py | 116 +++++++++++++++++++++++++ uv.toml | 4 + 3 files changed, 177 insertions(+) create mode 100644 .github/workflows/sync_uv_excludes.yml create mode 100644 scripts/generate_uv_excludes.py diff --git a/.github/workflows/sync_uv_excludes.yml b/.github/workflows/sync_uv_excludes.yml new file mode 100644 index 00000000..0fd7969d --- /dev/null +++ b/.github/workflows/sync_uv_excludes.yml @@ -0,0 +1,57 @@ +name: Sync uv excludes + +# Keeps the [exclude-newer-package] table in uv.toml in sync with the packages +# maintained in deepset-ai/haystack-core-integrations. Runs on a schedule and +# opens a pull request whenever a new integration is added or removed. + +on: + workflow_dispatch: # Activate this workflow manually + schedule: + - cron: "0 6 * * 1" # every Monday at 06:00 UTC + +permissions: + contents: write + pull-requests: write + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout tutorials + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Checkout haystack-core-integrations + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: deepset-ai/haystack-core-integrations + path: .haystack-core-integrations + sparse-checkout: integrations + sparse-checkout-cone-mode: false + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.11" + + - name: Regenerate uv.toml + run: | + python scripts/generate_uv_excludes.py \ + --integrations-repo .haystack-core-integrations \ + --uv-toml uv.toml + + - name: Create pull request + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + add-paths: uv.toml + branch: chore/sync-uv-excludes + delete-branch: true + commit-message: "chore: sync exclude-newer-package table with haystack-core-integrations" + title: "chore: sync exclude-newer-package table with haystack-core-integrations" + body: | + Automated update of the `[exclude-newer-package]` table in `uv.toml`. + + The packages maintained in + [deepset-ai/haystack-core-integrations](https://github.com/deepset-ai/haystack-core-integrations) + changed, so the list of first-party packages exempt from the + `exclude-newer` supply-chain cutoff has been regenerated by + `scripts/generate_uv_excludes.py`. + labels: dependencies diff --git a/scripts/generate_uv_excludes.py b/scripts/generate_uv_excludes.py new file mode 100644 index 00000000..53e5fb28 --- /dev/null +++ b/scripts/generate_uv_excludes.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Regenerate the [exclude-newer-package] table in uv.toml. + +The global ``exclude-newer`` cutoff in ``uv.toml`` skips package versions +published within the last few days as a supply-chain safeguard. First-party +packages should be exempt from that cutoff so freshly released versions can be +installed immediately. + +This script rewrites ``uv.toml`` so the ``[exclude-newer-package]`` table +always contains ``haystack-ai`` plus every package maintained in +https://github.com/deepset-ai/haystack-core-integrations . The integration +package names are read from the ``name`` field of each +``integrations/*/pyproject.toml`` in a local checkout of that repository. + +Usage: + python scripts/generate_uv_excludes.py \ + --integrations-repo /path/to/haystack-core-integrations \ + --uv-toml uv.toml [--check] + +With ``--check`` the script does not write anything and exits non-zero if the +file is out of date (useful in CI). +""" + +import argparse +import re +import sys +from pathlib import Path + +# Packages that are first-party but not part of haystack-core-integrations. +ALWAYS_INCLUDE = ["haystack-ai"] + +HEADER = """\ +# Exclude package versions published within the last 3 days to protect against supply chain +# attacks via compromised dependencies. +exclude-newer = "P3D" + +# first-party dependencies can be excluded from the global cutoff by adding entries below. +# This includes haystack-ai and all packages maintained in +# https://github.com/deepset-ai/haystack-core-integrations +# +# NOTE: the integration entries are generated by scripts/generate_uv_excludes.py +# and kept in sync by the "Sync uv excludes" GitHub Actions workflow. Edit the +# script rather than this list by hand. +[exclude-newer-package] +""" + +NAME_RE = re.compile(r'^name\s*=\s*"([^"]+)"', re.MULTILINE) + + +def normalize(name: str) -> str: + """Normalize a distribution name to PEP 503 form (lowercase, hyphens).""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def collect_integration_names(integrations_repo: Path) -> list[str]: + integrations_dir = integrations_repo / "integrations" + if not integrations_dir.is_dir(): + sys.exit(f"error: {integrations_dir} not found; is --integrations-repo correct?") + + names = set() + for pyproject in integrations_dir.glob("*/pyproject.toml"): + text = pyproject.read_text(encoding="utf-8") + match = NAME_RE.search(text) + if match: + names.add(normalize(match.group(1))) + if not names: + sys.exit(f"error: no package names found under {integrations_dir}") + return sorted(names) + + +def render(names: list[str]) -> str: + lines = [HEADER] + for pkg in ALWAYS_INCLUDE: + lines.append(f"{normalize(pkg)} = false\n") + for pkg in names: + if normalize(pkg) in {normalize(p) for p in ALWAYS_INCLUDE}: + continue + lines.append(f"{pkg} = false\n") + return "".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--integrations-repo", + type=Path, + required=True, + help="Path to a local checkout of deepset-ai/haystack-core-integrations", + ) + parser.add_argument("--uv-toml", type=Path, default=Path("uv.toml"), help="Path to uv.toml to write") + parser.add_argument( + "--check", + action="store_true", + help="Do not write; exit non-zero if uv.toml is out of date", + ) + args = parser.parse_args() + + names = collect_integration_names(args.integrations_repo) + content = render(names) + + current = args.uv_toml.read_text(encoding="utf-8") if args.uv_toml.exists() else "" + if current == content: + print(f"{args.uv_toml} is up to date ({len(names)} integration packages).") + return 0 + + if args.check: + print(f"{args.uv_toml} is OUT OF DATE. Run generate_uv_excludes.py to update it.", file=sys.stderr) + return 1 + + args.uv_toml.write_text(content, encoding="utf-8") + print(f"Wrote {args.uv_toml} with {len(names)} integration packages.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/uv.toml b/uv.toml index b26e525c..8458d85f 100644 --- a/uv.toml +++ b/uv.toml @@ -5,6 +5,10 @@ exclude-newer = "P3D" # first-party dependencies can be excluded from the global cutoff by adding entries below. # This includes haystack-ai and all packages maintained in # https://github.com/deepset-ai/haystack-core-integrations +# +# NOTE: the integration entries are generated by scripts/generate_uv_excludes.py +# and kept in sync by the "Sync uv excludes" GitHub Actions workflow. Edit the +# script rather than this list by hand. [exclude-newer-package] haystack-ai = false aimlapi-haystack = false