From 0aae2c98eb9ab6a14a716188fd636f2f02ef9a1b Mon Sep 17 00:00:00 2001 From: Alina Ryan Date: Thu, 15 Oct 2026 09:45:22 -0400 Subject: [PATCH 1/6] Add policy document conversion pipeline with VLM artifact interpretation Adds a new `docling-vlm-policy` KFP pipeline that combines Docling's standard structural parsing with a VLM served via KServe for classifying and interpreting non-text artifacts (risk matrices, decision trees, charts, forms, screenshots, diagrams) in policy documents. Each artifact type gets a specialized prompt producing structured Markdown, assembled inline with `` markers. Includes pipeline definition, interpret_policy_artifacts component, KServe InferenceService manifest, walkthrough notebook, and README. Co-Authored-By: Claude Opus 4.6 --- kubeflow-pipelines/README.md | 18 +- .../docling-vlm-policy/.gitignore | 3 + .../docling-vlm-policy/.pylintrc | 2 + .../docling-vlm-policy/README.md | 203 +++++++ .../kserve/vlm-inferenceservice.yaml | 56 ++ .../docling-vlm-policy/local_run.py | 71 +++ .../docling-vlm-policy/policy_components.py | 338 +++++++++++ .../policy_convert_pipeline.py | 129 ++++ .../docling-vlm-policy/requirements.txt | 4 + .../policy-document-conversion-vlm.ipynb | 563 ++++++++++++++++++ 10 files changed, 1383 insertions(+), 4 deletions(-) create mode 100644 kubeflow-pipelines/docling-vlm-policy/.gitignore create mode 100644 kubeflow-pipelines/docling-vlm-policy/.pylintrc create mode 100644 kubeflow-pipelines/docling-vlm-policy/README.md create mode 100644 kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml create mode 100644 kubeflow-pipelines/docling-vlm-policy/local_run.py create mode 100644 kubeflow-pipelines/docling-vlm-policy/policy_components.py create mode 100644 kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline.py create mode 100644 kubeflow-pipelines/docling-vlm-policy/requirements.txt create mode 100644 notebooks/use-cases/policy-document-conversion-vlm.ipynb diff --git a/kubeflow-pipelines/README.md b/kubeflow-pipelines/README.md index 03998d61..8d1fe805 100644 --- a/kubeflow-pipelines/README.md +++ b/kubeflow-pipelines/README.md @@ -6,10 +6,11 @@ In this section of the Data Processing repository, we provide reference **data-processing pipelines** for [Open Data Hub](https://github.com/opendatahub-io) / [Red Hat OpenShift AI](https://www.redhat.com/en/products/ai/openshift-ai), packaged as [Kubeflow Pipelines](https://www.kubeflow.org/docs/components/pipelines/) (KFP). -Two KFP pipelines are included: +Three KFP pipelines are included: - **[docling-standard](docling-standard)**: Standard Docling pipeline for standard conversions, OCR, table structure, enrichments. - **[docling-vlm](docling-vlm)**: Vision-Language-Model (VLM) pipeline supporting local or remote models. +- **[docling-vlm-policy](docling-vlm-policy)**: Policy document pipeline that combines standard Docling parsing with VLM-based artifact classification and type-specific interpretation (risk matrices, decision trees, charts, forms, screenshots, diagrams). ## 🚀 Key Features @@ -54,11 +55,20 @@ kubeflow-pipelines | |- requirements.txt | |- docling-vlm/ - |- vlm_components.py - |- vlm_convert_pipeline.py - |- vlm_convert_pipeline_compiled.yaml (generated) +| |- vlm_components.py +| |- vlm_convert_pipeline.py +| |- vlm_convert_pipeline_compiled.yaml (generated) +| |- local_run.py # Local testing script +| |- requirements.txt +| +|- docling-vlm-policy/ + |- policy_components.py + |- policy_convert_pipeline.py + |- policy_convert_pipeline_compiled.yaml (generated) |- local_run.py # Local testing script |- requirements.txt + |- kserve/ + |- vlm-inferenceservice.yaml ``` ## ✨ Getting Started diff --git a/kubeflow-pipelines/docling-vlm-policy/.gitignore b/kubeflow-pipelines/docling-vlm-policy/.gitignore new file mode 100644 index 00000000..d0ee3b17 --- /dev/null +++ b/kubeflow-pipelines/docling-vlm-policy/.gitignore @@ -0,0 +1,3 @@ +venv/ +__pycache__/ +*.pyc diff --git a/kubeflow-pipelines/docling-vlm-policy/.pylintrc b/kubeflow-pipelines/docling-vlm-policy/.pylintrc new file mode 100644 index 00000000..26984fd2 --- /dev/null +++ b/kubeflow-pipelines/docling-vlm-policy/.pylintrc @@ -0,0 +1,2 @@ +[MASTER] +disable=C0114,C0115,C0116 diff --git a/kubeflow-pipelines/docling-vlm-policy/README.md b/kubeflow-pipelines/docling-vlm-policy/README.md new file mode 100644 index 00000000..0b84fd9a --- /dev/null +++ b/kubeflow-pipelines/docling-vlm-policy/README.md @@ -0,0 +1,203 @@ +# VLM Policy Document Processing Pipeline + +Policy-document-specific [Kubeflow Pipeline](https://www.kubeflow.org/docs/components/pipelines/) (KFP) for [Open Data Hub](https://github.com/opendatahub-io) / [Red Hat OpenShift AI](https://www.redhat.com/en/products/ai/openshift-ai) that converts policy documents containing non-text artifacts (risk matrices, decision trees, compliance charts, forms, screenshots) into structured Markdown with high-fidelity artifact interpretations. + +## Architecture + +This pipeline combines Docling's standard structural document parsing with a Vision Language Model (VLM) served via KServe for targeted artifact interpretation: + +``` +import_pdfs → create_pdf_splits → download_docling_models (standard) + │ + ParallelFor(splits): + → docling_convert_standard (image_export_mode="referenced") + → interpret_policy_artifacts (VLM classify + interpret + assemble) + → [optional] docling_chunk +``` + +**Why standard Docling + external VLM?** The standard Docling pipeline provides high-quality structural parsing (headings, tables, numbered clauses, cross-references, reading order) and extracts images as separate files. An external VLM then classifies and interprets each extracted image with artifact-type-specific prompts, producing structured Markdown that replaces the image in the final output. + +### Data Flow + +1. **Import**: PDFs downloaded from HTTP URLs or S3-compatible storage +2. **Split**: File list divided into batches for parallel processing via KFP `ParallelFor` +3. **Convert**: Docling standard pipeline parses document structure, exports images as referenced PNGs +4. **Interpret**: Each image sent to VLM for classification → type-specific structured interpretation +5. **Assemble**: Image references in Markdown replaced with interpreted content wrapped in artifact markers +6. **Chunk** (optional): DoclingDocument JSON chunked with HybridChunker for RAG workflows + +### Artifact Type Classification + +Each extracted image is classified into one of these categories: + +| Type | Description | Output Format | +|------|-------------|---------------| +| `risk_matrix` | Risk assessment matrix or heat map | Markdown table with severity levels | +| `decision_tree` | Decision flowchart with branches | Nested Markdown list with branches/outcomes | +| `chart` | Bar, line, pie, scatter plots | Data description with values, trends, summary table | +| `form` | Forms, questionnaires, templates | Structured field-value pairs table | +| `screenshot` | UI screenshots | Step-by-step textual walkthrough | +| `diagram` | Architecture, process, org charts | Component/relationship description | +| `other` | Anything else | Detailed structured description | + +### Output Format + +Each interpreted artifact is wrapped in HTML comment markers for downstream identification: + +```markdown + +| | Low Impact | Medium Impact | High Impact | +|---|---|---|---| +| **High Likelihood** | Medium | High | Critical | +| **Medium Likelihood** | Low | Medium | High | +| **Low Likelihood** | Negligible | Low | Medium | + +``` + +### Output Files + +Per document, the pipeline produces: + +| File | Description | +|------|-------------| +| `{name}_policy.md` | Assembled Markdown with artifact interpretations inline | +| `{name}_artifacts.json` | Metadata: artifact type, interpretation, source image per artifact | +| `{name}.json` | Original DoclingDocument JSON (for HybridChunker / downstream consumers) | + +## Prerequisites + +- Access to a KFP 2.x instance (Red Hat OpenShift AI / Data Science Pipelines) +- A VLM served via KServe InferenceService with OpenAI-compatible API (see [Deploying the VLM](#deploying-the-vlm)) +- Kubernetes Secret with VLM endpoint configuration (see [Secret Configuration](#secret-configuration)) + +## Deploying the VLM + +Deploy a VLM as a KServe InferenceService using the provided manifest: + +```bash +# Optional: create a HuggingFace token secret if your model requires authentication +kubectl create secret generic hf-token --from-literal=token="hf_YOUR_TOKEN" + +# Deploy the InferenceService +kubectl apply -f kserve/vlm-inferenceservice.yaml +``` + +The default manifest deploys `ibm-granite/granite-vision-3.2-2b` using `registry.redhat.io/rhaiis/vllm-cuda-rhel9`. Adjust the `--model` argument for a different model (e.g., `ibm-granite/granite-vision-3.2-8b` for higher quality). + +Wait for the InferenceService to be ready: + +```bash +kubectl get inferenceservice policy-vlm -w +``` + +The VLM will be accessible in-cluster at: +``` +http://policy-vlm-predictor.NAMESPACE.svc.cluster.local:8080/v1/chat/completions +``` + +## Secret Configuration + +Create a Kubernetes Secret with the VLM endpoint configuration and optionally S3 credentials: + +```bash +# VLM endpoint only +kubectl create secret generic data-processing-docling-pipeline \ + --from-literal=REMOTE_MODEL_ENDPOINT_URL="http://policy-vlm-predictor.NAMESPACE.svc.cluster.local:8080" \ + --from-literal=REMOTE_MODEL_NAME="ibm-granite/granite-vision-3.2-2b" \ + --from-literal=REMOTE_MODEL_API_KEY="" + +# With S3 source (add these keys to the same secret) +kubectl create secret generic data-processing-docling-pipeline \ + --from-literal=REMOTE_MODEL_ENDPOINT_URL="http://policy-vlm-predictor.NAMESPACE.svc.cluster.local:8080" \ + --from-literal=REMOTE_MODEL_NAME="ibm-granite/granite-vision-3.2-2b" \ + --from-literal=REMOTE_MODEL_API_KEY="" \ + --from-literal=S3_ENDPOINT_URL="https://s3.us-east-2.amazonaws.com" \ + --from-literal=S3_ACCESS_KEY="REDACTED" \ + --from-literal=S3_SECRET_KEY="REDACTED" \ + --from-literal=S3_BUCKET="my-bucket" \ + --from-literal=S3_PREFIX="policy-documents" +``` + +## Installation + +Download the [compiled YAML file](policy_convert_pipeline_compiled.yaml?raw=1) and upload it on the _Import pipeline_ screen in the KFP UI. + +## Configuration Options + +### Document source + +- `pdf_filenames`: Comma-separated list of PDF file names to process. +- `pdf_base_url`: HTTP/S base URL where the PDF files are located. +- `pdf_from_s3`: Set to `True` to download from S3-compatible storage instead of URLs. + +### Conversion + +- `num_splits`: Number of parallel processing batches (increase for large corpora). +- `docling_num_threads`: Threads per Docling conversion container (default: `4`). +- `docling_timeout_per_document`: Timeout in seconds per document (default: `300`). +- `docling_ocr`: Enable OCR for scanned content (default: `True`). +- `docling_table_mode`: Table detection mode — `accurate` (default) or `fast`. + +### VLM interpretation + +- `vlm_max_tokens`: Maximum tokens for each VLM interpretation response (default: `1024`). +- `vlm_timeout`: Timeout in seconds per VLM API call (default: `120`). + +### Chunking + +- `docling_chunk_enabled`: Enable HybridChunker chunking (default: `False`). +- `docling_chunk_max_tokens`: Maximum tokens per chunk (default: `512`). +- `docling_chunk_merge_peers`: Merge adjacent small chunks (default: `True`). + +## Handling Large Corpora (200+ pages) + +For processing large policy document corpora: + +1. **Increase `num_splits`**: Set to 5-10 for 200+ page corpora to distribute work across parallel containers +2. **Increase `docling_timeout_per_document`**: Set to 600+ for complex documents +3. **Increase `vlm_timeout`**: Large or complex images may require longer VLM processing +4. **Resource tuning**: Adjust container memory/CPU limits in `policy_convert_pipeline.py` if needed + +## Sample Documents + +For testing, ~27 policy PDFs are available in the [`asago-ai/asago-examples`](https://github.com/asago-ai/asago-examples) repository under `asago-policy-mapper/policy_examples/` (Amadeus, Cisco, eBay, NHS, ICRC, NYC, etc.). + +## Downstream Integration: asago-policy-mapper + +The `asago-policy-mapper` currently calls `doc.export_to_markdown(image_mode=ImageRefMode.PLACEHOLDER, image_placeholder="")` which silently drops images. With this pipeline: + +- Use `{name}_policy.md` directly — it contains the full document text with VLM-interpreted artifacts inline +- The `{name}.json` (DoclingDocument) is still available for `HybridChunker`-based workflows +- Artifact markers (``) allow downstream consumers to identify and handle interpreted content programmatically + +## Quality Evaluation + +To evaluate interpretation quality: + +1. **Manual review**: Compare `_policy.md` output against original PDF for a sample of documents +2. **Artifact coverage**: Check `_artifacts.json` — `total_artifacts` should match the number of non-text elements in the source +3. **Classification accuracy**: Review artifact type assignments in `_artifacts.json` +4. **Structural preservation**: Verify headings, numbered clauses, cross-references, and tables are intact in the assembled Markdown + +## Local Testing + +```bash +pip install docker kfp +cd data-processing/kubeflow-pipelines/docling-vlm-policy +python local_run.py +``` + +Requires a Docker-compatible daemon and a VLM secret configured locally. + +## Compiling from Source + +```bash +git clone https://github.com/opendatahub-io/data-processing.git +cd data-processing/kubeflow-pipelines/docling-vlm-policy +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +python policy_convert_pipeline.py +``` + +This generates `policy_convert_pipeline_compiled.yaml`. diff --git a/kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml b/kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml new file mode 100644 index 00000000..66ae793a --- /dev/null +++ b/kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml @@ -0,0 +1,56 @@ +apiVersion: serving.kserve.io/v1beta1 +kind: InferenceService +metadata: + name: policy-vlm + annotations: + serving.kserve.io/deploymentMode: RawDeployment +spec: + predictor: + minReplicas: 1 + maxReplicas: 1 + containers: + - name: kserve-container + image: registry.redhat.io/rhaiis/vllm-cuda-rhel9:latest + command: + - python + - -m + - vllm.entrypoints.openai.api_server + args: + - --model + - ibm-granite/granite-vision-3.2-2b + - --port + - "8080" + - --max-model-len + - "4096" + - --trust-remote-code + env: + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token + key: token + optional: true + ports: + - containerPort: 8080 + protocol: TCP + resources: + requests: + nvidia.com/gpu: "1" + memory: "24Gi" + cpu: "4" + limits: + nvidia.com/gpu: "1" + memory: "48Gi" + cpu: "8" + readinessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 120 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8080 + initialDelaySeconds: 120 + periodSeconds: 30 diff --git a/kubeflow-pipelines/docling-vlm-policy/local_run.py b/kubeflow-pipelines/docling-vlm-policy/local_run.py new file mode 100644 index 00000000..24865a90 --- /dev/null +++ b/kubeflow-pipelines/docling-vlm-policy/local_run.py @@ -0,0 +1,71 @@ +import sys +from pathlib import Path +from typing import List + +# Add parent directories to Python path for imports +sys.path.insert(0, str(Path(__file__).parent.parent)) +sys.path.insert(0, str(Path(__file__).parent.parent / "docling-standard")) + +from common import ( + create_pdf_splits, + docling_chunk, + download_docling_models, + import_pdfs, +) +from kfp import dsl, local +from policy_components import interpret_policy_artifacts +from standard_components import docling_convert_standard + + +@dsl.component(base_image="python:3.11") +def take_first_split(splits: List[List[str]]) -> List[str]: + return splits[0] if splits else [] + + +@dsl.pipeline() +def policy_convert_pipeline_local(): + """ + Local pipeline for testing policy document conversion with VLM interpretation. + """ + importer = import_pdfs( + filenames="2305.03393v1-pg9.pdf", + base_url="https://github.com/docling-project/docling/raw/v2.43.0/tests/data/pdf", + ) + + pdf_splits = create_pdf_splits( + input_path=importer.outputs["output_path"], + num_splits=1, + ) + + artifacts = download_docling_models( + pipeline_type="standard", + remote_model_endpoint_enabled=False, + ) + + first_split = take_first_split(splits=pdf_splits.output) + + converter = docling_convert_standard( + input_path=importer.outputs["output_path"], + artifacts_path=artifacts.outputs["output_path"], + pdf_filenames=first_split.output, + image_export_mode="referenced", + ) + + interpreter = interpret_policy_artifacts( + input_path=converter.outputs["output_path"], + ) + + docling_chunk( + input_path=interpreter.outputs["output_path"], + max_tokens=512, + merge_peers=True, + ) + + +def main() -> None: + local.init(runner=local.DockerRunner()) + policy_convert_pipeline_local() + + +if __name__ == "__main__": + main() diff --git a/kubeflow-pipelines/docling-vlm-policy/policy_components.py b/kubeflow-pipelines/docling-vlm-policy/policy_components.py new file mode 100644 index 00000000..96884809 --- /dev/null +++ b/kubeflow-pipelines/docling-vlm-policy/policy_components.py @@ -0,0 +1,338 @@ +import sys +from pathlib import Path + +# Add the parent directory to Python path to find common +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from common.constants import DOCLING_BASE_IMAGE +from kfp import dsl + + +@dsl.component( + base_image=DOCLING_BASE_IMAGE, +) +def interpret_policy_artifacts( + input_path: dsl.Input[dsl.Artifact], + output_path: dsl.Output[dsl.Artifact], + vlm_secret_mount_path: str = "/mnt/secrets", + vlm_max_tokens: int = 1024, + vlm_timeout: int = 120, +): + """ + Classify and interpret non-text artifacts in policy documents using a VLM. + + Reads Docling conversion output (JSON, Markdown, and referenced image files), + sends each extracted image to a VLM for artifact-type classification and + type-specific structured interpretation, then assembles final Markdown with + interpreted content replacing image references wrapped in artifact-type markers. + + Args: + input_path: Path to directory containing Docling conversion output. + output_path: Path to output directory for assembled policy Markdown, + artifact metadata, and passthrough DoclingDocument JSON. + vlm_secret_mount_path: Path to mounted Kubernetes secret containing + REMOTE_MODEL_ENDPOINT_URL, REMOTE_MODEL_NAME, REMOTE_MODEL_API_KEY. + vlm_max_tokens: Maximum tokens for each VLM interpretation response. + vlm_timeout: Timeout in seconds per VLM API call. + """ + import base64 + import json + import os + import re + import shutil + import time + from pathlib import Path + + import requests + + CLASSIFICATION_PROMPT = ( + "You are analyzing an image extracted from a policy document. " + "Classify this image into exactly one of the following categories. " + "Respond with ONLY the category name, nothing else.\n\n" + "Categories:\n" + "- chart: Any type of chart, graph, or plot (bar, line, pie, scatter, etc.)\n" + "- risk_matrix: A risk assessment matrix or heat map showing likelihood vs impact\n" + "- decision_tree: A decision tree, flowchart showing decision logic with branches\n" + "- form: A form, questionnaire, or template with fields to fill in\n" + "- screenshot: A screenshot of a user interface or application\n" + "- diagram: An architecture diagram, process flow, org chart, or network diagram\n" + "- other: Any other type of image that doesn't fit the above categories" + ) + + INTERPRETATION_PROMPTS = { + "risk_matrix": ( + "Convert this risk matrix into structured Markdown. " + "Create a Markdown table that preserves all row headers, column headers, " + "and cell values. Include severity levels and any color coding described as " + "text labels (e.g., 'High Risk'). Add any legends, footnotes, or scoring " + "criteria below the table." + ), + "decision_tree": ( + "Convert this decision tree into a nested Markdown list representing the " + "complete decision logic. Use indentation to show hierarchy and branches. " + "Each decision point should show the question or condition. " + "Each branch should show the path taken (Yes/No, or named paths). " + "Each leaf node should show the final outcome or action. " + "Preserve all labels, conditions, thresholds, and outcomes exactly as shown." + ), + "chart": ( + "Describe this chart in structured Markdown. Include:\n" + "- **Chart type**: (bar, line, pie, etc.)\n" + "- **Title**: The chart title if visible\n" + "- **Axes**: X-axis and Y-axis labels and units\n" + "- **Data summary**: A Markdown table with the key data points and values\n" + "- **Trends**: Notable trends, peaks, or patterns in the data\n" + "- **Annotations**: Any callouts, labels, or annotations on the chart" + ), + "form": ( + "Convert this form into structured Markdown. " + "Create a Markdown table with 'Field' and 'Value/Description' columns. " + "Include all field labels and any pre-filled values. " + "For checkboxes, indicate [x] checked or [ ] unchecked. " + "For dropdowns, show the selected value if visible. " + "Include section headers, instructions, and help text." + ), + "screenshot": ( + "Describe this UI screenshot as a structured step-by-step walkthrough. " + "Identify the application or interface shown. " + "Describe visible elements: navigation, buttons, menus, form fields, " + "status indicators, data displays, and notifications. " + "Note the current state or action being performed. " + "If it shows a workflow or sequence, use numbered steps." + ), + "diagram": ( + "Describe this diagram in structured Markdown. " + "List all components and their labels. " + "Describe relationships and connections between components. " + "Note directional flows (arrows) and their labels. " + "Identify groupings, layers, or zones. " + "Use a nested list to represent the hierarchy, or a table for " + "components and connections. " + "Preserve any annotations, version numbers, or metadata shown." + ), + "other": ( + "Describe the content of this image in structured Markdown. " + "Preserve all visible text, numbers, labels, and structural information. " + "Use appropriate Markdown formatting — tables for tabular data, " + "lists for hierarchical or sequential content, and headings for sections. " + "Capture any legends, footnotes, or annotations." + ), + } + + VALID_TYPES = set(INTERPRETATION_PROMPTS.keys()) + + # --- Read VLM credentials from mounted secret --- + def read_secret(name): + fpath = os.path.join(vlm_secret_mount_path, name) + if not os.path.isfile(fpath): + raise ValueError( + f"Key {name} not found in secret mount {vlm_secret_mount_path}" + ) + with open(fpath) as f: + return f.read().strip() + + if not os.path.exists(vlm_secret_mount_path): + raise ValueError( + f"Secret mount path {vlm_secret_mount_path} does not exist. " + "Create a Kubernetes Secret with VLM endpoint configuration." + ) + + vlm_endpoint_url = read_secret("REMOTE_MODEL_ENDPOINT_URL") + vlm_model_name = read_secret("REMOTE_MODEL_NAME") + vlm_api_key = read_secret("REMOTE_MODEL_API_KEY") + + if not vlm_endpoint_url.endswith("/v1/chat/completions"): + vlm_endpoint_url = vlm_endpoint_url.rstrip("/") + if not vlm_endpoint_url.endswith("/v1"): + vlm_endpoint_url += "/v1" + vlm_endpoint_url += "/chat/completions" + + input_path_p = Path(input_path.path) + output_path_p = Path(output_path.path) + output_path_p.mkdir(parents=True, exist_ok=True) + + # --- VLM call helper with retries --- + def call_vlm(image_b64, prompt, max_tokens): + payload = { + "model": vlm_model_name, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{image_b64}" + }, + }, + ], + } + ], + "max_tokens": max_tokens, + } + headers = {} + if vlm_api_key: + headers["Authorization"] = f"Bearer {vlm_api_key}" + + for attempt in range(3): + try: + resp = requests.post( + vlm_endpoint_url, + json=payload, + headers=headers, + timeout=vlm_timeout, + ) + resp.raise_for_status() + return resp.json()["choices"][0]["message"]["content"] + except (requests.RequestException, KeyError, IndexError) as e: + if attempt < 2: + wait = 2 ** (attempt + 1) + print( + f" VLM call attempt {attempt + 1} failed: {e}. " + f"Retrying in {wait}s...", + flush=True, + ) + time.sleep(wait) + else: + raise RuntimeError( + f"VLM call failed after 3 attempts: {e}" + ) from e + + def resolve_image_path(img_ref, doc_name): + """Try multiple strategies to find the image file.""" + candidates = [ + input_path_p / img_ref, + input_path_p / doc_name / img_ref, + input_path_p / Path(img_ref).name, + ] + for candidate in candidates: + if candidate.exists(): + return candidate + found = list(input_path_p.rglob(Path(img_ref).name)) + if found: + return found[0] + return None + + # --- Process each document --- + md_files = sorted(input_path_p.glob("*.md")) + if not md_files: + print( + "interpret-policy-artifacts: no markdown files found, nothing to process", + flush=True, + ) + return + + print( + f"interpret-policy-artifacts: processing {len(md_files)} document(s)", + flush=True, + ) + + img_pattern = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)") + + for md_file in md_files: + doc_name = md_file.stem + print(f"interpret-policy-artifacts: processing {doc_name}", flush=True) + + md_content = md_file.read_text(encoding="utf-8") + matches = list(img_pattern.finditer(md_content)) + + if not matches: + print( + f" {doc_name}: no images found, copying markdown as-is", flush=True + ) + shutil.copy2(md_file, output_path_p / f"{doc_name}_policy.md") + json_src = input_path_p / f"{doc_name}.json" + if json_src.exists(): + shutil.copy2(json_src, output_path_p / f"{doc_name}.json") + continue + + print( + f" {doc_name}: found {len(matches)} image(s) to interpret", flush=True + ) + + artifacts_metadata = [] + assembled_md = md_content + + for idx, match in enumerate(matches): + img_ref = match.group(2) + full_match_text = match.group(0) + + img_path = resolve_image_path(img_ref, doc_name) + if img_path is None: + print( + f" {doc_name}: image {img_ref} not found, skipping", flush=True + ) + continue + + with open(img_path, "rb") as f: + img_b64 = base64.b64encode(f.read()).decode("utf-8") + + # Step 1: Classify artifact type + print( + f" {doc_name}: classifying image {idx + 1}/{len(matches)}...", + flush=True, + ) + raw_type = call_vlm(img_b64, CLASSIFICATION_PROMPT, max_tokens=20) + artifact_type = raw_type.strip().lower().replace(" ", "_") + if artifact_type not in VALID_TYPES: + artifact_type = "other" + print( + f" {doc_name}: image {idx + 1} classified as '{artifact_type}'", + flush=True, + ) + + # Step 2: Interpret with type-specific prompt + print( + f" {doc_name}: interpreting image {idx + 1} as {artifact_type}...", + flush=True, + ) + interpretation = call_vlm( + img_b64, + INTERPRETATION_PROMPTS[artifact_type], + max_tokens=vlm_max_tokens, + ) + + # Step 3: Build replacement with artifact markers + replacement = ( + f"\n" + f"{interpretation.strip()}\n" + f"" + ) + assembled_md = assembled_md.replace(full_match_text, replacement, 1) + + artifacts_metadata.append( + { + "index": idx, + "type": artifact_type, + "source_image": str(img_ref), + "interpretation": interpretation.strip(), + } + ) + + # Save assembled policy markdown + policy_md_path = output_path_p / f"{doc_name}_policy.md" + policy_md_path.write_text(assembled_md, encoding="utf-8") + print(f" {doc_name}: saved {policy_md_path.name}", flush=True) + + # Save artifact metadata + artifacts_json_path = output_path_p / f"{doc_name}_artifacts.json" + with open(artifacts_json_path, "w", encoding="utf-8") as f: + json.dump( + { + "document": doc_name, + "total_artifacts": len(artifacts_metadata), + "artifacts": artifacts_metadata, + }, + f, + indent=2, + ensure_ascii=False, + ) + print(f" {doc_name}: saved {artifacts_json_path.name}", flush=True) + + # Copy DoclingDocument JSON for downstream consumers and chunking + json_src = input_path_p / f"{doc_name}.json" + if json_src.exists(): + shutil.copy2(json_src, output_path_p / f"{doc_name}.json") + + print("interpret-policy-artifacts: done", flush=True) diff --git a/kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline.py b/kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline.py new file mode 100644 index 00000000..8bcce32a --- /dev/null +++ b/kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline.py @@ -0,0 +1,129 @@ +import sys +from pathlib import Path + +# Add parent directories to Python path for imports +sys.path.insert(0, str(Path(__file__).parent.parent)) +sys.path.insert(0, str(Path(__file__).parent.parent / "docling-standard")) + +from common import ( + create_pdf_splits, + docling_chunk, + download_docling_models, + import_pdfs, +) +from kfp import compiler, dsl +from policy_components import interpret_policy_artifacts +from standard_components import docling_convert_standard + + +@dsl.pipeline( + name="data-processing-docling-vlm-policy-pipeline", + description="Policy document conversion pipeline with artifact-type-specific VLM interpretation by the Data Processing Team", +) +def policy_convert_pipeline( + num_splits: int = 3, + pdf_from_s3: bool = False, + pdf_filenames: str = "2203.01017v2.pdf", + # URL source params + pdf_base_url: str = "https://github.com/docling-project/docling/raw/v2.43.0/tests/data/pdf", + # Docling conversion params + docling_num_threads: int = 4, + docling_timeout_per_document: int = 300, + docling_ocr: bool = True, + docling_table_mode: str = "accurate", + # VLM interpretation params + vlm_max_tokens: int = 1024, + vlm_timeout: int = 120, + # Chunking params + docling_chunk_enabled: bool = False, + docling_chunk_max_tokens: int = 512, + docling_chunk_merge_peers: bool = True, +): + from kfp import kubernetes # pylint: disable=import-outside-toplevel + + secret_mount_path = "/mnt/secrets" + + # Step 1: Import PDFs from URL or S3 + importer = import_pdfs( + filenames=pdf_filenames, + base_url=pdf_base_url, + from_s3=pdf_from_s3, + s3_secret_mount_path=secret_mount_path, + ) + importer.set_caching_options(False) + kubernetes.use_secret_as_volume( + importer, + secret_name="data-processing-docling-pipeline", + mount_path=secret_mount_path, + optional=True, + ) + + # Step 2: Split PDFs for parallel processing + pdf_splits = create_pdf_splits( + input_path=importer.outputs["output_path"], + num_splits=num_splits, + ) + + # Step 3: Download Docling models (standard pipeline for structural parsing) + artifacts = download_docling_models( + pipeline_type="standard", + remote_model_endpoint_enabled=False, + ) + artifacts.set_caching_options(False) + + with dsl.ParallelFor(pdf_splits.output) as pdf_split: + # Step 4: Convert PDFs with standard pipeline, exporting images as + # referenced PNGs so the VLM interpretation step can access them + converter = docling_convert_standard( + input_path=importer.outputs["output_path"], + artifacts_path=artifacts.outputs["output_path"], + pdf_filenames=pdf_split, + image_export_mode="referenced", + num_threads=docling_num_threads, + timeout_per_document=docling_timeout_per_document, + ocr=docling_ocr, + table_mode=docling_table_mode, + ) + converter.set_caching_options(False) + converter.set_memory_request("1G") + converter.set_memory_limit("6G") + converter.set_cpu_request("500m") + converter.set_cpu_limit("4") + + # Step 5: Classify and interpret artifacts with VLM, assemble + # policy-optimized Markdown with artifact-type markers + interpreter = interpret_policy_artifacts( + input_path=converter.outputs["output_path"], + vlm_secret_mount_path=secret_mount_path, + vlm_max_tokens=vlm_max_tokens, + vlm_timeout=vlm_timeout, + ) + interpreter.set_caching_options(False) + interpreter.set_memory_request("512M") + interpreter.set_memory_limit("4G") + interpreter.set_cpu_request("250m") + interpreter.set_cpu_limit("2") + kubernetes.use_secret_as_volume( + interpreter, + secret_name="data-processing-docling-pipeline", + mount_path=secret_mount_path, + ) + + # Step 6: Optionally chunk the DoclingDocument JSON output + with dsl.If(docling_chunk_enabled == True): # noqa: E712 + chunker = docling_chunk( + input_path=interpreter.outputs["output_path"], + max_tokens=docling_chunk_max_tokens, + merge_peers=docling_chunk_merge_peers, + ) + chunker.set_caching_options(False) + chunker.set_memory_request("512M") + chunker.set_memory_limit("2G") + chunker.set_cpu_request("250m") + chunker.set_cpu_limit("2") + + +if __name__ == "__main__": + output_yaml = "policy_convert_pipeline_compiled.yaml" + compiler.Compiler().compile(policy_convert_pipeline, output_yaml) + print(f"Docling VLM policy pipeline compiled to {output_yaml}") diff --git a/kubeflow-pipelines/docling-vlm-policy/requirements.txt b/kubeflow-pipelines/docling-vlm-policy/requirements.txt new file mode 100644 index 00000000..e0c16144 --- /dev/null +++ b/kubeflow-pipelines/docling-vlm-policy/requirements.txt @@ -0,0 +1,4 @@ +docling == 2.57.0 +kfp == 2.14.6 +kfp-kubernetes == 2.14.6 +boto3 == 1.40.52 diff --git a/notebooks/use-cases/policy-document-conversion-vlm.ipynb b/notebooks/use-cases/policy-document-conversion-vlm.ipynb new file mode 100644 index 00000000..cf52c14c --- /dev/null +++ b/notebooks/use-cases/policy-document-conversion-vlm.ipynb @@ -0,0 +1,563 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Policy Document Conversion with VLM Artifact Interpretation\n", + "\n", + "This notebook demonstrates high-fidelity conversion of **policy documents** containing non-text artifacts (risk matrices, decision trees, compliance charts, forms, screenshots) into structured Markdown on OpenShift AI.\n", + "\n", + "The approach combines:\n", + "1. **Docling standard pipeline** for structural document parsing (headings, tables, numbered clauses, reading order)\n", + "2. **VLM (Vision Language Model)** for classifying and interpreting extracted images with artifact-type-specific prompts\n", + "3. **Assembly** that replaces image references with structured Markdown interpretations wrapped in artifact-type markers\n", + "\n", + "This is the step-by-step walkthrough of the [`docling-vlm-policy` KFP pipeline](../../kubeflow-pipelines/docling-vlm-policy/)." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## 1. Installation\n", + "\n", + "Install required packages. Run once per session." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -qq docling requests" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "## 2. Deploy the VLM InferenceService\n", + "\n", + "The VLM must be deployed as a KServe InferenceService before running the pipeline. The manifest is provided at `kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml`.\n", + "\n", + "```bash\n", + "# Deploy Granite Vision via vLLM on KServe\n", + "kubectl apply -f ../../kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml\n", + "\n", + "# Wait for it to be ready\n", + "kubectl get inferenceservice policy-vlm -w\n", + "```\n", + "\n", + "Once ready, the endpoint is accessible in-cluster at:\n", + "```\n", + "http://policy-vlm-predictor..svc.cluster.local:8080/v1/chat/completions\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "## 3. Configuration\n", + "\n", + "### Set the VLM endpoint\n", + "\n", + "Point to your deployed VLM InferenceService. When running on OpenShift AI, use the in-cluster service URL." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "VLM_ENDPOINT_URL = os.getenv(\n", + " \"VLM_SERVICE_ENDPOINT_URL\",\n", + " \"http://policy-vlm-predictor.default.svc.cluster.local:8080/v1/chat/completions\",\n", + ")\n", + "VLM_MODEL_NAME = os.getenv(\"VLM_SERVICE_MODEL_NAME\", \"ibm-granite/granite-vision-3.2-2b\")\n", + "VLM_API_KEY = os.getenv(\"VLM_SERVICE_API_KEY\", \"\")\n", + "VLM_MAX_TOKENS = 1024\n", + "VLM_TIMEOUT = 120" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "### Set input documents\n", + "\n", + "For testing, use sample PDFs from the Docling test suite. For real policy documents, point to your own files or the policy examples in [`asago-ai/asago-examples`](https://github.com/asago-ai/asago-examples) under `asago-policy-mapper/policy_examples/`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "files = [\n", + " \"https://raw.githubusercontent.com/py-pdf/sample-files/refs/heads/main/003-pdflatex-image/pdflatex-image.pdf\",\n", + "]\n", + "\n", + "output_dir = Path(\"policy-conversion-output\")\n", + "output_dir.mkdir(parents=True, exist_ok=True)" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "## 4. Convert Documents with Docling\n", + "\n", + "Use the standard Docling pipeline for structural parsing. We set `image_mode=ImageRefMode.REFERENCED` so that images are exported as separate PNG files alongside the Markdown, which the VLM interpretation step needs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "from docling.datamodel.accelerator_options import AcceleratorDevice, AcceleratorOptions\n", + "from docling.datamodel.base_models import InputFormat\n", + "from docling.datamodel.pipeline_options import PdfPipelineOptions\n", + "from docling.document_converter import DocumentConverter, PdfFormatOption\n", + "from docling.pipeline.standard_pdf_pipeline import StandardPdfPipeline\n", + "from docling_core.types.doc import ImageRefMode\n", + "\n", + "pipeline_options = PdfPipelineOptions()\n", + "pipeline_options.generate_page_images = True\n", + "pipeline_options.do_table_structure = True\n", + "pipeline_options.do_ocr = True\n", + "pipeline_options.document_timeout = 600\n", + "pipeline_options.accelerator_options = AcceleratorOptions(\n", + " num_threads=4, device=AcceleratorDevice.AUTO\n", + ")\n", + "\n", + "converter = DocumentConverter(\n", + " format_options={\n", + " InputFormat.PDF: PdfFormatOption(\n", + " pipeline_options=pipeline_options,\n", + " pipeline_cls=StandardPdfPipeline,\n", + " )\n", + " }\n", + ")\n", + "\n", + "results = {}\n", + "for file in files:\n", + " print(f\"Converting {file}...\")\n", + " result = converter.convert(file)\n", + " doc = result.document\n", + " stem = Path(file).stem\n", + "\n", + " # Save JSON (DoclingDocument)\n", + " json_path = output_dir / f\"{stem}.json\"\n", + " doc.save_as_json(json_path, image_mode=ImageRefMode.REFERENCED)\n", + " print(f\" Saved JSON: {json_path}\")\n", + "\n", + " # Save Markdown with referenced images\n", + " md_path = output_dir / f\"{stem}.md\"\n", + " doc.save_as_markdown(md_path, image_mode=ImageRefMode.REFERENCED)\n", + " print(f\" Saved Markdown: {md_path}\")\n", + "\n", + " results[stem] = {\"document\": doc, \"md_path\": md_path, \"json_path\": json_path}\n", + "\n", + "print(f\"\\nConverted {len(results)} document(s)\")" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "## 5. Find Extracted Images\n", + "\n", + "Inspect the Markdown output to find image references that need VLM interpretation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "\n", + "IMG_PATTERN = re.compile(r\"!\\[([^\\]]*)\\]\\(([^)]+)\\)\")\n", + "\n", + "for stem, info in results.items():\n", + " md_content = info[\"md_path\"].read_text(encoding=\"utf-8\")\n", + " matches = list(IMG_PATTERN.finditer(md_content))\n", + " print(f\"{stem}: {len(matches)} image(s) found\")\n", + " for i, m in enumerate(matches):\n", + " print(f\" [{i}] {m.group(2)}\")" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, + "source": [ + "## 6. VLM Helper Function\n", + "\n", + "Define the helper to call the VLM via the OpenAI-compatible chat completions API with an image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "import base64\n", + "import requests\n", + "\n", + "\n", + "def call_vlm(image_b64: str, prompt: str, max_tokens: int = VLM_MAX_TOKENS) -> str:\n", + " \"\"\"Send an image + prompt to the VLM and return the text response.\"\"\"\n", + " payload = {\n", + " \"model\": VLM_MODEL_NAME,\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": prompt},\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": f\"data:image/png;base64,{image_b64}\"\n", + " },\n", + " },\n", + " ],\n", + " }\n", + " ],\n", + " \"max_tokens\": max_tokens,\n", + " }\n", + " headers = {}\n", + " if VLM_API_KEY:\n", + " headers[\"Authorization\"] = f\"Bearer {VLM_API_KEY}\"\n", + "\n", + " resp = requests.post(VLM_ENDPOINT_URL, json=payload, headers=headers, timeout=VLM_TIMEOUT)\n", + " resp.raise_for_status()\n", + " return resp.json()[\"choices\"][0][\"message\"][\"content\"]" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "## 7. Classify and Interpret Artifacts\n", + "\n", + "For each image extracted by Docling:\n", + "1. **Classify** it into an artifact type (risk matrix, decision tree, chart, form, screenshot, diagram, other)\n", + "2. **Interpret** it using a type-specific VLM prompt that produces structured Markdown" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "CLASSIFICATION_PROMPT = (\n", + " \"You are analyzing an image extracted from a policy document. \"\n", + " \"Classify this image into exactly one of the following categories. \"\n", + " \"Respond with ONLY the category name, nothing else.\\n\\n\"\n", + " \"Categories:\\n\"\n", + " \"- chart: Any type of chart, graph, or plot\\n\"\n", + " \"- risk_matrix: A risk assessment matrix or heat map\\n\"\n", + " \"- decision_tree: A decision tree or flowchart with branches\\n\"\n", + " \"- form: A form, questionnaire, or template\\n\"\n", + " \"- screenshot: A screenshot of a user interface\\n\"\n", + " \"- diagram: An architecture diagram, process flow, or org chart\\n\"\n", + " \"- other: Any other type of image\"\n", + ")\n", + "\n", + "INTERPRETATION_PROMPTS = {\n", + " \"risk_matrix\": (\n", + " \"Convert this risk matrix into structured Markdown. \"\n", + " \"Create a Markdown table that preserves all row headers, column headers, \"\n", + " \"and cell values. Include severity levels and any color coding described as \"\n", + " \"text labels (e.g., 'High Risk'). Add any legends or footnotes below the table.\"\n", + " ),\n", + " \"decision_tree\": (\n", + " \"Convert this decision tree into a nested Markdown list representing the \"\n", + " \"complete decision logic. Use indentation to show hierarchy and branches. \"\n", + " \"Preserve all labels, conditions, and outcomes exactly as shown.\"\n", + " ),\n", + " \"chart\": (\n", + " \"Describe this chart in structured Markdown. Include chart type, title, \"\n", + " \"axis labels, a data summary table with key values, and notable trends.\"\n", + " ),\n", + " \"form\": (\n", + " \"Convert this form into a Markdown table with 'Field' and 'Value/Description' \"\n", + " \"columns. Include all field labels, values, checkboxes, and instructions.\"\n", + " ),\n", + " \"screenshot\": (\n", + " \"Describe this UI screenshot as a structured step-by-step walkthrough. \"\n", + " \"Identify the application, describe visible elements, and note the current state.\"\n", + " ),\n", + " \"diagram\": (\n", + " \"Describe this diagram in structured Markdown. List all components, \"\n", + " \"their relationships, directional flows, and any annotations.\"\n", + " ),\n", + " \"other\": (\n", + " \"Describe the content of this image in structured Markdown. \"\n", + " \"Preserve all visible text, numbers, and structural information.\"\n", + " ),\n", + "}\n", + "\n", + "VALID_TYPES = set(INTERPRETATION_PROMPTS.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "all_artifacts = {}\n", + "\n", + "for stem, info in results.items():\n", + " md_content = info[\"md_path\"].read_text(encoding=\"utf-8\")\n", + " matches = list(IMG_PATTERN.finditer(md_content))\n", + "\n", + " if not matches:\n", + " print(f\"{stem}: no images, skipping\")\n", + " all_artifacts[stem] = []\n", + " continue\n", + "\n", + " artifacts = []\n", + " for idx, match in enumerate(matches):\n", + " img_ref = match.group(2)\n", + "\n", + " # Resolve image path\n", + " img_path = output_dir / img_ref\n", + " if not img_path.exists():\n", + " img_path = output_dir / Path(img_ref).name\n", + " if not img_path.exists():\n", + " candidates = list(output_dir.rglob(Path(img_ref).name))\n", + " img_path = candidates[0] if candidates else None\n", + "\n", + " if img_path is None or not img_path.exists():\n", + " print(f\" [{idx}] Image not found: {img_ref}\")\n", + " continue\n", + "\n", + " with open(img_path, \"rb\") as f:\n", + " img_b64 = base64.b64encode(f.read()).decode(\"utf-8\")\n", + "\n", + " # Classify\n", + " print(f\" [{idx}] Classifying...\")\n", + " raw_type = call_vlm(img_b64, CLASSIFICATION_PROMPT, max_tokens=20)\n", + " artifact_type = raw_type.strip().lower().replace(\" \", \"_\")\n", + " if artifact_type not in VALID_TYPES:\n", + " artifact_type = \"other\"\n", + " print(f\" [{idx}] Type: {artifact_type}\")\n", + "\n", + " # Interpret\n", + " print(f\" [{idx}] Interpreting as {artifact_type}...\")\n", + " interpretation = call_vlm(img_b64, INTERPRETATION_PROMPTS[artifact_type])\n", + " print(f\" [{idx}] Done ({len(interpretation)} chars)\")\n", + "\n", + " artifacts.append({\n", + " \"index\": idx,\n", + " \"type\": artifact_type,\n", + " \"source_image\": str(img_ref),\n", + " \"match_text\": match.group(0),\n", + " \"interpretation\": interpretation.strip(),\n", + " })\n", + "\n", + " all_artifacts[stem] = artifacts\n", + " print(f\"{stem}: processed {len(artifacts)} artifact(s)\")" + ] + }, + { + "cell_type": "markdown", + "id": "17", + "metadata": {}, + "source": [ + "## 8. Assemble Policy Markdown\n", + "\n", + "Replace each image reference in the Markdown with the VLM-generated interpretation, wrapped in `` / `` markers." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "for stem, info in results.items():\n", + " md_content = info[\"md_path\"].read_text(encoding=\"utf-8\")\n", + " artifacts = all_artifacts.get(stem, [])\n", + "\n", + " for artifact in artifacts:\n", + " replacement = (\n", + " f\"\\n\"\n", + " f\"{artifact['interpretation']}\\n\"\n", + " f\"\"\n", + " )\n", + " md_content = md_content.replace(artifact[\"match_text\"], replacement, 1)\n", + "\n", + " # Save assembled policy markdown\n", + " policy_path = output_dir / f\"{stem}_policy.md\"\n", + " policy_path.write_text(md_content, encoding=\"utf-8\")\n", + " print(f\"Saved: {policy_path}\")\n", + "\n", + " # Save artifact metadata\n", + " meta_path = output_dir / f\"{stem}_artifacts.json\"\n", + " with open(meta_path, \"w\") as f:\n", + " json.dump(\n", + " {\"document\": stem, \"total_artifacts\": len(artifacts), \"artifacts\": artifacts},\n", + " f, indent=2, ensure_ascii=False,\n", + " )\n", + " print(f\"Saved: {meta_path}\")" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "## 9. Review Output\n", + "\n", + "Inspect the assembled Markdown to verify artifact interpretations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import Markdown, display\n", + "\n", + "for stem in results:\n", + " policy_path = output_dir / f\"{stem}_policy.md\"\n", + " if policy_path.exists():\n", + " content = policy_path.read_text(encoding=\"utf-8\")\n", + " display(Markdown(f\"### {stem}_policy.md\\n\\n---\"))\n", + " display(Markdown(content[:3000])) # Show first 3000 chars\n", + " if len(content) > 3000:\n", + " print(f\"... ({len(content) - 3000} more characters)\")" + ] + }, + { + "cell_type": "markdown", + "id": "21", + "metadata": {}, + "source": [ + "## 10. Quality Evaluation\n", + "\n", + "Basic quality checks on the assembled output." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "for stem in results:\n", + " policy_path = output_dir / f\"{stem}_policy.md\"\n", + " meta_path = output_dir / f\"{stem}_artifacts.json\"\n", + "\n", + " if not policy_path.exists():\n", + " continue\n", + "\n", + " content = policy_path.read_text(encoding=\"utf-8\")\n", + " with open(meta_path) as f:\n", + " meta = json.load(f)\n", + "\n", + " # Count artifact markers\n", + " open_markers = content.count(\"\")\n", + "\n", + " # Check for remaining image references (should be zero)\n", + " remaining_imgs = len(IMG_PATTERN.findall(content))\n", + "\n", + " # Check structural elements\n", + " headings = len(re.findall(r\"^#+\\s\", content, re.MULTILINE))\n", + " tables = content.count(\"|---\")\n", + "\n", + " print(f\"=== {stem} ===\")\n", + " print(f\" Total artifacts interpreted: {meta['total_artifacts']}\")\n", + " print(f\" Artifact markers (open/close): {open_markers}/{close_markers}\")\n", + " print(f\" Remaining unprocessed images: {remaining_imgs}\")\n", + " print(f\" Headings preserved: {headings}\")\n", + " print(f\" Tables in output: {tables}\")\n", + " print(f\" Output length: {len(content)} chars\")\n", + " print()\n", + "\n", + " # Type distribution\n", + " type_counts = {}\n", + " for a in meta[\"artifacts\"]:\n", + " type_counts[a[\"type\"]] = type_counts.get(a[\"type\"], 0) + 1\n", + " if type_counts:\n", + " print(f\" Artifact types: {type_counts}\")" + ] + }, + { + "cell_type": "markdown", + "id": "23", + "metadata": {}, + "source": [ + "## Additional Resources\n", + "\n", + "- [Pipeline README](../../kubeflow-pipelines/docling-vlm-policy/README.md) \u2014 full architecture and deployment guide\n", + "- [KServe InferenceService manifest](../../kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml) \u2014 VLM deployment\n", + "- [Sample policy documents](https://github.com/asago-ai/asago-examples) \u2014 `asago-policy-mapper/policy_examples/`\n", + "- [Docling documentation](https://docling-project.github.io/docling/) \u2014 conversion pipeline reference\n", + "\n", + "For feedback, please [open an issue](https://github.com/opendatahub-io/odh-data-processing/issues)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbformat_minor": 5, + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 892277c0ec9f09d1a9ab52e82b2116e366869d17 Mon Sep 17 00:00:00 2001 From: Alina Ryan Date: Thu, 15 Oct 2026 09:49:52 -0400 Subject: [PATCH 2/6] Fix notebook validation: add parameters tag and strip outputs Add papermill 'parameters' tag to the input files cell and run nbstripout. Also add docling-vlm-policy Python files to Makefile ruff targets. Co-Authored-By: Claude Opus 4.6 --- Makefile | 3 ++- .../use-cases/policy-document-conversion-vlm.ipynb | 14 +++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 685c6faa..40a3a3db 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ ALL_NOTEBOOKS := $(USE_CASES) $(TUTORIALS) COMMON := $(wildcard kubeflow-pipelines/common/*.py) DOCLING_STANDARD_PIPELINE := $(wildcard kubeflow-pipelines/docling-standard/*.py) DOCLING_VLM_PIPELINE := $(wildcard kubeflow-pipelines/docling-vlm/*.py) -ALL_PYTHON_FILES := $(COMMON) $(DOCLING_STANDARD_PIPELINE) $(DOCLING_VLM_PIPELINE) +DOCLING_VLM_POLICY_PIPELINE := $(wildcard kubeflow-pipelines/docling-vlm-policy/*.py) +ALL_PYTHON_FILES := $(COMMON) $(DOCLING_STANDARD_PIPELINE) $(DOCLING_VLM_PIPELINE) $(DOCLING_VLM_POLICY_PIPELINE) format-python: @echo "Running ruff format..." diff --git a/notebooks/use-cases/policy-document-conversion-vlm.ipynb b/notebooks/use-cases/policy-document-conversion-vlm.ipynb index cf52c14c..e9eb3b00 100644 --- a/notebooks/use-cases/policy-document-conversion-vlm.ipynb +++ b/notebooks/use-cases/policy-document-conversion-vlm.ipynb @@ -105,7 +105,11 @@ "cell_type": "code", "execution_count": null, "id": "7", - "metadata": {}, + "metadata": { + "tags": [ + "parameters" + ] + }, "outputs": [], "source": [ "from pathlib import Path\n", @@ -531,10 +535,10 @@ "source": [ "## Additional Resources\n", "\n", - "- [Pipeline README](../../kubeflow-pipelines/docling-vlm-policy/README.md) \u2014 full architecture and deployment guide\n", - "- [KServe InferenceService manifest](../../kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml) \u2014 VLM deployment\n", - "- [Sample policy documents](https://github.com/asago-ai/asago-examples) \u2014 `asago-policy-mapper/policy_examples/`\n", - "- [Docling documentation](https://docling-project.github.io/docling/) \u2014 conversion pipeline reference\n", + "- [Pipeline README](../../kubeflow-pipelines/docling-vlm-policy/README.md) — full architecture and deployment guide\n", + "- [KServe InferenceService manifest](../../kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml) — VLM deployment\n", + "- [Sample policy documents](https://github.com/asago-ai/asago-examples) — `asago-policy-mapper/policy_examples/`\n", + "- [Docling documentation](https://docling-project.github.io/docling/) — conversion pipeline reference\n", "\n", "For feedback, please [open an issue](https://github.com/opendatahub-io/odh-data-processing/issues)." ] From f8c1baf6afbe8720b4cb6d9984fddb2c2062e5e9 Mon Sep 17 00:00:00 2001 From: Alina Ryan Date: Thu, 15 Oct 2026 09:59:04 -0400 Subject: [PATCH 3/6] Fix ruff formatting in policy_components.py Co-Authored-By: Claude Opus 4.6 --- .../docling-vlm-policy/policy_components.py | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/kubeflow-pipelines/docling-vlm-policy/policy_components.py b/kubeflow-pipelines/docling-vlm-policy/policy_components.py index 96884809..c741b36e 100644 --- a/kubeflow-pipelines/docling-vlm-policy/policy_components.py +++ b/kubeflow-pipelines/docling-vlm-policy/policy_components.py @@ -162,9 +162,7 @@ def call_vlm(image_b64, prompt, max_tokens): {"type": "text", "text": prompt}, { "type": "image_url", - "image_url": { - "url": f"data:image/png;base64,{image_b64}" - }, + "image_url": {"url": f"data:image/png;base64,{image_b64}"}, }, ], } @@ -195,9 +193,7 @@ def call_vlm(image_b64, prompt, max_tokens): ) time.sleep(wait) else: - raise RuntimeError( - f"VLM call failed after 3 attempts: {e}" - ) from e + raise RuntimeError(f"VLM call failed after 3 attempts: {e}") from e def resolve_image_path(img_ref, doc_name): """Try multiple strategies to find the image file.""" @@ -238,18 +234,14 @@ def resolve_image_path(img_ref, doc_name): matches = list(img_pattern.finditer(md_content)) if not matches: - print( - f" {doc_name}: no images found, copying markdown as-is", flush=True - ) + print(f" {doc_name}: no images found, copying markdown as-is", flush=True) shutil.copy2(md_file, output_path_p / f"{doc_name}_policy.md") json_src = input_path_p / f"{doc_name}.json" if json_src.exists(): shutil.copy2(json_src, output_path_p / f"{doc_name}.json") continue - print( - f" {doc_name}: found {len(matches)} image(s) to interpret", flush=True - ) + print(f" {doc_name}: found {len(matches)} image(s) to interpret", flush=True) artifacts_metadata = [] assembled_md = md_content @@ -260,9 +252,7 @@ def resolve_image_path(img_ref, doc_name): img_path = resolve_image_path(img_ref, doc_name) if img_path is None: - print( - f" {doc_name}: image {img_ref} not found, skipping", flush=True - ) + print(f" {doc_name}: image {img_ref} not found, skipping", flush=True) continue with open(img_path, "rb") as f: From a274e2fa58bd1136dedf28067f11f6b70182aa33 Mon Sep 17 00:00:00 2001 From: Alina Ryan Date: Thu, 5 Nov 2026 15:35:23 -0500 Subject: [PATCH 4/6] Address PR review feedback - Add compiled pipeline YAML and integrate into compile-kfp CI workflow - Guard notebook VLM cells with connectivity check and vlm_enabled flag so CI execution tests pass without a GPU endpoint - Move VLM config into the papermill parameters cell for CI overrides - Fall back to environment variables when Kubernetes secret mount is missing, enabling local runner and non-cluster testing - Write artifact metadata to metadata/ subdirectory to avoid chunker validation errors on non-DoclingDocument JSON files - Pin KServe vLLM image to 0.8.5 instead of floating on latest Co-Authored-By: Claude Opus 4.6 --- .github/workflows/compile-kfp.yml | 6 + .../docling-vlm-policy/README.md | 4 +- .../kserve/vlm-inferenceservice.yaml | 2 +- .../docling-vlm-policy/policy_components.py | 32 +- .../policy_convert_pipeline_compiled.yaml | 1233 +++++++++++++++++ .../policy-document-conversion-vlm.ipynb | 160 +-- 6 files changed, 1276 insertions(+), 161 deletions(-) create mode 100644 kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline_compiled.yaml diff --git a/.github/workflows/compile-kfp.yml b/.github/workflows/compile-kfp.yml index d516f04e..aed6d53a 100644 --- a/.github/workflows/compile-kfp.yml +++ b/.github/workflows/compile-kfp.yml @@ -6,6 +6,7 @@ on: paths: - "kubeflow-pipelines/docling-standard/**" - "kubeflow-pipelines/docling-vlm/**" + - "kubeflow-pipelines/docling-vlm-policy/**" - "kubeflow-pipelines/common/**" - ".github/workflows/compile-kfp.yml" push: @@ -13,6 +14,7 @@ on: paths: - "kubeflow-pipelines/docling-standard/**" - "kubeflow-pipelines/docling-vlm/**" + - "kubeflow-pipelines/docling-vlm-policy/**" - "kubeflow-pipelines/common/**" - ".github/workflows/compile-kfp.yml" permissions: @@ -33,6 +35,10 @@ jobs: dir: kubeflow-pipelines/docling-vlm cmd: python vlm_convert_pipeline.py out: vlm_convert_pipeline_compiled.yaml + - name: docling-vlm-policy + dir: kubeflow-pipelines/docling-vlm-policy + cmd: python policy_convert_pipeline.py + out: policy_convert_pipeline_compiled.yaml steps: - name: Checkout diff --git a/kubeflow-pipelines/docling-vlm-policy/README.md b/kubeflow-pipelines/docling-vlm-policy/README.md index 0b84fd9a..0894b6bd 100644 --- a/kubeflow-pipelines/docling-vlm-policy/README.md +++ b/kubeflow-pipelines/docling-vlm-policy/README.md @@ -61,7 +61,7 @@ Per document, the pipeline produces: | File | Description | |------|-------------| | `{name}_policy.md` | Assembled Markdown with artifact interpretations inline | -| `{name}_artifacts.json` | Metadata: artifact type, interpretation, source image per artifact | +| `metadata/{name}_artifacts.json` | Metadata: artifact type, interpretation, source image per artifact | | `{name}.json` | Original DoclingDocument JSON (for HybridChunker / downstream consumers) | ## Prerequisites @@ -82,7 +82,7 @@ kubectl create secret generic hf-token --from-literal=token="hf_YOUR_TOKEN" kubectl apply -f kserve/vlm-inferenceservice.yaml ``` -The default manifest deploys `ibm-granite/granite-vision-3.2-2b` using `registry.redhat.io/rhaiis/vllm-cuda-rhel9`. Adjust the `--model` argument for a different model (e.g., `ibm-granite/granite-vision-3.2-8b` for higher quality). +The default manifest deploys `ibm-granite/granite-vision-3.2-2b` using `registry.redhat.io/rhaiis/vllm-cuda-rhel9:0.8.5`. Adjust the `--model` argument for a different model (e.g., `ibm-granite/granite-vision-3.2-8b` for higher quality), and update the image tag to the latest available version from the [Red Hat container catalog](https://catalog.redhat.com/). Wait for the InferenceService to be ready: diff --git a/kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml b/kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml index 66ae793a..07cac0d5 100644 --- a/kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml +++ b/kubeflow-pipelines/docling-vlm-policy/kserve/vlm-inferenceservice.yaml @@ -10,7 +10,7 @@ spec: maxReplicas: 1 containers: - name: kserve-container - image: registry.redhat.io/rhaiis/vllm-cuda-rhel9:latest + image: registry.redhat.io/rhaiis/vllm-cuda-rhel9:0.8.5 command: - python - -m diff --git a/kubeflow-pipelines/docling-vlm-policy/policy_components.py b/kubeflow-pipelines/docling-vlm-policy/policy_components.py index c741b36e..2dce9f42 100644 --- a/kubeflow-pipelines/docling-vlm-policy/policy_components.py +++ b/kubeflow-pipelines/docling-vlm-policy/policy_components.py @@ -121,20 +121,25 @@ def interpret_policy_artifacts( VALID_TYPES = set(INTERPRETATION_PROMPTS.keys()) - # --- Read VLM credentials from mounted secret --- + # --- Read VLM credentials from mounted secret or environment variables --- def read_secret(name): fpath = os.path.join(vlm_secret_mount_path, name) - if not os.path.isfile(fpath): - raise ValueError( - f"Key {name} not found in secret mount {vlm_secret_mount_path}" - ) - with open(fpath) as f: - return f.read().strip() + if os.path.isfile(fpath): + with open(fpath) as f: + return f.read().strip() + env_val = os.environ.get(name) + if env_val is not None: + return env_val + raise ValueError( + f"Key {name} not found in secret mount {vlm_secret_mount_path} " + f"or environment variable {name}" + ) if not os.path.exists(vlm_secret_mount_path): - raise ValueError( - f"Secret mount path {vlm_secret_mount_path} does not exist. " - "Create a Kubernetes Secret with VLM endpoint configuration." + print( + f"Secret mount path {vlm_secret_mount_path} not found, " + "falling back to environment variables", + flush=True, ) vlm_endpoint_url = read_secret("REMOTE_MODEL_ENDPOINT_URL") @@ -305,8 +310,11 @@ def resolve_image_path(img_ref, doc_name): policy_md_path.write_text(assembled_md, encoding="utf-8") print(f" {doc_name}: saved {policy_md_path.name}", flush=True) - # Save artifact metadata - artifacts_json_path = output_path_p / f"{doc_name}_artifacts.json" + # Save artifact metadata in a subdirectory to avoid conflicts with + # the chunker, which globs *.json and validates as DoclingDocument + metadata_dir = output_path_p / "metadata" + metadata_dir.mkdir(parents=True, exist_ok=True) + artifacts_json_path = metadata_dir / f"{doc_name}_artifacts.json" with open(artifacts_json_path, "w", encoding="utf-8") as f: json.dump( { diff --git a/kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline_compiled.yaml b/kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline_compiled.yaml new file mode 100644 index 00000000..15d591f0 --- /dev/null +++ b/kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline_compiled.yaml @@ -0,0 +1,1233 @@ +# PIPELINE DEFINITION +# Name: data-processing-docling-vlm-policy-pipeline +# Description: Policy document conversion pipeline with artifact-type-specific VLM interpretation by the Data Processing Team +# Inputs: +# docling_chunk_enabled: bool [Default: False] +# docling_chunk_max_tokens: int [Default: 512.0] +# docling_chunk_merge_peers: bool [Default: True] +# docling_num_threads: int [Default: 4.0] +# docling_ocr: bool [Default: True] +# docling_table_mode: str [Default: 'accurate'] +# docling_timeout_per_document: int [Default: 300.0] +# num_splits: int [Default: 3.0] +# pdf_base_url: str [Default: 'https://github.com/docling-project/docling/raw/v2.43.0/tests/data/pdf'] +# pdf_filenames: str [Default: '2203.01017v2.pdf'] +# pdf_from_s3: bool [Default: False] +# vlm_max_tokens: int [Default: 1024.0] +# vlm_timeout: int [Default: 120.0] +components: + comp-condition-2: + dag: + tasks: + docling-chunk: + cachingOptions: {} + componentRef: + name: comp-docling-chunk + inputs: + artifacts: + input_path: + componentInputArtifact: pipelinechannel--interpret-policy-artifacts-output_path + parameters: + max_tokens: + componentInputParameter: pipelinechannel--docling_chunk_max_tokens + merge_peers: + componentInputParameter: pipelinechannel--docling_chunk_merge_peers + taskInfo: + name: docling-chunk + inputDefinitions: + artifacts: + pipelinechannel--interpret-policy-artifacts-output_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--docling_chunk_enabled: + parameterType: BOOLEAN + pipelinechannel--docling_chunk_max_tokens: + parameterType: NUMBER_INTEGER + pipelinechannel--docling_chunk_merge_peers: + parameterType: BOOLEAN + comp-create-pdf-splits: + executorLabel: exec-create-pdf-splits + inputDefinitions: + artifacts: + input_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path to the input directory containing PDF files. + parameters: + num_splits: + description: Number of splits to create. + parameterType: NUMBER_INTEGER + outputDefinitions: + parameters: + Output: + parameterType: LIST + comp-docling-chunk: + executorLabel: exec-docling-chunk + inputDefinitions: + artifacts: + input_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path to the input directory containing Docling JSON files + parameters: + max_tokens: + defaultValue: 512.0 + description: Maximum number of tokens per chunk + isOptional: true + parameterType: NUMBER_INTEGER + merge_peers: + defaultValue: true + description: Whether to merge smaller chunks at the same level + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + artifacts: + output_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-docling-convert-standard: + executorLabel: exec-docling-convert-standard + inputDefinitions: + artifacts: + artifacts_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path to the directory containing Docling models. + input_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path to the input directory containing PDF files. + parameters: + allow_external_plugins: + defaultValue: false + description: Whether or not to allow external plugins. + isOptional: true + parameterType: BOOLEAN + enrich_code: + defaultValue: false + description: Whether or not to enrich code. + isOptional: true + parameterType: BOOLEAN + enrich_formula: + defaultValue: false + description: Whether or not to enrich formula. + isOptional: true + parameterType: BOOLEAN + enrich_picture_classes: + defaultValue: false + description: Whether or not to enrich picture classes. + isOptional: true + parameterType: BOOLEAN + enrich_picture_description: + defaultValue: false + description: Whether or not to enrich picture description. + isOptional: true + parameterType: BOOLEAN + force_ocr: + defaultValue: false + description: Whether or not to force OCR. + isOptional: true + parameterType: BOOLEAN + image_export_mode: + defaultValue: embedded + description: Mode to export images. + isOptional: true + parameterType: STRING + num_threads: + defaultValue: 4.0 + description: Number of threads to use per document processing. + isOptional: true + parameterType: NUMBER_INTEGER + ocr: + defaultValue: true + description: Whether or not to use OCR if needed. + isOptional: true + parameterType: BOOLEAN + ocr_engine: + defaultValue: tesseract_cli + description: Engine to use for OCR. + isOptional: true + parameterType: STRING + pdf_backend: + defaultValue: dlparse_v4 + description: Backend to use for PDF processing. + isOptional: true + parameterType: STRING + pdf_filenames: + description: List of PDF file names to process. + parameterType: LIST + table_mode: + defaultValue: accurate + description: Mode to detect tables. + isOptional: true + parameterType: STRING + timeout_per_document: + defaultValue: 300.0 + description: Timeout per document processing. + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + output_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-download-docling-models: + executorLabel: exec-download-docling-models + inputDefinitions: + parameters: + pipeline_type: + defaultValue: standard + description: Type of pipeline (standard, vlm) + isOptional: true + parameterType: STRING + remote_model_endpoint_enabled: + defaultValue: false + description: Whether to download remote model endpoint models (VLM only) + isOptional: true + parameterType: BOOLEAN + outputDefinitions: + artifacts: + output_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-for-loop-1: + dag: + tasks: + condition-2: + componentRef: + name: comp-condition-2 + dependentTasks: + - interpret-policy-artifacts + inputs: + artifacts: + pipelinechannel--interpret-policy-artifacts-output_path: + taskOutputArtifact: + outputArtifactKey: output_path + producerTask: interpret-policy-artifacts + parameters: + pipelinechannel--docling_chunk_enabled: + componentInputParameter: pipelinechannel--docling_chunk_enabled + pipelinechannel--docling_chunk_max_tokens: + componentInputParameter: pipelinechannel--docling_chunk_max_tokens + pipelinechannel--docling_chunk_merge_peers: + componentInputParameter: pipelinechannel--docling_chunk_merge_peers + taskInfo: + name: condition-2 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--docling_chunk_enabled'] + == true + docling-convert-standard: + cachingOptions: {} + componentRef: + name: comp-docling-convert-standard + inputs: + artifacts: + artifacts_path: + componentInputArtifact: pipelinechannel--download-docling-models-output_path + input_path: + componentInputArtifact: pipelinechannel--import-pdfs-output_path + parameters: + image_export_mode: + runtimeValue: + constant: referenced + num_threads: + componentInputParameter: pipelinechannel--docling_num_threads + ocr: + componentInputParameter: pipelinechannel--docling_ocr + pdf_filenames: + componentInputParameter: pipelinechannel--create-pdf-splits-Output-loop-item + table_mode: + componentInputParameter: pipelinechannel--docling_table_mode + timeout_per_document: + componentInputParameter: pipelinechannel--docling_timeout_per_document + taskInfo: + name: docling-convert-standard + interpret-policy-artifacts: + cachingOptions: {} + componentRef: + name: comp-interpret-policy-artifacts + dependentTasks: + - docling-convert-standard + inputs: + artifacts: + input_path: + taskOutputArtifact: + outputArtifactKey: output_path + producerTask: docling-convert-standard + parameters: + vlm_max_tokens: + componentInputParameter: pipelinechannel--vlm_max_tokens + vlm_secret_mount_path: + runtimeValue: + constant: /mnt/secrets + vlm_timeout: + componentInputParameter: pipelinechannel--vlm_timeout + taskInfo: + name: interpret-policy-artifacts + inputDefinitions: + artifacts: + pipelinechannel--download-docling-models-output_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + pipelinechannel--import-pdfs-output_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + parameters: + pipelinechannel--create-pdf-splits-Output: + parameterType: LIST + pipelinechannel--create-pdf-splits-Output-loop-item: + parameterType: LIST + pipelinechannel--docling_chunk_enabled: + parameterType: BOOLEAN + pipelinechannel--docling_chunk_max_tokens: + parameterType: NUMBER_INTEGER + pipelinechannel--docling_chunk_merge_peers: + parameterType: BOOLEAN + pipelinechannel--docling_num_threads: + parameterType: NUMBER_INTEGER + pipelinechannel--docling_ocr: + parameterType: BOOLEAN + pipelinechannel--docling_table_mode: + parameterType: STRING + pipelinechannel--docling_timeout_per_document: + parameterType: NUMBER_INTEGER + pipelinechannel--vlm_max_tokens: + parameterType: NUMBER_INTEGER + pipelinechannel--vlm_timeout: + parameterType: NUMBER_INTEGER + comp-import-pdfs: + executorLabel: exec-import-pdfs + inputDefinitions: + parameters: + base_url: + description: Base URL of the PDF files. + parameterType: STRING + filenames: + description: List of PDF filenames to import. + parameterType: STRING + from_s3: + defaultValue: false + description: Whether or not to import from S3. + isOptional: true + parameterType: BOOLEAN + s3_secret_mount_path: + defaultValue: /mnt/secrets + description: Path to the secret mount path for the S3 credentials. + isOptional: true + parameterType: STRING + outputDefinitions: + artifacts: + output_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + comp-interpret-policy-artifacts: + executorLabel: exec-interpret-policy-artifacts + inputDefinitions: + artifacts: + input_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 + description: Path to directory containing Docling conversion output. + parameters: + vlm_max_tokens: + defaultValue: 1024.0 + description: Maximum tokens for each VLM interpretation response. + isOptional: true + parameterType: NUMBER_INTEGER + vlm_secret_mount_path: + defaultValue: /mnt/secrets + description: 'Path to mounted Kubernetes secret containing + + REMOTE_MODEL_ENDPOINT_URL, REMOTE_MODEL_NAME, REMOTE_MODEL_API_KEY.' + isOptional: true + parameterType: STRING + vlm_timeout: + defaultValue: 120.0 + description: Timeout in seconds per VLM API call. + isOptional: true + parameterType: NUMBER_INTEGER + outputDefinitions: + artifacts: + output_path: + artifactType: + schemaTitle: system.Artifact + schemaVersion: 0.0.1 +deploymentSpec: + executors: + exec-create-pdf-splits: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - create_pdf_splits + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef create_pdf_splits(\n input_path: dsl.Input[dsl.Artifact],\n\ + \ num_splits: int,\n) -> List[List[str]]:\n \"\"\"\n Create a list\ + \ of PDF splits.\n\n Args:\n input_path: Path to the input directory\ + \ containing PDF files.\n num_splits: Number of splits to create.\n\ + \ \"\"\"\n from pathlib import Path # pylint: disable=import-outside-toplevel\n\ + \n input_path_p = Path(input_path.path)\n\n all_pdfs = [path.name\ + \ for path in input_path_p.glob(\"*.pdf\")]\n all_splits = [all_pdfs[i::num_splits]\ + \ for i in range(num_splits)]\n filled_splits = list(filter(None, all_splits))\n\ + \ return filled_splits\n\n" + image: registry.access.redhat.com/ubi9/python-311:9.6-1755074620 + exec-docling-chunk: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - docling_chunk + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef docling_chunk(\n input_path: dsl.Input[dsl.Artifact],\n \ + \ output_path: dsl.Output[dsl.Artifact],\n max_tokens: int = 512,\n\ + \ merge_peers: bool = True,\n):\n \"\"\"\n Chunk Docling documents\ + \ using HybridChunker. Takes converted docling JSON files as input\n \ + \ and produces chunked JSONL files with semantic chunks suitable for RAG.\n\ + \n Output format is JSONL (one JSON object per line) for easy inspection\ + \ and streaming.\n\n Args:\n input_path: Path to the input directory\ + \ containing Docling JSON files\n output_path: Path to the output\ + \ directory for the chunked JSONL files\n max_tokens: Maximum number\ + \ of tokens per chunk\n merge_peers: Whether to merge smaller chunks\ + \ at the same level\n \"\"\"\n import json # pylint: disable=import-outside-toplevel\n\ + \ from datetime import datetime, timezone # pylint: disable=import-outside-toplevel\n\ + \ from pathlib import Path # pylint: disable=import-outside-toplevel\n\ + \n # HybridChunker = Docling's smart chunking class that combines:\n\ + \ # 1. Document structure awareness\n # 2. Token-based splitting\n\ + \ from docling.chunking import HybridChunker # pylint: disable=import-outside-toplevel\n\ + \ from docling_core.transforms.chunker.tokenizer.huggingface import (\n\ + \ HuggingFaceTokenizer,\n ) # pylint: disable=import-outside-toplevel\n\ + \ from docling_core.types import DoclingDocument # pylint: disable=import-outside-toplevel\n\ + \ from transformers import AutoTokenizer # pylint: disable=import-outside-toplevel\n\ + \n # Convert KFP artifact paths to Path objects\n input_path_p = Path(input_path.path)\n\ + \ output_path_p = Path(output_path.path)\n output_path_p.mkdir(parents=True,\ + \ exist_ok=True)\n\n # Initialize tokenizer for HybridChunker (new API)\n\ + \ # Using a lightweight sentence-transformer model for tokenization\n\ + \ EMBED_MODEL_ID = \"sentence-transformers/all-MiniLM-L6-v2\"\n try:\n\ + \ hf_tokenizer = AutoTokenizer.from_pretrained(\n EMBED_MODEL_ID,\n\ + \ resume_download=True,\n timeout=60,\n )\n\ + \ print(f\"docling-chunk: loaded tokenizer from {EMBED_MODEL_ID}\"\ + , flush=True)\n except Exception as e:\n print(f\"docling-chunk:\ + \ ERROR loading tokenizer: {e}\", flush=True)\n raise RuntimeError(\n\ + \ f\"Failed to load tokenizer model {EMBED_MODEL_ID}. \"\n \ + \ \"Ensure network access to HuggingFace Hub or pre-download the\ + \ model.\"\n ) from e\n\n tokenizer = HuggingFaceTokenizer(\n\ + \ tokenizer=hf_tokenizer,\n max_tokens=max_tokens,\n )\n\ + \n # Initialize Hybrid chunker with user-specified parameters\n #\ + \ tokenizer: The tokenizer wrapper to use for counting tokens (includes\ + \ max_tokens)\n # merge_peers: if true, smaller adjacent chunks will\ + \ be merged together\n chunker = HybridChunker(\n tokenizer=tokenizer,\n\ + \ merge_peers=merge_peers,\n )\n\n # Find all JSON files in\ + \ the input directory\n json_files = list(input_path_p.glob(\"*.json\"\ + ))\n if not json_files:\n print(f\"docling-chunk: No JSON files\ + \ found in {input_path_p}\", flush=True)\n return\n\n print(\n\ + \ f\"docling-chunk: processing {len(json_files)} files with max_tokens={max_tokens}\ + \ and merge_peers={merge_peers}\",\n flush=True,\n )\n\n #\ + \ Track processing results\n processed_count = 0\n skipped_files =\ + \ []\n\n # Process each file\n for json_file in json_files:\n \ + \ print(f\"docling-chunk: processing {json_file}\", flush=True)\n\n \ + \ # Load and validate the JSON file\n try:\n with\ + \ open(json_file, \"r\", encoding=\"utf-8\") as f:\n doc_data\ + \ = json.load(f)\n except json.JSONDecodeError as e:\n \ + \ print(\n f\"docling-chunk: skipping {json_file.name} -\ + \ invalid JSON: {e}\",\n flush=True,\n )\n \ + \ skipped_files.append((json_file.name, f\"invalid JSON: {e}\"))\n\ + \ continue\n\n # Parse the JSON data into a DoclingDocument\ + \ object\n # This validates that the JSON conforms to the DoclingDocument\ + \ schema\n try:\n doc = DoclingDocument.model_validate(doc_data)\n\ + \ except Exception as e:\n # Catches pydantic.ValidationError\ + \ and any other validation issues\n print(\n f\"\ + docling-chunk: skipping {json_file.name} - not a valid DoclingDocument:\ + \ {e}\",\n flush=True,\n )\n skipped_files.append((json_file.name,\ + \ f\"validation failed: {e}\"))\n continue\n\n # Chunk\ + \ the document using HybridChunker\n chunks = list(chunker.chunk(dl_doc=doc))\n\ + \n # Generate output filename: original_name_chunks.jsonl\n \ + \ output_filename = f\"{json_file.stem}_chunks.jsonl\"\n output_file\ + \ = output_path_p / output_filename\n\n # Get current timestamp in\ + \ ISO format\n timestamp = datetime.now(timezone.utc).isoformat()\n\ + \n # Chunking config (for reproducibility)\n chunking_config\ + \ = {\n \"max_tokens\": max_tokens,\n \"merge_peers\"\ + : merge_peers,\n \"tokenizer_model\": EMBED_MODEL_ID,\n \ + \ }\n\n # Write chunks as JSONL (one JSON object per line)\n \ + \ with open(output_file, \"w\", encoding=\"utf-8\") as f:\n \ + \ for idx, chunk in enumerate(chunks):\n # Get contextualized\ + \ text for this chunk\n chunk_text = chunker.contextualize(chunk=chunk)\n\ + \n # Build the chunk object\n chunk_obj =\ + \ {\n \"timestamp\": timestamp,\n \ + \ \"source_document\": json_file.name,\n \"chunk_index\"\ + : idx,\n \"chunking_config\": chunking_config,\n \ + \ \"text\": chunk_text,\n }\n\n \ + \ # Write as a single line of JSON\n f.write(json.dumps(chunk_obj,\ + \ ensure_ascii=False) + \"\\n\")\n\n print(\n f\"docling-chunk:\ + \ saved {len(chunks)} chunks to {output_filename}\",\n flush=True,\n\ + \ )\n processed_count += 1\n\n # Report summary\n print(\n\ + \ f\"docling-chunk: done - processed {processed_count}/{len(json_files)}\ + \ files\",\n flush=True,\n )\n if skipped_files:\n print(\n\ + \ f\"docling-chunk: skipped {len(skipped_files)} invalid files:\"\ + ,\n flush=True,\n )\n for filename, reason in skipped_files:\n\ + \ print(f\" - {filename}: {reason}\", flush=True)\n\n" + image: quay.io/fabianofranz/docling-ubi9:2.54.0 + resources: + cpuLimit: 2.0 + cpuRequest: 0.25 + memoryLimit: 2.0 + memoryRequest: 0.512 + resourceCpuLimit: '2' + resourceCpuRequest: 250m + resourceMemoryLimit: 2G + resourceMemoryRequest: 512M + exec-docling-convert-standard: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - docling_convert_standard + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef docling_convert_standard(\n input_path: dsl.Input[dsl.Artifact],\n\ + \ artifacts_path: dsl.Input[dsl.Artifact],\n output_path: dsl.Output[dsl.Artifact],\n\ + \ pdf_filenames: List[str],\n pdf_backend: str = \"dlparse_v4\",\n\ + \ image_export_mode: str = \"embedded\",\n table_mode: str = \"accurate\"\ + ,\n num_threads: int = 4,\n timeout_per_document: int = 300,\n \ + \ ocr: bool = True,\n force_ocr: bool = False,\n ocr_engine: str =\ + \ \"tesseract_cli\",\n allow_external_plugins: bool = False,\n enrich_code:\ + \ bool = False,\n enrich_formula: bool = False,\n enrich_picture_classes:\ + \ bool = False,\n enrich_picture_description: bool = False,\n):\n \ + \ \"\"\"\n Convert a list of PDF files to JSON and Markdown using Docling\ + \ (Standard Pipeline).\n\n Args:\n input_path: Path to the input\ + \ directory containing PDF files.\n artifacts_path: Path to the directory\ + \ containing Docling models.\n output_path: Path to the output directory\ + \ for converted JSON and Markdown files.\n pdf_filenames: List of\ + \ PDF file names to process.\n pdf_backend: Backend to use for PDF\ + \ processing.\n image_export_mode: Mode to export images.\n \ + \ table_mode: Mode to detect tables.\n num_threads: Number of threads\ + \ to use per document processing.\n timeout_per_document: Timeout\ + \ per document processing.\n ocr: Whether or not to use OCR if needed.\n\ + \ force_ocr: Whether or not to force OCR.\n ocr_engine: Engine\ + \ to use for OCR.\n allow_external_plugins: Whether or not to allow\ + \ external plugins.\n enrich_code: Whether or not to enrich code.\n\ + \ enrich_formula: Whether or not to enrich formula.\n enrich_picture_classes:\ + \ Whether or not to enrich picture classes.\n enrich_picture_description:\ + \ Whether or not to enrich picture description.\n \"\"\"\n from importlib\ + \ import import_module # pylint: disable=import-outside-toplevel\n from\ + \ pathlib import Path # pylint: disable=import-outside-toplevel\n\n \ + \ from docling.datamodel.accelerator_options import ( # pylint: disable=import-outside-toplevel\n\ + \ AcceleratorDevice,\n AcceleratorOptions,\n )\n from\ + \ docling.datamodel.base_models import (\n InputFormat,\n ) #\ + \ pylint: disable=import-outside-toplevel\n from docling.datamodel.pipeline_options\ + \ import ( # pylint: disable=import-outside-toplevel\n EasyOcrOptions,\n\ + \ OcrEngine,\n OcrMacOptions,\n PdfBackend,\n \ + \ PdfPipelineOptions,\n RapidOcrOptions,\n TableFormerMode,\n\ + \ TesseractCliOcrOptions,\n TesseractOcrOptions,\n )\n\ + \ from docling.document_converter import ( # pylint: disable=import-outside-toplevel\n\ + \ DocumentConverter,\n PdfFormatOption,\n )\n from docling.pipeline.standard_pdf_pipeline\ + \ import (\n StandardPdfPipeline,\n ) # pylint: disable=import-outside-toplevel\n\ + \ from docling_core.types.doc.base import (\n ImageRefMode,\n\ + \ ) # pylint: disable=import-outside-toplevel\n\n if not pdf_filenames:\n\ + \ raise ValueError(\n \"pdf_filenames must be provided\ + \ with the list of file names to process\"\n )\n\n allowed_pdf_backends\ + \ = {e.value for e in PdfBackend}\n if pdf_backend not in allowed_pdf_backends:\n\ + \ raise ValueError(\n f\"Invalid pdf_backend: {pdf_backend}.\ + \ Must be one of {sorted(allowed_pdf_backends)}\"\n )\n\n allowed_table_modes\ + \ = {e.value for e in TableFormerMode}\n if table_mode not in allowed_table_modes:\n\ + \ raise ValueError(\n f\"Invalid table_mode: {table_mode}.\ + \ Must be one of {sorted(allowed_table_modes)}\"\n )\n\n allowed_image_export_modes\ + \ = {e.value for e in ImageRefMode}\n if image_export_mode not in allowed_image_export_modes:\n\ + \ raise ValueError(\n f\"Invalid image_export_mode: {image_export_mode}.\ + \ Must be one of {sorted(allowed_image_export_modes)}\"\n )\n\n \ + \ if not allow_external_plugins:\n allowed_ocr_engines = {e.value\ + \ for e in OcrEngine}\n if ocr_engine not in allowed_ocr_engines:\n\ + \ raise ValueError(\n f\"Invalid ocr_engine: {ocr_engine}.\ + \ Must be one of {sorted(allowed_ocr_engines)}\"\n )\n\n #\ + \ Dictionary to map the engine name string to the corresponding class\n\ + \ ocr_engine_map = {\n \"easyocr\": EasyOcrOptions,\n \"\ + tesseract_cli\": TesseractCliOcrOptions,\n \"tesseract\": TesseractOcrOptions,\n\ + \ \"ocrmac\": OcrMacOptions,\n \"rapidocr\": RapidOcrOptions,\n\ + \ }\n\n input_path_p = Path(input_path.path)\n artifacts_path_p\ + \ = Path(artifacts_path.path)\n output_path_p = Path(output_path.path)\n\ + \ output_path_p.mkdir(parents=True, exist_ok=True)\n\n input_pdfs\ + \ = [input_path_p / name for name in pdf_filenames]\n print(\n \ + \ f\"docling-standard-convert: starting with backend='{pdf_backend}', files={len(input_pdfs)}\"\ + ,\n flush=True,\n )\n\n pipeline_options = PdfPipelineOptions()\n\ + \ pipeline_options.artifacts_path = artifacts_path_p\n pipeline_options.do_code_enrichment\ + \ = enrich_code\n pipeline_options.do_formula_enrichment = enrich_formula\n\ + \ pipeline_options.do_picture_classification = enrich_picture_classes\n\ + \ pipeline_options.do_picture_description = enrich_picture_description\n\ + \ pipeline_options.do_ocr = ocr\n if ocr and ocr_engine in ocr_engine_map:\n\ + \ OcrOptionsClass = ocr_engine_map[ocr_engine]\n ocr_options_instance\ + \ = OcrOptionsClass(force_full_page_ocr=force_ocr)\n pipeline_options.ocr_options\ + \ = ocr_options_instance\n pipeline_options.do_table_structure = True\n\ + \ pipeline_options.table_structure_options.do_cell_matching = True\n\ + \ pipeline_options.generate_page_images = True\n pipeline_options.table_structure_options.mode\ + \ = TableFormerMode(table_mode)\n pipeline_options.document_timeout =\ + \ float(timeout_per_document)\n pipeline_options.accelerator_options\ + \ = AcceleratorOptions(\n num_threads=num_threads, device=AcceleratorDevice.AUTO\n\ + \ )\n\n backend_to_impl = {\n PdfBackend.PYPDFIUM2.value: (\n\ + \ \"docling.backend.pypdfium2_backend\",\n \"PyPdfiumDocumentBackend\"\ + ,\n ),\n PdfBackend.DLPARSE_V1.value: (\n \"docling.backend.docling_parse_backend\"\ + ,\n \"DoclingParseDocumentBackend\",\n ),\n PdfBackend.DLPARSE_V2.value:\ + \ (\n \"docling.backend.docling_parse_v2_backend\",\n \ + \ \"DoclingParseV2DocumentBackend\",\n ),\n PdfBackend.DLPARSE_V4.value:\ + \ (\n \"docling.backend.docling_parse_v4_backend\",\n \ + \ \"DoclingParseV4DocumentBackend\",\n ),\n }\n\n module_name,\ + \ class_name = backend_to_impl[pdf_backend]\n backend_class = getattr(import_module(module_name),\ + \ class_name)\n\n doc_converter = DocumentConverter(\n format_options={\n\ + \ InputFormat.PDF: PdfFormatOption(\n pipeline_options=pipeline_options,\n\ + \ backend=backend_class,\n pipeline_cls=StandardPdfPipeline,\n\ + \ )\n }\n )\n\n results = doc_converter.convert_all(input_pdfs,\ + \ raises_on_error=True)\n\n for result in results:\n doc_filename\ + \ = result.input.file.stem\n\n output_json_path = output_path_p /\ + \ f\"{doc_filename}.json\"\n print(f\"docling-standard-convert: saving\ + \ {output_json_path}\", flush=True)\n result.document.save_as_json(\n\ + \ output_json_path, image_mode=ImageRefMode(image_export_mode)\n\ + \ )\n\n output_md_path = output_path_p / f\"{doc_filename}.md\"\ + \n print(f\"docling-standard-convert: saving {output_md_path}\",\ + \ flush=True)\n result.document.save_as_markdown(\n output_md_path,\ + \ image_mode=ImageRefMode(image_export_mode)\n )\n\n print(\"\ + docling-standard-convert: done\", flush=True)\n\n" + image: quay.io/fabianofranz/docling-ubi9:2.54.0 + resources: + cpuLimit: 4.0 + cpuRequest: 0.5 + memoryLimit: 6.0 + memoryRequest: 1.0 + resourceCpuLimit: '4' + resourceCpuRequest: 500m + resourceMemoryLimit: 6G + resourceMemoryRequest: 1G + exec-download-docling-models: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - download_docling_models + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef download_docling_models(\n output_path: dsl.Output[dsl.Artifact],\n\ + \ pipeline_type: str = \"standard\",\n remote_model_endpoint_enabled:\ + \ bool = False,\n):\n \"\"\"\n Download Docling models based on pipeline\ + \ type and configuration.\n\n This unified component handles model downloading\ + \ for different pipeline types:\n - standard : Download traditional Docling\ + \ models (layout, tableformer)\n - vlm : Download Docling VLM models\ + \ (smolvlm, smoldocling) for local inference\n When remote_model_endpoint_enabled=True,\ + \ downloads minimal models for remote inference\n\n Args:\n output_path:\ + \ Path to the output directory for Docling models\n pipeline_type:\ + \ Type of pipeline (standard, vlm)\n remote_model_endpoint_enabled:\ + \ Whether to download remote model endpoint models (VLM only)\n \"\"\"\ + \n from pathlib import Path # pylint: disable=import-outside-toplevel\n\ + \n from docling.utils.model_downloader import (\n download_models,\n\ + \ ) # pylint: disable=import-outside-toplevel\n\n output_path_p =\ + \ Path(output_path.path)\n output_path_p.mkdir(parents=True, exist_ok=True)\n\ + \n if pipeline_type == \"standard\":\n # Standard pipeline: download\ + \ traditional models\n download_models(\n output_dir=output_path_p,\n\ + \ progress=True,\n with_layout=True,\n \ + \ with_tableformer=True,\n with_easyocr=False,\n )\n \ + \ elif pipeline_type == \"vlm\" and remote_model_endpoint_enabled:\n \ + \ # VLM pipeline with remote model endpoint: Download minimal required\ + \ models\n # Only models set are what lives in fabianofranz repo\n\ + \ # TODO: figure out what needs to be downloaded or removed\n \ + \ download_models(\n output_dir=output_path_p,\n \ + \ progress=False,\n force=False,\n with_layout=True,\n\ + \ with_tableformer=True,\n with_code_formula=False,\n\ + \ with_picture_classifier=False,\n with_smolvlm=False,\n\ + \ with_smoldocling=False,\n with_smoldocling_mlx=False,\n\ + \ with_granite_vision=False,\n with_easyocr=False,\n\ + \ )\n elif pipeline_type == \"vlm\":\n # VLM pipeline with\ + \ local models: Download VLM models for local inference\n # TODO:\ + \ set models downloaded by model name passed into KFP pipeline ex: smoldocling\ + \ OR granite-vision\n download_models(\n output_dir=output_path_p,\n\ + \ with_smolvlm=True,\n with_smoldocling=True,\n \ + \ progress=False,\n force=False,\n with_layout=False,\n\ + \ with_tableformer=False,\n with_code_formula=False,\n\ + \ with_picture_classifier=False,\n with_smoldocling_mlx=False,\n\ + \ with_granite_vision=False,\n with_easyocr=False,\n\ + \ )\n else:\n raise ValueError(\n f\"Invalid\ + \ pipeline_type: {pipeline_type}. Must be 'standard' or 'vlm'\"\n \ + \ )\n\n" + image: quay.io/fabianofranz/docling-ubi9:2.54.0 + exec-import-pdfs: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - import_pdfs + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'boto3' 'requests'\ + \ && python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef import_pdfs(\n output_path: dsl.Output[dsl.Artifact],\n \ + \ filenames: str,\n base_url: str,\n from_s3: bool = False,\n \ + \ s3_secret_mount_path: str = \"/mnt/secrets\",\n):\n \"\"\"\n Import\ + \ PDF filenames (comma-separated) from specified URL or S3 bucket.\n\n \ + \ Args:\n filenames: List of PDF filenames to import.\n \ + \ base_url: Base URL of the PDF files.\n output_path: Path to the\ + \ output directory for the PDF files.\n from_s3: Whether or not to\ + \ import from S3.\n s3_secret_mount_path: Path to the secret mount\ + \ path for the S3 credentials.\n \"\"\"\n import os # pylint: disable=import-outside-toplevel\n\ + \ from pathlib import Path # pylint: disable=import-outside-toplevel\n\ + \n import boto3 # pylint: disable=import-outside-toplevel\n import\ + \ requests # pylint: disable=import-outside-toplevel\n\n filenames_list\ + \ = [name.strip() for name in filenames.split(\",\") if name.strip()]\n\ + \ if not filenames_list:\n raise ValueError(\n \"filenames\ + \ must contain at least one filename (comma-separated)\"\n )\n\n\ + \ output_path_p = Path(output_path.path)\n output_path_p.mkdir(parents=True,\ + \ exist_ok=True)\n\n if from_s3:\n if not os.path.exists(s3_secret_mount_path):\n\ + \ raise ValueError(\n f\"Secret for S3 should\ + \ be mounted in {s3_secret_mount_path}\"\n )\n\n s3_endpoint_url_secret\ + \ = \"S3_ENDPOINT_URL\"\n s3_endpoint_url_file_path = os.path.join(\n\ + \ s3_secret_mount_path, s3_endpoint_url_secret\n )\n \ + \ if os.path.isfile(s3_endpoint_url_file_path):\n with\ + \ open(s3_endpoint_url_file_path) as f:\n s3_endpoint_url\ + \ = f.read()\n else:\n raise ValueError(\n \ + \ f\"Key {s3_endpoint_url_secret} not defined in secret {s3_secret_mount_path}\"\ + \n )\n\n s3_access_key_secret = \"S3_ACCESS_KEY\"\n \ + \ s3_access_key_file_path = os.path.join(\n s3_secret_mount_path,\ + \ s3_access_key_secret\n )\n if os.path.isfile(s3_access_key_file_path):\n\ + \ with open(s3_access_key_file_path) as f:\n s3_access_key\ + \ = f.read()\n else:\n raise ValueError(\n \ + \ f\"Key {s3_access_key_secret} not defined in secret {s3_secret_mount_path}\"\ + \n )\n\n s3_secret_key_secret = \"S3_SECRET_KEY\"\n \ + \ s3_secret_key_file_path = os.path.join(\n s3_secret_mount_path,\ + \ s3_secret_key_secret\n )\n if os.path.isfile(s3_secret_key_file_path):\n\ + \ with open(s3_secret_key_file_path) as f:\n s3_secret_key\ + \ = f.read()\n else:\n raise ValueError(\n \ + \ f\"Key {s3_secret_key_secret} not defined in secret {s3_secret_mount_path}\"\ + \n )\n\n s3_bucket_secret = \"S3_BUCKET\"\n s3_bucket_file_path\ + \ = os.path.join(s3_secret_mount_path, s3_bucket_secret)\n if os.path.isfile(s3_bucket_file_path):\n\ + \ with open(s3_bucket_file_path) as f:\n s3_bucket\ + \ = f.read()\n else:\n raise ValueError(\n \ + \ f\"Key {s3_bucket_secret} not defined in secret {s3_secret_mount_path}\"\ + \n )\n\n s3_prefix_secret = \"S3_PREFIX\"\n s3_prefix_file_path\ + \ = os.path.join(s3_secret_mount_path, s3_prefix_secret)\n if os.path.isfile(s3_prefix_file_path):\n\ + \ with open(s3_prefix_file_path) as f:\n s3_prefix\ + \ = f.read()\n else:\n raise ValueError(\n \ + \ f\"Key {s3_prefix_secret} not defined in secret {s3_secret_mount_path}\"\ + \n )\n\n if not s3_endpoint_url:\n raise ValueError(\"\ + S3_ENDPOINT_URL must be provided\")\n\n if not s3_bucket:\n \ + \ raise ValueError(\"S3_BUCKET must be provided\")\n\n s3_client\ + \ = boto3.client(\n \"s3\",\n endpoint_url=s3_endpoint_url,\n\ + \ aws_access_key_id=s3_access_key,\n aws_secret_access_key=s3_secret_key,\n\ + \ )\n\n for filename in filenames_list:\n orig\ + \ = f\"{s3_prefix.rstrip('/')}/{filename.lstrip('/')}\"\n dest\ + \ = output_path_p / filename\n print(f\"import-test-pdfs: downloading\ + \ {orig} -> {dest} from s3\", flush=True)\n s3_client.download_file(s3_bucket,\ + \ orig, dest)\n else:\n if not base_url:\n raise ValueError(\"\ + base_url must be provided\")\n\n for filename in filenames_list:\n\ + \ url = f\"{base_url.rstrip('/')}/{filename.lstrip('/')}\"\n\ + \ dest = output_path_p / filename\n print(f\"import-test-pdfs:\ + \ downloading {url} -> {dest}\", flush=True)\n with requests.get(url,\ + \ stream=True, timeout=30) as resp:\n resp.raise_for_status()\n\ + \ with dest.open(\"wb\") as f:\n for chunk\ + \ in resp.iter_content(chunk_size=8192):\n if chunk:\n\ + \ f.write(chunk)\n\n print(\"import-test-pdfs:\ + \ done\", flush=True)\n\n" + image: registry.access.redhat.com/ubi9/python-311:9.6-1755074620 + exec-interpret-policy-artifacts: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - interpret_policy_artifacts + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef interpret_policy_artifacts(\n input_path: dsl.Input[dsl.Artifact],\n\ + \ output_path: dsl.Output[dsl.Artifact],\n vlm_secret_mount_path:\ + \ str = \"/mnt/secrets\",\n vlm_max_tokens: int = 1024,\n vlm_timeout:\ + \ int = 120,\n):\n \"\"\"\n Classify and interpret non-text artifacts\ + \ in policy documents using a VLM.\n\n Reads Docling conversion output\ + \ (JSON, Markdown, and referenced image files),\n sends each extracted\ + \ image to a VLM for artifact-type classification and\n type-specific\ + \ structured interpretation, then assembles final Markdown with\n interpreted\ + \ content replacing image references wrapped in artifact-type markers.\n\ + \n Args:\n input_path: Path to directory containing Docling conversion\ + \ output.\n output_path: Path to output directory for assembled policy\ + \ Markdown,\n artifact metadata, and passthrough DoclingDocument\ + \ JSON.\n vlm_secret_mount_path: Path to mounted Kubernetes secret\ + \ containing\n REMOTE_MODEL_ENDPOINT_URL, REMOTE_MODEL_NAME,\ + \ REMOTE_MODEL_API_KEY.\n vlm_max_tokens: Maximum tokens for each\ + \ VLM interpretation response.\n vlm_timeout: Timeout in seconds\ + \ per VLM API call.\n \"\"\"\n import base64\n import json\n \ + \ import os\n import re\n import shutil\n import time\n from\ + \ pathlib import Path\n\n import requests\n\n CLASSIFICATION_PROMPT\ + \ = (\n \"You are analyzing an image extracted from a policy document.\ + \ \"\n \"Classify this image into exactly one of the following categories.\ + \ \"\n \"Respond with ONLY the category name, nothing else.\\n\\\ + n\"\n \"Categories:\\n\"\n \"- chart: Any type of chart, graph,\ + \ or plot (bar, line, pie, scatter, etc.)\\n\"\n \"- risk_matrix:\ + \ A risk assessment matrix or heat map showing likelihood vs impact\\n\"\ + \n \"- decision_tree: A decision tree, flowchart showing decision\ + \ logic with branches\\n\"\n \"- form: A form, questionnaire, or\ + \ template with fields to fill in\\n\"\n \"- screenshot: A screenshot\ + \ of a user interface or application\\n\"\n \"- diagram: An architecture\ + \ diagram, process flow, org chart, or network diagram\\n\"\n \"\ + - other: Any other type of image that doesn't fit the above categories\"\ + \n )\n\n INTERPRETATION_PROMPTS = {\n \"risk_matrix\": (\n\ + \ \"Convert this risk matrix into structured Markdown. \"\n \ + \ \"Create a Markdown table that preserves all row headers, column\ + \ headers, \"\n \"and cell values. Include severity levels and\ + \ any color coding described as \"\n \"text labels (e.g., 'High\ + \ Risk'). Add any legends, footnotes, or scoring \"\n \"criteria\ + \ below the table.\"\n ),\n \"decision_tree\": (\n \ + \ \"Convert this decision tree into a nested Markdown list representing\ + \ the \"\n \"complete decision logic. Use indentation to show\ + \ hierarchy and branches. \"\n \"Each decision point should show\ + \ the question or condition. \"\n \"Each branch should show the\ + \ path taken (Yes/No, or named paths). \"\n \"Each leaf node\ + \ should show the final outcome or action. \"\n \"Preserve all\ + \ labels, conditions, thresholds, and outcomes exactly as shown.\"\n \ + \ ),\n \"chart\": (\n \"Describe this chart in structured\ + \ Markdown. Include:\\n\"\n \"- **Chart type**: (bar, line, pie,\ + \ etc.)\\n\"\n \"- **Title**: The chart title if visible\\n\"\ + \n \"- **Axes**: X-axis and Y-axis labels and units\\n\"\n \ + \ \"- **Data summary**: A Markdown table with the key data points\ + \ and values\\n\"\n \"- **Trends**: Notable trends, peaks, or\ + \ patterns in the data\\n\"\n \"- **Annotations**: Any callouts,\ + \ labels, or annotations on the chart\"\n ),\n \"form\": (\n\ + \ \"Convert this form into structured Markdown. \"\n \ + \ \"Create a Markdown table with 'Field' and 'Value/Description' columns.\ + \ \"\n \"Include all field labels and any pre-filled values.\ + \ \"\n \"For checkboxes, indicate [x] checked or [ ] unchecked.\ + \ \"\n \"For dropdowns, show the selected value if visible. \"\ + \n \"Include section headers, instructions, and help text.\"\n\ + \ ),\n \"screenshot\": (\n \"Describe this UI screenshot\ + \ as a structured step-by-step walkthrough. \"\n \"Identify the\ + \ application or interface shown. \"\n \"Describe visible elements:\ + \ navigation, buttons, menus, form fields, \"\n \"status indicators,\ + \ data displays, and notifications. \"\n \"Note the current state\ + \ or action being performed. \"\n \"If it shows a workflow or\ + \ sequence, use numbered steps.\"\n ),\n \"diagram\": (\n\ + \ \"Describe this diagram in structured Markdown. \"\n \ + \ \"List all components and their labels. \"\n \"Describe\ + \ relationships and connections between components. \"\n \"Note\ + \ directional flows (arrows) and their labels. \"\n \"Identify\ + \ groupings, layers, or zones. \"\n \"Use a nested list to represent\ + \ the hierarchy, or a table for \"\n \"components and connections.\ + \ \"\n \"Preserve any annotations, version numbers, or metadata\ + \ shown.\"\n ),\n \"other\": (\n \"Describe the\ + \ content of this image in structured Markdown. \"\n \"Preserve\ + \ all visible text, numbers, labels, and structural information. \"\n \ + \ \"Use appropriate Markdown formatting \u2014 tables for tabular\ + \ data, \"\n \"lists for hierarchical or sequential content,\ + \ and headings for sections. \"\n \"Capture any legends, footnotes,\ + \ or annotations.\"\n ),\n }\n\n VALID_TYPES = set(INTERPRETATION_PROMPTS.keys())\n\ + \n # --- Read VLM credentials from mounted secret or environment variables\ + \ ---\n def read_secret(name):\n fpath = os.path.join(vlm_secret_mount_path,\ + \ name)\n if os.path.isfile(fpath):\n with open(fpath)\ + \ as f:\n return f.read().strip()\n env_val = os.environ.get(name)\n\ + \ if env_val is not None:\n return env_val\n raise\ + \ ValueError(\n f\"Key {name} not found in secret mount {vlm_secret_mount_path}\ + \ \"\n f\"or environment variable {name}\"\n )\n\n \ + \ if not os.path.exists(vlm_secret_mount_path):\n print(\n \ + \ f\"Secret mount path {vlm_secret_mount_path} not found, \"\n \ + \ \"falling back to environment variables\",\n flush=True,\n\ + \ )\n\n vlm_endpoint_url = read_secret(\"REMOTE_MODEL_ENDPOINT_URL\"\ + )\n vlm_model_name = read_secret(\"REMOTE_MODEL_NAME\")\n vlm_api_key\ + \ = read_secret(\"REMOTE_MODEL_API_KEY\")\n\n if not vlm_endpoint_url.endswith(\"\ + /v1/chat/completions\"):\n vlm_endpoint_url = vlm_endpoint_url.rstrip(\"\ + /\")\n if not vlm_endpoint_url.endswith(\"/v1\"):\n vlm_endpoint_url\ + \ += \"/v1\"\n vlm_endpoint_url += \"/chat/completions\"\n\n input_path_p\ + \ = Path(input_path.path)\n output_path_p = Path(output_path.path)\n\ + \ output_path_p.mkdir(parents=True, exist_ok=True)\n\n # --- VLM call\ + \ helper with retries ---\n def call_vlm(image_b64, prompt, max_tokens):\n\ + \ payload = {\n \"model\": vlm_model_name,\n \ + \ \"messages\": [\n {\n \"role\": \"\ + user\",\n \"content\": [\n {\"\ + type\": \"text\", \"text\": prompt},\n {\n \ + \ \"type\": \"image_url\",\n \ + \ \"image_url\": {\"url\": f\"data:image/png;base64,{image_b64}\"},\n\ + \ },\n ],\n }\n\ + \ ],\n \"max_tokens\": max_tokens,\n }\n \ + \ headers = {}\n if vlm_api_key:\n headers[\"Authorization\"\ + ] = f\"Bearer {vlm_api_key}\"\n\n for attempt in range(3):\n \ + \ try:\n resp = requests.post(\n \ + \ vlm_endpoint_url,\n json=payload,\n \ + \ headers=headers,\n timeout=vlm_timeout,\n \ + \ )\n resp.raise_for_status()\n \ + \ return resp.json()[\"choices\"][0][\"message\"][\"content\"]\n \ + \ except (requests.RequestException, KeyError, IndexError) as e:\n\ + \ if attempt < 2:\n wait = 2 ** (attempt\ + \ + 1)\n print(\n f\" VLM call\ + \ attempt {attempt + 1} failed: {e}. \"\n f\"Retrying\ + \ in {wait}s...\",\n flush=True,\n \ + \ )\n time.sleep(wait)\n else:\n\ + \ raise RuntimeError(f\"VLM call failed after 3 attempts:\ + \ {e}\") from e\n\n def resolve_image_path(img_ref, doc_name):\n \ + \ \"\"\"Try multiple strategies to find the image file.\"\"\"\n \ + \ candidates = [\n input_path_p / img_ref,\n input_path_p\ + \ / doc_name / img_ref,\n input_path_p / Path(img_ref).name,\n\ + \ ]\n for candidate in candidates:\n if candidate.exists():\n\ + \ return candidate\n found = list(input_path_p.rglob(Path(img_ref).name))\n\ + \ if found:\n return found[0]\n return None\n\n\ + \ # --- Process each document ---\n md_files = sorted(input_path_p.glob(\"\ + *.md\"))\n if not md_files:\n print(\n \"interpret-policy-artifacts:\ + \ no markdown files found, nothing to process\",\n flush=True,\n\ + \ )\n return\n\n print(\n f\"interpret-policy-artifacts:\ + \ processing {len(md_files)} document(s)\",\n flush=True,\n )\n\ + \n img_pattern = re.compile(r\"!\\[([^\\]]*)\\]\\(([^)]+)\\)\")\n\n \ + \ for md_file in md_files:\n doc_name = md_file.stem\n print(f\"\ + interpret-policy-artifacts: processing {doc_name}\", flush=True)\n\n \ + \ md_content = md_file.read_text(encoding=\"utf-8\")\n matches\ + \ = list(img_pattern.finditer(md_content))\n\n if not matches:\n\ + \ print(f\" {doc_name}: no images found, copying markdown as-is\"\ + , flush=True)\n shutil.copy2(md_file, output_path_p / f\"{doc_name}_policy.md\"\ + )\n json_src = input_path_p / f\"{doc_name}.json\"\n \ + \ if json_src.exists():\n shutil.copy2(json_src, output_path_p\ + \ / f\"{doc_name}.json\")\n continue\n\n print(f\" {doc_name}:\ + \ found {len(matches)} image(s) to interpret\", flush=True)\n\n artifacts_metadata\ + \ = []\n assembled_md = md_content\n\n for idx, match in enumerate(matches):\n\ + \ img_ref = match.group(2)\n full_match_text = match.group(0)\n\ + \n img_path = resolve_image_path(img_ref, doc_name)\n \ + \ if img_path is None:\n print(f\" {doc_name}: image\ + \ {img_ref} not found, skipping\", flush=True)\n continue\n\ + \n with open(img_path, \"rb\") as f:\n img_b64\ + \ = base64.b64encode(f.read()).decode(\"utf-8\")\n\n # Step 1:\ + \ Classify artifact type\n print(\n f\" {doc_name}:\ + \ classifying image {idx + 1}/{len(matches)}...\",\n flush=True,\n\ + \ )\n raw_type = call_vlm(img_b64, CLASSIFICATION_PROMPT,\ + \ max_tokens=20)\n artifact_type = raw_type.strip().lower().replace(\"\ + \ \", \"_\")\n if artifact_type not in VALID_TYPES:\n \ + \ artifact_type = \"other\"\n print(\n \ + \ f\" {doc_name}: image {idx + 1} classified as '{artifact_type}'\",\n\ + \ flush=True,\n )\n\n # Step 2: Interpret\ + \ with type-specific prompt\n print(\n f\" {doc_name}:\ + \ interpreting image {idx + 1} as {artifact_type}...\",\n \ + \ flush=True,\n )\n interpretation = call_vlm(\n\ + \ img_b64,\n INTERPRETATION_PROMPTS[artifact_type],\n\ + \ max_tokens=vlm_max_tokens,\n )\n\n \ + \ # Step 3: Build replacement with artifact markers\n replacement\ + \ = (\n f\"\\n\"\n \ + \ f\"{interpretation.strip()}\\n\"\n f\"\"\n )\n assembled_md = assembled_md.replace(full_match_text,\ + \ replacement, 1)\n\n artifacts_metadata.append(\n \ + \ {\n \"index\": idx,\n \"type\"\ + : artifact_type,\n \"source_image\": str(img_ref),\n\ + \ \"interpretation\": interpretation.strip(),\n \ + \ }\n )\n\n # Save assembled policy markdown\n\ + \ policy_md_path = output_path_p / f\"{doc_name}_policy.md\"\n \ + \ policy_md_path.write_text(assembled_md, encoding=\"utf-8\")\n \ + \ print(f\" {doc_name}: saved {policy_md_path.name}\", flush=True)\n\ + \n # Save artifact metadata in a subdirectory to avoid conflicts\ + \ with\n # the chunker, which globs *.json and validates as DoclingDocument\n\ + \ metadata_dir = output_path_p / \"metadata\"\n metadata_dir.mkdir(parents=True,\ + \ exist_ok=True)\n artifacts_json_path = metadata_dir / f\"{doc_name}_artifacts.json\"\ + \n with open(artifacts_json_path, \"w\", encoding=\"utf-8\") as f:\n\ + \ json.dump(\n {\n \"document\"\ + : doc_name,\n \"total_artifacts\": len(artifacts_metadata),\n\ + \ \"artifacts\": artifacts_metadata,\n \ + \ },\n f,\n indent=2,\n ensure_ascii=False,\n\ + \ )\n print(f\" {doc_name}: saved {artifacts_json_path.name}\"\ + , flush=True)\n\n # Copy DoclingDocument JSON for downstream consumers\ + \ and chunking\n json_src = input_path_p / f\"{doc_name}.json\"\n\ + \ if json_src.exists():\n shutil.copy2(json_src, output_path_p\ + \ / f\"{doc_name}.json\")\n\n print(\"interpret-policy-artifacts: done\"\ + , flush=True)\n\n" + image: quay.io/fabianofranz/docling-ubi9:2.54.0 + resources: + cpuLimit: 2.0 + cpuRequest: 0.25 + memoryLimit: 4.0 + memoryRequest: 0.512 + resourceCpuLimit: '2' + resourceCpuRequest: 250m + resourceMemoryLimit: 4G + resourceMemoryRequest: 512M +pipelineInfo: + description: Policy document conversion pipeline with artifact-type-specific VLM + interpretation by the Data Processing Team + name: data-processing-docling-vlm-policy-pipeline +root: + dag: + tasks: + create-pdf-splits: + cachingOptions: + enableCache: true + componentRef: + name: comp-create-pdf-splits + dependentTasks: + - import-pdfs + inputs: + artifacts: + input_path: + taskOutputArtifact: + outputArtifactKey: output_path + producerTask: import-pdfs + parameters: + num_splits: + componentInputParameter: num_splits + taskInfo: + name: create-pdf-splits + download-docling-models: + cachingOptions: {} + componentRef: + name: comp-download-docling-models + inputs: + parameters: + pipeline_type: + runtimeValue: + constant: standard + remote_model_endpoint_enabled: + runtimeValue: + constant: false + taskInfo: + name: download-docling-models + for-loop-1: + componentRef: + name: comp-for-loop-1 + dependentTasks: + - create-pdf-splits + - download-docling-models + - import-pdfs + inputs: + artifacts: + pipelinechannel--download-docling-models-output_path: + taskOutputArtifact: + outputArtifactKey: output_path + producerTask: download-docling-models + pipelinechannel--import-pdfs-output_path: + taskOutputArtifact: + outputArtifactKey: output_path + producerTask: import-pdfs + parameters: + pipelinechannel--create-pdf-splits-Output: + taskOutputParameter: + outputParameterKey: Output + producerTask: create-pdf-splits + pipelinechannel--docling_chunk_enabled: + componentInputParameter: docling_chunk_enabled + pipelinechannel--docling_chunk_max_tokens: + componentInputParameter: docling_chunk_max_tokens + pipelinechannel--docling_chunk_merge_peers: + componentInputParameter: docling_chunk_merge_peers + pipelinechannel--docling_num_threads: + componentInputParameter: docling_num_threads + pipelinechannel--docling_ocr: + componentInputParameter: docling_ocr + pipelinechannel--docling_table_mode: + componentInputParameter: docling_table_mode + pipelinechannel--docling_timeout_per_document: + componentInputParameter: docling_timeout_per_document + pipelinechannel--vlm_max_tokens: + componentInputParameter: vlm_max_tokens + pipelinechannel--vlm_timeout: + componentInputParameter: vlm_timeout + parameterIterator: + itemInput: pipelinechannel--create-pdf-splits-Output-loop-item + items: + inputParameter: pipelinechannel--create-pdf-splits-Output + taskInfo: + name: for-loop-1 + import-pdfs: + cachingOptions: {} + componentRef: + name: comp-import-pdfs + inputs: + parameters: + base_url: + componentInputParameter: pdf_base_url + filenames: + componentInputParameter: pdf_filenames + from_s3: + componentInputParameter: pdf_from_s3 + s3_secret_mount_path: + runtimeValue: + constant: /mnt/secrets + taskInfo: + name: import-pdfs + inputDefinitions: + parameters: + docling_chunk_enabled: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + docling_chunk_max_tokens: + defaultValue: 512.0 + isOptional: true + parameterType: NUMBER_INTEGER + docling_chunk_merge_peers: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + docling_num_threads: + defaultValue: 4.0 + isOptional: true + parameterType: NUMBER_INTEGER + docling_ocr: + defaultValue: true + isOptional: true + parameterType: BOOLEAN + docling_table_mode: + defaultValue: accurate + isOptional: true + parameterType: STRING + docling_timeout_per_document: + defaultValue: 300.0 + isOptional: true + parameterType: NUMBER_INTEGER + num_splits: + defaultValue: 3.0 + isOptional: true + parameterType: NUMBER_INTEGER + pdf_base_url: + defaultValue: https://github.com/docling-project/docling/raw/v2.43.0/tests/data/pdf + isOptional: true + parameterType: STRING + pdf_filenames: + defaultValue: 2203.01017v2.pdf + isOptional: true + parameterType: STRING + pdf_from_s3: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + vlm_max_tokens: + defaultValue: 1024.0 + isOptional: true + parameterType: NUMBER_INTEGER + vlm_timeout: + defaultValue: 120.0 + isOptional: true + parameterType: NUMBER_INTEGER +schemaVersion: 2.1.0 +sdkVersion: kfp-2.14.6 +--- +platforms: + kubernetes: + deploymentSpec: + executors: + exec-import-pdfs: + secretAsVolume: + - mountPath: /mnt/secrets + optional: true + secretName: data-processing-docling-pipeline + secretNameParameter: + runtimeValue: + constant: data-processing-docling-pipeline + exec-interpret-policy-artifacts: + secretAsVolume: + - mountPath: /mnt/secrets + optional: false + secretName: data-processing-docling-pipeline + secretNameParameter: + runtimeValue: + constant: data-processing-docling-pipeline diff --git a/notebooks/use-cases/policy-document-conversion-vlm.ipynb b/notebooks/use-cases/policy-document-conversion-vlm.ipynb index e9eb3b00..62cc0ad5 100644 --- a/notebooks/use-cases/policy-document-conversion-vlm.ipynb +++ b/notebooks/use-cases/policy-document-conversion-vlm.ipynb @@ -78,18 +78,7 @@ "id": "5", "metadata": {}, "outputs": [], - "source": [ - "import os\n", - "\n", - "VLM_ENDPOINT_URL = os.getenv(\n", - " \"VLM_SERVICE_ENDPOINT_URL\",\n", - " \"http://policy-vlm-predictor.default.svc.cluster.local:8080/v1/chat/completions\",\n", - ")\n", - "VLM_MODEL_NAME = os.getenv(\"VLM_SERVICE_MODEL_NAME\", \"ibm-granite/granite-vision-3.2-2b\")\n", - "VLM_API_KEY = os.getenv(\"VLM_SERVICE_API_KEY\", \"\")\n", - "VLM_MAX_TOKENS = 1024\n", - "VLM_TIMEOUT = 120" - ] + "source": "import os\n\n# VLM config is set in the parameters cell below and can be\n# overridden by papermill or environment variables at runtime." }, { "cell_type": "markdown", @@ -111,16 +100,15 @@ ] }, "outputs": [], - "source": [ - "from pathlib import Path\n", - "\n", - "files = [\n", - " \"https://raw.githubusercontent.com/py-pdf/sample-files/refs/heads/main/003-pdflatex-image/pdflatex-image.pdf\",\n", - "]\n", - "\n", - "output_dir = Path(\"policy-conversion-output\")\n", - "output_dir.mkdir(parents=True, exist_ok=True)" - ] + "source": "from pathlib import Path\n\nfiles = [\n \"https://raw.githubusercontent.com/py-pdf/sample-files/refs/heads/main/003-pdflatex-image/pdflatex-image.pdf\",\n]\n\noutput_dir = Path(\"policy-conversion-output\")\noutput_dir.mkdir(parents=True, exist_ok=True)\n\n# VLM configuration — override via papermill or environment variables\nVLM_ENDPOINT_URL = os.getenv(\n \"VLM_SERVICE_ENDPOINT_URL\",\n \"http://policy-vlm-predictor.default.svc.cluster.local:8080/v1/chat/completions\",\n)\nVLM_MODEL_NAME = os.getenv(\"VLM_SERVICE_MODEL_NAME\", \"ibm-granite/granite-vision-3.2-2b\")\nVLM_API_KEY = os.getenv(\"VLM_SERVICE_API_KEY\", \"\")\nVLM_MAX_TOKENS = 1024\nVLM_TIMEOUT = 120\n\n# Set to False to skip VLM interpretation (e.g. in CI or without a GPU endpoint)\nvlm_enabled = True" + }, + { + "cell_type": "code", + "id": "50c22a3f", + "source": "import requests\n\nif vlm_enabled:\n try:\n base_url = VLM_ENDPOINT_URL.split(\"/v1/\")[0] if \"/v1/\" in VLM_ENDPOINT_URL else VLM_ENDPOINT_URL.rstrip(\"/\")\n resp = requests.get(f\"{base_url}/health\", timeout=5)\n resp.raise_for_status()\n print(f\"VLM endpoint reachable at {base_url}\")\n except Exception as e:\n print(f\"VLM endpoint not reachable ({e}), disabling VLM interpretation\")\n vlm_enabled = False\nelse:\n print(\"VLM interpretation disabled by parameter\")", + "metadata": {}, + "execution_count": null, + "outputs": [] }, { "cell_type": "markdown", @@ -341,63 +329,7 @@ "id": "16", "metadata": {}, "outputs": [], - "source": [ - "import json\n", - "\n", - "all_artifacts = {}\n", - "\n", - "for stem, info in results.items():\n", - " md_content = info[\"md_path\"].read_text(encoding=\"utf-8\")\n", - " matches = list(IMG_PATTERN.finditer(md_content))\n", - "\n", - " if not matches:\n", - " print(f\"{stem}: no images, skipping\")\n", - " all_artifacts[stem] = []\n", - " continue\n", - "\n", - " artifacts = []\n", - " for idx, match in enumerate(matches):\n", - " img_ref = match.group(2)\n", - "\n", - " # Resolve image path\n", - " img_path = output_dir / img_ref\n", - " if not img_path.exists():\n", - " img_path = output_dir / Path(img_ref).name\n", - " if not img_path.exists():\n", - " candidates = list(output_dir.rglob(Path(img_ref).name))\n", - " img_path = candidates[0] if candidates else None\n", - "\n", - " if img_path is None or not img_path.exists():\n", - " print(f\" [{idx}] Image not found: {img_ref}\")\n", - " continue\n", - "\n", - " with open(img_path, \"rb\") as f:\n", - " img_b64 = base64.b64encode(f.read()).decode(\"utf-8\")\n", - "\n", - " # Classify\n", - " print(f\" [{idx}] Classifying...\")\n", - " raw_type = call_vlm(img_b64, CLASSIFICATION_PROMPT, max_tokens=20)\n", - " artifact_type = raw_type.strip().lower().replace(\" \", \"_\")\n", - " if artifact_type not in VALID_TYPES:\n", - " artifact_type = \"other\"\n", - " print(f\" [{idx}] Type: {artifact_type}\")\n", - "\n", - " # Interpret\n", - " print(f\" [{idx}] Interpreting as {artifact_type}...\")\n", - " interpretation = call_vlm(img_b64, INTERPRETATION_PROMPTS[artifact_type])\n", - " print(f\" [{idx}] Done ({len(interpretation)} chars)\")\n", - "\n", - " artifacts.append({\n", - " \"index\": idx,\n", - " \"type\": artifact_type,\n", - " \"source_image\": str(img_ref),\n", - " \"match_text\": match.group(0),\n", - " \"interpretation\": interpretation.strip(),\n", - " })\n", - "\n", - " all_artifacts[stem] = artifacts\n", - " print(f\"{stem}: processed {len(artifacts)} artifact(s)\")" - ] + "source": "import json\n\nall_artifacts = {}\n\nif not vlm_enabled:\n print(\"VLM disabled — skipping artifact classification and interpretation\")\n for stem in results:\n all_artifacts[stem] = []\nelse:\n for stem, info in results.items():\n md_content = info[\"md_path\"].read_text(encoding=\"utf-8\")\n matches = list(IMG_PATTERN.finditer(md_content))\n\n if not matches:\n print(f\"{stem}: no images, skipping\")\n all_artifacts[stem] = []\n continue\n\n artifacts = []\n for idx, match in enumerate(matches):\n img_ref = match.group(2)\n\n # Resolve image path\n img_path = output_dir / img_ref\n if not img_path.exists():\n img_path = output_dir / Path(img_ref).name\n if not img_path.exists():\n candidates = list(output_dir.rglob(Path(img_ref).name))\n img_path = candidates[0] if candidates else None\n\n if img_path is None or not img_path.exists():\n print(f\" [{idx}] Image not found: {img_ref}\")\n continue\n\n with open(img_path, \"rb\") as f:\n img_b64 = base64.b64encode(f.read()).decode(\"utf-8\")\n\n # Classify\n print(f\" [{idx}] Classifying...\")\n raw_type = call_vlm(img_b64, CLASSIFICATION_PROMPT, max_tokens=20)\n artifact_type = raw_type.strip().lower().replace(\" \", \"_\")\n if artifact_type not in VALID_TYPES:\n artifact_type = \"other\"\n print(f\" [{idx}] Type: {artifact_type}\")\n\n # Interpret\n print(f\" [{idx}] Interpreting as {artifact_type}...\")\n interpretation = call_vlm(img_b64, INTERPRETATION_PROMPTS[artifact_type])\n print(f\" [{idx}] Done ({len(interpretation)} chars)\")\n\n artifacts.append({\n \"index\": idx,\n \"type\": artifact_type,\n \"source_image\": str(img_ref),\n \"match_text\": match.group(0),\n \"interpretation\": interpretation.strip(),\n })\n\n all_artifacts[stem] = artifacts\n print(f\"{stem}: processed {len(artifacts)} artifact(s)\")" }, { "cell_type": "markdown", @@ -415,33 +347,7 @@ "id": "18", "metadata": {}, "outputs": [], - "source": [ - "for stem, info in results.items():\n", - " md_content = info[\"md_path\"].read_text(encoding=\"utf-8\")\n", - " artifacts = all_artifacts.get(stem, [])\n", - "\n", - " for artifact in artifacts:\n", - " replacement = (\n", - " f\"\\n\"\n", - " f\"{artifact['interpretation']}\\n\"\n", - " f\"\"\n", - " )\n", - " md_content = md_content.replace(artifact[\"match_text\"], replacement, 1)\n", - "\n", - " # Save assembled policy markdown\n", - " policy_path = output_dir / f\"{stem}_policy.md\"\n", - " policy_path.write_text(md_content, encoding=\"utf-8\")\n", - " print(f\"Saved: {policy_path}\")\n", - "\n", - " # Save artifact metadata\n", - " meta_path = output_dir / f\"{stem}_artifacts.json\"\n", - " with open(meta_path, \"w\") as f:\n", - " json.dump(\n", - " {\"document\": stem, \"total_artifacts\": len(artifacts), \"artifacts\": artifacts},\n", - " f, indent=2, ensure_ascii=False,\n", - " )\n", - " print(f\"Saved: {meta_path}\")" - ] + "source": "for stem, info in results.items():\n md_content = info[\"md_path\"].read_text(encoding=\"utf-8\")\n artifacts = all_artifacts.get(stem, [])\n\n for artifact in artifacts:\n replacement = (\n f\"\\n\"\n f\"{artifact['interpretation']}\\n\"\n f\"\"\n )\n md_content = md_content.replace(artifact[\"match_text\"], replacement, 1)\n\n # Save assembled policy markdown\n policy_path = output_dir / f\"{stem}_policy.md\"\n policy_path.write_text(md_content, encoding=\"utf-8\")\n print(f\"Saved: {policy_path}\")\n\n # Save artifact metadata\n meta_dir = output_dir / \"metadata\"\n meta_dir.mkdir(parents=True, exist_ok=True)\n meta_path = meta_dir / f\"{stem}_artifacts.json\"\n with open(meta_path, \"w\") as f:\n json.dump(\n {\"document\": stem, \"total_artifacts\": len(artifacts), \"artifacts\": artifacts},\n f, indent=2, ensure_ascii=False,\n )\n print(f\"Saved: {meta_path}\")" }, { "cell_type": "markdown", @@ -488,45 +394,7 @@ "id": "22", "metadata": {}, "outputs": [], - "source": [ - "for stem in results:\n", - " policy_path = output_dir / f\"{stem}_policy.md\"\n", - " meta_path = output_dir / f\"{stem}_artifacts.json\"\n", - "\n", - " if not policy_path.exists():\n", - " continue\n", - "\n", - " content = policy_path.read_text(encoding=\"utf-8\")\n", - " with open(meta_path) as f:\n", - " meta = json.load(f)\n", - "\n", - " # Count artifact markers\n", - " open_markers = content.count(\"\")\n", - "\n", - " # Check for remaining image references (should be zero)\n", - " remaining_imgs = len(IMG_PATTERN.findall(content))\n", - "\n", - " # Check structural elements\n", - " headings = len(re.findall(r\"^#+\\s\", content, re.MULTILINE))\n", - " tables = content.count(\"|---\")\n", - "\n", - " print(f\"=== {stem} ===\")\n", - " print(f\" Total artifacts interpreted: {meta['total_artifacts']}\")\n", - " print(f\" Artifact markers (open/close): {open_markers}/{close_markers}\")\n", - " print(f\" Remaining unprocessed images: {remaining_imgs}\")\n", - " print(f\" Headings preserved: {headings}\")\n", - " print(f\" Tables in output: {tables}\")\n", - " print(f\" Output length: {len(content)} chars\")\n", - " print()\n", - "\n", - " # Type distribution\n", - " type_counts = {}\n", - " for a in meta[\"artifacts\"]:\n", - " type_counts[a[\"type\"]] = type_counts.get(a[\"type\"], 0) + 1\n", - " if type_counts:\n", - " print(f\" Artifact types: {type_counts}\")" - ] + "source": "for stem in results:\n policy_path = output_dir / f\"{stem}_policy.md\"\n meta_path = output_dir / \"metadata\" / f\"{stem}_artifacts.json\"\n\n if not policy_path.exists():\n continue\n\n content = policy_path.read_text(encoding=\"utf-8\")\n with open(meta_path) as f:\n meta = json.load(f)\n\n # Count artifact markers\n open_markers = content.count(\"\")\n\n # Check for remaining image references (should be zero when VLM is enabled)\n remaining_imgs = len(IMG_PATTERN.findall(content))\n\n # Check structural elements\n headings = len(re.findall(r\"^#+\\s\", content, re.MULTILINE))\n tables = content.count(\"|---\")\n\n print(f\"=== {stem} ===\")\n print(f\" VLM enabled: {vlm_enabled}\")\n print(f\" Total artifacts interpreted: {meta['total_artifacts']}\")\n print(f\" Artifact markers (open/close): {open_markers}/{close_markers}\")\n print(f\" Remaining unprocessed images: {remaining_imgs}\")\n print(f\" Headings preserved: {headings}\")\n print(f\" Tables in output: {tables}\")\n print(f\" Output length: {len(content)} chars\")\n print()\n\n # Type distribution\n type_counts = {}\n for a in meta[\"artifacts\"]:\n type_counts[a[\"type\"]] = type_counts.get(a[\"type\"], 0) + 1\n if type_counts:\n print(f\" Artifact types: {type_counts}\")" }, { "cell_type": "markdown", @@ -564,4 +432,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file From 8359c46f415f8216856362183007fa48726b09e0 Mon Sep 17 00:00:00 2001 From: Alina Ryan Date: Wed, 11 Nov 2026 13:30:42 -0500 Subject: [PATCH 5/6] Address alimaredia review feedback - Forward VLM env vars into DockerRunner via environment param so local_run.py actually works without Kubernetes secrets - Write metadata file for no-image documents so downstream consumers can distinguish "zero artifacts" from "output missing" - Track skipped images in metadata with reason field, add total_images count so artifact coverage is auditable from _artifacts.json alone - Update README local testing section with env var setup instructions Co-Authored-By: Claude Opus 4.6 --- .../docling-vlm-policy/README.md | 10 +++- .../docling-vlm-policy/local_run.py | 16 +++++- .../docling-vlm-policy/policy_components.py | 20 +++++++ .../policy_convert_pipeline_compiled.yaml | 53 +++++++++++-------- 4 files changed, 75 insertions(+), 24 deletions(-) diff --git a/kubeflow-pipelines/docling-vlm-policy/README.md b/kubeflow-pipelines/docling-vlm-policy/README.md index 0894b6bd..afa07cec 100644 --- a/kubeflow-pipelines/docling-vlm-policy/README.md +++ b/kubeflow-pipelines/docling-vlm-policy/README.md @@ -175,7 +175,7 @@ The `asago-policy-mapper` currently calls `doc.export_to_markdown(image_mode=Ima To evaluate interpretation quality: 1. **Manual review**: Compare `_policy.md` output against original PDF for a sample of documents -2. **Artifact coverage**: Check `_artifacts.json` — `total_artifacts` should match the number of non-text elements in the source +2. **Artifact coverage**: Check `_artifacts.json` — `total_images` is the number of images found in the document; `total_artifacts` is how many were successfully interpreted; `skipped` lists any images that could not be resolved 3. **Classification accuracy**: Review artifact type assignments in `_artifacts.json` 4. **Structural preservation**: Verify headings, numbered clauses, cross-references, and tables are intact in the assembled Markdown @@ -183,11 +183,17 @@ To evaluate interpretation quality: ```bash pip install docker kfp + +# Set VLM endpoint environment variables (forwarded into the Docker container) +export REMOTE_MODEL_ENDPOINT_URL="http://localhost:8080" +export REMOTE_MODEL_NAME="ibm-granite/granite-vision-3.2-2b" +export REMOTE_MODEL_API_KEY="" + cd data-processing/kubeflow-pipelines/docling-vlm-policy python local_run.py ``` -Requires a Docker-compatible daemon and a VLM secret configured locally. +Requires a Docker-compatible daemon and a reachable VLM endpoint. The `REMOTE_MODEL_*` environment variables are forwarded into the Docker container via `DockerRunner(environment=...)`. ## Compiling from Source diff --git a/kubeflow-pipelines/docling-vlm-policy/local_run.py b/kubeflow-pipelines/docling-vlm-policy/local_run.py index 24865a90..81fd1d7c 100644 --- a/kubeflow-pipelines/docling-vlm-policy/local_run.py +++ b/kubeflow-pipelines/docling-vlm-policy/local_run.py @@ -1,3 +1,4 @@ +import os import sys from pathlib import Path from typing import List @@ -16,6 +17,12 @@ from policy_components import interpret_policy_artifacts from standard_components import docling_convert_standard +VLM_ENV_KEYS = ( + "REMOTE_MODEL_ENDPOINT_URL", + "REMOTE_MODEL_NAME", + "REMOTE_MODEL_API_KEY", +) + @dsl.component(base_image="python:3.11") def take_first_split(splits: List[List[str]]) -> List[str]: @@ -63,7 +70,14 @@ def policy_convert_pipeline_local(): def main() -> None: - local.init(runner=local.DockerRunner()) + vlm_env = {k: os.environ[k] for k in VLM_ENV_KEYS if k in os.environ} + if not vlm_env: + print( + "Set REMOTE_MODEL_ENDPOINT_URL, REMOTE_MODEL_NAME, and " + "REMOTE_MODEL_API_KEY environment variables to configure the VLM " + "endpoint for local runs." + ) + local.init(runner=local.DockerRunner(environment=vlm_env)) policy_convert_pipeline_local() diff --git a/kubeflow-pipelines/docling-vlm-policy/policy_components.py b/kubeflow-pipelines/docling-vlm-policy/policy_components.py index 2dce9f42..41235c00 100644 --- a/kubeflow-pipelines/docling-vlm-policy/policy_components.py +++ b/kubeflow-pipelines/docling-vlm-policy/policy_components.py @@ -244,11 +244,26 @@ def resolve_image_path(img_ref, doc_name): json_src = input_path_p / f"{doc_name}.json" if json_src.exists(): shutil.copy2(json_src, output_path_p / f"{doc_name}.json") + metadata_dir = output_path_p / "metadata" + metadata_dir.mkdir(parents=True, exist_ok=True) + with open(metadata_dir / f"{doc_name}_artifacts.json", "w") as f: + json.dump( + { + "document": doc_name, + "total_images": 0, + "total_artifacts": 0, + "skipped": [], + "artifacts": [], + }, + f, + indent=2, + ) continue print(f" {doc_name}: found {len(matches)} image(s) to interpret", flush=True) artifacts_metadata = [] + skipped_metadata = [] assembled_md = md_content for idx, match in enumerate(matches): @@ -258,6 +273,9 @@ def resolve_image_path(img_ref, doc_name): img_path = resolve_image_path(img_ref, doc_name) if img_path is None: print(f" {doc_name}: image {img_ref} not found, skipping", flush=True) + skipped_metadata.append( + {"index": idx, "source_image": str(img_ref), "reason": "not found"} + ) continue with open(img_path, "rb") as f: @@ -319,7 +337,9 @@ def resolve_image_path(img_ref, doc_name): json.dump( { "document": doc_name, + "total_images": len(matches), "total_artifacts": len(artifacts_metadata), + "skipped": skipped_metadata, "artifacts": artifacts_metadata, }, f, diff --git a/kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline_compiled.yaml b/kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline_compiled.yaml index 15d591f0..120b06c6 100644 --- a/kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline_compiled.yaml +++ b/kubeflow-pipelines/docling-vlm-policy/policy_convert_pipeline_compiled.yaml @@ -993,19 +993,29 @@ deploymentSpec: , flush=True)\n shutil.copy2(md_file, output_path_p / f\"{doc_name}_policy.md\"\ )\n json_src = input_path_p / f\"{doc_name}.json\"\n \ \ if json_src.exists():\n shutil.copy2(json_src, output_path_p\ - \ / f\"{doc_name}.json\")\n continue\n\n print(f\" {doc_name}:\ + \ / f\"{doc_name}.json\")\n metadata_dir = output_path_p / \"\ + metadata\"\n metadata_dir.mkdir(parents=True, exist_ok=True)\n\ + \ with open(metadata_dir / f\"{doc_name}_artifacts.json\", \"\ + w\") as f:\n json.dump(\n {\n \ + \ \"document\": doc_name,\n \"total_images\"\ + : 0,\n \"total_artifacts\": 0,\n \ + \ \"skipped\": [],\n \"artifacts\": [],\n\ + \ },\n f,\n indent=2,\n\ + \ )\n continue\n\n print(f\" {doc_name}:\ \ found {len(matches)} image(s) to interpret\", flush=True)\n\n artifacts_metadata\ - \ = []\n assembled_md = md_content\n\n for idx, match in enumerate(matches):\n\ - \ img_ref = match.group(2)\n full_match_text = match.group(0)\n\ - \n img_path = resolve_image_path(img_ref, doc_name)\n \ - \ if img_path is None:\n print(f\" {doc_name}: image\ - \ {img_ref} not found, skipping\", flush=True)\n continue\n\ - \n with open(img_path, \"rb\") as f:\n img_b64\ - \ = base64.b64encode(f.read()).decode(\"utf-8\")\n\n # Step 1:\ - \ Classify artifact type\n print(\n f\" {doc_name}:\ - \ classifying image {idx + 1}/{len(matches)}...\",\n flush=True,\n\ - \ )\n raw_type = call_vlm(img_b64, CLASSIFICATION_PROMPT,\ - \ max_tokens=20)\n artifact_type = raw_type.strip().lower().replace(\"\ + \ = []\n skipped_metadata = []\n assembled_md = md_content\n\ + \n for idx, match in enumerate(matches):\n img_ref = match.group(2)\n\ + \ full_match_text = match.group(0)\n\n img_path =\ + \ resolve_image_path(img_ref, doc_name)\n if img_path is None:\n\ + \ print(f\" {doc_name}: image {img_ref} not found, skipping\"\ + , flush=True)\n skipped_metadata.append(\n \ + \ {\"index\": idx, \"source_image\": str(img_ref), \"reason\": \"not\ + \ found\"}\n )\n continue\n\n with\ + \ open(img_path, \"rb\") as f:\n img_b64 = base64.b64encode(f.read()).decode(\"\ + utf-8\")\n\n # Step 1: Classify artifact type\n print(\n\ + \ f\" {doc_name}: classifying image {idx + 1}/{len(matches)}...\"\ + ,\n flush=True,\n )\n raw_type = call_vlm(img_b64,\ + \ CLASSIFICATION_PROMPT, max_tokens=20)\n artifact_type = raw_type.strip().lower().replace(\"\ \ \", \"_\")\n if artifact_type not in VALID_TYPES:\n \ \ artifact_type = \"other\"\n print(\n \ \ f\" {doc_name}: image {idx + 1} classified as '{artifact_type}'\",\n\ @@ -1033,15 +1043,16 @@ deploymentSpec: \ exist_ok=True)\n artifacts_json_path = metadata_dir / f\"{doc_name}_artifacts.json\"\ \n with open(artifacts_json_path, \"w\", encoding=\"utf-8\") as f:\n\ \ json.dump(\n {\n \"document\"\ - : doc_name,\n \"total_artifacts\": len(artifacts_metadata),\n\ - \ \"artifacts\": artifacts_metadata,\n \ - \ },\n f,\n indent=2,\n ensure_ascii=False,\n\ - \ )\n print(f\" {doc_name}: saved {artifacts_json_path.name}\"\ - , flush=True)\n\n # Copy DoclingDocument JSON for downstream consumers\ - \ and chunking\n json_src = input_path_p / f\"{doc_name}.json\"\n\ - \ if json_src.exists():\n shutil.copy2(json_src, output_path_p\ - \ / f\"{doc_name}.json\")\n\n print(\"interpret-policy-artifacts: done\"\ - , flush=True)\n\n" + : doc_name,\n \"total_images\": len(matches),\n \ + \ \"total_artifacts\": len(artifacts_metadata),\n \ + \ \"skipped\": skipped_metadata,\n \"artifacts\"\ + : artifacts_metadata,\n },\n f,\n \ + \ indent=2,\n ensure_ascii=False,\n )\n\ + \ print(f\" {doc_name}: saved {artifacts_json_path.name}\", flush=True)\n\ + \n # Copy DoclingDocument JSON for downstream consumers and chunking\n\ + \ json_src = input_path_p / f\"{doc_name}.json\"\n if json_src.exists():\n\ + \ shutil.copy2(json_src, output_path_p / f\"{doc_name}.json\"\ + )\n\n print(\"interpret-policy-artifacts: done\", flush=True)\n\n" image: quay.io/fabianofranz/docling-ubi9:2.54.0 resources: cpuLimit: 2.0 From 19f37e4b379029634b4d518acd38bb769e6dba8c Mon Sep 17 00:00:00 2001 From: Alina Ryan Date: Wed, 11 Nov 2026 16:08:08 -0500 Subject: [PATCH 6/6] Fix Mergify config: replace removed commit_message_template commit_message_template was removed by Mergify upstream. Migrate to the declarative commit_message_format setting. Co-Authored-By: Claude Opus 4.6 --- .github/mergify.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/mergify.yml b/.github/mergify.yml index 0ac9879f..d06d57c2 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -13,10 +13,7 @@ pull_request_rules: actions: merge: method: squash - commit_message_template: | - {{ title }} (#{{ number }}) - - {{ body.split('Description')[0].strip() if body else '' }} + commit_message_format: title+body comment: message: | 🎉 **Auto-merged successfully!**