diff --git a/docs/spec/dva-processing.yaml b/docs/spec/dva-processing.yaml new file mode 100644 index 0000000..ec298cc --- /dev/null +++ b/docs/spec/dva-processing.yaml @@ -0,0 +1,291 @@ +openapi: 3.1.0 + +info: + title: DVA Processing + version: 0.2.0 + description: > + Stateless veracity-check engine. Evaluates data requirements expressed in + VLAs (Veracity Level Agreements) against supplied data and returns one + EvaluationResult per requirement. + +servers: + - url: http://localhost:5000 + +paths: + /evaluate: + post: + operationId: evaluate + summary: Evaluate a single requirement against data. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationRequest' + examples: + jq_non_empty_actor_name: + summary: jq requirement checking actor name is non-empty + value: + requirement: + implementation: >- + { success: (.actor.name | length > 0), details: "actor name non-empty" } + engine: JQ + data: + actor: + name: Jean Dupont + verb: + id: http://adlnet.gov/expapi/verbs/interacted + responses: + '200': + description: Evaluation completed (check `success`/`error` for outcome). + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResult' + '422': + description: Validation error for the request body. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '500': + description: >- + Evaluation engine error. The response body is still an + EvaluationResult with success=false and error describing the failure. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /evaluate-batch: + post: + operationId: evaluateBatch + summary: Evaluate all requirements in a VLA against data. + description: > + Iterates over every quality requirement declared in the VLA and + returns one EvaluationResult per requirement. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationBatchRequest' + examples: + xapi_statement_vla: + summary: VLA with two JQ quality requirements in a top-level list + value: + vla: + quality: + - engine: JQ + implementation: >- + { success: (.actor.name | length > 0), details: "actor name non-empty" } + - engine: JQ + implementation: >- + { success: (.verb.id | length > 0), details: "verb id non-empty" } + data: + actor: + name: Jean Dupont + verb: + id: http://adlnet.gov/expapi/verbs/interacted + responses: + '200': + description: >- + Array of per-requirement evaluation results, one per quality rule. + Always returns 200 — check each result's success/error fields. + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/EvaluationResult' + '422': + description: Validation error for the request body. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + + /evaluate/from-template: + post: + operationId: evaluateFromTemplate + summary: Fetch a VLA template, render with a model, and evaluate against data. + description: > + Fetches the template identified by `templateID`, renders it with + `templateModel`, evaluates the resulting requirement against `data`, + and returns the result. Field names are camelCase on the wire. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationFromTemplateRequest' + examples: + actor_name_template: + summary: Render an actor.name check template and evaluate passing data + value: + templateID: a5dee716-2129-4588-a1a2-04a4c2923a79 + templateModel: + field: actor.name + data: + actor: + name: Jean Dupont + verb: + id: http://adlnet.gov/expapi/verbs/interacted + template_not_found: + summary: Template not found (returns 404) + value: + templateID: 00000000-0000-0000-0000-000000000099 + templateModel: {} + data: {} + responses: + '200': + description: >- + Evaluation completed. Check success/error for the outcome. When + the template fails to render, success is false and error contains + the reason. + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluationResult' + '404': + description: The requested template was not found. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + '422': + description: Validation error for the request body. + content: + application/json: + schema: + $ref: '#/components/schemas/Error' + +components: + schemas: + QualityEngine: + type: string + enum: [SCHEMA, GREAT_EXPECTATIONS, JQ] + description: >- + The engine used to evaluate a requirement. SCHEMA performs structural + validation against a JSON schema, GREAT_EXPECTATIONS delegates to the + Great Expectations library, and JQ evaluates a jq expression whose output + must be `{ success: boolean, details: string }`. + + EvaluationRequest: + type: object + description: Request body for POST /evaluate. + required: [requirement, data] + properties: + requirement: + $ref: '#/components/schemas/Requirement' + data: + description: The data document to evaluate the requirement against. + additionalProperties: true + additionalProperties: false + + Requirement: + type: object + description: A single veracity requirement to evaluate. + required: [implementation, engine] + properties: + implementation: + type: string + description: >- + Engine-specific implementation. For JQ this is a jq expression whose + output must be an object `{ success: boolean, details: string }`. + engine: + $ref: '#/components/schemas/QualityEngine' + additionalProperties: false + + EvaluationBatchRequest: + type: object + description: Request body for POST /evaluate-batch. + required: [vla, data] + properties: + vla: + type: object + description: >- + The VLA document. Requirements may be declared as a top-level + `quality` list, under `schema` (a single object with a `quality` + list), or as a list of schema items each carrying a `quality` list. + additionalProperties: true + data: + description: The data document to evaluate all requirements against. + additionalProperties: true + additionalProperties: false + + EvaluationFromTemplateRequest: + type: object + description: >- + Request body for POST /evaluate/from-template. Field names are + camelCase on the wire. + required: [templateID, templateModel, data] + properties: + templateID: + type: string + format: uuid + description: UUID of the template to fetch and render. + templateModel: + type: object + description: >- + Model object used to render the fetched template. Keys are template + placeholders, values are their substitutions. + additionalProperties: true + data: + description: The data document to evaluate the rendered requirement against. + additionalProperties: true + additionalProperties: false + + EvaluationResult: + type: object + description: Outcome of evaluating a single requirement against data. + required: [timestamp, success] + properties: + engine: + description: >- + The engine that produced this result. Nullable (e.g. when evaluation + failed before an engine could be selected). + oneOf: + - $ref: '#/components/schemas/QualityEngine' + - type: 'null' + timestamp: + type: string + format: date-time + description: ISO-8601 timestamp at which evaluation completed. + success: + type: boolean + description: Whether the requirement was satisfied by the data. + details: + type: string + description: Engine-provided human-readable details about the outcome. + error: + type: string + description: >- + Non-null when the evaluation failed to run (e.g. invalid expression, + engine error). When set, `success` is false. + additionalProperties: false + + Error: + type: object + description: RFC 7807 problem document returned for 4xx/5xx responses. + required: [type, title] + properties: + type: + type: string + format: uri-reference + description: A URI reference identifying the error category. + default: about:blank + title: + type: string + description: A short human-readable summary of the error. + status: + type: integer + description: The HTTP status code generated by the origin server. + detail: + type: string + description: A human-readable explanation specific to this occurrence. + instance: + type: string + format: uri-reference + description: A URI reference identifying the specific occurrence. + additionalProperties: false \ No newline at end of file diff --git a/dva-processing/Dockerfile b/dva-processing/Dockerfile index bba2612..4513d4d 100644 --- a/dva-processing/Dockerfile +++ b/dva-processing/Dockerfile @@ -20,6 +20,10 @@ RUN \ # Copy app files COPY ./dva-processing/ /app/ +# Hand-written OpenAPI spec served at /swagger/openapi.yaml +COPY ./docs/spec/dva-processing.yaml /app/openapi.yaml +ENV DVA_PROCESSING_OPENAPI_FILE=/app/openapi.yaml + # Sync project RUN \ --mount=type=cache,target=/root/.cache/uv \ diff --git a/dva-processing/pyproject.toml b/dva-processing/pyproject.toml index 57bef18..5eb5750 100644 --- a/dva-processing/pyproject.toml +++ b/dva-processing/pyproject.toml @@ -7,6 +7,7 @@ authors = [{ name = "FTSRG", email = "bpeter@edu.bme.hu" }] license = "Apache-2.0" requires-python = ">=3.10" dependencies = [ + "chevron>=0.14", "fastapi~=0.136.3", "great-expectations>=1.3.3", "jsonpath-ng>=1.7.0", diff --git a/dva-processing/src/dva_processing/http.py b/dva-processing/src/dva_processing/http.py index 1f23941..04b6652 100644 --- a/dva-processing/src/dva_processing/http.py +++ b/dva-processing/src/dva_processing/http.py @@ -1,13 +1,79 @@ +import os + from fastapi import FastAPI, status from fastapi.exceptions import RequestValidationError -from fastapi.responses import Response +from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, Response from .log import get_logger -from .model import EvaluationResult -from .processing import EvaluationRequest, handle_eval_request +from .model import ( + EvaluateBatchRequest, + EvaluationFromTemplateRequest, + EvaluationResult, +) +from .processing import ( + EvaluationRequest, + TemplateNotFoundError, + handle_eval_batch_request, + handle_eval_from_template_request, + handle_eval_request, +) logger = get_logger() -app = FastAPI() + +_SWAGGER_UI_HTML = """\ + + + + DVA Processing Swagger UI + + + + +
+ + + + + +""" + +app = FastAPI( + docs_url=None, + redoc_url=None, + openapi_url=None, +) + + +@app.get("/swagger", response_class=HTMLResponse, include_in_schema=False) +def swagger_ui() -> HTMLResponse: + return HTMLResponse(content=_SWAGGER_UI_HTML) + + +@app.get( + "/swagger/openapi.yaml", + response_class=PlainTextResponse, + include_in_schema=False, +) +def swagger_spec() -> PlainTextResponse: + spec_path = os.environ.get("DVA_PROCESSING_OPENAPI_FILE", "/app/openapi.yaml") + try: + with open(spec_path, "r", encoding="utf-8") as fh: + content = fh.read() + except FileNotFoundError: + return PlainTextResponse( + content="# spec file not found", status_code=status.HTTP_404_NOT_FOUND + ) + return PlainTextResponse(content=content, media_type="application/yaml") @app.post("/evaluate") @@ -22,6 +88,38 @@ def process_request( return result +@app.post("/evaluate-batch", response_model=list[EvaluationResult]) +def process_batch(request: EvaluateBatchRequest): + logger.info( + "Received batch evaluation request", + vla_keys=list((request.vla or {}).keys()), + ) + return handle_eval_batch_request(request) + + +@app.post("/evaluate/from-template", response_model=EvaluationResult) +def process_from_template(request: EvaluationFromTemplateRequest, response: Response): + logger.info( + "Received evaluate-from-template request", template_id=request.template_id + ) + result = handle_eval_from_template_request(request) + if result.error is not None: + logger.warning("Error during evaluate-from-template", error=result.error) + return result + + +@app.exception_handler(TemplateNotFoundError) +def handle_template_not_found(exc: TemplateNotFoundError): + return JSONResponse( + status_code=status.HTTP_404_NOT_FOUND, + content={ + "type": "template_not_found", + "title": "Template not found", + "detail": str(exc), + }, + ) + + @app.exception_handler(RequestValidationError) def handle_validation_exception( request: EvaluationRequest, diff --git a/dva-processing/src/dva_processing/model.py b/dva-processing/src/dva_processing/model.py index afc82ea..c615733 100644 --- a/dva-processing/src/dva_processing/model.py +++ b/dva-processing/src/dva_processing/model.py @@ -2,7 +2,7 @@ from enum import StrEnum, auto from typing import Any, Optional -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator class CapitalStrEnum(StrEnum): @@ -27,6 +27,11 @@ class EvaluationRequest(BaseModel): data: Any +class EvaluateBatchRequest(BaseModel): + vla: dict[str, Any] + data: Any + + class EvaluationResult(BaseModel): engine: Optional[QualityEngine] timestamp: datetime @@ -35,6 +40,14 @@ class EvaluationResult(BaseModel): error: Optional[str] = None +class EvaluationFromTemplateRequest(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + template_id: Any = Field(alias="templateID") + template_model: dict[str, Any] = Field(alias="templateModel") + data: Any + + class AoVRequest(BaseModel): id: str exchangeID: str diff --git a/dva-processing/src/dva_processing/processing.py b/dva-processing/src/dva_processing/processing.py index 1cf57ca..64a882f 100644 --- a/dva-processing/src/dva_processing/processing.py +++ b/dva-processing/src/dva_processing/processing.py @@ -1,7 +1,9 @@ import json +from os import environ from typing import Any import psycopg as pg +import requests from .config import PG_PASS, PG_URL, PG_USER from .eval import eval_requirement @@ -10,8 +12,11 @@ AoVGenerationRequest, AoVGenerationRequestPayload, AoVRequest, + EvaluateBatchRequest, + EvaluationFromTemplateRequest, EvaluationRequest, EvaluationResult, + QualityEngine, Requirement, ) from .util import now @@ -33,6 +38,108 @@ def handle_eval_request(request: EvaluationRequest) -> EvaluationResult: ) +def _eval_one(data: Any, requirement_dict: dict[str, Any]) -> EvaluationResult: + try: + requirement = Requirement(**requirement_dict) + return eval_requirement(data, requirement) + except Exception as e: + logger.warning( + "An error was thrown during evaluation of a requirement; tolerating", + error=e, + ) + return EvaluationResult( + engine=None, timestamp=now(), success=False, error=str(e) + ) + + +def handle_eval_batch_request(request: EvaluateBatchRequest) -> list[EvaluationResult]: + vla: dict[str, Any] = request.vla or {} + + schema_items: list[Any] = [] + raw_schema = vla.get("schema") + if isinstance(raw_schema, list): + schema_items = raw_schema + elif isinstance(raw_schema, dict): + schema_items = [raw_schema] + + results: list[EvaluationResult] = [] + any_evaluations = False + + for schema_item in schema_items: + if not isinstance(schema_item, dict): + continue + quality = schema_item.get("quality") or [] + if not isinstance(quality, list): + continue + for requirement_dict in quality: + any_evaluations = True + results.append(_eval_one(request.data, requirement_dict)) + + if not any_evaluations: + top_quality = vla.get("quality") + if isinstance(top_quality, list): + for requirement_dict in top_quality: + any_evaluations = True + results.append(_eval_one(request.data, requirement_dict)) + elif isinstance(top_quality, dict): + any_evaluations = True + results.append(_eval_one(request.data, top_quality)) + + if not any_evaluations: + logger.warning("Nothing was evaluated from this VLA") + + return results + + +class TemplateNotFoundError(Exception): + def __init__(self, template_id: str) -> None: + self.template_id = template_id + super().__init__(f"Template {template_id} not found at the VLA Manager API") + + +def handle_eval_from_template_request( + request: EvaluationFromTemplateRequest, +) -> EvaluationResult: + vla_manager_url = environ.get("DVA_VLA_MANAGER_URL", "http://localhost:8000") + try: + template_id = request.template_id + resp = requests.get(f"{vla_manager_url}/template/{template_id}", timeout=10) + if resp.status_code == 404: + raise TemplateNotFoundError(template_id) + resp.raise_for_status() + template = resp.json() + except TemplateNotFoundError: + raise + except Exception as e: + return EvaluationResult( + engine=None, timestamp=now(), success=False, error=str(e) + ) + + em = template["evaluationMethod"] + try: + import chevron + + rendered = chevron.render( + em["implementationTemplate"], request.template_model + ) + except Exception as e: + return EvaluationResult( + engine=None, timestamp=now(), success=False, + error=f"Failed to render template: {e}", + ) + + try: + engine = QualityEngine(em["engine"].upper()) + except ValueError: + return EvaluationResult( + engine=None, timestamp=now(), success=False, + error=f"Unknown engine '{em['engine']}' in template", + ) + + requirement = Requirement(implementation=rendered, engine=engine) + return eval_requirement(request.data, requirement) + + def handle_aov_request(request: AoVRequest) -> AoVGenerationRequest: logger.debug("Handling an AoV request", request=request) contract: dict[str, Any] = request.contract