backend_cli_validator.py

from __future__ import annotations

import argparse
import json
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, Iterable, List, Tuple

from rich.console import Console
from rich.panel import Panel
from rich.table import Table

from app import predict_emotion


console = Console()


@dataclass(frozen=True)
class TestCase:
    name: str
    text: str
    expected_label: str


@dataclass
class CaseResult:
    case: TestCase
    predicted_label: str
    confidence: float
    pass_label: bool
    pass_confidence: bool
    passed: bool
    error: str | None = None


def default_test_cases() -> List[TestCase]:
    return [
        TestCase(
            name="Joy baseline",
            text=(
                "I feel so joyful, grateful, and excited today. "
                "Everything is amazing and I am genuinely happy."
            ),
            expected_label="joy",
        ),
        TestCase(
            name="Sadness baseline",
            text=(
                "I feel deeply sad, heartbroken, and empty. "
                "I miss the past and I cannot stop crying."
            ),
            expected_label="sadness",
        ),
        TestCase(
            name="Anger baseline",
            text=(
                "I am furious and angry right now, this is disgusting and completely "
                "unacceptable, I am outraged and mad."
            ),
            expected_label="anger",
        ),
        TestCase(
            name="Fear baseline",
            text=(
                "I am terrified and scared, my heart is racing and I feel panic and fear "
                "about what might happen."
            ),
            expected_label="fear",
        ),
        TestCase(
            name="Neutral baseline",
            text=(
                "The meeting starts at 10:00 AM, followed by a status update at 11:30 and "
                "documentation review at 2:00 PM."
            ),
            expected_label="neutral",
        ),
        TestCase(
            name="Surprise baseline",
            text=(
                "I am shocked and surprised, I never expected this to happen and I am "
                "completely speechless."
            ),
            expected_label="surprise",
        ),
        TestCase(
            name="Mixed emotion: sad then happy",
            text="I was sad but now I am happy",
            expected_label="joy",
        ),
        TestCase(
            name="Edge case: burned out",
            text="I am tired and done with everything",
            expected_label="sadness",
        ),
    ]


def load_extra_cases(json_path: Path) -> List[TestCase]:
    with json_path.open("r", encoding="utf-8") as f:
        payload = json.load(f)

    if not isinstance(payload, list):
        raise ValueError("Custom test-case file must contain a JSON list")

    extra_cases: List[TestCase] = []
    for idx, item in enumerate(payload, start=1):
        if not isinstance(item, dict):
            raise ValueError(f"Case #{idx} must be a JSON object")

        name = str(item.get("name", f"Custom case {idx}")).strip()
        text = str(item.get("text", "")).strip()
        expected = str(item.get("expected_label", "")).strip().lower()

        if not text:
            raise ValueError(f"Case '{name}' has empty text")
        if not expected:
            raise ValueError(f"Case '{name}' has empty expected_label")

        extra_cases.append(TestCase(name=name, text=text, expected_label=expected))

    return extra_cases


def validate_prediction_schema(result: Dict[str, Any]) -> Tuple[bool, str]:
    required_keys = {"label", "confidence", "probabilities"}
    missing = required_keys.difference(result.keys())
    if missing:
        return False, f"Missing keys: {sorted(missing)}"

    if not isinstance(result["label"], str):
        return False, "'label' must be a string"

    if not isinstance(result["confidence"], (int, float)):
        return False, "'confidence' must be numeric"

    probs = result["probabilities"]
    if not isinstance(probs, dict):
        return False, "'probabilities' must be a dictionary"

    if result["label"] not in probs:
        return False, "Predicted label is missing in probabilities"

    return True, "ok"


def evaluate_case(case: TestCase, min_confidence: float) -> CaseResult:
    try:
        output = predict_emotion(case.text)
        is_valid, message = validate_prediction_schema(output)
        if not is_valid:
            return CaseResult(
                case=case,
                predicted_label="<invalid>",
                confidence=0.0,
                pass_label=False,
                pass_confidence=False,
                passed=False,
                error=message,
            )

        predicted = str(output["label"]).lower().strip()
        confidence = float(output["confidence"])

        pass_label = predicted == case.expected_label
        pass_confidence = confidence > min_confidence
        passed = pass_label and pass_confidence

        return CaseResult(
            case=case,
            predicted_label=predicted,
            confidence=confidence,
            pass_label=pass_label,
            pass_confidence=pass_confidence,
            passed=passed,
        )
    except Exception as exc:  # pragma: no cover - defensive CLI behavior
        return CaseResult(
            case=case,
            predicted_label="<error>",
            confidence=0.0,
            pass_label=False,
            pass_confidence=False,
            passed=False,
            error=str(exc),
        )


def evaluate_suite(cases: Iterable[TestCase], min_confidence: float) -> List[CaseResult]:
    return [evaluate_case(case, min_confidence) for case in cases]


def print_results_table(results: List[CaseResult], min_confidence: float) -> None:
    table = Table(title="Emotion Backend Validation Results", show_lines=True)
    table.add_column("Case", style="cyan", no_wrap=False)
    table.add_column("Expected", style="magenta")
    table.add_column("Predicted", style="white")
    table.add_column("Confidence", justify="right")
    table.add_column("Label Check", justify="center")
    table.add_column(f"Conf > {min_confidence:.2f}", justify="center")
    table.add_column("Status", justify="center")

    for r in results:
        conf_text = f"{r.confidence:.4f}"
        label_check = "[green]PASS[/green]" if r.pass_label else "[red]FAIL[/red]"
        conf_check = "[green]PASS[/green]" if r.pass_confidence else "[red]FAIL[/red]"
        status = "[bold green]PASS[/bold green]" if r.passed else "[bold red]FAIL[/bold red]"

        if r.error:
            conf_text = "-"
            conf_check = "[red]FAIL[/red]"
            status = "[bold red]FAIL[/bold red]"

        predicted_text = r.predicted_label
        if r.error:
            predicted_text = f"{predicted_text} ({r.error})"

        table.add_row(
            r.case.name,
            r.case.expected_label,
            predicted_text,
            conf_text,
            label_check,
            conf_check,
            status,
        )

    console.print(table)


def print_summary(results: List[CaseResult]) -> int:
    total = len(results)
    passed = sum(1 for r in results if r.passed)
    failed = total - passed
    accuracy = (passed / total * 100.0) if total else 0.0

    summary = (
        f"Total cases: {total}\n"
        f"Passed: {passed}\n"
        f"Failed: {failed}\n"
        f"Accuracy: {accuracy:.2f}%"
    )

    style = "green" if failed == 0 else "yellow"
    console.print(Panel(summary, title="Summary", border_style=style))

    if failed:
        failed_table = Table(title="Failed Cases", show_lines=True)
        failed_table.add_column("Case", style="cyan")
        failed_table.add_column("Expected", style="magenta")
        failed_table.add_column("Predicted", style="white")
        failed_table.add_column("Confidence", justify="right")
        failed_table.add_column("Reason", style="red")

        for r in results:
            if r.passed:
                continue

            reasons: List[str] = []
            if r.error:
                reasons.append(r.error)
            if not r.pass_label:
                reasons.append("label mismatch")
            if not r.pass_confidence:
                reasons.append("low confidence")

            failed_table.add_row(
                r.case.name,
                r.case.expected_label,
                r.predicted_label,
                f"{r.confidence:.4f}",
                ", ".join(reasons),
            )

        console.print(failed_table)

    return failed


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="CLI validator for predict_emotion backend output"
    )
    parser.add_argument(
        "--min-confidence",
        type=float,
        default=0.5,
        help="Minimum required confidence for each test case",
    )
    parser.add_argument(
        "--cases-file",
        type=Path,
        default=None,
        help="Optional JSON file with additional test cases",
    )
    return parser.parse_args()


def main() -> int:
    args = parse_args()

    if args.min_confidence < 0.0 or args.min_confidence > 1.0:
        console.print("[bold red]--min-confidence must be between 0.0 and 1.0[/bold red]")
        return 2

    test_cases = default_test_cases()

    if args.cases_file is not None:
        if not args.cases_file.exists():
            console.print(f"[bold red]Cases file not found:[/bold red] {args.cases_file}")
            return 2
        extra_cases = load_extra_cases(args.cases_file)
        test_cases.extend(extra_cases)

    console.print(
        Panel(
            f"Running {len(test_cases)} test cases with minimum confidence > {args.min_confidence:.2f}",
            title="Emotion Backend CLI Validator",
            border_style="blue",
        )
    )

    results = evaluate_suite(test_cases, args.min_confidence)
    print_results_table(results, args.min_confidence)
    failed = print_summary(results)

    return 0 if failed == 0 else 1


if __name__ == "__main__":
    sys.exit(main())