Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Backend/db/migrations/003_post_mortem_fields.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
-- Campos para post-mortem editable/exportable por incidente.
ALTER TABLE incidents
ADD COLUMN IF NOT EXISTS post_mortem_md TEXT,
ADD COLUMN IF NOT EXISTS post_mortem_updated_at TIMESTAMPTZ;
100 changes: 99 additions & 1 deletion Backend/routers/incidents.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from typing import Literal, Optional

from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from fastapi.responses import JSONResponse, PlainTextResponse
from pydantic import BaseModel, Field

from auth import get_current_user
Expand All @@ -10,6 +11,12 @@
from services.agents.memory.incidents import query_incidents
from services.agents.memory.runbooks import query_runbooks
from services.langgraph_engine import run_langgraph_engine
from services.postmortem.formatters import format_incident_export_markdown
from services.postmortem.service import (
build_incident_export_payload,
generate_post_mortem_for_incident,
save_post_mortem,
)
from services.prometheus import get_container_metrics, get_postgres_metrics

router = APIRouter(prefix="/api/incidents", tags=["incidents"])
Expand Down Expand Up @@ -74,17 +81,108 @@ async def get_incident(incident_id: str, user=Depends(get_current_user)):
async def update_incident_status(
incident_id: str,
body: IncidentStatusUpdate,
background_tasks: BackgroundTasks,
user=Depends(get_current_user),
):
update_fields = {"status": body.status}
if body.status == "resolved":
from datetime import datetime, timezone
update_fields["resolved_at"] = datetime.now(tz=timezone.utc).isoformat()

response = (
supabase.table("incidents")
.update({"status": body.status})
.update(update_fields)
.eq("id", incident_id)
.execute()
)

if body.status == "resolved":
background_tasks.add_task(generate_post_mortem_for_incident, incident_id)

return response.data


class UpdatePostMortemRequest(BaseModel):
content: str = Field(..., min_length=1, max_length=100000)


@router.get("/{incident_id}/post-mortem")
async def get_post_mortem(incident_id: str, user=Depends(get_current_user)):
try:
response = (
supabase.table("incidents")
.select("id,status,post_mortem_md,post_mortem_updated_at")
.eq("id", incident_id)
.single()
.execute()
)
except Exception:
raise HTTPException(status_code=404, detail="Incidente no encontrado")

incident = response.data
if not incident:
raise HTTPException(status_code=404, detail="Incidente no encontrado")

content = (incident.get("post_mortem_md") or "").strip()
generated = False
if not content and incident.get("status") == "resolved":
generated_content = generate_post_mortem_for_incident(incident_id)
content = (generated_content or "").strip()
generated = bool(content)

return {
"incident_id": incident_id,
"status": incident.get("status"),
"content": content,
"updated_at": incident.get("post_mortem_updated_at"),
"generated": generated,
}


@router.put("/{incident_id}/post-mortem")
async def update_post_mortem(
incident_id: str,
body: UpdatePostMortemRequest,
user=Depends(get_current_user),
):
try:
exists = (
supabase.table("incidents")
.select("id")
.eq("id", incident_id)
.single()
.execute()
)
except Exception:
raise HTTPException(status_code=404, detail="Incidente no encontrado")

if not exists.data:
raise HTTPException(status_code=404, detail="Incidente no encontrado")

save_post_mortem(incident_id, body.content)
return {"incident_id": incident_id, "saved": True}


@router.get("/{incident_id}/export")
async def export_incident(
incident_id: str,
format: str = "json",
user=Depends(get_current_user),
):
payload = build_incident_export_payload(incident_id)
if not payload:
raise HTTPException(status_code=404, detail="Incidente no encontrado")

fmt = (format or "json").lower()
if fmt == "json":
return JSONResponse(content=payload)
if fmt == "markdown":
md = format_incident_export_markdown(payload)
return PlainTextResponse(content=md, media_type="text/markdown")

raise HTTPException(status_code=400, detail="Formato no soportado. Usa 'json' o 'markdown'.")


class CreateIncidentManual(BaseModel):
title: str = Field(..., min_length=1, max_length=200)
target: str = Field(..., min_length=1, max_length=100)
Expand Down
2 changes: 2 additions & 0 deletions Backend/services/alert_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import requests

from db.supabase_client import supabase
from services.postmortem.service import generate_post_mortem_for_incident

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -319,5 +320,6 @@ def _resolve_incident(target: str) -> None:
from services.incident_events import record_event
for row in response.data:
record_event(row["id"], "resolved")
generate_post_mortem_for_incident(row["id"])
except Exception as e:
logger.error(f"Error resolviendo incidente para '{target}': {e}")
11 changes: 11 additions & 0 deletions Backend/services/postmortem/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
from services.postmortem.service import (
build_incident_export_payload,
generate_post_mortem_for_incident,
save_post_mortem,
)

__all__ = [
"build_incident_export_payload",
"generate_post_mortem_for_incident",
"save_post_mortem",
]
122 changes: 122 additions & 0 deletions Backend/services/postmortem/formatters.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
from datetime import datetime
from typing import Any


def parse_iso(value: str | None) -> datetime | None:
if not value:
return None
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except Exception:
return None


def format_mttr(created_at: str | None, resolved_at: str | None) -> str:
start = parse_iso(created_at)
end = parse_iso(resolved_at)
if not start or not end:
return "No disponible"

delta = end - start
total_sec = int(delta.total_seconds())
if total_sec < 0:
return "No disponible"

mins, sec = divmod(total_sec, 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {sec}s"
if mins:
return f"{mins}m {sec}s"
return f"{sec}s"


def format_timeline_markdown(events: list[dict[str, Any]]) -> str:
if not events:
return "- Sin eventos registrados"

lines: list[str] = []
for event in events:
status = event.get("status", "unknown")
when = event.get("created_at") or event.get("updated_at") or ""
pretty_when = when.replace("T", " ").replace("+00:00", " UTC") if when else "sin fecha"
lines.append(f"- [{pretty_when}] `{status}`")
return "\n".join(lines)


def format_incident_export_markdown(payload: dict[str, Any]) -> str:
incident = payload.get("metadata", {})
evidence = payload.get("evidence", {})
timeline = payload.get("timeline", [])
decisions = payload.get("decisions", [])
actions = payload.get("actions", [])

lines = [
f"# Incident Export: {incident.get('title') or 'Sin título'}",
"",
"## Metadata",
f"- ID: `{incident.get('id', 'N/A')}`",
f"- Severity: `{incident.get('severity', 'N/A')}`",
f"- Status: `{incident.get('status', 'N/A')}`",
f"- Source type: `{incident.get('source_type', 'N/A')}`",
f"- Target: `{incident.get('target', 'N/A')}`",
f"- Created at: `{incident.get('created_at', 'N/A')}`",
f"- Resolved at: `{incident.get('resolved_at', 'N/A')}`",
f"- MTTR: `{incident.get('mttr', 'No disponible')}`",
"",
"## Evidence",
"### Metrics Snapshot",
"```json",
str(evidence.get("metrics_snapshot") or {}),
"```",
"",
"### Logs",
(evidence.get("logs") or "Sin logs")[:4000],
"",
"### Agent Reasoning",
evidence.get("agent_reasoning") or "Sin razonamiento disponible",
"",
"## Timeline",
]

if timeline:
for event in timeline:
lines.append(
f"- [{event.get('created_at', 'N/A')}] `{event.get('status', 'unknown')}`"
)
else:
lines.append("- Sin eventos")

lines.extend(["", "## Decisions"])
if decisions:
for decision in decisions:
detail = decision.get('detail', '')
lines.append(
f"- [{decision.get('at', 'N/A')}]"
f" {decision.get('type', 'decision')}: {detail}"
)
else:
lines.append("- Sin decisiones registradas")

lines.extend(["", "## Actions"])
if actions:
for action in actions:
lines.extend([
f"### {action.get('name', 'action')}",
f"- Command: `{action.get('command', 'N/A')}`",
f"- Executed at: `{action.get('executed_at', 'N/A')}`",
f"- Status: `{action.get('status', 'N/A')}`",
"- Output:",
"```",
(action.get("output") or "")[0:3000],
"```",
"",
])
else:
lines.append("- Sin acciones registradas")

post_mortem = payload.get("post_mortem")
if post_mortem:
lines.extend(["", "## Post-Mortem", post_mortem])

return "\n".join(lines).strip() + "\n"
92 changes: 92 additions & 0 deletions Backend/services/postmortem/generator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
import json
import logging
import os
from typing import Any

from langchain_core.messages import HumanMessage
from langchain_openai import ChatOpenAI

from services.postmortem.formatters import format_timeline_markdown
from services.postmortem.template import render_post_mortem_template

logger = logging.getLogger(__name__)


def _fallback_post_mortem(
incident: dict[str, Any], timeline: list[dict[str, Any]], mttr: str
) -> str:
resolution = (
incident.get("action_result")
or "Se estabilizó el servicio y quedó en estado resolved."
)[:700]
return render_post_mortem_template(
title=incident.get("title") or "Incidente",
summary=(incident.get("logs") or "No hay descripción disponible")[:700],
root_cause="Pendiente de validación manual por el equipo de guardia.",
timeline=format_timeline_markdown(timeline),
resolution=resolution,
mttr=mttr,
lessons_learned=(
"1. Mejorar observabilidad en componentes críticos.\n"
"2. Definir runbooks más específicos por tipo de incidente."
),
preventive_actions=(
"1. Agregar alertas tempranas con umbrales más conservadores.\n"
"2. Revisar capacidad y límites del recurso afectado."
),
)


def generate_post_mortem_with_llm(
incident: dict[str, Any],
timeline: list[dict[str, Any]],
mttr: str,
) -> str:
api_key = os.getenv("OPENAI_API_KEY")
if not api_key:
return _fallback_post_mortem(incident, timeline, mttr)

timeline_text = format_timeline_markdown(timeline)
payload = {
"title": incident.get("title"),
"severity": incident.get("severity"),
"status": incident.get("status"),
"target": incident.get("target"),
"source_type": incident.get("source_type"),
"logs": (incident.get("logs") or "")[:2500],
"agent_reasoning": (incident.get("agent_reasoning") or "")[:3500],
"action_result": (incident.get("action_result") or "")[:1500],
"action_error": (incident.get("action_error") or "")[:1000],
"timeline": timeline_text,
"mttr": mttr,
}

prompt = (
"Genera un post-mortem técnico en español en formato Markdown. "
"Responde SOLO con markdown y usando exactamente estas secciones:\n"
"# Post-Mortem: [Incident Title]\n"
"## Summary\n"
"## Root Cause\n"
"## Timeline\n"
"## Resolution\n"
"## MTTR\n"
"## Lessons Learned\n"
"## Preventive Actions\n\n"
"Debe ser concreto, accionable y no inventar datos fuera del contexto.\n\n"
f"Contexto JSON:\n{json.dumps(payload, ensure_ascii=False)}"
)

try:
llm = ChatOpenAI(
model=os.getenv("OPENAI_MODEL", "gpt-4o-mini"),
temperature=0.2,
api_key=api_key,
timeout=45,
max_retries=1,
)
response = llm.invoke([HumanMessage(content=prompt)])
content = (response.content or "").strip()
return content or _fallback_post_mortem(incident, timeline, mttr)
except Exception as exc:
logger.warning(f"[postmortem.generator] Fallback por error de LLM: {exc}")
return _fallback_post_mortem(incident, timeline, mttr)
Loading
Loading