Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Backend/db/migrations/003_post_mortem_fields.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
-- Campos para post-mortem editable/exportable por incidente.
ALTER TABLE incidents
ADD COLUMN IF NOT EXISTS post_mortem_md TEXT,
ADD COLUMN IF NOT EXISTS post_mortem_updated_at TIMESTAMPTZ;
100 changes: 99 additions & 1 deletion Backend/routers/incidents.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,20 @@

from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
from pydantic import BaseModel, Field
from fastapi.responses import JSONResponse, PlainTextResponse

from auth import get_current_user
from db.supabase_client import supabase
from models.incident import IncidentStatusUpdate
from services.agents.memory.incidents import query_incidents
from services.agents.memory.runbooks import query_runbooks
from services.langgraph_engine import run_langgraph_engine
from services.postmortem.formatters import format_incident_export_markdown
from services.postmortem.service import (
build_incident_export_payload,
generate_post_mortem_for_incident,
save_post_mortem,
)
from services.prometheus import get_container_metrics, get_postgres_metrics

router = APIRouter(prefix="/api/incidents", tags=["incidents"])
Expand Down Expand Up @@ -74,17 +81,108 @@ async def get_incident(incident_id: str, user=Depends(get_current_user)):
async def update_incident_status(
incident_id: str,
body: IncidentStatusUpdate,
background_tasks: BackgroundTasks,
user=Depends(get_current_user),
):
update_fields = {"status": body.status}
if body.status == "resolved":
from datetime import datetime, timezone
update_fields["resolved_at"] = datetime.now(tz=timezone.utc).isoformat()

response = (
supabase.table("incidents")
.update({"status": body.status})
.update(update_fields)
.eq("id", incident_id)
.execute()
)

if body.status == "resolved":
background_tasks.add_task(generate_post_mortem_for_incident, incident_id)

return response.data


class UpdatePostMortemRequest(BaseModel):
content: str = Field(..., min_length=1, max_length=100000)


@router.get("/{incident_id}/post-mortem")
async def get_post_mortem(incident_id: str, user=Depends(get_current_user)):
try:
response = (
supabase.table("incidents")
.select("id,status,post_mortem_md,post_mortem_updated_at")
.eq("id", incident_id)
.single()
.execute()
)
except Exception:
raise HTTPException(status_code=404, detail="Incidente no encontrado")

incident = response.data
if not incident:
raise HTTPException(status_code=404, detail="Incidente no encontrado")

content = (incident.get("post_mortem_md") or "").strip()
generated = False
if not content and incident.get("status") == "resolved":
generated_content = generate_post_mortem_for_incident(incident_id)
content = (generated_content or "").strip()
generated = bool(content)

return {
"incident_id": incident_id,
"status": incident.get("status"),
"content": content,
"updated_at": incident.get("post_mortem_updated_at"),
"generated": generated,
}


@router.put("/{incident_id}/post-mortem")
async def update_post_mortem(
incident_id: str,
body: UpdatePostMortemRequest,
user=Depends(get_current_user),
):
try:
exists = (
supabase.table("incidents")
.select("id")
.eq("id", incident_id)
.single()
.execute()
)
except Exception:
raise HTTPException(status_code=404, detail="Incidente no encontrado")

if not exists.data:
raise HTTPException(status_code=404, detail="Incidente no encontrado")

save_post_mortem(incident_id, body.content)
return {"incident_id": incident_id, "saved": True}


@router.get("/{incident_id}/export")
async def export_incident(
incident_id: str,
format: str = "json",
user=Depends(get_current_user),
):
payload = build_incident_export_payload(incident_id)
if not payload:
raise HTTPException(status_code=404, detail="Incidente no encontrado")

fmt = (format or "json").lower()
if fmt == "json":
return JSONResponse(content=payload)
if fmt == "markdown":
md = format_incident_export_markdown(payload)
return PlainTextResponse(content=md, media_type="text/markdown")

raise HTTPException(status_code=400, detail="Formato no soportado. Usa 'json' o 'markdown'.")


class CreateIncidentManual(BaseModel):
title: str = Field(..., min_length=1, max_length=200)
target: str = Field(..., min_length=1, max_length=100)
Expand Down
2 changes: 2 additions & 0 deletions Backend/services/alert_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import requests

from db.supabase_client import supabase
from services.postmortem.service import generate_post_mortem_for_incident

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -319,5 +320,6 @@ def _resolve_incident(target: str) -> None:
from services.incident_events import record_event
for row in response.data:
record_event(row["id"], "resolved")
generate_post_mortem_for_incident(row["id"])
except Exception as e:
logger.error(f"Error resolviendo incidente para '{target}': {e}")
11 changes: 11 additions & 0 deletions Backend/services/postmortem/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
from services.postmortem.service import (
build_incident_export_payload,
generate_post_mortem_for_incident,
save_post_mortem,
)

__all__ = [
"build_incident_export_payload",
"generate_post_mortem_for_incident",
"save_post_mortem",
]
120 changes: 120 additions & 0 deletions Backend/services/postmortem/formatters.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
from datetime import datetime, timezone
from typing import Any


def parse_iso(value: str | None) -> datetime | None:
if not value:
return None
try:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
except Exception:
return None


def format_mttr(created_at: str | None, resolved_at: str | None) -> str:
start = parse_iso(created_at)
end = parse_iso(resolved_at)
if not start or not end:
return "No disponible"

delta = end - start
total_sec = int(delta.total_seconds())
if total_sec < 0:
return "No disponible"

mins, sec = divmod(total_sec, 60)
hrs, mins = divmod(mins, 60)
if hrs:
return f"{hrs}h {mins}m {sec}s"
if mins:
return f"{mins}m {sec}s"
return f"{sec}s"


def format_timeline_markdown(events: list[dict[str, Any]]) -> str:
if not events:
return "- Sin eventos registrados"

lines: list[str] = []
for event in events:
status = event.get("status", "unknown")
when = event.get("created_at") or event.get("updated_at") or ""
pretty_when = when.replace("T", " ").replace("+00:00", " UTC") if when else "sin fecha"
lines.append(f"- [{pretty_when}] `{status}`")
return "\n".join(lines)


def format_incident_export_markdown(payload: dict[str, Any]) -> str:
incident = payload.get("metadata", {})
evidence = payload.get("evidence", {})
timeline = payload.get("timeline", [])
decisions = payload.get("decisions", [])
actions = payload.get("actions", [])

lines = [
f"# Incident Export: {incident.get('title') or 'Sin título'}",
"",
"## Metadata",
f"- ID: `{incident.get('id', 'N/A')}`",
f"- Severity: `{incident.get('severity', 'N/A')}`",
f"- Status: `{incident.get('status', 'N/A')}`",
f"- Source type: `{incident.get('source_type', 'N/A')}`",
f"- Target: `{incident.get('target', 'N/A')}`",
f"- Created at: `{incident.get('created_at', 'N/A')}`",
f"- Resolved at: `{incident.get('resolved_at', 'N/A')}`",
f"- MTTR: `{incident.get('mttr', 'No disponible')}`",
"",
"## Evidence",
"### Metrics Snapshot",
"```json",
str(evidence.get("metrics_snapshot") or {}),
"```",
"",
"### Logs",
(evidence.get("logs") or "Sin logs")[:4000],
"",
"### Agent Reasoning",
evidence.get("agent_reasoning") or "Sin razonamiento disponible",
"",
"## Timeline",
]

if timeline:
for event in timeline:
lines.append(
f"- [{event.get('created_at', 'N/A')}] `{event.get('status', 'unknown')}`"
)
else:
lines.append("- Sin eventos")

lines.extend(["", "## Decisions"])
if decisions:
for decision in decisions:
lines.append(
f"- [{decision.get('at', 'N/A')}] {decision.get('type', 'decision')}: {decision.get('detail', '')}"
)
else:
lines.append("- Sin decisiones registradas")

lines.extend(["", "## Actions"])
if actions:
for action in actions:
lines.extend([
f"### {action.get('name', 'action')}",
f"- Command: `{action.get('command', 'N/A')}`",
f"- Executed at: `{action.get('executed_at', 'N/A')}`",
f"- Status: `{action.get('status', 'N/A')}`",
"- Output:",
"```",
(action.get("output") or "")[0:3000],
"```",
"",
])
else:
lines.append("- Sin acciones registradas")

post_mortem = payload.get("post_mortem")
if post_mortem:
lines.extend(["", "## Post-Mortem", post_mortem])

return "\n".join(lines).strip() + "\n"
80 changes: 80 additions & 0 deletions Backend/services/postmortem/generator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import json
import logging
import os
from typing import Any

from langchain_core.messages import HumanMessage
from langchain_openai import ChatOpenAI

from services.postmortem.formatters import format_timeline_markdown
from services.postmortem.template import render_post_mortem_template

logger = logging.getLogger(__name__)


def _fallback_post_mortem(incident: dict[str, Any], timeline: list[dict[str, Any]], mttr: str) -> str:
return render_post_mortem_template(
title=incident.get("title") or "Incidente",
summary=(incident.get("logs") or "No hay descripción disponible")[:700],
root_cause="Pendiente de validación manual por el equipo de guardia.",
timeline=format_timeline_markdown(timeline),
resolution=(incident.get("action_result") or "Se estabilizó el servicio y quedó en estado resolved.")[:700],
mttr=mttr,
lessons_learned="1. Mejorar observabilidad en componentes críticos.\n2. Definir runbooks más específicos por tipo de incidente.",
preventive_actions="1. Agregar alertas tempranas con umbrales más conservadores.\n2. Revisar capacidad y límites del recurso afectado.",
)


def generate_post_mortem_with_llm(
incident: dict[str, Any],
timeline: list[dict[str, Any]],
mttr: str,
) -> str:
api_key = os.getenv("OPENAI_API_KEY")
if not api_key:
return _fallback_post_mortem(incident, timeline, mttr)

timeline_text = format_timeline_markdown(timeline)
payload = {
"title": incident.get("title"),
"severity": incident.get("severity"),
"status": incident.get("status"),
"target": incident.get("target"),
"source_type": incident.get("source_type"),
"logs": (incident.get("logs") or "")[:2500],
"agent_reasoning": (incident.get("agent_reasoning") or "")[:3500],
"action_result": (incident.get("action_result") or "")[:1500],
"action_error": (incident.get("action_error") or "")[:1000],
"timeline": timeline_text,
"mttr": mttr,
}

prompt = (
"Genera un post-mortem técnico en español en formato Markdown. "
"Responde SOLO con markdown y usando exactamente estas secciones:\n"
"# Post-Mortem: [Incident Title]\n"
"## Summary\n"
"## Root Cause\n"
"## Timeline\n"
"## Resolution\n"
"## MTTR\n"
"## Lessons Learned\n"
"## Preventive Actions\n\n"
"Debe ser concreto, accionable y no inventar datos fuera del contexto.\n\n"
f"Contexto JSON:\n{json.dumps(payload, ensure_ascii=False)}"
)

try:
llm = ChatOpenAI(
model=os.getenv("OPENAI_MODEL", "gpt-4o-mini"),
temperature=0.2,
api_key=api_key,
timeout=45,
max_retries=1,
)
response = llm.invoke([HumanMessage(content=prompt)])
content = (response.content or "").strip()
return content or _fallback_post_mortem(incident, timeline, mttr)
except Exception as exc:
logger.warning(f"[postmortem.generator] Fallback por error de LLM: {exc}")
return _fallback_post_mortem(incident, timeline, mttr)
33 changes: 33 additions & 0 deletions Backend/services/postmortem/knowledge_base.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import logging

from services.agents.memory.chroma_client import get_or_create_collection

logger = logging.getLogger(__name__)

_POST_MORTEM_COLLECTION = "postmortems-incidents"


def store_post_mortem_in_kb(incident: dict, content_md: str) -> None:
incident_id = str(incident.get("id") or "")
if not incident_id or not content_md.strip():
return

metadata = {
"incident_id": incident_id,
"title": incident.get("title") or "",
"severity": incident.get("severity") or "",
"status": incident.get("status") or "",
"target": incident.get("target") or "",
"source_type": incident.get("source_type") or "",
"resolved_at": incident.get("resolved_at") or "",
}

try:
collection = get_or_create_collection(_POST_MORTEM_COLLECTION)
collection.upsert(
ids=[incident_id],
documents=[content_md[:12000]],
metadatas=[metadata],
)
except Exception as exc:
logger.warning(f"[postmortem.kb] No se pudo guardar en ChromaDB {incident_id[:8]}: {exc}")
Loading
Loading