server.js

require("dotenv").config();
const express = require("express");
const multer = require("multer");
const pdfParse = require("pdf-parse");
const mammoth = require("mammoth");
const crypto = require("crypto");
const fs = require("fs/promises");
const path = require("path");
const mcpClient = require("./mcpClient");
const logger = require("./logger");

const app = express();
const PORT = process.env.PORT || 3000;
const ROOT = __dirname;
const STORAGE_DIR = path.join(ROOT, "storage");
const UPLOAD_DIR = path.join(STORAGE_DIR, "uploads");
const NOTES_PATH = path.join(STORAGE_DIR, "notes.json");
const SECRETS_PATH = path.join(STORAGE_DIR, "provider-secrets.json");

// In-memory cache to avoid repeated disk reads
let cachedNotes = [];
let cachedSecrets = {};

async function initInMemoryCache() {
  await ensureStorage();
  try {
    const raw = await fs.readFile(NOTES_PATH, "utf8");
    cachedNotes = JSON.parse(raw);
  } catch (e) {
    cachedNotes = [];
  }
  try {
    const raw = await fs.readFile(SECRETS_PATH, "utf8");
    cachedSecrets = JSON.parse(raw);
  } catch (e) {
    cachedSecrets = {};
  }
}

const upload = multer({
  storage: multer.memoryStorage(),
  limits: {
    files: 10,
    fileSize: 20 * 1024 * 1024
  }
});

const STOP_WORDS = new Set([
  "about", "after", "again", "against", "all", "also", "an", "and", "any", "are", "because", "been",
  "before", "being", "between", "both", "but", "can", "could", "did", "does", "doing", "during", "each",
  "few", "for", "from", "further", "had", "has", "have", "having", "her", "here", "hers", "him", "himself",
  "his", "how", "into", "its", "itself", "just", "more", "most", "not", "now", "off", "once", "only", "other",
  "our", "ours", "ourselves", "out", "over", "same", "should", "some", "such", "than", "that", "the", "their",
  "theirs", "them", "themselves", "then", "there", "these", "they", "this", "those", "through", "under", "until",
  "very", "was", "were", "what", "when", "where", "which", "while", "who", "will", "with", "would", "your",
  "yours", "yourself", "yourselves"
]);

app.use(express.json({ limit: "25mb" }));
app.use(express.urlencoded({ extended: true }));

// CORS & static serving
app.use((req, res, next) => {
  res.setHeader("Access-Control-Allow-Origin", "*");
  res.setHeader("Access-Control-Allow-Methods", "GET,POST,DELETE,OPTIONS");
  res.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization");
  if (req.method === "OPTIONS") {
    res.status(204).end();
    return;
  }
  next();
});
app.use(express.static(ROOT));

// ─── Routes ────────────────────────────────────────────────────────────────────

app.get("/api/health", async (_req, res) => {
  try {
    await ensureStorage();
    res.json({ ok: true, mcpOnline: mcpClient.isOnline() });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

app.get("/api/system-accounts", async (req, res) => {
  const fssync = require("fs");
  const pathlib = require("path");
  const { execSync } = require("child_process");

  const accounts = [];
  const osUser = process.env.USERNAME || "Google User";

  // 1. Try Git global config
  try {
    const gitEmail = execSync("git config --global user.email", { encoding: "utf8" }).trim();
    const gitName = execSync("git config --global user.name", { encoding: "utf8" }).trim();
    let displayName = gitName;
    if (!displayName || displayName === "mrpunithvenkatsain47139" || /\d/.test(displayName)) {
      displayName = osUser;
    }
    if (gitEmail && gitEmail.includes("@")) {
      accounts.push({ name: displayName || "Git User", email: gitEmail });
    }
  } catch (e) { /* ignore */ }

  // 2. Scan Google Chrome profiles
  const localAppData = process.env.LOCALAPPDATA;
  if (localAppData) {
    const chromeDir = pathlib.join(localAppData, "Google", "Chrome", "User Data");
    if (fssync.existsSync(chromeDir)) {
      const profiles = [
        "Default", "Profile 1", "Profile 2", "Profile 3", "Profile 4",
        "Profile 5", "Profile 6", "Profile 7", "Profile 8", "Profile 9",
        "Profile 10", "Profile 11", "Profile 12", "Profile 13", "Profile 14"
      ];
      profiles.forEach(p => {
        const prefPath = pathlib.join(chromeDir, p, "Preferences");
        if (fssync.existsSync(prefPath)) {
          try {
            const json = JSON.parse(fssync.readFileSync(prefPath, "utf8"));
            let email = null;
            let name = null;
            if (json.google && json.google.services && json.google.services.username) {
              email = json.google.services.username;
            }
            if (json.profile && json.profile.name) name = json.profile.name;
            if (email && email.includes("@")) {
              accounts.push({ name: name || osUser, email });
            }
          } catch (e) {}
        }
      });

      const defaultPref = pathlib.join(chromeDir, "Default", "Preferences");
      if (fssync.existsSync(defaultPref)) {
        try {
          const json = JSON.parse(fssync.readFileSync(defaultPref, "utf8"));
          if (json.profile && json.profile.info_cache) {
            Object.keys(json.profile.info_cache).forEach(key => {
              const item = json.profile.info_cache[key];
              if (item.user_name && item.user_name.includes("@")) {
                accounts.push({
                  name: item.name || item.given_name || osUser,
                  email: item.user_name
                });
              }
            });
          }
        } catch (e) {}
      }
    }
  }

  // De-duplicate case-insensitively
  const uniqueMap = new Map();
  accounts.forEach(acc => {
    const key = acc.email.toLowerCase().trim();
    if (!uniqueMap.has(key)) {
      uniqueMap.set(key, acc);
    }
  });

  res.json(Array.from(uniqueMap.values()));
});

app.get("/api/files", async (_req, res) => {
  try {
    const notes = await readNotes();
    res.json({ notes: notes.map(stripNoteForClient) });
  } catch (error) {
    console.error("Get files error:", error);
    res.status(500).json({ error: error.message });
  }
});

app.post("/api/files/upload", upload.array("files", 10), async (req, res) => {
  try {
    await ensureStorage();

    if (!req.files || req.files.length === 0) {
      res.status(400).json({ error: "No files were uploaded." });
      return;
    }

    const existingNotes = await readNotes();
    const uploaded = [];

    let apiKey = process.env.GEMINI_API_KEY;
    if (!apiKey) {
      const secrets = await readSecrets();
      apiKey = secrets.gemini;
    }

    for (const file of req.files) {
      const id = crypto.randomUUID();
      const extension = path.extname(file.originalname) || guessExtension(file.mimetype);
      const savedName = `${id}${extension}`;
      const absoluteFilePath = path.join(UPLOAD_DIR, savedName);

      await fs.writeFile(absoluteFilePath, file.buffer);

      const extractedText = await extractText(file);
      const analysis = analyzeText(extractedText, file.originalname);

      let quizzes = analysis.quizzes;
      if (apiKey) {
        try {
          console.log(`Generating AI quizzes for note: ${file.originalname}`);
          const aiQuizzes = await generateQuizzesWithAI(extractedText, file.originalname, apiKey);
          if (aiQuizzes && aiQuizzes.length > 0) {
            quizzes = aiQuizzes;
          }
        } catch (err) {
          console.error(`Failed to generate AI quizzes for ${file.originalname}, falling back to local extractor:`, err);
        }
      }

      const note = {
        id,
        name: file.originalname,
        size: file.size,
        mimeType: file.mimetype,
        createdAt: new Date().toISOString(),
        storedFile: savedName,
        extractedText,
        summary: analysis.summary,
        topics: analysis.topics,
        dates: analysis.dates,
        snippets: analysis.snippets,
        quizzes
      };

      // Index chunks to Elasticsearch via MCP if online
      if (mcpClient.isOnline()) {
        const notesIndex = getScopedIndexName("notes", req);
        const chunks = note.snippets || splitIntoChunks(extractedText);
        logger.logAgentDecision("Uploading document chunks to Elastic via MCP", { 
          filename: file.originalname, 
          chunksCount: chunks.length,
          index: notesIndex
        });
        
        for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
          const chunkText = chunks[chunkIndex];
          const chunkDoc = {
            filename: file.originalname,
            uploadTimestamp: note.createdAt,
            documentType: file.mimetype,
            chunkId: `${id}-chunk-${chunkIndex}`,
            content: chunkText,
            topicLabels: note.topics || [],
            userId: req.headers["x-user-uid"] || "global",
            noteId: id
          };
          await indexDocumentMCP(notesIndex, chunkDoc, `${id}-chunk-${chunkIndex}`);
        }
      }

      existingNotes.unshift(note);
      uploaded.push(stripNoteForClient(note));
    }

    await writeNotes(existingNotes);
    res.status(201).json({ notes: uploaded });
  } catch (error) {
    console.error("Upload error:", error);
    res.status(500).json({ error: "Failed to upload files: " + error.message });
  }
});

app.post("/api/notes/index", async (req, res) => {
  try {
    const note = req.body;
    if (!note || !note.id) {
      return res.status(400).json({ error: "Note object is required." });
    }

    // Index chunks to Elasticsearch via MCP if online
    if (mcpClient.isOnline()) {
      const notesIndex = getScopedIndexName("notes", req);
      const chunks = note.snippets || splitIntoChunks(note.extractedText || "");
      logger.logAgentDecision("Indexing document chunks to Elastic via MCP (Firebase bridge)", { 
        filename: note.name, 
        chunksCount: chunks.length,
        index: notesIndex
      });
      
      for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
        const chunkText = chunks[chunkIndex];
        const chunkDoc = {
          filename: note.name,
          uploadTimestamp: note.createdAt || new Date().toISOString(),
          documentType: note.mimeType || "text/plain",
          chunkId: `${note.id}-chunk-${chunkIndex}`,
          content: chunkText,
          topicLabels: note.topics || [],
          userId: req.headers["x-user-uid"] || "global",
          noteId: note.id
        };
        await indexDocumentMCP(notesIndex, chunkDoc, `${note.id}-chunk-${chunkIndex}`);
      }
    }
    res.json({ ok: true });
  } catch (error) {
    logger.logError("Failed to index note via bridge", error);
    res.status(500).json({ error: error.message });
  }
});

app.delete("/api/files/:id", async (req, res) => {
  try {
    await ensureStorage();
    const notes = await readNotes();
    const noteIndex = notes.findIndex((note) => note.id === req.params.id);

    if (noteIndex === -1) {
      res.status(404).json({ error: "Note not found." });
      return;
    }

    const [note] = notes.splice(noteIndex, 1);
    const absoluteFilePath = path.join(UPLOAD_DIR, note.storedFile);

    await fs.rm(absoluteFilePath, { force: true });
    await writeNotes(notes);

    // Delete note chunks from Elasticsearch via MCP
    if (mcpClient.isOnline()) {
      const notesIndex = getScopedIndexName("notes", req);
      try {
        await mcpClient.callTool("delete_by_query", {
          index: notesIndex,
          query: {
            match: {
              noteId: req.params.id
            }
          }
        });
        logger.logAgentDecision("Deleted note chunks from Elasticsearch", { noteId: req.params.id, index: notesIndex });
      } catch (err) {
        logger.logError(`Failed to delete note chunks from index ${notesIndex}`, err);
      }
    }

    res.json({ ok: true });
  } catch (error) {
    console.error("Delete note error:", error);
    res.status(500).json({ error: "Failed to delete note: " + error.message });
  }
});

app.post("/api/ask", async (req, res) => {
  try {
    await ensureStorage();

    const question = String(req.body.question || "").trim();
    const noteIds = Array.isArray(req.body.noteIds) ? req.body.noteIds.map(String) : [];
    const depth = String(req.body.depth || "balanced").trim();

    if (!question) {
      res.status(400).json({ error: "Question is required." });
      return;
    }

    const notes = await readNotes();
    const selectedNotes = noteIds.length > 0 ? notes.filter((note) => noteIds.includes(note.id)) : notes;

    if (selectedNotes.length === 0) {
      res.status(400).json({ error: "Upload and select at least one note first." });
      return;
    }

    let relevantChunks = [];
    let isMcpUsed = false;

    if (mcpClient.isOnline()) {
      const notesIndex = getScopedIndexName("notes", req);
      const startTime = Date.now();
      logger.logAgentDecision("Attempting Elastic MCP search for grounding context", { question, index: notesIndex });
      
      const esResults = await searchDocumentsMCP(notesIndex, question, ["content"], 8);
      if (esResults && esResults.length > 0) {
        relevantChunks = esResults.map((doc) => ({
          noteId: doc.noteId || "mcp-note",
          noteName: doc.filename || "Note",
          chunk: doc.content || doc.chunk || "",
          score: 1
        }));
        isMcpUsed = true;
        logger.logRetrievalPerformance("search_documents", Date.now() - startTime, esResults.length);
      }
    }

    if (!isMcpUsed) {
      logger.logAgentDecision("Falling back to local note chunk retrieval engine", { question });
      relevantChunks = retrieveRelevantChunks(question, selectedNotes);
    }

    const citations = buildCitations(relevantChunks, selectedNotes);

    let apiKey = process.env.GEMINI_API_KEY;
    if (!apiKey) {
      const secrets = await readSecrets();
      apiKey = secrets.gemini;
    }

    let answer = "";
    let provider = "Local grounded mode";
    let model = "local-grounded";
    let mode = "local";

    if (apiKey) {
      provider = "Gemini Answering (AI Mode)";
      mode = "gemini-api";

      const eligibleModels = await getEligibleModels(apiKey);
      let success = false;
      let lastError = null;

      for (const currentModel of eligibleModels) {
        try {
          console.log(`Attempting backend Gemini call with model: ${currentModel}`);
          answer = await callGeminiBackend(question, relevantChunks, apiKey, depth, currentModel);
          model = currentModel;
          success = true;
          break;
        } catch (err) {
          console.error(`Gemini call failed for model ${currentModel}: ${err.message}`);
          lastError = err;
          continue;
        }
      }

      if (!success) {
        console.error("All Gemini backend model attempts failed, falling back to local synthesis.");
        model = "local-fallback";
        answer = `[AI Answering failed for all models. Last error: ${lastError ? lastError.message : "Unknown error"}]. Falling back to local synthesis:\n\n` +
                 localGroundedAnswer(question, selectedNotes, relevantChunks, depth);
      }
    } else {
      answer = localGroundedAnswer(question, selectedNotes, relevantChunks, depth);
    }

    res.json({ answer, citations, provider, model, mode });
  } catch (error) {
    console.error("Ask query error:", error);
    res.status(500).json({ error: "Failed to process research query: " + error.message });
  }
});

app.post("/api/plan/generate", async (req, res) => {
  try {
    await ensureStorage();

    const prompt = String(req.body.prompt || "").trim();
    const noteIds = Array.isArray(req.body.noteIds) ? req.body.noteIds.map(String) : [];
    const calendarEvents = Array.isArray(req.body.calendarEvents) ? req.body.calendarEvents : [];
    const isReplan = !!req.body.isReplan;
    const todayStr = String(req.body.todayStr || "").trim();

    if (!prompt) {
      res.status(400).json({ error: "Planner prompt is required." });
      return;
    }

    const notes = await readNotes();
    const selectedNotes = noteIds.length > 0 ? notes.filter((note) => noteIds.includes(note.id)) : notes;

    let notesSummary = "";
    let isMcpUsed = false;

    if (mcpClient.isOnline()) {
      const notesIndex = getScopedIndexName("notes", req);
      const searchQuery = prompt || selectedNotes.map(n => n.name).join(" ");
      const startTime = Date.now();
      logger.logAgentDecision("Attempting Elastic MCP search for planner grounding", { searchQuery, index: notesIndex });

      const esResults = await searchDocumentsMCP(notesIndex, searchQuery, ["content"], 10);
      if (esResults && esResults.length > 0) {
        notesSummary = esResults.map(doc => {
          return `Note Name: ${doc.filename || "Study Material"}\nContent Chunk: ${doc.content || ""}\nTopics: ${(doc.topicLabels || []).join(", ")}`;
        }).join("\n\n");
        isMcpUsed = true;
        logger.logRetrievalPerformance("search_documents (planner)", Date.now() - startTime, esResults.length);
      }
    }

    if (!isMcpUsed) {
      logger.logAgentDecision("Falling back to local note configurations for study planner", { noteIds });
      notesSummary = selectedNotes.map((note) => {
        return `Note Name: ${note.name}\nSummary: ${note.summary || "No summary available."}\nTopics: ${(note.topics || []).join(", ") || "No topics extracted."}\nDates: ${(note.dates || []).join(", ") || "None"}`;
      }).join("\n\n");
    }

    const calendarSummary = calendarEvents.map(e => `- ${e.title} on ${e.day} from ${e.start} to ${e.end}`).join("\n");

    let apiKey = process.env.GEMINI_API_KEY;
    if (!apiKey) {
      const secrets = await readSecrets();
      apiKey = secrets.gemini;
    }

    let plan = [];
    let provider = "Local planner fallback";
    let model = "local-grounded";

    if (apiKey) {
      provider = "Gemini Planner (AI Mode)";
      const eligibleModels = await getEligibleModels(apiKey);
      let success = false;
      let lastError = null;

      for (const currentModel of eligibleModels) {
        try {
          console.log(`Attempting backend Gemini planner call with model: ${currentModel}`);
          const rawResponse = await callGeminiPlannerBackend(prompt, notesSummary, calendarSummary, apiKey, currentModel, isReplan, todayStr);

          let cleanedText = rawResponse.trim();
          if (cleanedText.startsWith("```")) {
            cleanedText = cleanedText.replace(/^```[a-zA-Z]*\n/, "").replace(/\n```$/, "");
          }
          cleanedText = cleanedText.trim();

          plan = JSON.parse(cleanedText);
          model = currentModel;
          success = true;
          break;
        } catch (err) {
          console.error(`Gemini planner call failed for model ${currentModel}: ${err.message}`);
          lastError = err;
          continue;
        }
      }

      if (!success) {
        throw lastError || new Error("Failed to generate plan with Gemini");
      }
    } else {
      throw new Error("No Gemini API key configured on backend.");
    }

    res.json({ plan, provider, model });
  } catch (error) {
    console.error("Plan generate query error:", error);
    res.status(500).json({ error: "Failed to generate study plan: " + error.message });
  }
});

app.post("/api/calendar/parse", async (req, res) => {
  try {
    await ensureStorage();
    const text = String(req.body.text || "").trim();
    if (!text) {
      res.status(400).json({ error: "Calendar text is required." });
      return;
    }

    let apiKey = process.env.GEMINI_API_KEY;
    if (!apiKey) {
      const secrets = await readSecrets();
      apiKey = secrets.gemini;
    }

    let events = [];
    if (apiKey) {
      const eligibleModels = await getEligibleModels(apiKey);
      let success = false;
      let lastError = null;

      const prompt = `Analyze the following schedule text and extract all planned events.
Format them as a JSON array of objects, where each object represents an event with the following fields:
- "title": Concise event title (e.g., "Chemistry Lab")
- "day": The day of the week (e.g., "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday")
- "start": Start time in 24-hour HH:MM format (e.g., "14:00")
- "end": End time in 24-hour HH:MM format (e.g., "16:00")
- "color": A hex color string for the event (use standard Google colors: "#1a73e8" for blue, "#ea4335" for red, "#fbbc05" for yellow, "#34a853" for green)

If no duration/end time is specified, assume the event lasts 1 hour.
Respond ONLY with the raw JSON array. Do not include markdown formatting or explanations.

Schedule Text:
"""
${text}
"""`;

      for (const currentModel of eligibleModels) {
        try {
          const controller = new AbortController();
          const timeoutId = setTimeout(() => controller.abort(), 20000);

          const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${currentModel}:generateContent?key=${apiKey}`, {
            method: "POST",
            headers: { "Content-Type": "application/json" },
            body: JSON.stringify({
              contents: [{ parts: [{ text: prompt }] }],
              generationConfig: { temperature: 0.1 }
            }),
            signal: controller.signal
          });

          clearTimeout(timeoutId);
          const data = await response.json();
          if (!response.ok) {
            throw new Error(data.error?.message || "Gemini API call failed");
          }

          const rawResponse = data.candidates?.[0]?.content?.parts?.[0]?.text || "[]";
          let cleanedText = rawResponse.trim();
          if (cleanedText.startsWith("```")) {
            cleanedText = cleanedText.replace(/^```[a-zA-Z]*\n/, "").replace(/\n```$/, "");
          }
          cleanedText = cleanedText.trim();
          events = JSON.parse(cleanedText);
          success = true;
          break;
        } catch (err) {
          console.error(`Calendar parsing failed with model ${currentModel}: ${err.message}`);
          lastError = err;
          continue;
        }
      }
      if (!success) throw lastError || new Error("Failed to parse calendar text with Gemini");
    } else {
      throw new Error("No Gemini API key configured on backend.");
    }

    res.json({ events });
  } catch (error) {
    console.error("Calendar parse error:", error);
    res.status(500).json({ error: "Failed to parse calendar schedule: " + error.message });
  }
});

app.post("/api/quiz", async (req, res) => {
  try {
    await ensureStorage();
    const noteIds = Array.isArray(req.body.noteIds) ? req.body.noteIds.map(String) : [];
    const notes = await readNotes();
    const selectedNotes = noteIds.length > 0 ? notes.filter((note) => noteIds.includes(note.id)) : notes;

    let allQuizzes = [];
    for (const note of selectedNotes) {
      if (note.quizzes && note.quizzes.length > 0) {
        allQuizzes.push(...note.quizzes);
      }
    }
    allQuizzes.sort(() => Math.random() - 0.5);

    res.json({ quizzes: allQuizzes });
  } catch (error) {
    console.error("Quiz generation error:", error);
    res.status(500).json({ error: "Failed to generate quiz: " + error.message });
  }
});

app.post("/api/quiz/generate", async (req, res) => {
  try {
    await ensureStorage();
    const noteIds = Array.isArray(req.body.noteIds) ? req.body.noteIds.map(String) : [];
    const prompt = String(req.body.prompt || "").trim();

    const notes = await readNotes();
    const selectedNotes = noteIds.length > 0 ? notes.filter((note) => noteIds.includes(note.id)) : notes;

    if (selectedNotes.length === 0) {
      res.status(400).json({ error: "Select at least one note first." });
      return;
    }

    let combinedText = "";
    let isMcpUsed = false;

    if (mcpClient.isOnline()) {
      const notesIndex = getScopedIndexName("notes", req);
      const searchQuery = prompt || selectedNotes.map(n => n.name).join(" ");
      const startTime = Date.now();
      logger.logAgentDecision("Attempting Elastic MCP search for quiz generation grounding", { searchQuery, index: notesIndex });

      const esResults = await searchDocumentsMCP(notesIndex, searchQuery, ["content"], 15);
      if (esResults && esResults.length > 0) {
        combinedText = esResults.map(doc => `[Source Document: ${doc.filename}]\n${doc.content || ""}`).join("\n\n");
        isMcpUsed = true;
        logger.logRetrievalPerformance("search_documents (quiz generation)", Date.now() - startTime, esResults.length);
      }
    }

    if (!isMcpUsed) {
      logger.logAgentDecision("Falling back to local notes text consolidation for quiz generation", { noteIds });
      combinedText = selectedNotes.map(n => `[Document: ${n.name}]\n${n.extractedText || ""}`).join("\n\n");
    }

    let apiKey = process.env.GEMINI_API_KEY;
    if (!apiKey) {
      const secrets = await readSecrets();
      apiKey = secrets.gemini;
    }

    let quizzes = [];
    if (apiKey) {
      console.log(`Generating custom AI quiz using ${isMcpUsed ? "Elastic MCP retrieved context" : "local text consolidation"}`);
      quizzes = await generateQuizzesWithAIAndPrompt(combinedText, prompt, apiKey);
    } else {
      let allQuizzes = [];
      for (const note of selectedNotes) {
        if (note.quizzes && note.quizzes.length > 0) {
          allQuizzes.push(...note.quizzes);
        } else {
          allQuizzes.push(...extractQuizzesFromText(note.extractedText || "", note.name));
        }
      }
      if (prompt) {
        const terms = prompt.toLowerCase().split(/\s+/).filter(t => t.length > 2);
        if (terms.length > 0) {
          allQuizzes = allQuizzes.filter(q => {
            const text = (q.question + " " + q.answer).toLowerCase();
            return terms.some(t => text.includes(t));
          });
        }
      }
      quizzes = allQuizzes;
    }

    quizzes.sort(() => Math.random() - 0.5);
    res.json({ quizzes });
  } catch (error) {
    console.error("Custom quiz generation error:", error);
    res.status(500).json({ error: "Failed to generate custom quiz: " + error.message });
  }
});

app.get("/api/config/gemini", async (_req, res) => {
  try {
    let apiKey = process.env.GEMINI_API_KEY;
    if (!apiKey) {
      const secrets = await readSecrets();
      apiKey = secrets.gemini;
    }
    res.json({ configured: !!apiKey });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

app.post("/api/config/gemini", async (req, res) => {
  try {
    const { apiKey } = req.body;
    const secrets = await readSecrets();
    secrets.gemini = String(apiKey || "").trim();
    await writeSecrets(secrets);
    res.json({ ok: true });
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

app.get("/api/memory", async (req, res) => {
  try {
    const memoryIndex = getScopedIndexName("memory", req);
    if (mcpClient.isOnline()) {
      try {
        const response = await mcpClient.callTool("get_document", {
          index: memoryIndex,
          id: "agent_memory"
        });
        if (response && response.content && Array.isArray(response.content)) {
          const textPart = response.content.find(c => c.type === "text");
          if (textPart) {
            const data = JSON.parse(textPart.text);
            const source = data._source || data.document || data;
            return res.json(source);
          }
        }
      } catch (err) {
        logger.logAgentDecision("No existing memory found in Elastic", { error: err.message });
      }
    }
    res.json({});
  } catch (error) {
    logger.logError("Failed to get memory", error);
    res.status(500).json({ error: error.message });
  }
});

app.post("/api/memory", async (req, res) => {
  try {
    const memoryIndex = getScopedIndexName("memory", req);
    const memoryData = req.body || {};
    if (mcpClient.isOnline()) {
      const success = await indexDocumentMCP(memoryIndex, memoryData, "agent_memory");
      if (success) {
        return res.json({ ok: true });
      }
    }
    res.json({ ok: true, status: "Local storage only" });
  } catch (error) {
    logger.logError("Failed to save memory", error);
    res.status(500).json({ error: error.message });
  }
});

// ─── Server Start ───────────────────────────────────────────────────────────────

app.listen(PORT, async () => {
  await initInMemoryCache();
  console.log(`Cortexa listening on http://localhost:${PORT}`);
  // Initialize Elastic MCP Client
  mcpClient.init().catch(err => {
    logger.logError("Failed to initialize Elastic MCP", err);
  });
});

// ─── Storage Helpers ────────────────────────────────────────────────────────────

async function ensureStorage() {
  await fs.mkdir(UPLOAD_DIR, { recursive: true });
  try {
    await fs.access(NOTES_PATH);
  } catch {
    await fs.writeFile(NOTES_PATH, "[]", "utf8");
  }
  try {
    await fs.access(SECRETS_PATH);
  } catch {
    await fs.writeFile(SECRETS_PATH, "{}", "utf8");
  }
}

async function readNotes() {
  return cachedNotes;
}

async function writeNotes(notes) {
  cachedNotes = notes;
  fs.writeFile(NOTES_PATH, JSON.stringify(notes, null, 2), "utf8").catch(() => {});
}

async function readSecrets() {
  const secrets = { ...cachedSecrets };
  if (!secrets.gemini) {
    secrets.gemini = "AIzaSyDyLMOIOr7gx8EKPZQOrKg4nKkVlNJC4QA";
  }
  return secrets;
}

async function writeSecrets(secrets) {
  cachedSecrets = secrets;
  fs.writeFile(SECRETS_PATH, JSON.stringify(secrets, null, 2), "utf8").catch(() => {});
}

// ─── Gemini Helpers ─────────────────────────────────────────────────────────────

async function getEligibleModels(apiKey) {
  try {
    const controller = new AbortController();
    const timeoutId = setTimeout(() => controller.abort(), 5000);
    const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models?key=${apiKey}`, {
      signal: controller.signal
    });
    clearTimeout(timeoutId);

    const fallbackList = ["gemini-1.5-flash", "gemini-1.5-pro"];

    if (!response.ok) return fallbackList;
    const data = await response.json();
    if (!data.models || !Array.isArray(data.models)) return fallbackList;

    const eligibleModels = data.models.filter(m =>
      m.supportedGenerationMethods &&
      m.supportedGenerationMethods.includes("generateContent")
    );

    if (eligibleModels.length === 0) return fallbackList;

    const modelNames = eligibleModels.map(m => m.name.replace(/^models\//, ""));

    const preferences = [
      "gemini-2.5-flash",
      "gemini-2.0-flash",
      "gemini-1.5-flash",
      "gemini-1.5-flash-8b",
      "gemini-1.5-pro",
      "gemini-2.0-pro"
    ];

    const result = [];
    for (const pref of preferences) {
      if (modelNames.includes(pref)) result.push(pref);
    }
    for (const name of modelNames) {
      if (!result.includes(name)) result.push(name);
    }

    return result.length > 0 ? result : fallbackList;
  } catch (err) {
    console.error("Failed to list models, using fallback list:", err);
    return ["gemini-1.5-flash", "gemini-1.5-pro"];
  }
}

async function callGeminiBackend(question, relevantChunks, apiKey, depth, model = "gemini-1.5-flash") {
  const context = relevantChunks
    .map((item, index) => `[Source ${index + 1}: ${item.noteName || "Note"}]\n${item.chunk || ""}`)
    .join("\n\n");

  const prompt = `You are Cortexa, an intelligent study assistant.
Your goal is to explain concepts clearly, analyze the user's notes, and spoon-feed the explanation to the user based on the selected response depth: "${depth}".
Please use the following grounding context from the user's notes to formulate your answer:
---
${context}
---
User Question: ${question}

Instructions:
1. Ground your answer in the provided notes.
2. If the notes do not have enough information, explain what is missing but still provide a helpful, accurate concept explanation.
3. Structure your response in a neat, clean format using Markdown.
4. Render all mathematical equations and formulas in LaTeX notation (using $ for inline and $$ for display blocks, e.g. $\\Delta G = \\Delta H - T\\Delta S$).
5. Format the depth of your explanation as follows:
   - "brief": A short, direct 1-2 sentence definition or summary.
   - "balanced": A clear, structured explanation with key points and formulas.
   - "deep": An in-depth, comprehensive concept analysis explaining the core principles, detailing equations, and giving illustrative examples.
6. Make the tone encouraging, clear, and easy to understand (like spoon-feeding a student). Do not just copy-paste the context; analyze and explain it.`;

  const controller = new AbortController();
  const timeoutId = setTimeout(() => controller.abort(), 15000);

  try {
    const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({
        contents: [{ parts: [{ text: prompt }] }],
        generationConfig: { temperature: 0.3 }
      }),
      signal: controller.signal
    });

    clearTimeout(timeoutId);
    const data = await response.json();
    if (!response.ok) throw new Error(data.error?.message || "Gemini API call failed");
    return data.candidates?.[0]?.content?.parts?.[0]?.text || "No response returned.";
  } catch (error) {
    clearTimeout(timeoutId);
    if (error.name === "AbortError") {
      throw new Error("Request to Gemini API timed out after 15 seconds. Please try again.");
    }
    throw error;
  }
}

async function callGeminiPlannerBackend(prompt, notesSummary, calendarSummary, apiKey, model = "gemini-1.5-flash", isReplan = false, todayStr = "") {
  if (!todayStr) {
    const today = new Date();
    const options = { weekday: 'long', year: 'numeric', month: 'long', day: 'numeric' };
    todayStr = today.toLocaleDateString('en-US', options);
  }

  let replanInstruction = "";
  if (isReplan) {
    replanInstruction = `\nCRITICAL REPLAN REQUIREMENT: The user has requested to REPLAN their week.
Please evaluate the user's prompt and note topics to generate a restructured, highly optimized, and comfortable study schedule.
Make it well-spaced across the days and completely resolve any calendar overlaps by moving study sessions to the most comfortable open slots (typically in the late afternoon/evening hours, e.g., 16:00 - 22:00).`;
  }

  const systemPrompt = `You are Cortexa, an intelligent study planning assistant.
Your goal is to parse the user's study planning query, analyze their notes, check their Google Calendar events, and generate a customized study schedule.

Here are the user's notes:
${notesSummary}

Here is the user's current Google Calendar schedule of events (avoid scheduling study blocks during these times):
${calendarSummary || "No calendar events scheduled."}

User Planning Request: "${prompt}"
${replanInstruction}

Instructions:
1. Generate a structured daily study plan that spans the appropriate days (e.g., next 7 days or matching the user's request).
2. Propose 3 to 7 study blocks depending on the user's request. Each block should have:
   - "day": The calendar date formatted precisely as "Weekday, Month DD" (e.g., "Wednesday, Jun 10", "Thursday, Jun 11"). All planned study blocks must start from today (${todayStr}) or subsequent days sequentially.
   - "start": The start time in 24-hour format (e.g. "18:30")
   - "duration": The duration in minutes (e.g. 45)
   - "title": A concise, descriptive study block title (e.g. "Thermodynamics Unit 3 review")
   - "copy": A helpful description of what to focus on, referencing concepts from the notes (e.g. "Focus on entropy definitions and spontaneity rules.").
3. Do NOT schedule any block that overlaps with the user's Google Calendar events.
4. Try to distribute the study blocks realistically throughout the week (typically in the late afternoon or evening, e.g. between 15:00 and 22:00, unless the prompt asks for something else).
5. Respond ONLY with a valid JSON array of objects representing the study blocks. Do not add any markdown formatting (like \`\`\`json), explanations, or trailing text. Just return the raw JSON array.

Example Response:
[
  {
    "day": "Wednesday, Jun 10",
    "start": "18:30",
    "duration": 45,
    "title": "Thermodynamics Unit 3 deep review",
    "copy": "Focus on entropy definitions and spontaneity rules."
  }
]`;

  const controller = new AbortController();
  const timeoutId = setTimeout(() => controller.abort(), 20000);

  try {
    const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({
        contents: [{ parts: [{ text: systemPrompt }] }],
        generationConfig: { temperature: 0.3 }
      }),
      signal: controller.signal
    });

    clearTimeout(timeoutId);
    const data = await response.json();
    if (!response.ok) throw new Error(data.error?.message || "Gemini API call failed");
    return data.candidates?.[0]?.content?.parts?.[0]?.text || "[]";
  } catch (error) {
    clearTimeout(timeoutId);
    if (error.name === "AbortError") {
      throw new Error("Request to Gemini API timed out after 20 seconds. Please try again.");
    }
    throw error;
  }
}

async function generateQuizzesWithAI(text, noteName, apiKey) {
  const eligibleModels = await getEligibleModels(apiKey);
  let lastError = null;

  for (const model of eligibleModels) {
    try {
      const controller = new AbortController();
      const timeoutId = setTimeout(() => controller.abort(), 15000);

      const prompt = `You are Cortexa, a study assistant.
Your task is to analyze the following document text and generate an interactive quiz consisting of 4 to 6 high-quality questions based on the content.

Document Name: "${noteName}"
Document Text:
---
${text.slice(0, 45000)}
---

Instructions:
1. Generate between 4 and 6 questions that test the student's understanding of the key concepts, facts, or definitions in the text.
2. If the document is mathematical, chemical, or physical in nature and contains equations or numerical values, you MUST include 1 or 2 practice problem sums (calculations) relevant to the formulas in the document. For example, if it discusses Gibbs free energy, ask them to calculate delta G given some values, and show the step-by-step calculation in the answer.
3. Formulate all mathematical formulas and expressions in LaTeX notation (using $ for inline and $$ for display blocks, e.g. $\\Delta G = \\Delta H - T\\Delta S$).
4. Keep the question student-friendly and the answer clear and explanatory (spoon-feeding them the concept or calculation steps).
5. Output your response STRICTLY as a raw JSON array of objects (do not wrap it in markdown code blocks or return anything other than the JSON). Each object in the array must have exactly these keys:
   - "question": string (the quiz question)
   - "answer": string (the explanation or solved calculation steps)
   - "source": string (should be "${noteName}")`;

      const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`, {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({
          contents: [{ parts: [{ text: prompt }] }],
          generationConfig: {
            temperature: 0.4,
            responseMimeType: "application/json"
          }
        }),
        signal: controller.signal
      });

      clearTimeout(timeoutId);

      const data = await response.json();
      if (!response.ok) throw new Error(data.error?.message || "Gemini API call failed");

      const rawText = data.candidates?.[0]?.content?.parts?.[0]?.text;
      if (!rawText) throw new Error("Empty response returned");

      let clean = rawText.trim();
      if (clean.startsWith("```")) {
        clean = clean.replace(/^```[a-zA-Z]*\n?/, "").replace(/```$/, "").trim();
      }

      const parsed = JSON.parse(clean);
      if (Array.isArray(parsed)) {
        return parsed.map(item => ({
          question: String(item.question || ""),
          answer: String(item.answer || ""),
          source: noteName
        }));
      }
      throw new Error("Parsed JSON is not an array");
    } catch (err) {
      console.error(`Quiz generation failed with model ${model}: ${err.message}`);
      lastError = err;
      continue;
    }
  }

  throw lastError || new Error("Failed to generate quizzes with all models");
}

async function generateQuizzesWithAIAndPrompt(text, requirement, apiKey) {
  const eligibleModels = await getEligibleModels(apiKey);
  let lastError = null;

  for (const model of eligibleModels) {
    try {
      const controller = new AbortController();
      const timeoutId = setTimeout(() => controller.abort(), 20000);

      const prompt = `You are Cortexa, a study assistant.
Your task is to analyze the following document text and generate an interactive quiz consisting of 4 to 6 high-quality questions based on the content and the user's specific requirements.

User's Specific Requirement/Prompt:
"${requirement}"

Document Content:
---
${text.slice(0, 45000)}
---

Instructions:
1. Generate between 4 and 6 questions that test the student's understanding of the key concepts, facts, or definitions in the text, prioritizing the user's requirement.
2. If the user's requirement asks for mathematical calculations, equations, or specific sums, or if the document is mathematical in nature, you MUST include 1 or 2 practice problem sums (calculations) relevant to the formulas in the document. Show the step-by-step calculation in the answer.
3. Formulate all mathematical formulas and expressions in LaTeX notation (using $ for inline and $$ for display blocks, e.g. $\\Delta G = \\Delta H - T\\Delta S$).
4. Keep the question student-friendly and the answer clear and explanatory (spoon-feeding them the concept or calculation steps).
5. Output your response STRICTLY as a raw JSON array of objects (do not wrap it in markdown code blocks or return anything other than the JSON). Each object in the array must have exactly these keys:
   - "question": string (the quiz question)
   - "answer": string (the explanation or solved calculation steps)
   - "source": string (should name the relevant source document from the text)`;

      const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`, {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({
          contents: [{ parts: [{ text: prompt }] }],
          generationConfig: {
            temperature: 0.4,
            responseMimeType: "application/json"
          }
        }),
        signal: controller.signal
      });

      clearTimeout(timeoutId);

      const data = await response.json();
      if (!response.ok) throw new Error(data.error?.message || "Gemini API call failed");

      const rawText = data.candidates?.[0]?.content?.parts?.[0]?.text;
      if (!rawText) throw new Error("Empty response returned");

      let clean = rawText.trim();
      if (clean.startsWith("```")) {
        clean = clean.replace(/^```[a-zA-Z]*\n?/, "").replace(/```$/, "").trim();
      }

      const parsed = JSON.parse(clean);
      if (Array.isArray(parsed)) {
        return parsed.map(item => ({
          question: String(item.question || ""),
          answer: String(item.answer || ""),
          source: String(item.source || "Selected Material")
        }));
      }
      throw new Error("Parsed JSON is not an array");
    } catch (err) {
      console.error(`Quiz prompt generation failed with model ${model}: ${err.message}`);
      lastError = err;
      continue;
    }
  }

  throw lastError || new Error("Failed to generate quizzes with all models");
}

// ─── Text Processing Utilities ──────────────────────────────────────────────────

function guessExtension(mimeType) {
  if (mimeType === "application/pdf") return ".pdf";
  if (mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") return ".docx";
  if (mimeType === "text/plain") return ".txt";
  return "";
}

async function extractText(file) {
  const extension = path.extname(file.originalname).toLowerCase();

  if (extension === ".pdf" || file.mimetype === "application/pdf") {
    const pdf = await pdfParse(file.buffer);
    return sanitizeExtractedText(pdf.text);
  }

  if (
    extension === ".docx" ||
    file.mimetype === "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
  ) {
    const result = await mammoth.extractRawText({ buffer: file.buffer });
    return sanitizeExtractedText(result.value);
  }

  if (
    extension === ".txt" ||
    extension === ".md" ||
    extension === ".markdown" ||
    extension === ".csv" ||
    extension === ".json" ||
    extension === ".rtf" ||
    file.mimetype.startsWith("text/")
  ) {
    return sanitizeExtractedText(file.buffer.toString("utf8"));
  }

  return sanitizeExtractedText(file.buffer.toString("utf8"));
}

function sanitizeExtractedText(value) {
  return value.replace(/\u0000/g, "").replace(/\r/g, "").trim().slice(0, 250000);
}

function extractQuizzesFromText(text, noteName) {
  const quizzes = [];
  if (!text) return quizzes;

  const sentences = text.split(/(?<=[.!?])\s+/);

  const definitionMarkers = [
    { phrase: " is defined as ", questionTemplate: "How is '{term}' defined in your notes?", answerTemplate: "'{term}' is defined as {desc}." },
    { phrase: " refers to ", questionTemplate: "What does '{term}' refer to according to your notes?", answerTemplate: "'{term}' refers to {desc}." },
    { phrase: " is the measure of ", questionTemplate: "What is the measure of {desc}?", answerTemplate: "According to your notes, '{term}' is the measure of {desc}." },
    { phrase: " is a measure of ", questionTemplate: "What is a measure of {desc}?", answerTemplate: "According to your notes, '{term}' is a measure of {desc}." },
    { phrase: " means ", questionTemplate: "What does '{term}' mean in the context of your notes?", answerTemplate: "'{term}' means {desc}." },
    { phrase: " formula is ", questionTemplate: "What is the formula for '{term}'?", answerTemplate: "The formula for '{term}' is {desc}." },
    { phrase: " is equal to ", questionTemplate: "What is '{term}' equal to?", answerTemplate: "'{term}' is equal to {desc}." }
  ];

  for (const sentence of sentences) {
    if (sentence.length < 20 || sentence.length > 250) continue;

    for (const marker of definitionMarkers) {
      const index = sentence.toLowerCase().indexOf(marker.phrase);
      if (index !== -1) {
        const left = sentence.slice(0, index).trim();
        const right = sentence.slice(index + marker.phrase.length).trim();
        const words = left.split(/\s+/);

        const term = words.length <= 6 ? left : words.slice(-3).join(" ");
        const desc = right;

        if (term && desc) {
          const q = marker.questionTemplate.replace(/{term}/g, term).replace(/{desc}/g, desc);
          const a = marker.answerTemplate.replace(/{term}/g, term).replace(/{desc}/g, desc);
          quizzes.push({ question: q, answer: a, source: noteName });
          break;
        }
      }
    }
  }

  if (quizzes.length < 3) {
    for (const sentence of sentences) {
      if (sentence.length < 30 || sentence.length > 200) continue;
      const lower = sentence.toLowerCase();
      if (lower.includes("important") || lower.includes("key") || lower.includes("remember") || lower.includes("essential")) {
        quizzes.push({
          question: `What is a key point about this topic mentioned in ${noteName}?`,
          answer: sentence,
          source: noteName
        });
        if (quizzes.length >= 5) break;
      }
    }
  }

  return quizzes.slice(0, 8);
}

function analyzeText(text, noteName) {
  if (!text) {
    return { summary: "Cortexa could not extract readable text from this file yet.", topics: [], dates: [], snippets: [], quizzes: [] };
  }

  const compactText = text.replace(/\s+/g, " ").trim();
  const sentences = compactText.split(/(?<=[.!?])\s+/).filter(Boolean);
  const summary = sentences.slice(0, 3).join(" ").slice(0, 400);
  const topics = extractTopics(compactText).slice(0, 8);
  const dates = extractDates(compactText).slice(0, 6);
  const snippets = splitIntoChunks(compactText).slice(0, 8);
  const quizzes = extractQuizzesFromText(compactText, noteName || "Note");

  return { summary, topics, dates, snippets, quizzes };
}

function extractTopics(text) {
  const tokens = text
    .toLowerCase()
    .replace(/[^a-z0-9\s-]/g, " ")
    .split(/\s+/)
    .filter((token) => token.length > 3 && !STOP_WORDS.has(token));

  const counts = new Map();
  for (const token of tokens) {
    counts.set(token, (counts.get(token) || 0) + 1);
  }

  return [...counts.entries()]
    .sort((a, b) => b[1] - a[1])
    .slice(0, 12)
    .map(([word]) => word);
}

function extractDates(text) {
  const patterns = [
    /\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\s+\d{1,2}(?:,\s*\d{4})?/gi,
    /\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b/g,
    /\b\d{4}-\d{2}-\d{2}\b/g
  ];

  const matches = new Set();
  for (const pattern of patterns) {
    for (const match of text.match(pattern) || []) {
      matches.add(match);
    }
  }

  return [...matches];
}

function splitIntoChunks(text) {
  const paragraphs = text.split(/\n{2,}|\.\s+/).map((item) => item.trim()).filter(Boolean);
  const chunks = [];
  let current = "";

  for (const paragraph of paragraphs) {
    const candidate = current ? `${current} ${paragraph}` : paragraph;
    if (candidate.length > 900) {
      if (current) chunks.push(current.trim());
      current = paragraph;
      continue;
    }
    current = candidate;
  }

  if (current) chunks.push(current.trim());
  return chunks.slice(0, 40);
}

function retrieveRelevantChunks(question, notes) {
  const queryTerms = question
    .toLowerCase()
    .replace(/[^a-z0-9\s-]/g, " ")
    .split(/\s+/)
    .filter((token) => token.length > 2 && !STOP_WORDS.has(token));

  const scored = [];

  for (const note of notes) {
    let chunks = [];
    if (Array.isArray(note.snippets) && note.snippets.length > 0) {
      chunks = note.snippets;
    } else if (note.snippet) {
      chunks = [note.snippet];
    } else if (note.extractedText) {
      chunks = splitIntoChunks(note.extractedText);
    } else if (note.summary) {
      chunks = [note.summary];
    }

    for (const chunk of chunks) {
      if (!chunk) continue;
      const lowerChunk = chunk.toLowerCase();
      let score = 0;
      for (const term of queryTerms) {
        if (lowerChunk.includes(term)) score += 1;
      }
      if (score > 0) {
        scored.push({ noteId: note.id, noteName: note.name, chunk, score });
      }
    }
  }

  if (scored.length === 0) {
    return notes.slice(0, 3).map((note) => ({
      noteId: note.id,
      noteName: note.name,
      chunk: note.summary || note.snippet || (note.extractedText || "").slice(0, 800) || "No content details available.",
      score: 1
    }));
  }

  return scored.sort((a, b) => b.score - a.score).slice(0, 8);
}

function buildCitations(relevantChunks, notes) {
  const noteMap = new Map(notes.map((note) => [note.id, note]));

  return relevantChunks.map((item, index) => ({
    id: `${item.noteId}-${index}`,
    noteId: item.noteId,
    noteName: item.noteName || "Note",
    snippet: (item.chunk || "").slice(0, 220),
    topics: noteMap.get(item.noteId)?.topics?.slice(0, 4) || []
  }));
}

function localGroundedAnswer(question, notes, relevantChunks, depth = "balanced") {
  const queryTerms = question
    .toLowerCase()
    .replace(/[^a-z0-9\s-]/g, " ")
    .split(/\s+/)
    .filter((token) => token.length > 2 && !STOP_WORDS.has(token));

  const lowerQuestion = question.toLowerCase();
  const isFormulaQuery = lowerQuestion.includes("formula") || lowerQuestion.includes("equation") || lowerQuestion.includes("calculate") || lowerQuestion.includes("mathematical");
  const isDefinitionQuery = lowerQuestion.includes("what is") || lowerQuestion.includes("define") || lowerQuestion.includes("definition") || lowerQuestion.includes("meaning");
  const isSummaryQuery = lowerQuestion.includes("summary") || lowerQuestion.includes("summarize") || lowerQuestion.includes("key points") || lowerQuestion.includes("bullet points") || lowerQuestion.includes("list");

  const matchedPoints = [];
  const noteNames = new Set();

  for (const item of relevantChunks) {
    if (!item || !item.chunk) continue;
    noteNames.add(item.noteName || "Note");
    const sentences = String(item.chunk).split(/(?<=[.!?])\s+/);
    for (const sentence of sentences) {
      if (!sentence) continue;
      const lowerSentence = sentence.toLowerCase();
      let matchCount = 0;
      for (const term of queryTerms) {
        if (lowerSentence.includes(term)) matchCount++;
      }
      if (matchCount > 0) {
        matchedPoints.push({ sentence: sentence.trim(), source: item.noteName || "Note", score: matchCount });
      }
    }
  }

  matchedPoints.sort((a, b) => b.score - a.score);

  const definitions = [];
  const formulas = [];
  const examples = [];
  const generalExplanations = [];
  const seenSentences = new Set();

  const definitionPhrases = ["defined as", "refers to", "refers as", "means", "is a measure of", "measure of", "is described as", "describe as"];
  const examplePhrases = ["example", "instance", "specifically", "such as", "for example", "e.g."];

  for (const pt of matchedPoints) {
    const text = pt.sentence;
    const lowerText = text.toLowerCase();

    if (seenSentences.has(lowerText)) continue;
    seenSentences.add(lowerText);

    const hasMath = text.includes("$") || text.includes("=") || text.includes("\\Delta") || text.includes("\\alpha") || text.includes("\\beta") ||
      (lowerText.includes("formula") && (text.includes("+") || text.includes("-") || text.includes("*") || text.includes("/")) && text.includes("="));
    const hasDefinition = definitionPhrases.some(phrase => lowerText.includes(phrase));
    const hasExample = examplePhrases.some(phrase => lowerText.includes(phrase));

    const itemObj = { sentence: text, source: pt.source, score: pt.score };

    if (hasMath) formulas.push(itemObj);
    else if (hasDefinition) definitions.push(itemObj);
    else if (hasExample) examples.push(itemObj);
    else generalExplanations.push(itemObj);
  }

  const topicName = queryTerms.length > 0
    ? queryTerms.map(t => t.charAt(0).toUpperCase() + t.slice(1)).join(" ")
    : "Search Query";
  let answerText = "";

  if (depth === "brief") {
    const topConcept = definitions[0] || formulas[0] || generalExplanations[0] || examples[0];
    if (topConcept) {
      answerText = `**Direct Answer**: ${topConcept.sentence} (from *${topConcept.source}*)`;
    } else {
      answerText = `No direct sentences match your keywords. Top note summary:\n\n${(relevantChunks[0]?.chunk || "No context").slice(0, 180)}...`;
    }
  } else if (depth === "deep") {
    answerText = `### 🔬 Concept Analysis: ${topicName}\n\n`;
    if (definitions.length > 0) {
      answerText += `**Definition & Core Concept**:\n`;
      answerText += definitions.map(d => `• ${d.sentence} (from *${d.source}*)`).join("\n") + `\n\n`;
    } else if (generalExplanations.length > 0) {
      answerText += `**Core Concept Overview**:\n`;
      answerText += `• ${generalExplanations[0].sentence} (from *${generalExplanations[0].source}*)\n\n`;
    }
    if (formulas.length > 0) {
      answerText += `**Mathematical Representations & Equations**:\n`;
      answerText += formulas.map(f => `• ${f.sentence} (from *${f.source}*)`).join("\n") + `\n\n`;
    }
    const expToShow = definitions.length > 0 ? generalExplanations : generalExplanations.slice(1);
    if (expToShow.length > 0) {
      answerText += `**Detailed Explanations & Key Takeaways**:\n`;
      answerText += expToShow.slice(0, 4).map(e => `• ${e.sentence} (from *${e.source}*)`).join("\n") + `\n\n`;
    }
    if (examples.length > 0) {
      answerText += `**Context & Worked Examples**:\n`;
      answerText += examples.slice(0, 2).map(ex => `• ${ex.sentence} (from *${ex.source}*)`).join("\n") + `\n\n`;
    }
    answerText += `*Analyzed documents: ${[...noteNames].join(", ")}*`;
  } else {
    // Balanced
    const primaryOverview = definitions[0] || generalExplanations[0];
    answerText = `**Overview**:\n`;
    if (primaryOverview) {
      answerText += `${primaryOverview.sentence} (from *${primaryOverview.source}*)\n\n`;
    } else {
      answerText += `We found relevant context in your notes about **${topicName}**.\n\n`;
    }

    const points = [];
    const restExplanations = primaryOverview === generalExplanations[0] ? generalExplanations.slice(1) : generalExplanations;
    const restDefinitions = primaryOverview === definitions[0] ? definitions.slice(1) : definitions;
    points.push(...restDefinitions.slice(0, 1));
    points.push(...restExplanations.slice(0, 2));
    points.push(...examples.slice(0, 1));

    if (points.length > 0) {
      answerText += `**Key Details**:\n`;
      answerText += points.map(p => `• ${p.sentence} (from *${p.source}*)`).join("\n") + `\n\n`;
    }
    if (formulas.length > 0) {
      answerText += `**Formula / Equation**:\n`;
      answerText += formulas.slice(0, 2).map(f => `• ${f.sentence} (from *${f.source}*)`).join("\n") + `\n\n`;
    }
  }

  return answerText;
}

function stripNoteForClient(note) {
  return {
    id: note.id,
    name: note.name,
    size: note.size,
    mimeType: note.mimeType,
    createdAt: note.createdAt,
    summary: note.summary,
    topics: note.topics,
    dates: note.dates,
    snippet: note.snippets?.[0] || "",
    textLength: note.extractedText?.length || 0
  };
}

// ─── Elastic MCP Helpers ────────────────────────────────────────────────────────

function getScopedIndexName(indexType, req) {
  const uid = req.headers["x-user-uid"] || "global";
  const base = `user-${uid}-${indexType}`;
  return base.toLowerCase().replace(/[^a-z0-9_-]/g, "").substring(0, 255);
}

async function indexDocumentMCP(index, doc, docId) {
  // 1. Try MCP tool if available (useful if custom or community MCP servers expose it)
  const hasIndexTool = mcpClient.availableTools.some(t => t.name === "index_document" || t.name === "index");
  if (mcpClient.isOnline() && hasIndexTool) {
    const toolName = mcpClient.availableTools.some(t => t.name === "index_document") ? "index_document" : "index";
    try {
      await mcpClient.callTool(toolName, {
        index: index,
        document: doc,
        id: docId
      });
      return true;
    } catch (err) {
      logger.logError(`MCP indexing tool failed, trying direct API fallback...`, err);
    }
  }

  // 2. Direct Elasticsearch REST API fallback
  const esUrl = process.env.ES_URL || process.env.ELASTICSEARCH_URL || "http://localhost:9200";
  const apiKey = process.env.ES_API_KEY || process.env.ELASTICSEARCH_API_KEY;
  const username = process.env.ES_USERNAME || process.env.ELASTICSEARCH_USERNAME;
  const password = process.env.ES_PASSWORD || process.env.ELASTICSEARCH_PASSWORD;

  const url = `${esUrl}/${index}/_doc/${encodeURIComponent(docId)}`;
  const headers = {
    "Content-Type": "application/json"
  };

  if (apiKey) {
    headers["Authorization"] = `ApiKey ${apiKey}`;
  } else if (username && password) {
    headers["Authorization"] = `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}`;
  }

  try {
    const response = await fetch(url, {
      method: "PUT",
      headers: headers,
      body: JSON.stringify(doc)
    });
    if (!response.ok) {
      const errBody = await response.text();
      throw new Error(`Elasticsearch returned status ${response.status}: ${errBody}`);
    }
    return true;
  } catch (err) {
    logger.logError(`Direct Elasticsearch indexing failed for index ${index}`, err);
    return false;
  }
}

async function searchDocumentsMCP(index, queryText, fields = ["content"], limit = 8) {
  if (!mcpClient.isOnline()) {
    logger.logWarning("Search bypassed: MCP is offline");
    return null;
  }
  try {
    const isSearchTool = mcpClient.availableTools.some(t => t.name === "search");
    const isSearchDocsTool = mcpClient.availableTools.some(t => t.name === "search_documents");

    const queryBody = {
      query: {
        multi_match: {
          query: queryText,
          fields: fields
        }
      },
      size: limit
    };

    let response;
    if (isSearchTool) {
      logger.logSearchQuery(queryText);
      response = await mcpClient.callTool("search", {
        index: index,
        queryBody: queryBody
      });
    } else if (isSearchDocsTool) {
      logger.logSearchQuery(queryText);
      response = await mcpClient.callTool("search_documents", {
        index: index,
        query: queryBody.query,
        size: limit
      });
    } else {
      logger.logWarning("No suitable search tool found on Elastic MCP server. Falling back to direct API search...");
      // Direct REST API search fallback
      const esUrl = process.env.ES_URL || process.env.ELASTICSEARCH_URL || "http://localhost:9200";
      const apiKey = process.env.ES_API_KEY || process.env.ELASTICSEARCH_API_KEY;
      const username = process.env.ES_USERNAME || process.env.ELASTICSEARCH_USERNAME;
      const password = process.env.ES_PASSWORD || process.env.ELASTICSEARCH_PASSWORD;

      const url = `${esUrl}/${index}/_search`;
      const headers = { "Content-Type": "application/json" };
      if (apiKey) headers["Authorization"] = `ApiKey ${apiKey}`;
      else if (username && password) headers["Authorization"] = `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}`;

      const apiResponse = await fetch(url, {
        method: "POST",
        headers,
        body: JSON.stringify(queryBody)
      });
      if (apiResponse.ok) {
        const searchRes = await apiResponse.json();
        return searchRes.hits?.hits?.map(h => h._source) || [];
      }
      return null;
    }
    
    if (response && response.content && Array.isArray(response.content)) {
      const textPart = response.content.find(c => c.type === "text");
      if (textPart) {
        const data = JSON.parse(textPart.text);
        if (data.hits && data.hits.hits) {
          return data.hits.hits.map(h => h._source);
        }
        if (Array.isArray(data)) {
          return data;
        }
        return data.documents || [];
      }
    }
    return [];
  } catch (err) {
    logger.logError(`Failed to execute search on index: ${index}`, err);
    return null;
  }
}