agent_net.cpp

// agent_net.cpp —— curl / SSE 流式解析 / 网页抓取与文本提取
#include "agent_internal.h"

// ============================================================
// HTTP 回调
// ============================================================
size_t WriteToFileCallback(void *ptr, size_t size, size_t nmemb, FILE *stream)
{
    return fwrite(ptr, size, nmemb, stream);
}
std::string downloadFile(const std::string &url, const std::string &savePath)
{
    // 防止静默覆盖已存在的文件；如需覆盖请先删除或换用其它路径。
    // 路径是 UTF-8：必须转宽字符后用 _wstat64/_wfopen，否则中文路径解析失败。
    const std::wstring wideSavePath = utf8ToWide(savePath);
    struct _stat64 st;
    if (_wstat64(wideSavePath.c_str(), &st) == 0)
        return "FAIL 下载: 目标文件已存在，已拒绝覆盖: " + savePath;

    CURL *curl = curl_easy_init();
    if (!curl) return "FAIL 下载: 无法初始化 CURL";
    FILE *file = _wfopen(wideSavePath.c_str(), L"wb");
    if (!file) { curl_easy_cleanup(curl); return "FAIL 下载: 无法创建文件"; }
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteToFileCallback);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, file);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 60L);
    CURLcode res = curl_easy_perform(curl);
    fclose(file);
    curl_easy_cleanup(curl);
    if (res == CURLE_OK) return "OK 下载: " + savePath;
    remove(savePath.c_str());
    return "FAIL 下载: " + std::string(curl_easy_strerror(res));
}

// ============================================================
// 流式响应（SSE）解析 — 实时打印正文并累积 tool_calls
// ============================================================
size_t StreamWriteCallback(void *ptr, size_t size, size_t nmemb, void *userdata)
{
    StreamContext *ctx = static_cast<StreamContext *>(userdata);
    size_t total = size * nmemb;
    try
    {
        const char *data = static_cast<const char *>(ptr);
        ctx->raw.append(data, total);
        ctx->lineBuffer.append(data, total);

        size_t pos;
        while ((pos = ctx->lineBuffer.find('\n')) != std::string::npos)
        {
            std::string line = ctx->lineBuffer.substr(0, pos);
            ctx->lineBuffer.erase(0, pos + 1);
            if (!line.empty() && line.back() == '\r')
                line.pop_back();

            if (line.rfind("data:", 0) != 0)
                continue;

            std::string payload = line.substr(5);
            size_t first = payload.find_first_not_of(' ');
            if (first == std::string::npos)
                continue;
            payload = payload.substr(first);
            if (payload == "[DONE]")
                continue;

            json chunk = json::parse(payload);
            if (!chunk.contains("choices") || chunk["choices"].empty())
                continue;
            const json &delta = chunk["choices"][0]["delta"];

            if (delta.contains("content") && delta["content"].is_string())
            {
                std::string text = delta["content"].get<std::string>();
                if (!ctx->contentStarted)
                {
                    ctx->contentStarted = true;
                    if (!g_assistantStartEmitted)
                    {
                        g_assistantStartEmitted = true;
                        std::cout << SIGNAL_ASSISTANT_START << std::endl;
                    }
                }
                ctx->content += text;
                std::cout << text << std::flush; // 流式打印正文
            }

            if (delta.contains("tool_calls") && delta["tool_calls"].is_array())
            {
                for (const auto &tc : delta["tool_calls"])
                {
                    int index = tc.value("index", 0);
                    if (index < 0 || index > 512)
                        continue; // 防御畸形 index，避免 resize 抛异常
                    if (index >= static_cast<int>(ctx->toolCallAcc.size()))
                        ctx->toolCallAcc.resize(index + 1);
                    ToolCallAcc &acc = ctx->toolCallAcc[index];
                    if (tc.contains("id") && tc["id"].is_string())
                        acc.id = tc["id"].get<std::string>();
                    if (tc.contains("function"))
                    {
                        const json &fn = tc["function"];
                        if (fn.contains("name") && fn["name"].is_string())
                            acc.name = fn["name"].get<std::string>();
                        if (fn.contains("arguments") && fn["arguments"].is_string())
                            acc.arguments += fn["arguments"].get<std::string>();
                    }
                }
            }
        }
    }
    catch (...) { /* 整个回调体都在捕获范围内：任何 std::string/json 异常（如 bad_alloc）都不会逃逸出 curl 的 C 回调 */ }
    return total;
}

// ============================================================
// 网络搜索与文本处理辅助
// ============================================================
std::string urlEncode(const std::string &value)
{
    std::ostringstream out;
    for (unsigned char c : value)
    {
        if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~')
            out << c;
        else
            out << '%' << std::hex << std::uppercase << std::setw(2) << std::setfill('0') << (int)c;
    }
    return out.str();
}

// 删除 <script>/<style>/注释 整块内容，避免脚本与样式混进搜索结果
void eraseElementBlocks(std::string &text, const std::string &openTag, const std::string &closeTag)
{
    size_t pos = 0;
    while ((pos = text.find(openTag, pos)) != std::string::npos)
    {
        size_t end = text.find(closeTag, pos + openTag.size());
        if (end == std::string::npos)
        {
            text.erase(pos);
            return;
        }
        text.erase(pos, end + closeTag.size() - pos);
    }
}

// 单趟实体解码：原先按 &amp; → &lt; → … 顺序反复替换，会把 &amp;lt; 解码两次
std::string decodeHtmlEntities(const std::string &text)
{
    static const std::pair<const char *, const char *> kEntities[] = {
        {"&amp;", "&"}, {"&lt;", "<"}, {"&gt;", ">"}, {"&quot;", "\""},
        {"&#39;", "'"}, {"&apos;", "'"}, {"&nbsp;", " "}, {"&#160;", " "}
    };
    std::string out;
    out.reserve(text.size());
    for (size_t i = 0; i < text.size();)
    {
        bool matched = false;
        if (text[i] == '&')
        {
            for (const auto &e : kEntities)
            {
                const size_t len = std::char_traits<char>::length(e.first);
                if (text.compare(i, len, e.first) == 0)
                {
                    out += e.second;
                    i += len;
                    matched = true;
                    break;
                }
            }
        }
        if (!matched)
            out.push_back(text[i++]);
    }
    return out;
}

std::string stripHtmlTags(std::string text)
{
    eraseElementBlocks(text, "<script", "</script>");
    eraseElementBlocks(text, "<style", "</style>");
    eraseElementBlocks(text, "<!--", "-->");

    std::string out;
    out.reserve(text.size());
    bool inTag = false;
    for (char c : text)
    {
        if (c == '<') { inTag = true; continue; }
        if (c == '>') { inTag = false; continue; }
        if (!inTag) out.push_back(c);
    }
    return decodeHtmlEntities(out);
}

std::string extractSearchResults(const std::string &html)
{
    std::string text = stripHtmlTags(html);
    std::string compact;
    compact.reserve(text.size());
    bool prevSpace = false;
    for (char c : text)
    {
        if (c == '\n' || c == '\r' || c == '\t')
            c = ' ';
        if (c == ' ' && prevSpace)
            continue;
        compact.push_back(c);
        prevSpace = (c == ' ');
    }
    if (compact.length() > 8000)
        compact = compact.substr(0, 8000) + "\n...(截断)";
    return "搜索结果（尽力提取，可能不完整）：\n" + compact;
}

size_t StringWriteCallback(void *ptr, size_t size, size_t nmemb, void *userdata)
{
    std::string *out = static_cast<std::string *>(userdata);
    size_t total = size * nmemb;
    try
    {
        out->append(static_cast<const char *>(ptr), total);
    }
    catch (...)
    {
        return 0; // 通知 curl 传输失败，避免异常逃逸出 C 回调
    }
    return total;
}

// 从响应头里取 charset
size_t HeaderWriteCallback(char *buffer, size_t size, size_t nitems, void *userdata)
{
    const size_t total = size * nitems;
    std::string *charset = static_cast<std::string *>(userdata);
    std::string line(buffer, total);
    std::string lower = line;
    std::transform(lower.begin(), lower.end(), lower.begin(),
                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
    size_t pos = lower.find("charset=");
    if (pos != std::string::npos)
    {
        size_t start = pos + 8;
        while (start < line.size() && (line[start] == '"' || line[start] == '\'' || line[start] == ' '))
            ++start;
        size_t end = start;
        while (end < line.size() && (std::isalnum((unsigned char)line[end]) ||
                                     line[end] == '-' || line[end] == '_' || line[end] == '.'))
            ++end;
        if (end > start) *charset = line.substr(start, end - start);
    }
    return total;
}

std::string findCharsetInText(const std::string &text)
{
    std::string head = text.substr(0, text.size() < 4096 ? text.size() : 4096);
    std::string lower = head;
    std::transform(lower.begin(), lower.end(), lower.begin(),
                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
    size_t pos = lower.find("charset=");
    if (pos == std::string::npos) return "";
    size_t start = pos + 8;
    while (start < head.size() && (head[start] == '"' || head[start] == '\'' || head[start] == ' '))
        ++start;
    size_t end = start;
    while (end < head.size() && (std::isalnum((unsigned char)head[end]) ||
                                 head[end] == '-' || head[end] == '_' || head[end] == '.'))
        ++end;
    return end > start ? head.substr(start, end - start) : "";
}

// 把声明字符集的字节串转成 UTF-8；无法识别时原样返回。
// 原先直接当 UTF-8 使用，中文站点（GBK/GB18030）必然乱码。
std::string toUtf8Text(const std::string &bytes, std::string charset)
{
    std::transform(charset.begin(), charset.end(), charset.begin(),
                   [](unsigned char c) { return static_cast<char>(std::tolower(c)); });
    if (charset.empty() || charset == "utf-8" || charset == "utf8") return bytes;

    UINT cp = 0;
    if (charset == "gbk" || charset == "gb2312" || charset == "gb18030" || charset == "cp936")
        cp = 936;
    else if (charset == "big5" || charset == "cp950")
        cp = 950;
    else if (charset == "shift_jis" || charset == "shift-jis" || charset == "sjis" || charset == "cp932")
        cp = 932;
    else if (charset == "euc-kr" || charset == "cp949")
        cp = 949;
    else if (charset == "iso-8859-1" || charset == "latin1" || charset == "windows-1252")
        cp = 28591;
    if (cp == 0) return bytes;

    const int wlen = MultiByteToWideChar(cp, 0, bytes.c_str(), static_cast<int>(bytes.size()), nullptr, 0);
    if (wlen <= 0) return bytes;
    std::wstring wide(static_cast<size_t>(wlen), L'\0');
    MultiByteToWideChar(cp, 0, bytes.c_str(), static_cast<int>(bytes.size()), &wide[0], wlen);
    return wstringToUtf8(wide);
}

std::string fetchUrlText(const std::string &url)
{
    std::string result;
    std::string charset;
    CURL *curl = curl_easy_init();
    if (!curl) return "";
    struct curl_slist *headers = nullptr;
    headers = curl_slist_append(headers, "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) YCodeAgent");
    curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, StringWriteCallback);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &result);
    curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, HeaderWriteCallback);
    curl_easy_setopt(curl, CURLOPT_HEADERDATA, &charset);
    curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, ""); // 允许并自动解压 gzip/deflate/br
    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_TIMEOUT, 30L);
    CURLcode res = curl_easy_perform(curl);
    long httpCode = 0;
    curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpCode);
    curl_slist_free_all(headers);
    curl_easy_cleanup(curl);
    if (res != CURLE_OK) return "";
    if (httpCode >= 400) return ""; // 404/500 页面不再被当成正文交给模型

    if (charset.empty())
        charset = findCharsetInText(result);
    return toUtf8Text(result, charset);
}