Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/applications/mne_analyze_studio/core/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,16 @@ target_link_libraries(${PROJECT_NAME} PUBLIC

target_compile_definitions(${PROJECT_NAME} PRIVATE MNE_ANALYZE_STUDIO_CORE_LIBRARY)

# llm-retry: header-only retry/backoff for the LLM planner requests
include(FetchContent)
FetchContent_Declare(llm_retry
GIT_REPOSITORY https://github.com/Mattbusel/llm-retry.git
GIT_TAG v0.3.1
GIT_SHALLOW TRUE
SOURCE_SUBDIR include)
FetchContent_MakeAvailable(llm_retry)
target_include_directories(${PROJECT_NAME} PRIVATE ${llm_retry_SOURCE_DIR}/include)

if(NOT BUILD_SHARED_LIBS)
target_compile_definitions(${PROJECT_NAME} PUBLIC STATICBUILD)
endif()
Expand Down
26 changes: 22 additions & 4 deletions src/applications/mne_analyze_studio/core/llmtoolplanner.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
#include <QNetworkRequest>
#include <QUrl>

#define LLM_RETRY_IMPLEMENTATION
#include <llm_retry.hpp>

using namespace MNEANALYZESTUDIO;

namespace
Expand Down Expand Up @@ -390,10 +393,25 @@ LlmPlanResult LlmToolPlanner::plan(const QString& userCommand,
request.setRawHeader("X-GitHub-Api-Version", "2022-11-28");
}

QNetworkReply* reply = m_networkAccessManager->post(request, QJsonDocument(payload).toJson(QJsonDocument::Compact));
QEventLoop loop;
connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit);
loop.exec();
// 408, 429, 5xx and 529 are retried with backoff; any other error falls through on the first reply.
QNetworkReply* reply = nullptr;
try {
llm::with_retry<int>([&]() {
if (reply) {
reply->deleteLater();
}
reply = m_networkAccessManager->post(request, QJsonDocument(payload).toJson(QJsonDocument::Compact));
QEventLoop loop;
connect(reply, &QNetworkReply::finished, &loop, &QEventLoop::quit);
loop.exec();
if (reply->error() != QNetworkReply::NoError) {
throw llm::LLMError{reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(), reply->errorString().toStdString(), true};
}
return 0;
});
} catch (const llm::LLMError&) {
// Not retryable or out of attempts: the last reply is reported below.
}

const QByteArray responseBytes = reply->readAll();
result.rawResponse = QString::fromUtf8(responseBytes);
Expand Down
34 changes: 32 additions & 2 deletions src/testframes/test_studio_core/test_studio_core.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,15 @@ namespace
class FakeLlmEndpoint : public QTcpServer
{
public:
FakeLlmEndpoint(int status, QByteArray body)
FakeLlmEndpoint(int status, QByteArray body, int firstStatus = 0)
: m_status(status)
, m_firstStatus(firstStatus)
, m_body(std::move(body))
{
connect(this, &QTcpServer::newConnection, this, [this]() {
QTcpSocket* socket = nextPendingConnection();
m_request.clear();
++m_requests;
connect(socket, &QTcpSocket::readyRead, socket, [this, socket]() {
m_request += socket->readAll();
const int headerEnd = m_request.indexOf("\r\n\r\n");
Expand All @@ -83,7 +86,7 @@ class FakeLlmEndpoint : public QTcpServer
const QByteArray length = header("content-length");
if (m_request.size() - headerEnd - 4 < length.toInt())
return;
socket->write("HTTP/1.1 " + QByteArray::number(m_status) + " X\r\nContent-Type: application/json\r\nContent-Length: " + QByteArray::number(m_body.size()) + "\r\nConnection: close\r\n\r\n" + m_body);
socket->write("HTTP/1.1 " + QByteArray::number(m_requests == 1 && m_firstStatus ? m_firstStatus : m_status) + " X\r\nContent-Type: application/json\r\nContent-Length: " + QByteArray::number(m_body.size()) + "\r\nConnection: close\r\n\r\n" + m_body);
socket->disconnectFromHost();
});
});
Expand All @@ -110,8 +113,15 @@ class FakeLlmEndpoint : public QTcpServer
return QJsonDocument::fromJson(m_request.mid(m_request.indexOf("\r\n\r\n") + 4)).object();
}

int requests() const
{
return m_requests;
}

private:
int m_status;
int m_firstStatus;
int m_requests = 0;
QByteArray m_body;
QByteArray m_request;
};
Expand Down Expand Up @@ -156,6 +166,7 @@ private slots:
void planner_mockPlansFollowTheRequest();
void planner_talksToEachProvider_data();
void planner_talksToEachProvider();
void planner_retriesRateLimit();
void agentChatDockWidget_conversationAndConfirmations();
void llmSettingsDialog_ruleBasedAndProfileFlow();
void workflowMiniMapWidget_graphRenderingAndActivation();
Expand Down Expand Up @@ -515,6 +526,25 @@ void TestStudioCore::planner_talksToEachProvider()

//=============================================================================================================

void TestStudioCore::planner_retriesRateLimit()
{
const QString plan = R"({"summary":"One step","steps":[{"tool_name":"view.raw.goto","description":"Go","arguments":{"sample":5}}]})";
const QJsonObject body{{"choices", QJsonArray{QJsonObject{{"message", QJsonObject{{"content", plan}}}}}}};
FakeLlmEndpoint endpoint(200, QJsonDocument(body).toJson(QJsonDocument::Compact), 429);
QVERIFY(endpoint.isListening());
LlmToolPlanner planner;
planner.setConfiguration(LlmPlannerConfig{"", "", endpoint.url(), "secret", "model-1"});

// A 429 on the first attempt is retried instead of failing the plan
const LlmPlanResult result = planner.plan("go to sample 5", QJsonArray{QJsonObject{{"name", "view.raw.goto"}}}, QJsonObject{});
QCOMPARE(endpoint.requests(), 2);
QCOMPARE(result.httpStatusCode, 200);
QVERIFY2(result.success, qPrintable(result.errorMessage));
QCOMPARE(result.plannedCommands, (QStringList{"tools.call view.raw.goto {\"sample\":5}"}));
}

//=============================================================================================================

void TestStudioCore::agentChatDockWidget_conversationAndConfirmations()
{
AgentChatDockWidget widget;
Expand Down