Skip to content

fix(onboarding): prevent first report output leaks #324

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hijzy merged 1 commit into from
Aug 31, 2026
+176 −53
Merged

fix(onboarding): prevent first report output leaks #324

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
144 changes: 100 additions & 44 deletions App/backend/src/services/onboarding-insight-service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ const MAX_BALANCED_QUERIES = 84;
const MAX_PREFERENCE_LLM_QUERIES = 24;
const DEFAULT_LLM_TIMEOUT_MS = 90_000;
const DEFAULT_LLM_MAX_TOKENS = 2_000;
const MEMMY_ACCOUNT_AGENT_CHAT_THINKING_BUDGET = 500;
const MAX_GENERATED_OUTPUT_CHARS = 12_000;
const GENERATED_REPORT_OPEN = "<memmy_report>";
const GENERATED_REPORT_CLOSE = "</memmy_report>";
Expand Down Expand Up @@ -1003,10 +1002,19 @@ function parseGeneratedFirstReport(
}

function findGeneratedReportOpen(output: string): { index: number; marker: string } | null {
if (output.startsWith(GENERATED_REPORT_ALIAS_OPEN)) {
return { index: 0, marker: GENERATED_REPORT_ALIAS_OPEN };
let first: { index: number; marker: string } | null = null;
for (const marker of GENERATED_REPORT_OPEN_MARKERS) {
let index = output.indexOf(marker);
while (index >= 0) {
const lineStart = output.lastIndexOf("\n", index - 1) + 1;
if (!output.slice(lineStart, index).trim() && (!first || index < first.index)) {
first = { index, marker };
break;
}
index = output.indexOf(marker, index + marker.length);
}
}
return findFirstGeneratedMarker(output, [GENERATED_REPORT_OPEN]);
return first;
}

function findGeneratedTaskContext(
Expand Down Expand Up @@ -1237,8 +1245,10 @@ function renderFallbackTrajectory(input: {
}

class FirstReportStreamParser {
private mode: "prefix" | "report" | "hidden" | "plain" = "prefix";
private mode: "prefix" | "report" | "hidden" = "prefix";
private buffer = "";
private visibleSource = "";
private emittedVisibleChars = 0;

push(delta: string): string[] {
if (this.mode === "hidden") {
Expand All @@ -1247,56 +1257,47 @@ class FirstReportStreamParser {
this.buffer += delta;
if (this.mode === "prefix") {
const candidate = this.buffer.trimStart();
const reportOpen = findLeadingGeneratedMarker(candidate, GENERATED_REPORT_OPEN_MARKERS);
if (!candidate || (!reportOpen && isGeneratedMarkerPrefix(candidate, GENERATED_REPORT_OPEN_MARKERS))) {
if (!candidate) {
return [];
}
const reportOpen = findGeneratedReportOpen(candidate);
if (!reportOpen) {
this.mode = "plain";
return this.drainVisibleText([
GENERATED_TASK_CONTEXT_OPEN,
...GENERATED_REPORT_CLOSE_MARKERS,
GENERATED_JSON_FENCE_OPEN,
GENERATED_NAKED_JSON_OPEN
]);
return [];
}
this.mode = "report";
this.buffer = candidate.slice(reportOpen.length);
this.buffer = candidate.slice(reportOpen.index + reportOpen.marker.length);
}
return this.mode === "plain"
? this.drainVisibleText([
GENERATED_TASK_CONTEXT_OPEN,
...GENERATED_REPORT_CLOSE_MARKERS,
GENERATED_JSON_FENCE_OPEN,
GENERATED_NAKED_JSON_OPEN
])
: this.drainVisibleText([
...GENERATED_REPORT_CLOSE_MARKERS,
GENERATED_TASK_CONTEXT_OPEN,
GENERATED_JSON_FENCE_OPEN,
GENERATED_NAKED_JSON_OPEN
]);
return this.drainVisibleText([
...GENERATED_REPORT_CLOSE_MARKERS,
GENERATED_TASK_CONTEXT_OPEN,
GENERATED_JSON_FENCE_OPEN,
GENERATED_NAKED_JSON_OPEN
]);
}

finish(): string[] {
if (this.mode === "prefix" || this.mode === "report" || this.mode === "plain") {
if (this.mode === "prefix") {
this.buffer = "";
this.visibleSource = "";
return [];
}
if (this.mode === "report") {
const remainder = this.buffer;
this.buffer = "";
const aliasBoundary = findGeneratedTaskContextAliasBoundary(remainder);
if (aliasBoundary) {
const report = remainder.slice(0, aliasBoundary.index);
return report ? [report] : [];
return this.visibleText(report, true);
}
const internalMarkers = [
...(this.mode === "prefix" ? GENERATED_REPORT_OPEN_MARKERS : []),
...GENERATED_REPORT_CLOSE_MARKERS,
GENERATED_TASK_CONTEXT_OPEN,
GENERATED_TASK_CONTEXT_ALIAS_OPEN,
GENERATED_JSON_FENCE_OPEN,
GENERATED_NAKED_JSON_OPEN
];
const isPartialInternalMarker = isGeneratedMarkerPrefix(remainder, internalMarkers);
return remainder && !isPartialInternalMarker ? [remainder] : [];
return remainder && !isPartialInternalMarker ? this.visibleText(remainder, true) : [];
}
return [];
}
Expand All @@ -1314,18 +1315,29 @@ class FirstReportStreamParser {
const report = this.buffer.slice(0, boundary.index);
this.buffer = "";
this.mode = "hidden";
return report ? [report] : [];
return this.visibleText(report, true);
}
if (boundary) {
const report = this.buffer.slice(0, boundary.index);
this.buffer = this.buffer.slice(boundary.index);
return report ? [report] : [];
return this.visibleText(report);
}
const retainedMarkers = [...delimiters, GENERATED_TASK_CONTEXT_ALIAS_OPEN];
const retainedChars = Math.max(...retainedMarkers.map((marker) => matchingDelimiterSuffixLength(this.buffer, marker)));
const report = this.buffer.slice(0, this.buffer.length - retainedChars);
this.buffer = this.buffer.slice(this.buffer.length - retainedChars);
return report ? [report] : [];
return this.visibleText(report);
}

private visibleText(text: string, finished = false): string[] {
this.visibleSource += text;
const sanitized = stripRawHtmlTags(this.visibleSource, true);
const delta = sanitized.slice(this.emittedVisibleChars);
this.emittedVisibleChars = sanitized.length;
if (finished) {
this.visibleSource = "";
}
return delta ? [delta] : [];
}
}

Expand Down Expand Up @@ -1371,10 +1383,6 @@ function findFirstGeneratedMarker(
return first;
}

function findLeadingGeneratedMarker(value: string, markers: readonly string[]): string | null {
return markers.find((marker) => value.startsWith(marker)) ?? null;
}

function isGeneratedMarkerPrefix(value: string, markers: readonly string[]): boolean {
return markers.some((marker) => marker.startsWith(value));
}
Expand Down Expand Up @@ -1928,6 +1936,7 @@ function buildLlmMessages(input: OnboardingInsightGenerationInput): Array<{ role
"『最近项目记忆』说明最新会话来自哪个 Agent、用户目标、已做事项、已验证结果、当前状态、仍待处理内容。workspacePath 有值时必须写清项目具体路径。只写当前有效结论，不展开冗长历史。",
"『接下来可以做』只列证据支持且尚未完成的 0-3 条待办，按执行顺序排列。第一条应是当前最小且可立即执行的下一步；任务已完成或没有明确待办时，直接说明暂时没有明确待办，不要补通用建议。",
"正文长度要求：中文 300-500 字，英文 180-300 words。重点是准确提炼最近一个项目现场，不要扩展成跨项目年度总结。",
"报告正文只允许使用 Markdown，不得包含任何原始 HTML 标签或样式。不要输出思考过程、执行计划、要求确认、Prompt 复述或起草说明。",
"你必须一次输出两个区块，严格使用以下顺序和标签；标签前后不要添加其他文字：",
`${GENERATED_REPORT_OPEN}\n这里放给用户看的 Markdown 报告正文\n${GENERATED_REPORT_CLOSE}`,
`${GENERATED_TASK_CONTEXT_OPEN}\n这里放一个合法 JSON 对象\n${GENERATED_TASK_CONTEXT_CLOSE}`,
Expand Down Expand Up @@ -2089,10 +2098,7 @@ function openAiCompatibleThinkingControlFields(
provider === "memmy_account" &&
model.includes("agent_chat")
) {
return {
enable_thinking: true,
thinking_budget: MEMMY_ACCOUNT_AGENT_CHAT_THINKING_BUDGET
};
return { enable_thinking: false };
}

if (provider === "dashscope" || baseUrl.includes("dashscope") || model.includes("qwen")) {
Expand Down Expand Up @@ -2323,10 +2329,60 @@ function extractLlmDelta(body: unknown): string | null {
}

function sanitizeGeneratedReport(report: string | null): string | null {
const trimmed = stripActionCopyFromReport(report ?? "").trim();
const trimmed = stripActionCopyFromReport(stripRawHtmlTags(report ?? "")).trim();
return trimmed ? trimmed.slice(0, 4_000) : null;
}

function stripRawHtmlTags(value: string, dropTrailingPartial = false): string {
let output = "";
let codeTicks = 0;
for (let index = 0; index < value.length;) {
if (value[index] === "`") {
let end = index + 1;
while (value[end] === "`") {
end += 1;
}
const ticks = end - index;
if (!codeTicks) {
codeTicks = ticks;
} else if (ticks >= codeTicks) {
codeTicks = 0;
}
output += value.slice(index, end);
index = end;
continue;
}
if (codeTicks || value[index] !== "<") {
output += value[index];
index += 1;
continue;
}
if (value.startsWith("<!--", index)) {
const commentEnd = value.indexOf("-->", index + 4);
if (commentEnd < 0) {
return dropTrailingPartial ? output : `${output}${value.slice(index)}`;
}
index = commentEnd + 3;
continue;
}
const tag = /^<\/?[A-Za-z][A-Za-z0-9-]*(?:\s[^>\n]*|\/?)>/.exec(value.slice(index));
if (tag) {
index += tag[0].length;
continue;
}
const remainder = value.slice(index);
if (dropTrailingPartial && (
remainder === "<" || remainder === "</" || remainder === "<!" || remainder === "<!-" ||
/^<\/?[A-Za-z][A-Za-z0-9-]*(?:\s[^>\n]*)?$/.test(remainder)
)) {
return output;
}
output += "<";
index += 1;
}
return output;
}

function stripActionCopyFromReport(report: string): string {
const reportBody = report.split(/\[\s*MEMMY_ACTIONS_JSON\s*\]/i, 1)[0] ?? report;
const paragraphs = reportBody
Expand Down
85 changes: 76 additions & 9 deletions App/backend/src/services/tests/onboarding-insight-service.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,9 +461,9 @@ describe("onboarding insight service", () => {
throw new Error("generateReport not used");
},
async *streamReport() {
yield "Hi，";
yield "<memmy_report>Hi，";
yield "我已经开始读你的最近任务。\r\n";
yield "## 接下来可以做\n1. 先验证记忆已完成摘要和索引。";
yield "## 接下来可以做\n1. 先验证记忆已完成摘要和索引。</memmy_report>";
}
},
memoryWriter: { write },
Expand Down Expand Up @@ -506,6 +506,71 @@ describe("onboarding insight service", () => {
}));
});

it("drops model planning text before the report envelope from the stream and final report", async () => {
const reportText = "Hi Jiang，\n\n## 你的偏好\n- 使用中文。";
const service = createOnboardingInsightService({
samplers: [sampler("codex", "Codex", [query("codex", "1", "生成初见报告")])],
reportGenerator: {
async generateReport() {
throw new Error("generateReport not used");
},
async *streamReport() {
yield "好的，我会严格按照你的要求，不暴露 homePathName。\n";
yield `<memmy_report>${reportText}`;
yield "</memmy_report>";
}
},
now: () => 100
});

const events = await collectStreamEvents(service.streamReport({ locale: "zh-CN" }));
const visibleText = events
.filter((event): event is { type: "chunk"; delta: string } =>
Boolean(event && typeof event === "object" && (event as { type?: unknown }).type === "chunk"))
.map((event) => event.delta)
.join("");
const done = events.find((event) =>
event && typeof event === "object" && (event as { type?: unknown }).type === "done"
) as { response: { reportMarkdown: string } } | undefined;

expect(visibleText).toBe(reportText);
expect(visibleText).not.toContain("严格按照你的要求");
expect(visibleText).not.toContain("homePathName");
expect(done?.response.reportMarkdown).toBe(reportText);
});

it("removes raw HTML split across streamed report chunks while preserving its text", async () => {
const service = createOnboardingInsightService({
samplers: [sampler("codex", "Codex", [query("codex", "1", "生成初见报告")])],
reportGenerator: {
async generateReport() {
throw new Error("generateReport not used");
},
async *streamReport() {
yield "<memmy_report>Hi Jiang，\n\n<span sty";
yield "le=\"color:grey\"><span style=\"color:#888\">以上内容依据现有证据整理";
yield "</";
yield "span></span>\n\n## 接下来可以做\n暂时没有明确待办。</memmy_report>";
}
},
now: () => 100
});

const events = await collectStreamEvents(service.streamReport({ locale: "zh-CN" }));
const visibleText = events
.filter((event): event is { type: "chunk"; delta: string } =>
Boolean(event && typeof event === "object" && (event as { type?: unknown }).type === "chunk"))
.map((event) => event.delta)
.join("");
const done = events.find((event) =>
event && typeof event === "object" && (event as { type?: unknown }).type === "done"
) as { response: { reportMarkdown: string } } | undefined;

expect(visibleText).toContain("以上内容依据现有证据整理");
expect(visibleText).not.toContain("<span");
expect(done?.response.reportMarkdown).toBe(visibleText);
});

it("does not wait for the Memory service before completing the first-login report", async () => {
let finishWrite = () => undefined;
const write = vi.fn(() => new Promise<void>((resolve) => {
Expand Down Expand Up @@ -707,7 +772,7 @@ describe("onboarding insight service", () => {
) as { response: { reportMarkdown: string } } | undefined;

expect(report.reportMarkdown).toBe(reportText);
expect(visibleText).toBe(reportText);
expect(visibleText).toBe("");
expect(done?.response.reportMarkdown).toBe(reportText);
}
);
Expand Down Expand Up @@ -739,7 +804,7 @@ describe("onboarding insight service", () => {
event && typeof event === "object" && (event as { type?: unknown }).type === "done"
) as { response: { reportMarkdown: string } } | undefined;

expect(visibleText).toBe(reportText);
expect(visibleText).toBe("");
expect(done?.response.reportMarkdown).toBe(reportText);
});

Expand Down Expand Up @@ -795,7 +860,7 @@ describe("onboarding insight service", () => {
throw new Error("generateReport not used");
},
async *streamReport() {
yield "## 最近项目记忆\n正文先展示。";
yield "<memmy_report>## 最近项目记忆\n正文先展示。";
yield "\n{";
yield `${JSON.stringify(taskContext).slice(1)}`;
}
Expand Down Expand Up @@ -832,9 +897,9 @@ describe("onboarding insight service", () => {
throw new Error("generateReport not used");
},
async *streamReport() {
yield "报告包含[";
yield "<memmy_report>报告包含[";
yield "普通说明]，";
yield "仍然应该正常显示。";
yield "仍然应该正常显示。</memmy_report>";
}
},
now: () => 100
Expand Down Expand Up @@ -995,8 +1060,8 @@ describe("onboarding insight service", () => {
const body = JSON.parse(String(fetchImpl.mock.calls[0]?.[1]?.body));
expect(body.model).toBe("agent_chat");
expect(body.max_tokens).toBe(2000);
expect(body.enable_thinking).toBe(true);
expect(body.thinking_budget).toBe(500);
expect(body.enable_thinking).toBe(false);
expect(body).not.toHaveProperty("thinking_budget");
expect(body).not.toHaveProperty("reasoning_effort");
expect(body.messages[0].content).not.toContain("保持 4-6 个短段落");
expect(body.messages[0].content).toContain("latestConversation 是所有已扫描 Agent 中时间最新的一个会话");
Expand All @@ -1010,6 +1075,8 @@ describe("onboarding insight service", () => {
expect(body.messages[0].content).toContain("不得把名字替换成“这个线索”");
expect(body.messages[0].content).toContain("有值时要自然说明用户最近更常用中文还是英文");
expect(body.messages[0].content).toContain("不要生成按钮、行动卡片、CTA");
expect(body.messages[0].content).toContain("不得包含任何原始 HTML 标签或样式");
expect(body.messages[0].content).toContain("不要输出思考过程、执行计划、要求确认、Prompt 复述或起草说明");
expect(body.messages[0].content).not.toContain("[MEMMY_ACTIONS_JSON]");
const userPayload = JSON.parse(String(body.messages[1].content));
expect(userPayload.reportGoal.primary).toBe("user_preferences_latest_project_memory_and_actionable_todos");
Expand Down
Loading