Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion server/utils/markdown.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,14 @@ export type UserSettings = {

const SLUG_MAX_LENGTH = 80;
const FALLBACK_SLUG = "untitled";
// Combining diacritical marks left behind after NFKD decomposition (e.g. the
// accent split off from "é"). Dropping them folds accented Latin to plain ASCII
// ("café" -> "cafe") instead of deleting the whole letter.
const COMBINING_MARKS_PATTERN = /\p{M}/gu;
// Everything a slug may not contain. Non-Latin scripts (Cyrillic, CJK, Arabic)
// survive NFKD, so this strips them entirely and the title falls back to
// FALLBACK_SLUG rather than leaking undecomposable characters into the filename.
const NON_SLUG_CHAR_PATTERN = /[^a-z0-9\s-]/g;

const turndown = new TurndownService({
headingStyle: "atx",
Expand All @@ -51,8 +59,10 @@ export function convertHtmlToMarkdown(html: string): string {

export function titleToSlug(title: string): string {
const slug = title
.normalize("NFKD")
.replace(COMBINING_MARKS_PATTERN, "")
.toLowerCase()
.replace(/[^a-z0-9\s-]/g, "")
.replace(NON_SLUG_CHAR_PATTERN, "")
.trim()
.replace(/\s+/g, "-")
.replace(/-+/g, "-")
Expand Down
24 changes: 24 additions & 0 deletions tests/server/utils/markdown.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,30 @@ describe("titleToSlug", () => {
expect(titleToSlug("-start")).toBe("start");
expect(titleToSlug("end-")).toBe("end");
});

it("returns fallback slug for non-Latin-script titles", () => {
expect(titleToSlug("Привет мир")).toBe("untitled");
expect(titleToSlug("日本語のタイトル")).toBe("untitled");
expect(titleToSlug("مرحبا بالعالم")).toBe("untitled");
});

it("folds accented Latin letters to their ASCII base", () => {
expect(titleToSlug("Café Meeting")).toBe("cafe-meeting");
expect(titleToSlug("Zürich Naïve")).toBe("zurich-naive");
});

it("keeps ASCII produced by compatibility decomposition", () => {
expect(titleToSlug("№5 Meeting")).toBe("no5-meeting");
});

it("keeps ASCII words when non-Latin characters are interspersed", () => {
expect(titleToSlug("Deploy 部署 v2")).toBe("deploy-v2");
});

it("treats non-ASCII whitespace as a word separator", () => {
expect(titleToSlug("Hello\u00A0World")).toBe("hello-world");
expect(titleToSlug("Deploy\u3000v2")).toBe("deploy-v2");
});
});

describe("buildFilename", () => {
Expand Down